diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e709101..bdfbe63e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,10 +89,12 @@ jobs: setup_envtest="$(go env GOPATH)/bin/setup-envtest" echo "KUBEBUILDER_ASSETS=$("$setup_envtest" use 1.31.0 -p path)" >> "$GITHUB_ENV" - - name: Set up Helm for uninstall lifecycle tests + - name: Set up Helm for lifecycle and CLI integration tests uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Test + env: + BREWLET_REQUIRE_CLI_INTEGRATION: "true" run: make test admission: diff --git a/core/README.md b/core/README.md index 0a88ddd6..0d26f891 100644 --- a/core/README.md +++ b/core/README.md @@ -34,6 +34,29 @@ go -C core build ./... go -C core test ./... ``` +## Kubernetes operations + +`brewlet k8s` provides JDK/launcher inventory, profile and application inspection, +control-plane/node status, readiness checks, fresh Helm installation, and safe +profile additions. It uses installed `kubectl`/Helm binaries and existing +kubeconfig credentials rather than embedding a Kubernetes client. + +```bash +brewlet k8s --context staging jdk list +brewlet k8s profile list +brewlet k8s status +brewlet k8s inspect app orders --namespace my-team +``` + +`jdk add` and `launcher add` update live profiles by default; use `--dry-run` +for client-validated previews or `--dry-run=server` for server validation. +Proposed YAML/JSON is printed only after validation succeeds; a failed dry run +returns an error without rendering output. Helm and GitOps-owned profiles must +be changed at their source; client dry-run mode's `--values` and `--file` options +support offline generation without overwriting the input. See the +[CLI reference](../docs/cli-reference.md#brewlet-k8s) for installation, source +validation, conditional updates, output formats, and permissions. + ## Managed dependency bundles Platform teams can publish an approved dependency lock and deterministic diff --git a/core/cmd/brewlet/main.go b/core/cmd/brewlet/main.go index 2c087687..7d3b639c 100644 --- a/core/cmd/brewlet/main.go +++ b/core/cmd/brewlet/main.go @@ -13,6 +13,7 @@ package main import ( + "context" "crypto/ecdsa" "encoding/json" "flag" @@ -25,8 +26,7 @@ import ( "time" "github.com/microsoft/brewlet/internal/artifact" - "github.com/microsoft/brewlet/internal/doctor" - "github.com/microsoft/brewlet/internal/inventory" + "github.com/microsoft/brewlet/internal/kube" "github.com/microsoft/brewlet/internal/runtime" ) @@ -55,6 +55,8 @@ func main() { err = cmdJDKs(os.Args[2:]) case "doctor": err = cmdDoctor(os.Args[2:]) + case "k8s": + err = kube.Run(context.Background(), os.Args[2:], os.Stdout, os.Stderr) case "version", "--version": fmt.Println(version) return @@ -84,6 +86,7 @@ USAGE: brewlet bundle [--store DIR] [--cpu N] [--memory M] [--uid UID] [--gid GID] [--jdk-root DIR] [--launcher NAME] [--launcher-root DIR] [--out DIR] brewlet jdks [--output table|wide|json] [--kubeconfig FILE] [--context CTX] [--selector SEL] brewlet doctor [--namespace NS] [--output table|json] [--kubeconfig FILE] [--context CTX] + brewlet k8s inventory, status, inspection, installation and profile updates (see k8s --help) brewlet version is name:tag, e.g. demo/hello:1.0.0 @@ -800,106 +803,11 @@ func parseProcessIDFlag(name, value string) (uint32, error) { return uint32(id), nil } -// cmdJDKs lists the JDKs available across the cluster (vendor, major/minor -// version, architecture) by reading the inventory each Brewlet node advertises. -// It shells out to `kubectl get nodes -o json` and renders the result, so it -// needs a working kubectl/kubeconfig but no in-process Kubernetes client. +// Preserve the original entry points as aliases for the Kubernetes group. func cmdJDKs(args []string) error { - fs := flag.NewFlagSet("jdks", flag.ExitOnError) - output := fs.String("output", "table", "output format: table (distinct JDKs), wide (per node), or json") - kubeconfig := fs.String("kubeconfig", "", "path to kubeconfig (default: kubectl's own resolution)") - context := fs.String("context", "", "kubeconfig context to use") - selector := fs.String("selector", "", "label selector to filter nodes (kubectl -l)") - if err := fs.Parse(args); err != nil { - return err - } - - kubeArgs := []string{"get", "nodes", "-o", "json"} - if *kubeconfig != "" { - kubeArgs = append(kubeArgs, "--kubeconfig", *kubeconfig) - } - if *context != "" { - kubeArgs = append(kubeArgs, "--context", *context) - } - if *selector != "" { - kubeArgs = append(kubeArgs, "-l", *selector) - } - - cmd := exec.Command("kubectl", kubeArgs...) - var stderr strings.Builder - cmd.Stderr = &stderr - out, err := cmd.Output() - if err != nil { - if msg := strings.TrimSpace(stderr.String()); msg != "" { - return fmt.Errorf("kubectl get nodes failed: %s", msg) - } - return fmt.Errorf("running kubectl (is it installed and configured?): %w", err) - } - - nodes, err := inventory.ParseNodes(out) - if err != nil { - return err - } - - switch *output { - case "table": - inventory.RenderTable(os.Stdout, nodes) - case "wide": - inventory.RenderByNode(os.Stdout, nodes) - case "json": - return inventory.RenderJSON(os.Stdout, nodes) - default: - return fmt.Errorf("unknown --output %q (want table, wide, or json)", *output) - } - return nil + return kube.Run(context.Background(), append([]string{"jdk", "list"}, args...), os.Stdout, os.Stderr) } func cmdDoctor(args []string) error { - fs := flag.NewFlagSet("doctor", flag.ExitOnError) - namespace := fs.String("namespace", "default", "namespace where the developer will create JavaApplication resources") - output := fs.String("output", "table", "output format: table or json") - kubeconfig := fs.String("kubeconfig", "", "path to kubeconfig (default: kubectl's own resolution)") - context := fs.String("context", "", "kubeconfig context to use") - if err := fs.Parse(args); err != nil { - return err - } - - executor := func(kubectlArgs ...string) ([]byte, error) { - cmd := exec.Command("kubectl", kubectlArgs...) - out, err := cmd.CombinedOutput() - if err != nil { - if _, lookupErr := exec.LookPath("kubectl"); lookupErr != nil { - return out, fmt.Errorf("kubectl is not installed or not on PATH") - } - } - return out, err - } - report := doctor.Run(executor, doctor.Options{ - Kubeconfig: *kubeconfig, - Context: *context, - Namespace: *namespace, - }) - - switch *output { - case "table": - for _, check := range report.Checks { - fmt.Printf("[%-4s] %-20s %s\n", strings.ToUpper(string(check.Status)), check.Name, check.Detail) - if check.Remediation != "" && check.Status != doctor.Pass { - fmt.Printf(" fix: %s\n", check.Remediation) - } - } - case "json": - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - if err := enc.Encode(report); err != nil { - return err - } - default: - return fmt.Errorf("unknown --output %q (want table or json)", *output) - } - - if !report.OK() { - return fmt.Errorf("doctor found one or more blocking checks") - } - return nil + return kube.Run(context.Background(), append([]string{"doctor"}, args...), os.Stdout, os.Stderr) } diff --git a/core/go.mod b/core/go.mod index 7c061666..9ece9f91 100644 --- a/core/go.mod +++ b/core/go.mod @@ -14,6 +14,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/runtime-spec v1.3.0 github.com/prometheus/client_golang v1.24.1 + go.yaml.in/yaml/v3 v3.0.4 google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 ) diff --git a/core/go.sum b/core/go.sum index f11d079b..546621af 100644 --- a/core/go.sum +++ b/core/go.sum @@ -164,6 +164,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -230,6 +232,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/core/internal/kube/cli.go b/core/internal/kube/cli.go new file mode 100644 index 00000000..82208a56 --- /dev/null +++ b/core/internal/kube/cli.go @@ -0,0 +1,274 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "strings" + "time" + + "github.com/microsoft/brewlet/internal/doctor" + "github.com/microsoft/brewlet/internal/inventory" +) + +const help = `Brewlet Kubernetes operations + +USAGE: + brewlet k8s [connection flags] [flags] + +COMMANDS: + jdk list advertised node JDKs (table, wide, json) + launcher list advertised optional launchers (table, wide, json) + profile list desired inventories and profile readiness + profile inspect NAME profile configuration and its claimed nodes + inspect app NAME application, owned workloads, pods and events + status control-plane rollouts, profiles and node failures + doctor developer and cluster readiness checks + install --version X.Y.Z -f FILE install the released Helm chart on a fresh cluster + jdk add --profile NAME --distribution NAME --feature N --image REF --java-home PATH + launcher add --profile NAME --name NAME --image REF --path PATH + +COMMON FLAGS (before the command or after its full name): + --kubeconfig FILE kubeconfig path (otherwise kubectl/Helm defaults) + --context NAME context to use without changing kubeconfig + --namespace NAME application namespace (current context); doctor defaults to default; install to brewlet + --timeout 30s per-kubectl-command deadline + +READ FLAGS: + --output table|json|yaml default table (inventory also supports wide, not yaml) + --selector LABELS node selector for inventory + --system-namespace NAME status control-plane namespace (default brewlet) + +PROFILE ADD FLAGS: + --profile NAME existing target profile + --image REF digest-pinned runtime source + JDK: --distribution NAME --feature N --java-home PATH + Launcher: --name NAME --path PATH + --output yaml|json default yaml; prints the full updated declaration + --replace explicitly replace an existing source with the same identity + +ADD: + Updates the live profile by default, even when stdout is redirected. + --dry-run validate locally and print the proposed declaration (same as client) + --dry-run=client local validation only; no patch is sent to the API server + --dry-run=server validate through the API server and print the result without saving + Failed dry runs return an error and print no rendered YAML/JSON. + Helm/GitOps-owned live profiles cannot be patched. Edit their source of truth. + +OFFLINE INPUT (requires --dry-run or --dry-run=client): + By default add reads the live profile. These options read files instead: + --file FILE read a single NodeProfile YAML/JSON file instead of the cluster + --values FILE read complete Helm values and emit updated values + Input files are never overwritten. + +INSTALL FLAGS: + --values FILE, -f FILE complete Helm values (repeatable, at least one required) + --release NAME Helm release name (default brewlet) + --wait-timeout 5m Helm rollout deadline (node provisioning is separate) + --dry-run render chart manifests locally without installing + +Installation and profile changes never create a cluster or debug a workload. +` + +// Run dispatches the Kubernetes command group. Reads and generated declarations +// go to stdout; mutation notices and diagnostic hints go to stderr. +func Run(ctx context.Context, args []string, out, stderr io.Writer) error { + return run(ctx, args, out, stderr, execute) +} + +func commonFlags(fs *flag.FlagSet, opts *options) { + fs.StringVar(&opts.kubeconfig, "kubeconfig", opts.kubeconfig, "kubeconfig path") + fs.StringVar(&opts.context, "context", opts.context, "Kubernetes context") + fs.StringVar(&opts.namespace, "namespace", opts.namespace, "namespace (defaults to current context)") + fs.DurationVar(&opts.timeout, "timeout", opts.timeout, "per-command kubectl timeout") +} + +func run(ctx context.Context, args []string, out, stderr io.Writer, exec executor) error { + opts := options{timeout: 30 * time.Second, systemNamespace: "brewlet"} + root := flag.NewFlagSet("k8s", flag.ContinueOnError) + root.SetOutput(stderr) + // Keep usage/validation failures off stdout so redirected previews stay empty. + root.Usage = func() {} + commonFlags(root, &opts) + if err := root.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + _, err := fmt.Fprint(out, help) + return err + } + return err + } + args = root.Args() + if len(args) == 0 || args[0] == "help" { + _, err := fmt.Fprint(out, help) + return err + } + command := args[0] + args = args[1:] + if command == "jdk" || command == "launcher" || command == "profile" || command == "inspect" { + if len(args) == 0 || args[0] == "--help" || args[0] == "-h" { + _, err := fmt.Fprint(out, help) + return err + } + command += " " + args[0] + args = args[1:] + } + fs := flag.NewFlagSet("k8s "+command, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = root.Usage + commonFlags(fs, &opts) + var update updateOptions + var install installOptions + switch command { + case "jdk list", "launcher list": + fs.StringVar(&opts.output, "output", "table", "table, wide, or json") + fs.StringVar(&opts.selector, "selector", "", "node label selector") + case "status": + fs.StringVar(&opts.systemNamespace, "system-namespace", "brewlet", "Brewlet control-plane namespace") + fallthrough + case "profile list", "profile inspect", "inspect app", "doctor": + fs.StringVar(&opts.output, "output", "table", "table, json, or yaml") + case "jdk add", "launcher add": + update.flags(fs, command == "jdk add") + fs.StringVar(&opts.output, "output", "yaml", "yaml or json") + case "install": + install.flags(fs) + default: + return fmt.Errorf("unknown Kubernetes command %q; see brewlet k8s --help", command) + } + pos, err := parseFlags(fs, args) + if errors.Is(err, flag.ErrHelp) { + _, err := fmt.Fprint(out, help) + return err + } + if err != nil { + return err + } + expected := 0 + if command == "profile inspect" || command == "inspect app" { + expected = 1 + } + if len(pos) != expected { + return fmt.Errorf("%s expects %d positional arguments; see brewlet k8s --help", command, expected) + } + if opts.timeout <= 0 { + return fmt.Errorf("--timeout must be positive") + } + if expected == 1 { + if err := validateName(pos[0], 253); err != nil { + return err + } + } + allowed := []string{"table", "json", "yaml"} + switch command { + case "jdk list", "launcher list": + allowed = []string{"table", "wide", "json"} + case "jdk add", "launcher add": + allowed = []string{"yaml", "json"} + case "install": + allowed = []string{""} + } + if !oneOf(opts.output, allowed...) { + return fmt.Errorf("invalid --output %q (want %s)", opts.output, strings.Join(allowed, ", ")) + } + c := &client{ctx: ctx, opts: opts, exec: exec, out: out, err: stderr} + switch command { + case "jdk list": + return c.jdks() + case "launcher list": + return c.launchers() + case "profile list": + return c.profiles() + case "profile inspect": + return c.inspectProfile(pos[0]) + case "inspect app": + return c.inspectApp(pos[0]) + case "status": + return c.status() + case "doctor": + return c.doctor() + case "jdk add", "launcher add": + return c.updateProfile(update) + case "install": + return c.install(install) + } + panic("unreachable command") +} + +func parseFlags(fs *flag.FlagSet, args []string) ([]string, error) { + var pos []string + for len(args) > 0 { + if args[0] == "--" { + return append(pos, args[1:]...), nil + } + if err := fs.Parse(args); err != nil { + return nil, err + } + args = fs.Args() + if len(args) > 0 { + pos = append(pos, args[0]) + args = args[1:] + } + } + return pos, nil +} + +func oneOf(value string, choices ...string) bool { + for _, choice := range choices { + if value == choice { + return true + } + } + return false +} + +func (c *client) jdks() error { + objects, err := c.list("nodes", c.nodeArgs()...) + if err != nil { + return err + } + raw, err := json.Marshal(struct { + Items []object `json:"items"` + }{objects}) + if err != nil { + return err + } + nodes, err := inventory.ParseNodes(raw) + if err != nil { + return err + } + switch c.opts.output { + case "table": + inventory.RenderTable(c.out, nodes) + case "wide": + inventory.RenderByNode(c.out, nodes) + case "json": + return inventory.RenderJSON(c.out, nodes) + } + return nil +} + +func (c *client) doctor() error { + report := doctor.Run(func(args ...string) ([]byte, error) { + return c.kubectl(nil, args...) + }, doctor.Options{Context: c.opts.context, Namespace: c.opts.namespace}) + if c.opts.output == "table" { + for _, check := range report.Checks { + fmt.Fprintf(c.out, "[%-4s] %-20s %s\n", strings.ToUpper(string(check.Status)), check.Name, check.Detail) + if check.Remediation != "" && check.Status != doctor.Pass { + fmt.Fprintf(c.out, " fix: %s\n", check.Remediation) + } + } + } else if err := encode(c.out, report, c.opts.output); err != nil { + return err + } + if !report.OK() { + return fmt.Errorf("doctor found one or more blocking checks") + } + return nil +} diff --git a/core/internal/kube/cli_test.go b/core/internal/kube/cli_test.go new file mode 100644 index 00000000..723e4438 --- /dev/null +++ b/core/internal/kube/cli_test.go @@ -0,0 +1,942 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const testImage = "docker.io/library/eclipse-temurin@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const secondImage = "docker.io/library/eclipse-temurin@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +func rawJSON(t *testing.T, value any) []byte { + t.Helper() + raw, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return raw +} + +func objectJSON(t *testing.T, raw string) object { + t.Helper() + var obj object + if err := json.Unmarshal([]byte(raw), &obj); err != nil { + t.Fatal(err) + } + return obj +} + +func listJSON(t *testing.T, items ...object) []byte { + t.Helper() + if items == nil { + items = []object{} + } + return rawJSON(t, struct { + Items []object `json:"items"` + }{items}) +} + +func fixtureProfile() string { + return `{ + "apiVersion":"node.brewlet.sh/v1alpha1","kind":"NodeProfile", + "metadata":{"name":"workers","uid":"profile-uid","resourceVersion":"42","generation":3, + "labels":{"team":"platform"},"finalizers":["node.brewlet.sh/cleanup"]}, + "spec":{"nodePool":{"names":["java"]},"jdks":[{"distribution":"temurin","feature":21, + "source":{"image":"` + testImage + `","javaHome":"/opt/java/openjdk"}}], + "launchers":[{"name":"jaz","source":{"image":"` + testImage + `","path":"/usr/bin/jaz"}}], + "rollout":{"maxUnavailable":1},"appCDS":{"regenerationEnabled":true}}, + "status":{"observedGeneration":3,"assignedNodes":1,"readyNodes":1, + "conditions":[{"type":"Ready","status":"True","observedGeneration":3,"reason":"AllNodesProvisioned"}]} + }` +} + +func addArgs(extra ...string) []string { + return append([]string{"jdk", "add", "--profile", "workers", "--distribution", "temurin", + "--feature", "25", "--image", testImage, "--java-home", "/opt/java/openjdk"}, extra...) +} + +func dryRunArgs(extra ...string) []string { + return addArgs(append([]string{"--dry-run"}, extra...)...) +} + +func runTest(t *testing.T, args []string, execute executor) (string, string, error) { + t.Helper() + var out, stderr bytes.Buffer + err := run(context.Background(), args, &out, &stderr, execute) + return out.String(), stderr.String(), err +} + +func noExecution(t *testing.T) executor { + t.Helper() + return func(_ context.Context, program string, args []string, _ []byte) ([]byte, error) { + t.Fatalf("unexpected execution: %s %v", program, args) + return nil, nil + } +} + +func writeFixture(t *testing.T, value string) string { + t.Helper() + file := filepath.Join(t.TempDir(), "input.yaml") + if err := os.WriteFile(file, []byte(value), 0o600); err != nil { + t.Fatal(err) + } + return file +} + +func hasArgs(args []string, want ...string) bool { + for i := range args { + if i+len(want) <= len(args) && reflect.DeepEqual(args[i:i+len(want)], want) { + return true + } + } + return false +} + +func flagValue(t *testing.T, args []string, flag string) string { + t.Helper() + for i, arg := range args { + if arg == flag && i+1 < len(args) { + return args[i+1] + } + } + t.Fatalf("missing %s in %v", flag, args) + return "" +} + +func TestCommandValidationBeforeExecution(t *testing.T) { + cases := [][]string{ + {"unknown"}, {"jdk", "remove"}, {"debug", "app", "orders"}, + {"jdk", "render"}, {"launcher", "render"}, + {"status", "extra"}, {"inspect", "app"}, {"profile", "inspect", "--bad"}, + {"profile", "inspect", "../bad"}, {"jdk", "list", "--output", "yaml"}, + {"status", "--output", "bogus"}, {"status", "--timeout", "0"}, + {"jdk", "list", "--timeout", "-1s"}, {"launcher", "list", "--oops"}, + addArgs("--apply"), addArgs("--file", "missing"), addArgs("--values", "missing"), + dryRunArgs("--values", "missing", "--file", "missing"), + dryRunArgs("--apply"), addArgs("--output", "table"), dryRunArgs("--output", "table"), + addArgs("--dry-run=invalid"), addArgs("--dry-run=false"), addArgs("--dry-run="), + addArgs("--dry-run=server", "--file", "missing"), addArgs("--dry-run=server", "--values", "missing"), + addArgs("--feature", "0"), addArgs("--feature", "2147483648"), + addArgs("--distribution", "../jdk"), addArgs("--distribution", "com.example"), + addArgs("--image", "temurin:21"), addArgs("--image", "docker.io/library/eclipse-temurin:21@sha256:"+strings.Repeat("a", 64)), + addArgs("--java-home", "/"), addArgs("--java-home", "/opt/../jdk"), + {"launcher", "add", "--profile", "workers", "--name", "java", "--image", testImage, "--path", "/usr/bin/java"}, + {"install", "--version", "latest", "-f", "missing"}, + {"install", "--version", "1.2.3"}, + {"install", "--version", "1.2.3", "-f", "missing", "--wait-timeout", "-1s"}, + } + for _, args := range cases { + t.Run(strings.Join(args, " "), func(t *testing.T) { + if out, _, err := runTest(t, args, noExecution(t)); err == nil || out != "" { + t.Fatalf("expected usage/validation error with empty stdout, got %q, %v", out, err) + } + }) + } + for _, args := range [][]string{nil, {"--help"}, {"jdk", "--help"}, {"jdk", "add", "--help"}, {"launcher", "add", "--help"}} { + out, _, err := runTest(t, args, noExecution(t)) + if err != nil || !strings.Contains(out, "Brewlet Kubernetes operations") { + t.Fatalf("help = %q, %v", out, err) + } + } +} + +func TestInventoryAndConnectionFlags(t *testing.T) { + node := objectJSON(t, `{"kind":"Node","metadata":{"name":"worker-a","annotations":{ + "brewlet.sh/jdks-info":"[{\"distribution\":\"temurin\",\"vendor\":\"Adoptium\",\"feature\":21,\"version\":\"21.0.5\",\"arch\":\"amd64\"}]", + "brewlet.sh/launchers":"jaz,jaz, custom"}}}`) + for _, command := range []string{"jdk", "launcher"} { + t.Run(command, func(t *testing.T) { + out, _, err := runTest(t, []string{"--context", "staging", command, "list", + "--kubeconfig", "/config with spaces", "--selector", "pool=java", "--timeout", "2s", "--output", "json"}, + func(ctx context.Context, program string, args []string, _ []byte) ([]byte, error) { + if program != "kubectl" || !hasArgs(args, "--context", "staging") || + !hasArgs(args, "--kubeconfig", "/config with spaces") || !hasArgs(args, "--selector", "pool=java") || + !hasArgs(args, "--request-timeout", "2s") { + t.Fatalf("connection/selector not forwarded: %s %v", program, args) + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > 2*time.Second { + t.Fatal("missing command deadline") + } + return listJSON(t, node), nil + }) + if err != nil || !json.Valid([]byte(out)) || !strings.Contains(out, "worker-a") { + t.Fatalf("inventory = %s, %v", out, err) + } + if command == "launcher" { + var rows []launcherInventory + if err := json.Unmarshal([]byte(out), &rows); err != nil || len(rows) != 2 || len(rows[1].Nodes) != 1 { + t.Fatalf("launcher deduplication: %s %v", out, err) + } + } + }) + } + for _, format := range []string{"table", "wide"} { + out, _, err := runTest(t, []string{"launcher", "list", "--output", format}, + func(context.Context, string, []string, []byte) ([]byte, error) { return listJSON(t, node), nil }) + if err != nil || !strings.Contains(out, "jaz") || !strings.Contains(out, "LAUNCHER") { + t.Fatalf("launcher %s = %s %v", format, out, err) + } + } +} + +func TestReadErrorsAreNotEmptySuccess(t *testing.T) { + for _, command := range [][]string{{"profile", "list"}, {"jdk", "list"}, {"launcher", "list"}, {"status"}} { + for _, response := range []string{`broken`, `{}`, `{"items":null}`} { + if _, _, err := runTest(t, command, func(context.Context, string, []string, []byte) ([]byte, error) { + return []byte(response), nil + }); err == nil { + t.Fatalf("%v accepted %s", command, response) + } + } + if _, _, err := runTest(t, command, func(context.Context, string, []string, []byte) ([]byte, error) { + return nil, errors.New("Forbidden: RBAC denied") + }); err == nil || !strings.Contains(err.Error(), "Forbidden") { + t.Fatalf("%v hid API failure: %v", command, err) + } + } +} + +func TestProfileInspectionAndStaleReadiness(t *testing.T) { + profile := objectJSON(t, fixtureProfile()) + for _, mutate := range []func(*object){ + func(o *object) { o.Status.ObservedGeneration-- }, + func(o *object) { o.Status.Conditions[0].ObservedGeneration-- }, + func(o *object) { o.Metadata.DeletionTimestamp = "2026-09-22T00:00:00Z" }, + } { + obj := objectJSON(t, fixtureProfile()) + mutate(&obj) + if ready, _ := readyCondition(obj); ready { + t.Fatal("stale/terminating profile reported ready") + } + } + node := objectJSON(t, `{"kind":"Node","metadata":{"name":"worker-a","uid":"node-uid", + "labels":{"brewlet.sh/owner-uid":"profile-uid","brewlet.sh/owner-node-uid":"node-uid","brewlet.sh/runtime":"ready"}, + "annotations":{"brewlet.sh/owner-name":"workers","brewlet.sh/profile-generation":"3","brewlet.sh/jdks":"temurin-21"}}, + "spec":{},"status":{"conditions":[{"type":"Ready","status":"True"}]}}`) + foreign := node + foreign.Metadata.UID = "recreated-node" + out, _, err := runTest(t, []string{"profile", "inspect", "workers", "--output", "json"}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "get", profilesResource) { + return rawJSON(t, profile), nil + } + if !hasArgs(args, "--selector", "brewlet.sh/owner-uid=profile-uid") { + t.Fatalf("profile node query not scoped: %v", args) + } + return listJSON(t, node, foreign), nil + }) + if err != nil { + t.Fatal(err) + } + var report struct { + Profile profileSummary `json:"profile"` + Nodes []nodeSummary `json:"nodes"` + } + if err := json.Unmarshal([]byte(out), &report); err != nil || len(report.Nodes) != 1 || !report.Profile.Ready { + t.Fatalf("profile report = %s %v", out, err) + } +} + +func healthyDeployment(t *testing.T) object { + return objectJSON(t, `{"kind":"Deployment","metadata":{"name":"brewlet-operator","generation":2}, + "spec":{"replicas":1},"status":{"observedGeneration":2,"replicas":1,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1}}`) +} + +func TestStatusDistinguishesOldReplicasAndMissingComponents(t *testing.T) { + for _, state := range []string{"ready", "old replicas", "operator missing", "admission missing", "node failed", "no nodes", "no profiles"} { + t.Run(state, func(t *testing.T) { + dep := healthyDeployment(t) + if state == "old replicas" { + dep.Status.UpdatedReplicas = 0 + } + node := objectJSON(t, `{"kind":"Node","metadata":{"name":"worker","labels":{"brewlet.sh/runtime":"ready"}}, + "spec":{},"status":{"conditions":[{"type":"Ready","status":"True"}]}}`) + if state == "node failed" { + node.Metadata.Annotations = map[string]string{"brewlet.sh/provision-error": "jdk-copy-failed"} + } + out, _, err := runTest(t, []string{"status", "--output", "json", "--system-namespace", "runtime-system"}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + switch { + case hasArgs(args, "get", "deployments"): + if !hasArgs(args, "--namespace", "runtime-system") { + t.Fatal("wrong component namespace") + } + if state == "operator missing" { + return listJSON(t), nil + } + if state == "admission missing" { + return listJSON(t, dep), nil + } + admission := healthyDeployment(t) + admission.Metadata.Name = "brewlet-admission" + return listJSON(t, dep, admission), nil + case hasArgs(args, "get", profilesResource): + if state == "no profiles" { + return listJSON(t), nil + } + return listJSON(t, objectJSON(t, fixtureProfile())), nil + default: + if state == "no nodes" { + return listJSON(t), nil + } + return listJSON(t, node), nil + } + }) + var report statusReport + if decodeErr := json.Unmarshal([]byte(out), &report); decodeErr != nil { + t.Fatal(decodeErr, out) + } + if report.Healthy != (state == "ready") || (err == nil) != report.Healthy { + t.Fatalf("status %s: %s, %v", state, out, err) + } + if state == "admission missing" && report.Components[1].Present { + t.Fatal("absent admission should be explicit") + } + }) + } +} + +func TestAppInspectionFollowsOwnershipAndOmitsEnvironment(t *testing.T) { + app := objectJSON(t, `{"apiVersion":"apps.brewlet.sh/v1alpha1","kind":"JavaApplication", + "metadata":{"name":"orders","namespace":"team","uid":"app-uid","generation":3}, + "spec":{"artifact":{"image":"registry.example/orders@sha256:abc"},"jvm":{"version":21},"env":[{"name":"TOKEN","value":"secret-value"}]}, + "status":{"observedGeneration":3,"conditions":[{"type":"Ready","status":"True","observedGeneration":3,"reason":"Reconciled"}]}}`) + dep := healthyDeployment(t) + dep.Metadata.Name, dep.Metadata.UID = "orders", "dep-uid" + dep.Metadata.OwnerReferences = []ownerReference{{UID: "app-uid", Controller: true}} + dep.Status.UpdatedReplicas = 0 + rs := objectJSON(t, `{"kind":"ReplicaSet","metadata":{"name":"orders-rs","uid":"rs-uid", + "ownerReferences":[{"uid":"dep-uid","controller":true}]}}`) + pod := objectJSON(t, `{"kind":"Pod","metadata":{"name":"orders-pod","uid":"pod-uid", + "ownerReferences":[{"uid":"rs-uid","controller":true}],"annotations":{"brewlet.sh/jdk":"21"}}, + "spec":{"nodeName":"worker","containers":[{"env":[{"name":"TOKEN","value":"secret-value"}]}]}, + "status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"ImagePullBackOff"}}}]}}`) + foreign := objectJSON(t, `{"kind":"Pod","metadata":{"name":"foreign","uid":"foreign-uid","ownerReferences":[{"uid":"other-app","controller":true}]},"spec":{}}`) + event := objectJSON(t, `{"involvedObject":{"kind":"Pod","name":"orders-pod","uid":"pod-uid"},"reason":"Failed","message":"Cannot pull image","type":"Warning"}`) + foreignEvent := objectJSON(t, `{"involvedObject":{"kind":"Pod","name":"foreign","uid":"foreign-uid"},"message":"unrelated event"}`) + out, _, err := runTest(t, []string{"inspect", "app", "orders", "--namespace", "team", "--output", "json"}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "--namespace", "team") { + t.Fatalf("missing namespace: %v", args) + } + switch { + case hasArgs(args, "get", appsResource): + return rawJSON(t, app), nil + case hasArgs(args, "get", "events"): + return listJSON(t, event, foreignEvent), nil + default: + return listJSON(t, pod, foreign, rs, dep), nil + } + }) + if err != nil { + t.Fatal(err) + } + var report appReport + if err := json.Unmarshal([]byte(out), &report); err != nil { + t.Fatal(err) + } + if report.Ready || report.JDKRequest != "21" || len(report.Pods) != 1 || len(report.Events) != 1 || + len(report.Pods[0].Problems) != 1 || strings.Contains(out, "secret-value") || strings.Contains(out, "foreign") { + t.Fatalf("unsafe or incorrect app inspection: %s", out) + } +} + +func TestOfflineProfileAndValuesUpdates(t *testing.T) { + profile := writeFixture(t, fixtureProfile()) + out, _, err := runTest(t, dryRunArgs("--file", profile, "--output", "json"), noExecution(t)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "resourceVersion") || strings.Contains(out, "finalizers") || + strings.Contains(out, `"status"`) || !strings.Contains(out, `"maxUnavailable"`) || + !strings.Contains(out, `"regenerationEnabled"`) || !strings.Contains(out, `"jaz"`) { + t.Fatalf("manifest did not preserve configuration/strip live metadata: %s", out) + } + doc, err := decodeDocument([]byte(out)) + if err != nil { + t.Fatal(err) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + t.Fatal(err) + } + var jdks []jdkSource + if err := json.Unmarshal(spec["jdks"], &jdks); err != nil || len(jdks) != 2 { + t.Fatalf("inventory: %s %v", spec["jdks"], err) + } + original, err := os.ReadFile(profile) + if err != nil || string(original) != fixtureProfile() { + t.Fatal("source file was overwritten") + } + values := writeFixture(t, `namespace: brewlet +provisioner: + pools: [java] + jdks: [] + rollout: {maxUnavailable: 1} +profiles: + - name: workers + pools: [batch] + jdks: [] + - name: unrelated + pools: [other] + jdks: [] +metrics: {enabled: true} +`) + for _, name := range []string{"default", "workers"} { + out, _, err := runTest(t, dryRunArgs("--values", values, "--profile", name), noExecution(t)) + if err != nil || !strings.Contains(out, "metrics:") || !strings.Contains(out, "name: unrelated") || !strings.Contains(out, "feature: 25") { + t.Fatalf("values update: %s, %v", out, err) + } + } + for _, invalid := range []string{"a: 1\na: 2", "spec: {}\n---\nspec: {}", "null", "provisioner: ["} { + if out, _, err := runTest(t, dryRunArgs("--file", writeFixture(t, invalid)), noExecution(t)); err == nil || out != "" { + t.Fatalf("invalid source %s must fail without output, got %q, %v", invalid, out, err) + } + } +} + +func TestInventoryReplacementAndLimits(t *testing.T) { + file := writeFixture(t, fixtureProfile()) + out, _, err := runTest(t, dryRunArgs("--file", file, "--feature", "21", "--output", "json"), noExecution(t)) + if err != nil || strings.Count(out, `"distribution"`) != 1 { + t.Fatalf("identical add is not idempotent: %s %v", out, err) + } + args := dryRunArgs("--file", file, "--feature", "21", "--image", secondImage) + if _, _, err := runTest(t, args, noExecution(t)); err == nil || !strings.Contains(err.Error(), "--replace") { + t.Fatalf("implicit replacement allowed: %v", err) + } + out, _, err = runTest(t, append(args, "--replace"), noExecution(t)) + if err != nil || !strings.Contains(out, secondImage) { + t.Fatalf("explicit replacement failed: %s %v", out, err) + } + var entries []jdkSource + for n := 1; n <= 32; n++ { + jdk := jdkSource{Distribution: "temurin", Feature: int64(n)} + jdk.Source.Image, jdk.Source.JavaHome = testImage, "/opt/java" + entries = append(entries, jdk) + } + spec := document{"jdks": rawJSON(t, entries)} + u := updateOptions{jdk: true, distribution: "temurin", feature: 99, image: testImage, path: "/opt/java"} + field, entry, err := u.entry() + if err != nil { + t.Fatal(err) + } + if err := updateInventory(spec, field, entry, false); err == nil { + t.Fatal("allowed inventory larger than CRD maxItems") + } + spec["jdks"] = rawJSON(t, []jdkSource{entries[0], entries[0]}) + if err := updateInventory(spec, field, entry, false); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatal("duplicate inventory was not rejected", err) + } +} + +func TestManagedProfilesCannotBePatched(t *testing.T) { + cases := []string{ + `"labels":{"app.kubernetes.io/managed-by":"Helm"}`, + `"annotations":{"meta.helm.sh/release-name":"brewlet"}`, + `"annotations":{"argocd.argoproj.io/tracking-id":"apps:profile/workers"}`, + `"labels":{"kustomize.toolkit.fluxcd.io/name":"runtime"}`, + `"ownerReferences":[{"uid":"controller","controller":true}]`, + `"managedFields":[{"manager":"custom-gitops","fieldsV1":{"f:spec":{}}}]`, + } + for _, ownership := range cases { + t.Run(ownership, func(t *testing.T) { + raw := strings.Replace(fixtureProfile(), `"labels":{"team":"platform"}`, ownership, 1) + executor := func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "get", profilesResource) || !hasArgs(args, "--show-managed-fields=true") { + t.Fatalf("attempted mutation or missed ownership fields: %v", args) + } + return []byte(raw), nil + } + for _, args := range [][]string{addArgs(), addArgs("--dry-run=server")} { + if out, _, err := runTest(t, args, executor); err == nil || !strings.Contains(err.Error(), "managed by") || out != "" { + t.Fatalf("managed profile must fail without output, got %q, %v", out, err) + } + } + out, stderr, err := runTest(t, dryRunArgs(), executor) + if err != nil || !strings.Contains(stderr, "source of truth") || out == "" { + t.Fatalf("managed declaration must remain available: %s %s %v", out, stderr, err) + } + }) + } +} + +func TestAddMutatesByDefaultAndDryRunUsesConditionalPatch(t *testing.T) { + for _, mode := range []string{"apply", "dry-run", "conflict"} { + t.Run(mode, func(t *testing.T) { + var patchPath string + calls := 0 + args := addArgs("--output", "json") + if mode == "dry-run" { + args = append(args, "--dry-run=server") + } + out, stderr, err := runTest(t, args, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + calls++ + if hasArgs(args, "get", profilesResource) { + return []byte(fixtureProfile()), nil + } + if !hasArgs(args, "patch", profilesResource, "workers") || !hasArgs(args, "--type=json") { + t.Fatalf("unexpected update: %v", args) + } + if hasArgs(args, "--dry-run=server") != (mode == "dry-run") { + t.Fatalf("wrong dry-run mode: %v", args) + } + patchPath = flagValue(t, args, "--patch-file") + info, err := os.Stat(patchPath) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("patch file is not private: %v", err) + } + raw, err := os.ReadFile(patchPath) + if err != nil { + t.Fatal(err) + } + var patch []struct { + Op, Path string + Value json.RawMessage + } + if err := json.Unmarshal(raw, &patch); err != nil || len(patch) != 3 { + t.Fatalf("patch: %s %v", raw, err) + } + if patch[0].Op != "test" || patch[0].Path != "/metadata/uid" || string(patch[0].Value) != `"profile-uid"` || + patch[1].Op != "test" || patch[1].Path != "/metadata/resourceVersion" || string(patch[1].Value) != `"42"` || + patch[2].Op != "add" || patch[2].Path != "/spec/jdks" { + t.Fatalf("patch is not identity/version fenced and narrow: %s", raw) + } + if mode == "conflict" { + return nil, errors.New("Conflict: resourceVersion changed") + } + doc, err := decodeDocument([]byte(fixtureProfile())) + if err != nil { + t.Fatal(err) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + t.Fatal(err) + } + spec["jdks"] = patch[2].Value + doc["spec"] = rawJSON(t, spec) + return rawJSON(t, doc), nil + }) + if calls != 2 { + t.Fatalf("unexpected retries: %d", calls) + } + if mode == "conflict" { + if err == nil || !strings.Contains(err.Error(), "Conflict") || out != "" || stderr != "" { + t.Fatalf("conflict concealed: %s %s %v", out, stderr, err) + } + } else { + if err != nil || !json.Valid([]byte(out)) || !strings.Contains(out, `"feature": 25`) { + t.Fatalf("add result: %s %s %v", out, stderr, err) + } + want := "Profile update accepted; provisioning is asynchronous." + if mode == "dry-run" { + want = "Server dry run succeeded; no inventory was changed." + } + if !strings.Contains(stderr, want) { + t.Fatalf("missing mutation/preview distinction: %s", stderr) + } + } + if _, err := os.Stat(patchPath); !os.IsNotExist(err) { + t.Fatal("temporary patch file leaked", err) + } + }) + } +} + +func TestFailedServerDryRunDoesNotRender(t *testing.T) { + for _, kind := range []string{"jdk", "launcher"} { + for _, format := range []string{"yaml", "json"} { + for _, failure := range []string{"read", "validation", "conflict", "decode", "metadata"} { + t.Run(kind+"/"+format+"/"+failure, func(t *testing.T) { + args := addArgs("--dry-run=server", "--output", format) + if kind == "launcher" { + args = []string{"launcher", "add", "--profile", "workers", "--name", "custom", + "--image", testImage, "--path", "/usr/bin/custom", "--dry-run=server", "--output", format} + } + calls := 0 + var patchFile string + out, stderr, err := runTest(t, args, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + calls++ + if hasArgs(args, "get", profilesResource) { + if failure == "read" { + return []byte(fixtureProfile()), errors.New("Forbidden: cannot read profile") + } + return []byte(fixtureProfile()), nil + } + if !hasArgs(args, "patch", profilesResource, "workers") || !hasArgs(args, "--dry-run=server") { + t.Fatalf("dry run attempted a non-preview operation: %v", args) + } + patchFile = flagValue(t, args, "--patch-file") + switch failure { + case "validation": + return []byte(fixtureProfile()), errors.New("admission rejected the source") + case "conflict": + return []byte(fixtureProfile()), errors.New("Conflict: resourceVersion changed") + case "decode": + return []byte("invalid JSON"), nil + case "metadata": + return []byte(`{"metadata":false}`), nil + default: + t.Fatal("unexpected patch after failed read") + return nil, nil + } + }) + if err == nil || out != "" || strings.Contains(stderr, "succeeded") || strings.Contains(stderr, "accepted") { + t.Fatalf("failed dry run rendered output or claimed success: stdout=%q stderr=%q err=%v", out, stderr, err) + } + wantCalls := 2 + if failure == "read" { + wantCalls = 1 + } + if calls != wantCalls { + t.Fatalf("unexpected fallback/retry after failure: got %d calls, want %d", calls, wantCalls) + } + if patchFile != "" { + if _, err := os.Stat(patchFile); !os.IsNotExist(err) { + t.Fatal("failed dry run left a temporary patch file", err) + } + } + }) + } + } + } +} + +func TestClientDryRunModesAndFailures(t *testing.T) { + file := writeFixture(t, fixtureProfile()) + for _, flag := range []string{"--dry-run", "--dry-run=client"} { + for _, format := range []string{"yaml", "json"} { + t.Run(flag+"/"+format, func(t *testing.T) { + out, _, err := runTest(t, addArgs(flag, "--file", file, "--output", format), noExecution(t)) + if err != nil || out == "" || !strings.Contains(out, "25") { + t.Fatalf("client preview failed: %s %v", out, err) + } + for _, invalid := range [][]string{ + {"--image", "temurin:25"}, + {"--feature", "21", "--image", secondImage}, + {"--file", filepath.Join(t.TempDir(), "missing.yaml")}, + } { + args := append(addArgs(flag, "--file", file, "--output", format), invalid...) + out, stderr, err := runTest(t, args, noExecution(t)) + if err == nil || out != "" || strings.Contains(stderr, "succeeded") { + t.Fatalf("failed client dry run rendered output: %q %q %v", out, stderr, err) + } + } + out, _, err = runTest(t, addArgs(flag, "--output", format), + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "get", profilesResource) { + t.Fatalf("client dry run attempted a write: %v", args) + } + return []byte(fixtureProfile()), nil + }) + if err != nil || out == "" { + t.Fatalf("live-profile client preview failed: %q %v", out, err) + } + }) + } + } +} + +func TestFailedValuesDryRunDoesNotRender(t *testing.T) { + values := writeFixture(t, `provisioner: + jdks: [] +profiles: + - name: workers + jdks: [] + - name: workers + jdks: [] +`) + out, stderr, err := runTest(t, dryRunArgs("--values", values), noExecution(t)) + if err == nil || !strings.Contains(err.Error(), "duplicate profile") || out != "" || stderr != "" { + t.Fatalf("partially processed values must not be rendered: %q %q %v", out, stderr, err) + } +} + +func TestKubectlLocalPatchContract(t *testing.T) { + if _, err := exec.LookPath("kubectl"); err != nil { + t.Skip("kubectl not installed") + } + for _, stale := range []bool{false, true} { + t.Run(fmt.Sprint(stale), func(t *testing.T) { + snapshot := fixtureProfile() + if stale { + snapshot = strings.Replace(snapshot, `"resourceVersion":"42"`, `"resourceVersion":"43"`, 1) + } + file := writeFixture(t, snapshot) + out, _, err := runTest(t, addArgs("--output", "json"), + func(ctx context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "get", profilesResource) { + return []byte(fixtureProfile()), nil + } + return execute(ctx, "kubectl", []string{"patch", "--local", "-f", file, "--type=json", + "--patch-file", flagValue(t, args, "--patch-file"), "-o", "json"}, nil) + }) + if stale { + if err == nil { + t.Fatal("kubectl accepted a stale resourceVersion") + } + } else if err != nil || !strings.Contains(out, `"feature": 25`) || !strings.Contains(out, `"maxUnavailable": 1`) { + t.Fatalf("kubectl rejected or lost fields: %s %v", out, err) + } + }) + } +} + +func TestInstallDelegatesToPinnedHelmChart(t *testing.T) { + file := writeFixture(t, "provisioner:\n pools: [workers]\n jdks: []\n") + for _, dryRun := range []bool{false, true} { + args := []string{"--context", "staging", "install", "--kubeconfig", "/kubeconfig", "--version", "0.1.0-rc.1", + "-f", file, "--values", file, "--namespace", "runtime-system", "--wait-timeout", "2m"} + if dryRun { + args = append(args, "--dry-run") + } + out, _, err := runTest(t, args, + func(ctx context.Context, program string, args []string, _ []byte) ([]byte, error) { + if program == "kubectl" { + if dryRun || !hasArgs(args, "get", "customresourcedefinitions") || + !hasArgs(args, profilesResource, appsResource, "--ignore-not-found") { + t.Fatalf("unexpected install preflight: %v", args) + } + return listJSON(t), nil + } + if program != "helm" || !hasArgs(args, "--version", "0.1.0-rc.1") || + !hasArgs(args, "--namespace", "runtime-system") || !hasArgs(args, "--set-string", "namespace=runtime-system") || + !hasArgs(args, "--kube-context", "staging") || !hasArgs(args, "--kubeconfig", "/kubeconfig") { + t.Fatalf("unsafe install command: %s %v", program, args) + } + if dryRun { + if !hasArgs(args, "template", "brewlet", chart) || hasArgs(args, "--wait") { + t.Fatalf("dry-run should only template: %v", args) + } + } else if !hasArgs(args, "install", "brewlet", chart) || !hasArgs(args, "--wait") { + t.Fatalf("install did not use Helm: %v", args) + } + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) < 2*time.Minute { + t.Fatal("Helm deadline does not account for rollout time") + } + return []byte("helm output\n"), nil + }) + if err != nil || out != "helm output\n" { + t.Fatalf("install: %s %v", out, err) + } + } + out, stderr, err := runTest(t, []string{"install", "--version", "1.2.3", "-f", file, "--dry-run"}, + func(context.Context, string, []string, []byte) ([]byte, error) { + return []byte("partially rendered Helm manifests"), errors.New("Helm release failed") + }) + if err == nil || !strings.Contains(err.Error(), "Helm release failed") || out != "" || stderr != "" { + t.Fatalf("Helm dry-run failure must not render partial output: %q %q %v", out, stderr, err) + } + _, _, err = runTest(t, []string{"install", "--version", "1.2.3", "-f", file}, + func(_ context.Context, program string, _ []string, _ []byte) ([]byte, error) { + if program != "kubectl" { + t.Fatal("attempted to install over existing CRDs") + } + return listJSON(t, objectJSON(t, `{"kind":"CustomResourceDefinition","metadata":{"name":"nodeprofiles.node.brewlet.sh"}}`)), nil + }) + if err == nil || !strings.Contains(err.Error(), "CRDs already exist") { + t.Fatal("existing CRDs were not protected", err) + } +} + +func TestExecuteErrorsAndCancellation(t *testing.T) { + if _, err := execute(context.Background(), "brewlet-test-nonexistent-program", nil, nil); err == nil { + t.Fatal("missing binary should fail") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := execute(ctx, "kubectl", []string{"version", "--client"}, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation not propagated: %v", err) + } +} + +func TestDoctorDelegatesChecksAndPropagatesFailure(t *testing.T) { + for _, fail := range []bool{false, true} { + for _, format := range []string{"table", "json", "yaml"} { + out, _, err := runTest(t, []string{"doctor", "--context", "staging", "--namespace", "team", "--output", format}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "--context", "staging") { + t.Fatalf("context missing: %v", args) + } + switch { + case hasArgs(args, "config", "current-context"): + return []byte("staging"), nil + case hasArgs(args, "get", "nodes"): + return []byte(`{"items":[{"metadata":{"name":"node","labels":{"brewlet.sh/runtime":"ready"}, + "annotations":{"brewlet.sh/jdks":"temurin-21"}},"spec":{}, + "status":{"nodeInfo":{"containerRuntimeVersion":"containerd://2.0.0"}}}]}`), nil + case hasArgs(args, "auth", "can-i"): + if !hasArgs(args, "-n", "team") { + t.Fatal("doctor used the wrong namespace") + } + if fail { + return []byte("no"), nil + } + return []byte("yes"), nil + default: + return []byte("ok"), nil + } + }) + if (err != nil) != fail || !strings.Contains(out, "developer-rbac") { + t.Fatalf("doctor fail=%t format=%s: %s %v", fail, format, out, err) + } + } + } +} + +func TestProfileTableIncludesDesiredInventory(t *testing.T) { + out, _, err := runTest(t, []string{"profile", "list"}, + func(context.Context, string, []string, []byte) ([]byte, error) { + return listJSON(t, objectJSON(t, fixtureProfile())), nil + }) + for _, text := range []string{"DESIRED JDKS", "DESIRED LAUNCHERS", "temurin-21", "jaz", "AllNodesProvisioned"} { + if !strings.Contains(out, text) { + t.Fatalf("profile table lacks %q: %s", text, out) + } + } + if err != nil { + t.Fatal(err) + } +} + +func TestLauncherAdditionAndTerminatingProfile(t *testing.T) { + profile := fixtureProfile() + args := []string{"launcher", "add", "--profile", "workers", "--name", "custom", + "--image", testImage, "--path", "/usr/bin/custom", "--output", "json"} + for _, mode := range []string{"add", "server", "client", "bare"} { + t.Run(mode, func(t *testing.T) { + command := append([]string(nil), args...) + if mode == "bare" { + command = append(command, "--dry-run") + } else if mode != "add" { + command = append(command, "--dry-run="+mode) + } + patches := 0 + out, _, err := runTest(t, command, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "get", profilesResource) { + return []byte(profile), nil + } + if mode == "client" || mode == "bare" || !hasArgs(args, "patch", profilesResource, "workers") { + t.Fatalf("unexpected command: %v", args) + } + patches++ + if hasArgs(args, "--dry-run=server") != (mode == "server") { + t.Fatalf("incorrect launcher mutation/preview mode: %v", args) + } + raw, err := os.ReadFile(flagValue(t, args, "--patch-file")) + if err != nil { + t.Fatal(err) + } + var patch []struct { + Op, Path string + Value json.RawMessage + } + if err := json.Unmarshal(raw, &patch); err != nil || len(patch) != 3 || + patch[0].Path != "/metadata/uid" || patch[1].Path != "/metadata/resourceVersion" || + patch[2].Path != "/spec/launchers" || patch[2].Op != "add" { + t.Fatalf("launcher patch is not conditional and narrow: %s %v", raw, err) + } + doc, err := decodeDocument([]byte(profile)) + if err != nil { + t.Fatal(err) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + t.Fatal(err) + } + spec["launchers"] = patch[2].Value + doc["spec"] = rawJSON(t, spec) + return rawJSON(t, doc), nil + }) + if err != nil || !strings.Contains(out, `"name": "custom"`) || !strings.Contains(out, `"name": "jaz"`) || + !strings.Contains(out, `"feature": 21`) { + t.Fatalf("launcher %s: %s %v", mode, out, err) + } + if (mode == "add" || mode == "server") && patches != 1 { + t.Fatalf("launcher add did not issue exactly one patch: %d", patches) + } + }) + } + for _, field := range []string{`"deletionTimestamp":"2026-09-22T00:00:00Z"`, `"namespace":"forbidden"`} { + invalid := strings.Replace(profile, `"generation":3`, `"generation":3,`+field, 1) + if _, _, err := runTest(t, args, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "patch") { + t.Fatal("mutated an invalid/terminating profile") + } + return []byte(invalid), nil + }); err == nil { + t.Fatal("invalid/terminating profile accepted") + } + } +} + +func TestHelmPreviewAndValuesContract(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("Helm not installed") + } + localChart, err := filepath.Abs(filepath.Join("..", "..", "..", "kubernetes", "charts", "brewlet")) + if err != nil { + t.Fatal(err) + } + values := writeFixture(t, `provisioner: + pools: [java-workers] + jdks: + - distribution: temurin + feature: 21 + source: + image: `+testImage+` + javaHome: /opt/java/openjdk + rollout: {maxUnavailable: 1} +profiles: + - name: batch + pools: [batch-workers] + jdks: + - distribution: temurin + feature: 21 + source: + image: `+testImage+` + javaHome: /opt/java/openjdk +`) + updated, _, err := runTest(t, dryRunArgs("--values", values, "--profile", "default"), noExecution(t)) + if err != nil { + t.Fatal(err) + } + nextValues := writeFixture(t, updated) + out, _, err := runTest(t, []string{"install", "--version", "1.2.3", "--values", nextValues, "--namespace", "runtime-system", "--dry-run"}, + func(ctx context.Context, program string, args []string, input []byte) ([]byte, error) { + if program != "helm" || args[0] != "template" { + t.Fatalf("preview attempted cluster access: %s %v", program, args) + } + for i, arg := range args { + if arg == chart { + args[i] = localChart + } + } + return execute(ctx, program, args, input) + }) + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ + "kind: NodeProfile", "kind: CustomResourceDefinition", "runtime-system", + "java-workers", "batch-workers", "feature: 21", "feature: 25", testImage, "maxUnavailable: 1", + } { + if !strings.Contains(out, text) { + t.Fatalf("Helm render missing %q", text) + } + } +} diff --git a/core/internal/kube/client.go b/core/internal/kube/client.go new file mode 100644 index 00000000..509b7cf9 --- /dev/null +++ b/core/internal/kube/client.go @@ -0,0 +1,233 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package kube provides Brewlet-specific Kubernetes operations through kubectl +// and Helm, using the caller's existing credentials and context. +package kube + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "go.yaml.in/yaml/v3" +) + +const profilesResource = "nodeprofiles.node.brewlet.sh" +const appsResource = "javaapplications.apps.brewlet.sh" + +type options struct { + kubeconfig, context, namespace, systemNamespace string + output, selector string + timeout time.Duration +} + +type executor func(context.Context, string, []string, []byte) ([]byte, error) + +type client struct { + ctx context.Context + opts options + exec executor + out io.Writer + err io.Writer +} + +func execute(ctx context.Context, program string, args []string, input []byte) ([]byte, error) { + cmd := exec.CommandContext(ctx, program, args...) + cmd.WaitDelay = time.Second + cmd.Stdin = bytes.NewReader(input) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if ctx.Err() != nil { + return nil, fmt.Errorf("%s: %w", program, ctx.Err()) + } + if err != nil { + return nil, fmt.Errorf("%s: %w: %s", program, err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +func (c *client) kubectl(input []byte, args ...string) ([]byte, error) { + base := []string{"--request-timeout", c.opts.timeout.String()} + if c.opts.kubeconfig != "" { + base = append(base, "--kubeconfig", c.opts.kubeconfig) + } + if c.opts.context != "" { + base = append(base, "--context", c.opts.context) + } + ctx, cancel := context.WithTimeout(c.ctx, c.opts.timeout) + defer cancel() + return c.exec(ctx, "kubectl", append(base, args...), input) +} + +func (c *client) get(resource, name string, extra ...string) (object, error) { + args := []string{"get", resource, name, "-o", "json"} + raw, err := c.kubectl(nil, append(args, extra...)...) + if err != nil { + return object{}, err + } + var obj object + if err := json.Unmarshal(raw, &obj); err != nil { + return obj, fmt.Errorf("decode %s: %w", resource, err) + } + if obj.Metadata.Name != name { + return obj, fmt.Errorf("get %s %q returned unexpected object %q", resource, name, obj.Metadata.Name) + } + obj.raw = raw + return obj, nil +} + +func (c *client) list(resource string, extra ...string) ([]object, error) { + args := append([]string{"get", resource, "-o", "json"}, extra...) + raw, err := c.kubectl(nil, args...) + if err != nil { + return nil, err + } + var list struct { + Items []object `json:"items"` + } + if err := json.Unmarshal(raw, &list); err != nil { + return nil, fmt.Errorf("decode %s list: %w", resource, err) + } + if list.Items == nil { + return nil, fmt.Errorf("%s response is not a Kubernetes list", resource) + } + return list.Items, nil +} + +func (c *client) namespaceArgs() []string { + if c.opts.namespace == "" { + return nil + } + return []string{"--namespace", c.opts.namespace} +} + +func (c *client) nodeArgs() []string { + if c.opts.selector == "" { + return nil + } + return []string{"--selector", c.opts.selector} +} + +func encode(w io.Writer, value any, format string) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + if format == "yaml" { + var document any + if err := yaml.Unmarshal(data, &document); err != nil { + return err + } + enc := yaml.NewEncoder(w) + enc.SetIndent(2) + defer enc.Close() + return enc.Encode(document) + } + _, err = fmt.Fprintln(w, string(data)) + return err +} + +type metadata struct { + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` + UID string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + Generation int64 `json:"generation"` + DeletionTimestamp string `json:"deletionTimestamp,omitempty"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + OwnerReferences []ownerReference `json:"ownerReferences"` + ManagedFields []struct { + Manager string `json:"manager"` + Subresource string `json:"subresource"` + Fields json.RawMessage `json:"fieldsV1"` + } `json:"managedFields"` +} + +type ownerReference struct { + UID string `json:"uid"` + Controller bool `json:"controller"` +} + +type condition struct { + Type string `json:"type"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +type object struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata metadata `json:"metadata"` + Spec json.RawMessage `json:"spec"` + Status struct { + ObservedGeneration int64 `json:"observedGeneration"` + Conditions []condition `json:"conditions"` + AssignedNodes int `json:"assignedNodes"` + ReadyNodes int `json:"readyNodes"` + Replicas int `json:"replicas"` + UpdatedReplicas int `json:"updatedReplicas"` + ReadyReplicas int `json:"readyReplicas"` + AvailableReplicas int `json:"availableReplicas"` + Phase string `json:"phase"` + ContainerStatuses []struct { + Name string `json:"name"` + Ready bool `json:"ready"` + RestartCount int `json:"restartCount"` + State struct { + Waiting *struct { + Reason string `json:"reason"` + } `json:"waiting"` + Terminated *struct { + Reason string `json:"reason"` + ExitCode int `json:"exitCode"` + } `json:"terminated"` + } `json:"state"` + } `json:"containerStatuses"` + } `json:"status"` + InvolvedObject struct { + UID string `json:"uid"` + Name string `json:"name"` + Kind string `json:"kind"` + } `json:"involvedObject"` + Type string `json:"type"` + Reason string `json:"reason"` + Message string `json:"message"` + raw json.RawMessage +} + +func readyCondition(obj object) (bool, string) { + if obj.Metadata.DeletionTimestamp != "" { + return false, "Terminating" + } + if obj.Status.ObservedGeneration != obj.Metadata.Generation { + return false, "NotObserved" + } + for _, cond := range obj.Status.Conditions { + if cond.Type == "Ready" { + if cond.ObservedGeneration != obj.Metadata.Generation { + return false, "NotObserved" + } + return cond.Status == "True", cond.Reason + } + } + return false, "Unknown" +} + +func ownedBy(obj object, uids map[string]bool) bool { + for _, owner := range obj.Metadata.OwnerReferences { + if owner.Controller && owner.UID != "" && uids[owner.UID] { + return true + } + } + return false +} diff --git a/core/internal/kube/install.go b/core/internal/kube/install.go new file mode 100644 index 00000000..9ae40f58 --- /dev/null +++ b/core/internal/kube/install.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "context" + "flag" + "fmt" + "regexp" + "time" +) + +const chart = "oci://ghcr.io/microsoft/charts/brewlet" + +type valuesFiles []string + +func (v *valuesFiles) String() string { return fmt.Sprint([]string(*v)) } +func (v *valuesFiles) Set(value string) error { + *v = append(*v, value) + return nil +} + +type installOptions struct { + version, release string + values valuesFiles + waitTimeout time.Duration + dryRun bool +} + +func (i *installOptions) flags(fs *flag.FlagSet) { + fs.StringVar(&i.version, "version", "", "required exact Helm chart release version") + fs.StringVar(&i.release, "release", "brewlet", "Helm release name") + fs.Var(&i.values, "values", "complete Helm values file (repeatable)") + fs.Var(&i.values, "f", "complete Helm values file (repeatable)") + fs.DurationVar(&i.waitTimeout, "wait-timeout", 5*time.Minute, "Helm rollout timeout") + fs.BoolVar(&i.dryRun, "dry-run", false, "render manifests locally instead of installing") +} + +var chartVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`) + +func (c *client) install(i installOptions) error { + if !chartVersion.MatchString(i.version) { + return fmt.Errorf("--version must be an exact chart version, such as 0.1.0 (not latest or a range)") + } + if err := validateName(i.release, 53); err != nil { + return fmt.Errorf("--release: %w", err) + } + if i.waitTimeout <= 0 { + return fmt.Errorf("--wait-timeout must be positive") + } + if len(i.values) == 0 { + return fmt.Errorf("at least one --values/-f file is required: explicitly configure node pools and digest-pinned runtime sources, or disable defaultProfile") + } + for _, file := range i.values { + if _, err := readDocument(file); err != nil { + return err + } + } + ns := c.opts.namespace + if ns == "" { + ns = "brewlet" + } + if err := validateToken(ns, 63); err != nil { + return fmt.Errorf("--namespace: %w", err) + } + if !i.dryRun { + existing, err := c.list("customresourcedefinitions", profilesResource, appsResource, "--ignore-not-found") + if err != nil { + return fmt.Errorf("check for existing Brewlet CRDs: %w", err) + } + if len(existing) != 0 { + return fmt.Errorf("Brewlet CRDs already exist; install is fresh-install-only. Follow the documented CRD migration and Helm upgrade procedure") + } + } + args := []string{"install", i.release, chart, "--create-namespace", "--wait", "--timeout", i.waitTimeout.String()} + if i.dryRun { + args = []string{"template", i.release, chart, "--include-crds"} + } + args = append(args, "--version", i.version, "--namespace", ns) + if c.opts.kubeconfig != "" { + args = append(args, "--kubeconfig", c.opts.kubeconfig) + } + if c.opts.context != "" { + args = append(args, "--kube-context", c.opts.context) + } + for _, file := range i.values { + args = append(args, "--values", file) + } + // The chart uses Values.namespace for resources, not Release.Namespace. + args = append(args, "--set-string", "namespace="+ns) + ctx, cancel := context.WithTimeout(c.ctx, i.waitTimeout+c.opts.timeout) + defer cancel() + result, err := c.exec(ctx, "helm", args, nil) + if err != nil { + return err + } + if _, err := c.out.Write(result); err != nil { + return err + } + if !i.dryRun { + fmt.Fprintf(c.err, "Helm release is ready. Node provisioning is separate; run brewlet k8s status --system-namespace %s with the same kubeconfig/context and inspect advertised JDK/launcher inventory.\n", ns) + } + return nil +} diff --git a/core/internal/kube/read.go b/core/internal/kube/read.go new file mode 100644 index 00000000..1cd4bd29 --- /dev/null +++ b/core/internal/kube/read.go @@ -0,0 +1,486 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "text/tabwriter" +) + +type launcherInventory struct { + Name string `json:"name"` + Nodes []string `json:"nodes"` +} + +func (c *client) launchers() error { + nodes, err := c.list("nodes", c.nodeArgs()...) + if err != nil { + return err + } + byName := map[string][]string{} + for _, node := range nodes { + seen := map[string]bool{} + for _, name := range strings.Split(node.Metadata.Annotations["brewlet.sh/launchers"], ",") { + name = strings.TrimSpace(name) + if name != "" && !seen[name] { + byName[name] = append(byName[name], node.Metadata.Name) + seen[name] = true + } + } + } + rows := []launcherInventory{} + for name, nodes := range byName { + sort.Strings(nodes) + rows = append(rows, launcherInventory{Name: name, Nodes: nodes}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Name < rows[j].Name }) + if c.opts.output == "json" { + return encode(c.out, rows, "json") + } + w := tabwriter.NewWriter(c.out, 0, 4, 2, ' ', 0) + if c.opts.output == "wide" { + fmt.Fprintln(w, "LAUNCHER\tNODE") + for _, row := range rows { + for _, node := range row.Nodes { + fmt.Fprintf(w, "%s\t%s\n", row.Name, node) + } + } + } else { + fmt.Fprintln(w, "LAUNCHER\tNODES") + for _, row := range rows { + fmt.Fprintf(w, "%s\t%d\n", row.Name, len(row.Nodes)) + } + } + return w.Flush() +} + +type profileSummary struct { + Name string `json:"name"` + Generation int64 `json:"generation"` + ObservedGeneration int64 `json:"observedGeneration"` + Ready bool `json:"ready"` + Reason string `json:"reason"` + AssignedNodes int `json:"assignedNodes"` + ReadyNodes int `json:"readyNodes"` + ManagedBy string `json:"managedBy,omitempty"` + Spec json.RawMessage `json:"spec"` + Conditions []condition `json:"conditions"` +} + +func summarizeProfile(obj object) profileSummary { + ready, reason := readyCondition(obj) + return profileSummary{ + Name: obj.Metadata.Name, Generation: obj.Metadata.Generation, + ObservedGeneration: obj.Status.ObservedGeneration, + Ready: ready, Reason: reason, AssignedNodes: obj.Status.AssignedNodes, + ReadyNodes: obj.Status.ReadyNodes, ManagedBy: managedBy(obj.Metadata), + Spec: obj.Spec, Conditions: obj.Status.Conditions, + } +} + +func (c *client) profiles() error { + objects, err := c.list(profilesResource) + if err != nil { + return err + } + rows := []profileSummary{} + for _, obj := range objects { + rows = append(rows, summarizeProfile(obj)) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Name < rows[j].Name }) + if c.opts.output != "table" { + return encode(c.out, rows, c.opts.output) + } + return c.profileTable(rows) +} + +func (c *client) profileTable(rows []profileSummary) error { + w := tabwriter.NewWriter(c.out, 0, 4, 2, ' ', 0) + fmt.Fprintln(w, "PROFILE\tPOOLS\tDESIRED JDKS\tDESIRED LAUNCHERS\tREADY\tNODES\tREASON") + for _, row := range rows { + var spec struct { + NodePool struct { + Names []string `json:"names"` + } `json:"nodePool"` + JDKs []jdkSource `json:"jdks"` + Launchers []launcherSource `json:"launchers"` + } + if err := json.Unmarshal(row.Spec, &spec); err != nil { + return fmt.Errorf("decode profile %s spec: %w", row.Name, err) + } + jdks, launchers := []string{}, []string{} + for _, jdk := range spec.JDKs { + jdks = append(jdks, fmt.Sprintf("%s-%d", jdk.Distribution, jdk.Feature)) + } + for _, launcher := range spec.Launchers { + launchers = append(launchers, launcher.Name) + } + pools := strings.Join(spec.NodePool.Names, ",") + if pools == "" { + pools = "(all eligible nodes)" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%t\t%d/%d\t%s\n", + row.Name, pools, strings.Join(jdks, ","), strings.Join(launchers, ","), + row.Ready, row.ReadyNodes, row.AssignedNodes, row.Reason) + } + return w.Flush() +} + +type nodeSummary struct { + Name string `json:"name"` + Profile string `json:"profile,omitempty"` + ProfileGeneration string `json:"profileGeneration,omitempty"` + RuntimeReady bool `json:"runtimeReady"` + NodeReady bool `json:"nodeReady"` + Unschedulable bool `json:"unschedulable"` + ProvisionState string `json:"provisionState,omitempty"` + Error string `json:"error,omitempty"` + Message string `json:"message,omitempty"` + JDKs string `json:"advertisedJdks,omitempty"` + Launchers string `json:"advertisedLaunchers,omitempty"` +} + +func summarizeNodes(objects []object, profileUID string) ([]nodeSummary, error) { + nodes := []nodeSummary{} + for _, obj := range objects { + labels, ann := obj.Metadata.Labels, obj.Metadata.Annotations + if profileUID != "" { + if labels["brewlet.sh/owner-uid"] != profileUID || labels["brewlet.sh/owner-node-uid"] != obj.Metadata.UID { + continue + } + } else if labels["brewlet.sh/runtime"] == "" && labels["brewlet.sh/owner-uid"] == "" && + ann["brewlet.sh/profile"] == "" && ann["brewlet.sh/provision-error"] == "" { + continue + } + var spec struct { + Unschedulable bool `json:"unschedulable"` + } + if err := json.Unmarshal(obj.Spec, &spec); err != nil { + return nil, fmt.Errorf("decode node %s: %w", obj.Metadata.Name, err) + } + nodeReady := false + for _, cond := range obj.Status.Conditions { + if cond.Type == "Ready" { + nodeReady = cond.Status == "True" + } + } + nodes = append(nodes, nodeSummary{ + Name: obj.Metadata.Name, Profile: ann["brewlet.sh/owner-name"], + ProfileGeneration: ann["brewlet.sh/profile-generation"], + RuntimeReady: labels["brewlet.sh/runtime"] == "ready", NodeReady: nodeReady, + Unschedulable: spec.Unschedulable, ProvisionState: labels["brewlet.sh/provision-state"], + Error: ann["brewlet.sh/provision-error"], Message: ann["brewlet.sh/provision-error-message"], + JDKs: ann["brewlet.sh/jdks"], Launchers: ann["brewlet.sh/launchers"], + }) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].Name < nodes[j].Name }) + return nodes, nil +} + +func (c *client) nodeTable(nodes []nodeSummary) error { + w := tabwriter.NewWriter(c.out, 0, 4, 2, ' ', 0) + fmt.Fprintln(w, "NODE\tPROFILE\tGENERATION\tRUNTIME READY\tNODE READY\tCORDONED\tPROVISIONING\tERROR") + for _, node := range nodes { + fmt.Fprintf(w, "%s\t%s\t%s\t%t\t%t\t%t\t%s\t%s\n", node.Name, node.Profile, + node.ProfileGeneration, node.RuntimeReady, node.NodeReady, node.Unschedulable, + node.ProvisionState, node.Error) + } + return w.Flush() +} + +func (c *client) inspectProfile(name string) error { + profile, err := c.get(profilesResource, name) + if err != nil { + return err + } + if profile.Metadata.UID == "" { + return fmt.Errorf("profile %s has no UID", name) + } + objects, err := c.list("nodes", "--selector", "brewlet.sh/owner-uid="+profile.Metadata.UID) + if err != nil { + return err + } + nodes, err := summarizeNodes(objects, profile.Metadata.UID) + if err != nil { + return err + } + report := struct { + Profile profileSummary `json:"profile"` + Nodes []nodeSummary `json:"nodes"` + }{summarizeProfile(profile), nodes} + format := c.opts.output + if format == "table" { + format = "yaml" + } + return encode(c.out, report, format) +} + +type deploymentSummary struct { + Name string `json:"name"` + Present bool `json:"present"` + Ready bool `json:"ready"` + Generation int64 `json:"generation"` + ObservedGeneration int64 `json:"observedGeneration"` + Desired int `json:"desired"` + Updated int `json:"updated"` + Available int `json:"available"` + Conditions []condition `json:"conditions,omitempty"` +} + +func summarizeDeployment(obj object) (deploymentSummary, error) { + var spec struct { + Replicas *int `json:"replicas"` + } + if err := json.Unmarshal(obj.Spec, &spec); err != nil { + return deploymentSummary{}, fmt.Errorf("decode deployment %s: %w", obj.Metadata.Name, err) + } + desired := 1 + if spec.Replicas != nil { + desired = *spec.Replicas + } + s := obj.Status + ready := obj.Metadata.DeletionTimestamp == "" && s.ObservedGeneration == obj.Metadata.Generation && + s.Replicas == desired && s.UpdatedReplicas == desired && s.ReadyReplicas == desired && + s.AvailableReplicas == desired + return deploymentSummary{ + Name: obj.Metadata.Name, Present: true, Ready: ready, + Generation: obj.Metadata.Generation, ObservedGeneration: s.ObservedGeneration, + Desired: desired, Updated: s.UpdatedReplicas, Available: s.AvailableReplicas, Conditions: s.Conditions, + }, nil +} + +type statusReport struct { + Healthy bool `json:"healthy"` + Components []deploymentSummary `json:"components"` + Profiles []profileSummary `json:"profiles"` + Nodes []nodeSummary `json:"nodes"` +} + +func (c *client) status() error { + deployments, err := c.list("deployments", "--namespace", c.opts.systemNamespace) + if err != nil { + return err + } + profiles, err := c.list(profilesResource) + if err != nil { + return err + } + objects, err := c.list("nodes") + if err != nil { + return err + } + nodes, err := summarizeNodes(objects, "") + if err != nil { + return err + } + report := statusReport{Healthy: true, Profiles: []profileSummary{}, Nodes: nodes} + for _, name := range []string{"brewlet-operator", "brewlet-admission"} { + component := deploymentSummary{Name: name} + for _, obj := range deployments { + if obj.Metadata.Name == name { + component, err = summarizeDeployment(obj) + if err != nil { + return err + } + break + } + } + report.Components = append(report.Components, component) + if !component.Ready { + report.Healthy = false + } + } + for _, obj := range profiles { + profile := summarizeProfile(obj) + report.Profiles = append(report.Profiles, profile) + report.Healthy = report.Healthy && profile.Ready + } + sort.Slice(report.Profiles, func(i, j int) bool { return report.Profiles[i].Name < report.Profiles[j].Name }) + readyNodes := 0 + for _, node := range nodes { + if node.Error != "" || !node.RuntimeReady || !node.NodeReady { + report.Healthy = false + } + if node.RuntimeReady && node.NodeReady && !node.Unschedulable { + readyNodes++ + } + } + report.Healthy = report.Healthy && len(profiles) > 0 && readyNodes > 0 + if c.opts.output == "table" { + for _, component := range report.Components { + fmt.Fprintf(c.out, "%s: present=%t ready=%t updated=%d/%d available=%d\n", + component.Name, component.Present, component.Ready, component.Updated, component.Desired, component.Available) + } + if err := c.profileTable(report.Profiles); err != nil { + return err + } + if err := c.nodeTable(report.Nodes); err != nil { + return err + } + } else if err := encode(c.out, report, c.opts.output); err != nil { + return err + } + if !report.Healthy { + return fmt.Errorf("Brewlet is not ready: inspect component rollouts, profile conditions and node failures (an intentionally disabled admission deployment is also reported as not ready)") + } + return nil +} + +type podSummary struct { + Name string `json:"name"` + Node string `json:"node,omitempty"` + Phase string `json:"phase"` + Ready bool `json:"ready"` + Restarts int `json:"restarts"` + Problems []string `json:"problems"` + JDKRequest string `json:"jdkRequest,omitempty"` + Launcher string `json:"launcher,omitempty"` +} + +type eventSummary struct { + Object string `json:"object"` + Type string `json:"type"` + Reason string `json:"reason"` + Message string `json:"message"` +} + +type appReport struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Ready bool `json:"ready"` + Reason string `json:"reason"` + Image string `json:"image"` + JDKRequest string `json:"jdkRequest"` + Launcher string `json:"launcher"` + Arch []string `json:"arch,omitempty"` + Conditions []condition `json:"conditions"` + Deployments []deploymentSummary `json:"deployments"` + Pods []podSummary `json:"pods"` + Events []eventSummary `json:"events"` +} + +func (c *client) inspectApp(name string) error { + app, err := c.get(appsResource, name, c.namespaceArgs()...) + if err != nil { + return err + } + if app.Metadata.UID == "" || app.Metadata.Namespace == "" { + return fmt.Errorf("application %s has no UID or namespace", name) + } + nsArgs := []string{"--namespace", app.Metadata.Namespace} + selector := "app.kubernetes.io/name=" + name + ",app.kubernetes.io/managed-by=brewlet-operator" + workloads, err := c.list("deployments,replicasets,pods", append(nsArgs, "--selector", selector)...) + if err != nil { + return err + } + events, err := c.list("events", nsArgs...) + if err != nil { + return err + } + var spec struct { + Artifact struct { + Image string `json:"image"` + } `json:"artifact"` + JVM struct { + Version int `json:"version"` + Distribution string `json:"distribution"` + Launcher string `json:"launcher"` + } `json:"jvm"` + Arch []string `json:"arch"` + } + if err := json.Unmarshal(app.Spec, &spec); err != nil { + return fmt.Errorf("decode application spec: %w", err) + } + ready, reason := readyCondition(app) + report := appReport{ + Name: name, Namespace: app.Metadata.Namespace, Ready: ready, Reason: reason, + Image: spec.Artifact.Image, Launcher: spec.JVM.Launcher, Arch: spec.Arch, + Conditions: app.Status.Conditions, Deployments: []deploymentSummary{}, Pods: []podSummary{}, Events: []eventSummary{}, + } + if report.Launcher == "" { + report.Launcher = "java" + } + if spec.JVM.Version > 0 { + report.JDKRequest = fmt.Sprint(spec.JVM.Version) + if spec.JVM.Distribution != "" { + report.JDKRequest = spec.JVM.Distribution + "-" + report.JDKRequest + } + } + uids := map[string]bool{app.Metadata.UID: true} + // Traverse controller ownership, not just labels: old ReplicaSets belong, + // but a similarly labelled Pod from another application does not. + for _, kind := range []string{"Deployment", "ReplicaSet", "Pod"} { + for _, obj := range workloads { + if obj.Kind != kind || !ownedBy(obj, uids) || obj.Metadata.UID == "" { + continue + } + uids[obj.Metadata.UID] = true + switch kind { + case "Deployment": + deployment, err := summarizeDeployment(obj) + if err != nil { + return err + } + report.Deployments = append(report.Deployments, deployment) + if report.Ready && !deployment.Ready { + report.Reason = "DeploymentNotReady" + } + report.Ready = report.Ready && deployment.Ready + case "Pod": + var podSpec struct { + NodeName string `json:"nodeName"` + } + if err := json.Unmarshal(obj.Spec, &podSpec); err != nil { + return fmt.Errorf("decode pod %s: %w", obj.Metadata.Name, err) + } + pod := podSummary{ + Name: obj.Metadata.Name, Node: podSpec.NodeName, Phase: obj.Status.Phase, + Problems: []string{}, JDKRequest: obj.Metadata.Annotations["brewlet.sh/jdk"], + Launcher: obj.Metadata.Annotations["brewlet.sh/launcher"], + } + for _, cond := range obj.Status.Conditions { + if cond.Type == "Ready" { + pod.Ready = cond.Status == "True" && obj.Metadata.DeletionTimestamp == "" + } + } + for _, container := range obj.Status.ContainerStatuses { + pod.Restarts += container.RestartCount + if container.State.Waiting != nil { + pod.Problems = append(pod.Problems, container.Name+": "+container.State.Waiting.Reason) + } + if term := container.State.Terminated; term != nil && term.ExitCode != 0 { + pod.Problems = append(pod.Problems, fmt.Sprintf("%s: %s (exit %d)", container.Name, term.Reason, term.ExitCode)) + } + } + report.Pods = append(report.Pods, pod) + } + } + } + report.Ready = report.Ready && len(report.Deployments) > 0 + if len(report.Deployments) == 0 { + report.Reason = "DeploymentMissing" + } + for _, event := range events { + if uids[event.InvolvedObject.UID] { + report.Events = append(report.Events, eventSummary{ + Object: event.InvolvedObject.Kind + "/" + event.InvolvedObject.Name, + Type: event.Type, Reason: event.Reason, Message: event.Message, + }) + } + } + sort.Slice(report.Deployments, func(i, j int) bool { return report.Deployments[i].Name < report.Deployments[j].Name }) + sort.Slice(report.Pods, func(i, j int) bool { return report.Pods[i].Name < report.Pods[j].Name }) + sort.Slice(report.Events, func(i, j int) bool { + a, b := report.Events[i], report.Events[j] + return a.Object+a.Reason+a.Message < b.Object+b.Reason+b.Message + }) + format := c.opts.output + if format == "table" { + format = "yaml" + } + return encode(c.out, report, format) +} diff --git a/core/internal/kube/update.go b/core/internal/kube/update.go new file mode 100644 index 00000000..6fa0fbc5 --- /dev/null +++ b/core/internal/kube/update.go @@ -0,0 +1,507 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "regexp" + "strings" + + "github.com/microsoft/brewlet/internal/sourcepolicy" + "go.yaml.in/yaml/v3" +) + +type document map[string]json.RawMessage + +type jdkSource struct { + Distribution string `json:"distribution"` + Feature int64 `json:"feature"` + Source struct { + Image string `json:"image"` + JavaHome string `json:"javaHome"` + } `json:"source"` +} + +type launcherSource struct { + Name string `json:"name"` + Source struct { + Image string `json:"image"` + Path string `json:"path"` + } `json:"source"` +} + +type updateOptions struct { + profile, file, values, distribution, name, image, path string + feature int64 + jdk, replace bool + dryRun dryRunMode +} + +type dryRunMode string + +const ( + dryRunClient dryRunMode = "client" + dryRunServer dryRunMode = "server" +) + +func (m *dryRunMode) String() string { return string(*m) } +func (m *dryRunMode) IsBoolFlag() bool { return true } +func (m *dryRunMode) Set(value string) error { + switch value { + case "true", "client": + *m = dryRunClient + case "server": + *m = dryRunServer + default: + return fmt.Errorf("--dry-run accepts client or server; omit the flag to apply changes") + } + return nil +} + +func (u *updateOptions) flags(fs *flag.FlagSet, jdk bool) { + u.jdk = jdk + fs.StringVar(&u.profile, "profile", "", "explicit target NodeProfile name") + fs.StringVar(&u.file, "file", "", "read a NodeProfile YAML/JSON file (requires --dry-run=client)") + fs.StringVar(&u.values, "values", "", "read complete Helm values (requires --dry-run=client)") + fs.Var(&u.dryRun, "dry-run", "validate and print without saving: client (default when bare) or server") + fs.StringVar(&u.image, "image", "", "fully qualified, tagless SHA-256 digest reference") + fs.BoolVar(&u.replace, "replace", false, "replace a source with the same inventory identity") + if jdk { + fs.StringVar(&u.distribution, "distribution", "", "JDK distribution token") + fs.Int64Var(&u.feature, "feature", 0, "JDK feature version") + fs.StringVar(&u.path, "java-home", "", "absolute JDK home inside the image") + } else { + fs.StringVar(&u.name, "name", "", "optional launcher name (not java)") + fs.StringVar(&u.path, "path", "", "absolute launcher binary path inside the image") + } +} + +var dnsLabel = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +func validateName(value string, max int) error { + if value == "" || len(value) > max { + return fmt.Errorf("name %q must contain 1-%d characters", value, max) + } + for _, part := range strings.Split(value, ".") { + if len(part) > 63 || !dnsLabel.MatchString(part) { + return fmt.Errorf("name %q must be a lowercase DNS name", value) + } + } + return nil +} + +func validateToken(value string, max int) error { + if len(value) > max || !dnsLabel.MatchString(value) { + return fmt.Errorf("%q must be a lowercase DNS label of at most %d characters", value, max) + } + return nil +} + +func validateSource(image, path string) error { + if len(image) > 2048 || len(path) > 1024 { + return fmt.Errorf("source image must be at most 2048 characters and source path at most 1024") + } + if err := sourcepolicy.ValidateDigestReference(image); err != nil { + return err + } + return sourcepolicy.ValidateAbsolutePath(path) +} + +func (u updateOptions) entry() (string, json.RawMessage, error) { + if err := validateSource(u.image, u.path); err != nil { + return "", nil, err + } + if u.jdk { + if err := validateToken(u.distribution, 48); err != nil { + return "", nil, err + } + if u.feature < 1 || u.feature > 2147483647 { + return "", nil, fmt.Errorf("--feature must be between 1 and 2147483647") + } + jdk := jdkSource{Distribution: u.distribution, Feature: u.feature} + jdk.Source.Image, jdk.Source.JavaHome = u.image, u.path + raw, err := json.Marshal(jdk) + return "jdks", raw, err + } + if err := validateToken(u.name, 54); err != nil { + return "", nil, err + } + if u.name == "java" { + return "", nil, fmt.Errorf("java is supplied by each JDK and must not be added as a launcher") + } + launcher := launcherSource{Name: u.name} + launcher.Source.Image, launcher.Source.Path = u.image, u.path + raw, err := json.Marshal(launcher) + return "launchers", raw, err +} + +func entryIdentity(field string, raw json.RawMessage) (string, error) { + if field == "jdks" { + var entry jdkSource + if err := json.Unmarshal(raw, &entry); err != nil { + return "", err + } + if err := validateToken(entry.Distribution, 48); err != nil { + return "", err + } + if entry.Feature < 1 || entry.Feature > 2147483647 { + return "", fmt.Errorf("invalid JDK feature %d", entry.Feature) + } + return fmt.Sprintf("%s-%d", entry.Distribution, entry.Feature), nil + } + var entry launcherSource + if err := json.Unmarshal(raw, &entry); err != nil { + return "", err + } + if err := validateToken(entry.Name, 54); err != nil { + return "", err + } + if entry.Name == "java" { + return "", fmt.Errorf("java must not appear in the launcher inventory") + } + return entry.Name, nil +} + +func updateInventory(spec document, field string, entry json.RawMessage, replace bool) error { + entries := []json.RawMessage{} + if raw, exists := spec[field]; exists { + if err := json.Unmarshal(raw, &entries); err != nil { + return fmt.Errorf("decode %s: %w", field, err) + } + } + key, err := entryIdentity(field, entry) + if err != nil { + return err + } + found := false + seen := map[string]bool{} + for i, existing := range entries { + id, err := entryIdentity(field, existing) + if err != nil { + return fmt.Errorf("invalid existing %s entry: %w", field, err) + } + if seen[id] { + return fmt.Errorf("duplicate %s entry %q; repair the source configuration first", field, id) + } + seen[id] = true + if id != key { + continue + } + found = true + var oldValue, newValue any + if err := json.Unmarshal(existing, &oldValue); err != nil { + return err + } + if err := json.Unmarshal(entry, &newValue); err != nil { + return err + } + oldJSON, err := json.Marshal(oldValue) + if err != nil { + return err + } + newJSON, err := json.Marshal(newValue) + if err != nil { + return err + } + if !bytes.Equal(oldJSON, newJSON) && !replace { + return fmt.Errorf("%s %q already exists with a different source; use --replace deliberately", field, key) + } + entries[i] = entry + } + if !found { + entries = append(entries, entry) + } + if len(entries) > 32 { + return fmt.Errorf("%s inventory cannot exceed 32 entries", field) + } + raw, err := json.Marshal(entries) + if err != nil { + return err + } + spec[field] = raw + return nil +} + +func readDocument(file string) (document, error) { + raw, err := os.ReadFile(file) + if err != nil { + return nil, err + } + dec := yaml.NewDecoder(bytes.NewReader(raw)) + var value map[string]any + if err := dec.Decode(&value); err != nil { + return nil, fmt.Errorf("decode %s: %w", file, err) + } + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + if err != nil { + return nil, fmt.Errorf("decode %s: %w", file, err) + } + return nil, fmt.Errorf("%s must contain exactly one YAML/JSON document", file) + } + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("convert %s to JSON: %w", file, err) + } + return decodeDocument(data) +} + +func decodeDocument(raw []byte) (document, error) { + var doc document + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, err + } + if doc == nil { + return nil, fmt.Errorf("expected a non-null object") + } + return doc, nil +} + +func managedBy(meta metadata) string { + if meta.Labels["app.kubernetes.io/managed-by"] != "" && + !oneOf(strings.ToLower(meta.Labels["app.kubernetes.io/managed-by"]), "brewlet", "kubectl") { + return meta.Labels["app.kubernetes.io/managed-by"] + } + for _, fields := range []map[string]string{meta.Labels, meta.Annotations} { + for _, key := range []string{ + "meta.helm.sh/release-name", "meta.helm.sh/release-namespace", + "argocd.argoproj.io/instance", "argocd.argoproj.io/tracking-id", + "kustomize.toolkit.fluxcd.io/name", "helm.toolkit.fluxcd.io/name", + } { + if fields[key] != "" { + return key + "=" + fields[key] + } + } + } + if len(meta.OwnerReferences) > 0 { + return "an owning Kubernetes resource" + } + for _, field := range meta.ManagedFields { + if field.Subresource != "" { + continue + } + var fields document + // Unknown ownership is not permission to overwrite a profile. + if err := json.Unmarshal(field.Fields, &fields); err != nil { + return "unrecognized field ownership" + } + if _, ownsSpec := fields["f:spec"]; ownsSpec && + !oneOf(field.Manager, "brewlet", "kubectl", "kubectl-client-side-apply", "kubectl-edit", "kubectl-patch") { + return "field manager " + field.Manager + } + } + return "" +} + +func (c *client) updateProfile(u updateOptions) error { + if err := validateName(u.profile, 253); err != nil { + return fmt.Errorf("--profile: %w", err) + } + if u.file != "" && u.values != "" { + return fmt.Errorf("--file and --values are mutually exclusive") + } + if u.dryRun != dryRunClient && (u.file != "" || u.values != "") { + return fmt.Errorf("--file and --values require --dry-run=client (or bare --dry-run)") + } + field, entry, err := u.entry() + if err != nil { + return err + } + if u.values != "" { + return c.updateValues(u, field, entry) + } + var doc document + var live object + if u.file != "" { + doc, err = readDocument(u.file) + } else { + live, err = c.get(profilesResource, u.profile, "--show-managed-fields=true") + if err == nil { + doc, err = decodeDocument(live.raw) + } + } + if err != nil { + return err + } + raw, err := json.Marshal(doc) + if err != nil { + return err + } + var profile object + if err := json.Unmarshal(raw, &profile); err != nil { + return fmt.Errorf("decode profile: %w", err) + } + if profile.APIVersion != "node.brewlet.sh/v1alpha1" || profile.Kind != "NodeProfile" || + profile.Metadata.Name != u.profile || profile.Metadata.Namespace != "" { + return fmt.Errorf("expected cluster-scoped node.brewlet.sh/v1alpha1 NodeProfile %q", u.profile) + } + if profile.Metadata.DeletionTimestamp != "" { + return fmt.Errorf("profile %q is terminating; refusing to change its inventory", u.profile) + } + owner := managedBy(profile.Metadata) + if u.dryRun != dryRunClient && owner != "" { + return fmt.Errorf("profile %q is managed by %s; edit its source of truth (use add --dry-run --values FILE for Helm), not the live object", u.profile, owner) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + return fmt.Errorf("decode profile spec: %w", err) + } + if err := updateInventory(spec, field, entry, u.replace); err != nil { + return err + } + doc["spec"], err = json.Marshal(spec) + if err != nil { + return err + } + if u.dryRun != dryRunClient { + if live.Metadata.UID == "" || live.Metadata.ResourceVersion == "" { + return fmt.Errorf("live profile has no UID/resourceVersion; cannot update safely") + } + patch := []struct { + Op string `json:"op"` + Path string `json:"path"` + Value any `json:"value"` + }{ + {"test", "/metadata/uid", live.Metadata.UID}, + {"test", "/metadata/resourceVersion", live.Metadata.ResourceVersion}, + {"add", "/spec/" + field, spec[field]}, + } + input, err := json.Marshal(patch) + if err != nil { + return err + } + patchFile, err := os.CreateTemp("", "brewlet-profile-patch-*.json") + if err != nil { + return err + } + defer os.Remove(patchFile.Name()) + if _, err := patchFile.Write(input); err != nil { + patchFile.Close() + return err + } + if err := patchFile.Close(); err != nil { + return err + } + args := []string{"patch", profilesResource, u.profile, "--type=json", "--patch-file", patchFile.Name(), "--field-manager=brewlet", "-o", "json"} + if u.dryRun == dryRunServer { + args = append(args, "--dry-run=server") + } + result, err := c.kubectl(nil, args...) + if err != nil { + return fmt.Errorf("profile update failed (concurrent changes require re-reading and reviewing the profile): %w", err) + } + doc, err = decodeDocument(result) + if err != nil { + return fmt.Errorf("decode updated profile: %w", err) + } + } + if err := cleanManifest(doc); err != nil { + return err + } + if err := encode(c.out, doc, c.opts.output); err != nil { + return err + } + switch u.dryRun { + case dryRunServer: + fmt.Fprintln(c.err, "Server dry run succeeded; no inventory was changed.") + case dryRunClient: + fmt.Fprintln(c.err, "Client dry run succeeded; no inventory was changed. Server validation was not performed.") + if owner != "" { + fmt.Fprintf(c.err, "Profile is managed by %s. Update its source of truth; for Helm use add --dry-run --values FILE.\n", owner) + } + default: + fmt.Fprintln(c.err, "Profile update accepted; provisioning is asynchronous. Inspect the profile and advertised node inventory before using the new source.") + } + return nil +} + +func cleanManifest(doc document) error { + meta, err := decodeDocument(doc["metadata"]) + if err != nil { + return err + } + for _, key := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "deletionTimestamp", "managedFields", "selfLink", "finalizers"} { + delete(meta, key) + } + if raw, exists := meta["annotations"]; exists && string(raw) != "null" { + annotations, err := decodeDocument(raw) + if err != nil { + return err + } + delete(annotations, "kubectl.kubernetes.io/last-applied-configuration") + meta["annotations"], err = json.Marshal(annotations) + if err != nil { + return err + } + } + doc["metadata"], err = json.Marshal(meta) + delete(doc, "status") + return err +} + +func (c *client) updateValues(u updateOptions, field string, entry json.RawMessage) error { + doc, err := readDocument(u.values) + if err != nil { + return err + } + if u.profile == "default" { + var defaultProfile struct { + Enabled *bool `json:"enabled"` + } + if raw, exists := doc["defaultProfile"]; exists { + if err := json.Unmarshal(raw, &defaultProfile); err != nil { + return err + } + if defaultProfile.Enabled != nil && !*defaultProfile.Enabled { + return fmt.Errorf("defaultProfile.enabled is false; edit the enabled profile's source instead") + } + } + spec, err := decodeDocument(doc["provisioner"]) + if err != nil { + return fmt.Errorf("values must contain the complete provisioner configuration: %w", err) + } + if err := updateInventory(spec, field, entry, u.replace); err != nil { + return err + } + doc["provisioner"], err = json.Marshal(spec) + if err != nil { + return err + } + } else { + var profiles []document + if err := json.Unmarshal(doc["profiles"], &profiles); err != nil { + return fmt.Errorf("values must contain profiles: %w", err) + } + found := false + for _, profile := range profiles { + var name string + if err := json.Unmarshal(profile["name"], &name); err != nil { + return err + } + if name == u.profile { + if found { + return fmt.Errorf("duplicate profile %q in values", name) + } + found = true + if err := updateInventory(profile, field, entry, u.replace); err != nil { + return err + } + } + } + if !found { + return fmt.Errorf("profile %q is not present in the values file", u.profile) + } + doc["profiles"], err = json.Marshal(profiles) + if err != nil { + return err + } + } + return encode(c.out, doc, c.opts.output) +} diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 38a2bf44..9b13c984 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -25,6 +25,7 @@ brewlet run [flags] pull + launch java -jar on this node brewlet bundle [flags] emit an OCI runc bundle (the shim path) brewlet jdks [flags] list JDKs available across the cluster brewlet doctor [flags] diagnose cluster and developer readiness +brewlet k8s [flags] manage and inspect Brewlet on Kubernetes brewlet version print the CLI version ``` @@ -50,7 +51,7 @@ brewlet keygen --private FILE --public FILE ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--private` | *(required)* | Output path for the PKCS#8 PEM private key. | | `--public` | *(required)* | Output path for the SubjectPublicKeyInfo PEM public key. | @@ -77,7 +78,7 @@ brewlet dependency-bundle \ ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--name` | *(required)* | Stable bundle name. | | `--version` | *(required)* | Bundle version. | | `--source-bom` | *(required)* | Approved Maven BOM in `groupId:artifactId:version` form. | @@ -126,7 +127,7 @@ brewlet push [--format image|artifact] [--store DIR] [--config FILE] ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--format` | `image` | Delivery format: `image` (standard, kubelet-pullable OCI image — the production `runtimeClassName: brewlet` pod image) or `artifact` (native Brewlet OCI artifact for local OCI-layout / CLI / bundle workflows, not the production Kubernetes pod image path). See [runnable-image delivery](runnable-image.md). | | `--store` | `./oci` | OCI layout directory to write the artifact into. | | `--config` | *(none)* | Path to a `jvm-config.json` to embed verbatim (overrides the generated one). See the [launch config schema](building-and-publishing.md#2-the-launch-config). | @@ -187,7 +188,7 @@ brewlet inspect [--store DIR] ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--store` | `./oci` | OCI layout directory to read from. | | `--trusted-public-key` | *(none)* | ECDSA P-256 public key used to verify managed provenance or attestation; requires `--trusted-signer-identity`. | | `--trusted-signer-identity` | *(none)* | Expected signed identity. For a bundle this is its publisher; for a final image this is its application builder. Requires `--trusted-public-key`. | @@ -211,7 +212,7 @@ brewlet run [--store DIR] [--jdk-root DIR] [--launcher NAME] [--appcds-reg ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--store` | `./oci` | OCI layout directory to read from. | | `--jdk-root` | *(none)* | Node JDK home to launch with. When unset, falls back to `BREWLET_JDK_HOME`, then `JAVA_HOME`, then `java` on `PATH`. | | `--launcher` | `java` | Launcher binary **name** under the selected JDK (or a compatible node-installed launcher name such as `jaz`), resolved on `PATH`. It is a lowercase DNS-1123 token, never a path: separators, `..` and absolute paths are rejected. | @@ -242,7 +243,7 @@ brewlet bundle [--store DIR] [--cpu N] [--memory M] [--uid UID] [--gid GID ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--store` | `./oci` | OCI layout directory to read from. | | `--cpu` | *(unlimited)* | CPU limit, e.g. `2` or `500m` → sandbox `cpu.max`. Minimum `10m` (`0.01` CPU), matching Linux's minimum quota for the 100ms period. | | `--memory` | *(unlimited)* | Memory limit, e.g. `512Mi` or `1Gi` → sandbox `memory.max`. | @@ -281,7 +282,7 @@ brewlet jdks [--output table|wide|json] [--kubeconfig FILE] [--context CTX] [--s ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--output` | `table` | `table` = distinct JDKs across the fleet with a node count; `wide` = one row per node; `json` = machine-readable aggregation (e.g. to drive a CI matrix). | | `--kubeconfig` | *(kubectl default)* | Path to a kubeconfig file. | | `--context` | *(current)* | kubeconfig context to use. | @@ -332,14 +333,302 @@ for CI and platform handoff automation. --- -## `brewlet version` +## `brewlet k8s` + +Brewlet-specific Kubernetes operations, using your installed `kubectl` and +existing kubeconfig credentials. Installation also requires Helm. No command +creates a cluster, changes your current context, directly modifies a node, or +opens a debugger. `brewlet jdks` and `brewlet doctor` remain compatibility +aliases for `brewlet k8s jdk list` and `brewlet k8s doctor`. + +```text +brewlet k8s jdk list +brewlet k8s launcher list +brewlet k8s profile list +brewlet k8s profile inspect NAME +brewlet k8s inspect app NAME +brewlet k8s status +brewlet k8s doctor +brewlet k8s install --version X.Y.Z --values FILE +brewlet k8s jdk add --profile NAME --distribution NAME --feature N --image REF --java-home PATH +brewlet k8s launcher add --profile NAME --name NAME --image REF --path PATH +``` + +**Command verbs describe their effects:** `add` updates the live profile and +`install` installs Brewlet. `--dry-run` previews those operations without saving +changes and prints the proposed YAML/JSON only after validation succeeds. +Failed dry runs exit nonzero with an error on stderr and no rendered output on +stdout. `list`, `inspect`, `status`, and `doctor` are read-only. Redirecting +stdout to a file does not change these semantics. + +### Connections and output + +Common flags may precede the command or follow its full name, for example +`brewlet k8s --context staging profile inspect java-workers --output json`. +Flags may also follow positional names. + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--kubeconfig FILE` | Tool default | Kubeconfig passed to every Kubernetes/Helm invocation. | +| `--context NAME` | Current context | Explicit context, without modifying kubeconfig. | +| `--namespace NAME` | Context namespace | Application inspection namespace. `doctor` preserves its original `default` namespace; installation defaults to `brewlet`. Profiles and node inventories are cluster-scoped. | +| `--timeout DURATION` | `30s` | Deadline for each kubectl invocation, including credential helpers. | +| `--output FORMAT` | Command-dependent | Read commands default to `table` and support `json`/`yaml`. JDK/launcher inventory supports `table`/`wide`/`json` instead. `add`, including dry runs, defaults to `yaml` and also supports `json`. Installation prints Helm output after a successful command. | -Print the release version embedded in the binary: +Inspection's default output is a structured YAML report rather than a flattened +table. All commands propagate tool/API failures; RBAC-denied reads are not +reported as empty inventories. Output goes to stdout and mutation notices to +stderr. + +### Inventory, status, and inspection ```bash -brewlet version +brewlet k8s jdk list --output wide +brewlet k8s launcher list --selector brewlet.sh/runtime=ready +brewlet k8s profile list --output json +brewlet k8s profile inspect java-workers +brewlet k8s status --system-namespace brewlet --output json +brewlet k8s inspect app orders --namespace my-team +brewlet k8s doctor --namespace my-team +``` + +JDK inventory reuses the existing [JDK aggregation](#brewlet-jdks), including +legacy annotation fallback. Launcher inventory aggregates optional launcher +names from `brewlet.sh/launchers`; the JDK's implicit `java` launcher is not a +separate entry. Both inventories describe **node-advertised state**, not a +catalog, a live probe of node files, or proof that a particular Pod uses a JDK. +Use `--selector` to restrict nodes, such as to runtime-ready nodes. + +Profile listing shows **desired** JDK/launcher inventories, pool selection, +readiness, and node counts. Profile inspection adds the full spec, conditions, +field-management information, and claimed node summaries. Node identity and +ownership must match the profile's UID. Missing or stale observed generations +never count as a current Ready condition. + +`status` reports the operator and admission **Deployment rollout state** in +`--system-namespace` (default `brewlet`), profile conditions, and node readiness +and provisioning errors. It exits nonzero if either component is absent or +unready, any profile/node is unready, or there are no profiles or no uncordoned, +runtime-ready, Kubernetes-ready nodes. An intentionally disabled admission +Deployment therefore also produces a nonzero result. This is a conservative +rollout snapshot, not a webhook TLS, certificate, endpoint, or scheduling probe. + +Application inspection follows controller ownership from JavaApplication to +Deployment, ReplicaSets, and Pods, and includes their events. Matching labels +alone do not establish ownership. Reports distinguish the **requested JDK** +from the actual per-replica runtime, which Brewlet does not yet report. They +show Pod placement, restarts, and container failure reasons without dumping +environment variables or Secrets. Condition/event messages are operator-provided +text; review them before sharing a report. Inspection succeeds even for an +unready resource; use `status` or `doctor` for readiness exit codes. + +Read permissions are required for the objects each command inspects. +Application inspection lists Deployments, ReplicaSets, Pods, and Events in its +namespace; profile inspection reads the profile and lists Nodes. `status` lists +control-plane Deployments, NodeProfiles, and Nodes. + +### Installation + +`--values FILE` takes a **Helm values file that you create**, not a Brewlet CLI +configuration file. `values.yaml` is an example filename, not a file generated +by the command or automatically loaded from this repository. You may name it +anything and pass its path with `--values` or `-f`. + +#### Draft your values file + +For a first installation, save the following as `values.yaml`. This is a +minimal configuration for one existing worker pool and one JDK; other settings +inherit the selected chart release's defaults. + +```yaml +provisioner: + pools: + - java-workers + jdks: + - distribution: temurin + feature: 21 + source: + image: docker.io/library/eclipse-temurin@sha256:<64-lowercase-hex> + javaHome: /opt/java/openjdk ``` +Replace the example choices before running the command: + +| Field | What to supply | +|---|---| +| `provisioner.pools` | Names of existing node pools Brewlet may provision, not individual node names. The CLI does not create these pools. Replace `java-workers` with a pool you control. | +| `provisioner.jdks[].distribution` | A lowercase inventory name for the JDK, such as `temurin` or `microsoft`. It does not select or download an image automatically. | +| `provisioner.jdks[].feature` | The Java feature version actually supplied by your source image, such as `21` or `25`. | +| `provisioner.jdks[].source.image` | The fully qualified, tagless image reference pinned to the approved manifest or multi-platform index digest. Replace `<64-lowercase-hex>` with all 64 lowercase hexadecimal characters from that digest. | +| `provisioner.jdks[].source.javaHome` | The absolute JDK home **inside the source image**, not your workstation's `JAVA_HOME`. Confirm that `/bin/java` exists in that image. | + +The Temurin entry is an example, not a built-in runtime recommendation. Choose +an image that supports every architecture in the selected pool; see the +[JDK source model](jdk-management.md#source-model) for source requirements. +Pool labels are detected automatically for supported providers. For bare-metal +or nonstandard labels, set `provisioner.poolKey` as described in +[pool configuration](configuration.md#where-the-provisioner-may-run). + +The chart deliberately leaves `provisioner.pools` and `provisioner.jdks` empty: +they are required while `defaultProfile.enabled` is true. Copying the chart's +default file unchanged will therefore fail. You do not need to copy every +setting into your own file; include your pool, complete JDK inventory, and any +other overrides you need. Additional JDKs are entries in the same `jdks` list. +Optional launchers belong in `provisioner.launchers`; omit that list to use the +JDK's implicit `java` launcher. + +The [full Helm values reference](configuration.md#helm-chart-values) and +[annotated chart defaults](https://github.com/microsoft/brewlet/blob/main/kubernetes/charts/brewlet/values.yaml) +describe optional settings, including launchers, mirrors, tolerations, rollout +policy, and additional profiles. Use settings supported by the chart version +you select. For GitOps-managed profiles, you may instead disable +`defaultProfile.enabled` and supply your own NodeProfiles; see +[installation and profile ownership](installation.md#helm-recommended). + +#### Preview and install + +After saving and reviewing `values.yaml`, replace `x.y.z` with the exact chart +release you want and `evaluation` with your intended kubeconfig context: + +```bash +# Set this to the exact approved chart release, not "latest". +RELEASE_VERSION=x.y.z + +# Download and render the chart; does not contact the Kubernetes API. +brewlet k8s install --version "$RELEASE_VERSION" --values values.yaml --dry-run + +# Install into a deliberately selected fresh cluster. +brewlet k8s install --context evaluation --version "$RELEASE_VERSION" \ + --values values.yaml --namespace brewlet +``` + +Preview validates chart rendering, not source image contents or node readiness. +If it fails, fix the reported values error before installing. Installation is +privileged and modifies the selected hosts; review the +[cluster prerequisites](installation.md#prerequisites) first. Control-plane +nodes and tainted pools require deliberate configuration, not an implicit opt-in. + +The CLI invokes `helm install` against +`oci://ghcr.io/microsoft/charts/brewlet`, creates the namespace, and waits for +chart resources. `--release` defaults to `brewlet`; `--wait-timeout` defaults to +`5m`. The Helm process deadline includes that duration plus `--timeout` for +setup. `--values`/`-f` may be repeated in Helm precedence order. The CLI sets the +chart's `namespace` value to the selected release namespace so resources and the +release cannot accidentally land in different namespaces. + +Installation refuses existing Brewlet CRDs and Helm refuses an existing release. +It does **not** upgrade a release or migrate CRDs: follow +[Upgrading](installation.md#upgrading) for those operations, including clusters +retaining CRDs after uninstall. Preview uses `helm template --include-crds`; it +still needs chart registry access, and it is not server-side validation. + +Helm readiness is not node provisioning completion. Follow installation with +`brewlet k8s status` and inventory inspection using the same kubeconfig/context +and the installation namespace as `--system-namespace`. Sources are never chosen for you, +control-plane provisioning is never implicitly enabled, and no cleanup or +uninstall is attempted after a failed installation. + +### Safe JDK and launcher additions + +`jdk add` and `launcher add` **update the live NodeProfile by default**. They +always select an existing profile and return its updated declaration. The +source image must be a fully qualified, tagless SHA-256 digest reference; paths +and inventory names follow the [JDK](jdk-management.md#source-model) and +[launcher](launchers.md#launcher-names-are-tokens-not-paths) source contracts. + +| Command mode | Effect | +| --- | --- | +| `jdk add` / `launcher add` | Persist the inventory change to the live profile. | +| `add --dry-run` | Validate locally and print the proposed declaration; equivalent to `--dry-run=client`. | +| `add --dry-run=client` | Validate the requested source and inventory change locally without sending a patch to the API server. | +| `add --dry-run=server` | Validate the conditional patch through the API server and print the result without saving it. | +| `add --dry-run --file FILE` | Read a source NodeProfile and validate/generate its proposed update offline. | +| `add --dry-run --values FILE` | Read complete Helm values and validate/generate their proposed update offline. | + +Bare `--dry-run` and `--dry-run=client` still read the live profile unless +`--file` or `--values` is supplied. They do not validate registry contents, node +compatibility, or server admission policy. Use `--dry-run=server` when you need +API-server validation. Invalid modes are rejected; omit `--dry-run` entirely +to apply changes. + +**Validation happens before rendering.** A failed source check, file read, +inventory update, cluster read, server dry run, or result decode returns an error +without printing a manifest. There is no fallback from a failed server dry run +to client-generated output. Shell redirection may still create or truncate its +destination file before the command runs; always redirect to a different file +and check the exit status. + +Replace the digest placeholders with approved 64-character lowercase digests: + +```bash +# Add a JDK to an unmanaged live profile. This changes the cluster. +brewlet k8s jdk add --profile java-workers \ + --distribution temurin --feature 25 \ + --image 'docker.io/library/eclipse-temurin@sha256:<64-lowercase-hex>' \ + --java-home /opt/java/openjdk + +# Validate a launcher addition through the API server without saving it. +brewlet k8s launcher add --profile java-workers \ + --name jaz \ + --image 'mcr.microsoft.com/openjdk/jdk@sha256:<64-lowercase-hex>' \ + --path /usr/bin/jaz --dry-run=server + +# Validate and print a GitOps change entirely offline from its source manifest. +brewlet k8s launcher add --dry-run --profile java-workers --file profile.yaml \ + --name jaz \ + --image 'mcr.microsoft.com/openjdk/jdk@sha256:<64-lowercase-hex>' \ + --path /usr/bin/jaz > profile-next.yaml + +# Generate proposed values for a Helm-owned profile; does not deploy them. +brewlet k8s jdk add --dry-run --profile default --values values.yaml \ + --distribution temurin --feature 25 \ + --image 'docker.io/library/eclipse-temurin@sha256:<64-lowercase-hex>' \ + --java-home /opt/java/openjdk > values-next.yaml +``` + +`--file` accepts one NodeProfile YAML/JSON document whose name matches +`--profile`. `--values` accepts one complete Helm values document: `default` +updates `provisioner`, while other names select an existing entry in `profiles`. +Other inventories, profiles, rollout policies, and chart settings are retained. +`--file` and `--values` are mutually exclusive and require client dry-run mode +(bare `--dry-run` or `--dry-run=client`). They cannot be used for a real update +or a server dry run. Input files are never overwritten by the CLI; redirect to +a **different** file, review the diff, and commit the desired source change. +YAML formatting and comments are not retained. For layered Helm values, use the +complete inventory-owning values, not a partial list overlay: Helm replaces +arrays rather than merging entries. + +Adding an identical entry is idempotent. Replacing the source for an existing +distribution/feature or launcher name requires `--replace`. Duplicate identities +and inventories above the CRD's 32-entry limit are rejected. Local generation +validates the supplied entry, not registry contents, node architecture +compatibility, or the cluster's full admission policy. + +For an **unmanaged live profile**, append `--dry-run=server` to `add` to validate a +conditional patch without saving; remove `--dry-run` to persist it. +There is no `--apply` flag. **`add ... > result.yaml` still changes the cluster**; +use `add ... --dry-run > proposal.yaml` for a client-validated preview. + +Live application rejects terminating profiles, Helm/Argo CD/Flux ownership +markers, owner references, and unrecognized spec field managers. It never +forces ownership. The patch changes only the selected inventory and tests both +the profile UID and resource version; a concurrent change fails without a +blind retry. These ownership checks also apply to `add --dry-run=server`. Use +client dry-run mode to prepare Helm/GitOps source changes and deploy them through +the owning system instead of bypassing those guards. Generation strips +server-owned status and lifecycle metadata from the emitted manifest; the actual +API patch leaves them untouched. + +A successful patch means **the declaration was accepted**, not that every node +has installed the source. Inspect the profile, its node generations, and +advertised inventory before deploying workloads that require the new runtime. + +--- + +## `brewlet version` + +Print the release version embedded in the binarybrewlet version + Source builds without release linker flags print `dev`. --- @@ -347,7 +636,7 @@ Source builds without release linker flags print `dev`. ## Environment variables | Variable | Used by | Meaning | -|---|---|---| +| --- | --- | --- | | `JAVA_HOME` | `run`, `push --appcds` | Default node JDK home if `--jdk-root` is unset (`run`); default training JVM if `--appcds-java` is unset (`push --appcds`). | | `BREWLET_JDK_HOME` | `run` | Overrides `JAVA_HOME` for JDK resolution. | | `BREWLET_STORE_ROOT` | shim (`layout` resolver) | OCI layout root used by the local layout resolver. | diff --git a/docs/configuration.md b/docs/configuration.md index 8d065312..60a06b54 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -64,7 +64,7 @@ with `--set key=value` or a values file. | `provisioner.includeControlPlane` | `false` | Allow the default profile onto control-plane nodes. Off by default, and only needed on single-node clusters such as kind or Docker Desktop, which label their node as the control plane without tainting it. | | `provisioner.tolerations` | `[]` | Tolerations for the default profile's provisioner pods. Standard Kubernetes toleration entries; each must name a `key`. Nothing is tolerated implicitly. | | `provisioner.jdks` | `[]` (**required**) | JDK entries with `distribution`, `feature`, digest-pinned `source.image`, and absolute `source.javaHome` ([§JDK management](jdk-management.md#source-model)). Brewlet ships no built-in runtime catalog, so rendering fails when `defaultProfile.enabled` is `true` and this is empty. | -| `provisioner.launchers` | structured `jaz` example | Optional launcher entries with `name`, digest-pinned `source.image`, and absolute `source.path` ([§Launchers](launchers.md#helm-example-jaz)). Empty = vanilla `java` only. | +| `provisioner.launchers` | `[]` | Optional launcher entries with `name`, digest-pinned `source.image`, and absolute `source.path` ([§Launchers](launchers.md#helm-example-jaz)). Empty = vanilla `java` only. | | `provisioner.appCDS.regenerationEnabled` | `false` | Authorize node-side AppCDS regeneration for the chart-managed default `NodeProfile`. | | `provisioner.rollout.maxUnavailable` | `null` | Bounds the default profile's provisioner DaemonSet rolling update. `null` keeps the DaemonSet default. | | `provisioner.rollout.validate` | `true` | Gate node readiness on post-install JDK smoke tests and staged-launcher executable checks. Arbitrary launchers are not executed as a probe. Renders the provisioner `BREWLET_VALIDATE` env. | diff --git a/docs/installation.md b/docs/installation.md index 33e012c3..95e54ac3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -212,6 +212,12 @@ digests. To reproduce a specific release, add `--version x.y.z`, replacing `x.y.z` with that release number. For an existing installation, follow [Upgrading](#upgrading) to update matching CRDs and retain your chosen values. +For a fresh cluster, the CLI also provides +[`brewlet k8s install`](cli-reference.md#installation), a version-pinned Helm +installation wrapper. Put both your pools and runtime sources in a complete +values file; preview with `--dry-run` before installing. This command refuses +existing Brewlet CRDs rather than silently skipping their migration. + `provisioner.poolKey` pins the node label the pool names are matched on. Leave it unset on AKS, EKS, and GKE, where the well-known provider label is auto-detected; set it explicitly on bare metal or kubeadm. diff --git a/docs/jdk-management.md b/docs/jdk-management.md index ebf8ec0d..b6d4061f 100644 --- a/docs/jdk-management.md +++ b/docs/jdk-management.md @@ -47,6 +47,15 @@ Mutable tags are intentionally rejected. A tag may resolve to different bytes between reviews or nodes; a digest identifies the exact OCI manifest or multi-platform index the administrator approved. +The CLI's [`brewlet k8s jdk add`](cli-reference.md#safe-jdk-and-launcher-additions) +updates an unmanaged live NodeProfile by default. `--dry-run` validates locally +and prints the proposed change; `--dry-run=server` validates through the API +server without saving. Failed dry runs print no rendered output. Use client +dry-run mode with `--file` or `--values` for offline source changes. +Live additions and server dry runs reject Helm/GitOps ownership. Use +`brewlet k8s profile list` for desired inventory and `brewlet k8s jdk list` for +node-advertised inventory; they are not interchangeable. + --- ## Helm examples: Temurin and Microsoft diff --git a/docs/launchers.md b/docs/launchers.md index 135c02d5..eefe1990 100644 --- a/docs/launchers.md +++ b/docs/launchers.md @@ -21,6 +21,15 @@ When a workload omits its launcher or requests `java`, Brewlet executes: Do not add `java` to `spec.launchers`. It comes from every declared JDK and is a reserved launcher name. +Use `brewlet k8s launcher list` to aggregate optional launchers advertised by +nodes, or `--output wide` for per-node rows. The +[`launcher add` workflow](cli-reference.md#safe-jdk-and-launcher-additions) +updates the live NodeProfile by default. `--dry-run` validates locally and prints +the proposal; `--dry-run=server` validates through the API server without saving. +If validation fails, no rendered output is printed. Client dry-run mode also +supports offline `--file` and `--values` inputs. The CLI never installs a +launcher directly onto a node; the operator provisions it after a profile update. + ## Launcher names are tokens, not paths A launcher name identifies a node-installed launcher layer directory diff --git a/integration-tests/README.md b/integration-tests/README.md index 78eb7db8..565bf70f 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -53,3 +53,21 @@ an exercised capability fails. The suite covers: | 15 | live opt-in Prometheus metrics through Helm, provisioner, shim, and exporter | both + fixtures | See [AGENTS.md](AGENTS.md) for cluster requirements, cleanup, and troubleshooting. + +## Kubernetes CLI API integration + +The process/API integration suite lives in the existing Kubernetes Go module +so it can reuse envtest and the production controllers: + +```bash +make -C kubernetes test-cli-integration +``` + +It builds the CLI and invokes real kubectl/Helm against a fresh API server and +etcd using only private fixture kubeconfigs. No Docker daemon, existing cluster, +or tier reset is used. Missing prerequisites fail this explicit target. The +Kubernetes pull-request CI job runs the same tests. + +See [CLI integration tests](../kubernetes/README.md#cli-integration-tests) for +coverage and limitations. In particular, fixture readiness and node inventory +are simulated; this is not coverage of host provisioning or JVM execution. diff --git a/kubernetes/Makefile b/kubernetes/Makefile index 58f53078..e29a52a3 100644 --- a/kubernetes/Makefile +++ b/kubernetes/Makefile @@ -10,7 +10,7 @@ ENVTEST_K8S_VERSION ?= 1.31.0 SETUP_ENVTEST_VERSION ?= v0.0.0-20250308055145-5fe7bb3edc86 SETUP_ENVTEST ?= $(shell $(GO) env GOPATH)/bin/setup-envtest -.PHONY: all fmt-check vet build test test-envtest ci operator-build admission-build operator-image admission-image helm-lint helm-template helm-digest-check helm-scheduling-check helm-admission-check helm-lifecycle-check helm-check print-setup-envtest-version clean +.PHONY: all fmt-check vet build test test-envtest test-cli-integration ci operator-build admission-build operator-image admission-image helm-lint helm-template helm-digest-check helm-scheduling-check helm-admission-check helm-lifecycle-check helm-check print-setup-envtest-version clean all: build @@ -40,6 +40,10 @@ print-setup-envtest-version: test-envtest: $(SETUP_ENVTEST) KUBEBUILDER_ASSETS="$$($(SETUP_ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GO) test -race ./... +test-cli-integration: $(SETUP_ENVTEST) + KUBEBUILDER_ASSETS="$${KUBEBUILDER_ASSETS:-$$($(SETUP_ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)}" \ + BREWLET_REQUIRE_CLI_INTEGRATION=true $(GO) test -race -count=1 -v ./internal/cli + ci: fmt-check vet build test-envtest helm-check operator-build: diff --git a/kubernetes/README.md b/kubernetes/README.md index 34cd87d0..82530907 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -104,6 +104,38 @@ make -C kubernetes test-envtest make -C kubernetes helm-check ``` +### CLI integration tests + +With Go and Helm installed, run the built CLI against an isolated API server: + +```bash +make -C kubernetes test-cli-integration +``` + +This target obtains the pinned envtest assets (kube-apiserver, etcd, and kubectl) +and runs `internal/cli` without contacting your configured cluster. It builds +`core/cmd/brewlet`, uses real kubectl and Helm subprocesses, and installs the +shipped CRDs in a fresh disposable API server with RBAC enabled. Its private +kubeconfigs deliberately have an invalid default context so explicit connection +and namespace handling are exercised. The existing Kubernetes CI test job runs +the suite with prerequisites required, rather than silently skipping it. + +Coverage includes inventory/status/doctor and compatibility aliases; profile +and application inspection; persistent JDK/launcher additions and replacement; +non-persisting client/server dry runs; failed dry runs with empty stdout; +admission and RBAC rejection; Helm ownership and offline values; stale +resource-version/recreated-UID conflicts; and the existing-CRD install guard. +Real profile and application reconciliation verifies that CLI writes feed the +controller's desired resources. Helm renders generated values using the local +chart, without downloading a released chart. + +These are API/process integration tests, not node-runtime E2E. Fixture node +inventory and readiness statuses are explicitly simulated: envtest has no +kubelet, scheduler, or Deployment controller. The suite does not prove a fresh +`brewlet k8s install` completes, download JDK images, execute privileged +provisioners, or launch a JVM. See the +[E2E runbook](../integration-tests/AGENTS.md) for the separate live-node tiers. + Build the component images from the repository root so the image also receives the shared license and notice-generation inputs: diff --git a/kubernetes/internal/cli/integration_test.go b/kubernetes/internal/cli/integration_test.go new file mode 100644 index 00000000..0b8c1536 --- /dev/null +++ b/kubernetes/internal/cli/integration_test.go @@ -0,0 +1,684 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cli_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + nodeapi "brewlet-operator/api/nodeprofile/v1alpha1" + appapi "brewlet-operator/api/v1alpha1" + "brewlet-operator/internal/controller" + + admissionv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +const sourceImage = "registry.example.com/jdk@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const replacementImage = "registry.example.com/jdk@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +type fixture struct { + ctx context.Context + api client.Client + scheme *runtime.Scheme + config *rest.Config + server *envtest.Environment + binary, helm string + kubeconfig, work string + env []string +} + +type result struct { + stdout, stderr string + code int +} + +// These are API/process integration tests, not node-runtime E2E: envtest has no +// kubelet or workload controllers. Only fixture readiness/inventory is simulated. +func TestCLIIntegration(t *testing.T) { + assets := os.Getenv("KUBEBUILDER_ASSETS") + if assets == "" { + if os.Getenv("BREWLET_REQUIRE_CLI_INTEGRATION") == "true" { + t.Fatal("KUBEBUILDER_ASSETS is required; run make test-cli-integration") + } + t.Skip("requires envtest assets; run make test-cli-integration") + } + assets, err := filepath.Abs(assets) + must(t, err) + for _, name := range []string{"kube-apiserver", "etcd", "kubectl"} { + info, err := os.Stat(filepath.Join(assets, name)) + if err != nil || info.IsDir() { + t.Fatalf("missing envtest binary %s: %v", name, err) + } + } + helm, err := exec.LookPath("helm") + must(t, err) + work := t.TempDir() + binary := filepath.Join(work, "brewlet") + buildCtx, cancelBuild := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelBuild() + build := exec.CommandContext(buildCtx, "go", "-C", "../../../core", "build", "-o", binary, "./cmd/brewlet") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build CLI: %v\n%s", err, out) + } + scheme := runtime.NewScheme() + must(t, clientgoscheme.AddToScheme(scheme)) + must(t, nodeapi.AddToScheme(scheme)) + must(t, appapi.AddToScheme(scheme)) + server := &envtest.Environment{ + UseExistingCluster: ptr.To(false), + BinaryAssetsDirectory: assets, + CRDDirectoryPaths: []string{"../../charts/brewlet/crds"}, + ErrorIfCRDPathMissing: true, + } + server.ControlPlane.GetAPIServer().Configure().Set("authorization-mode", "RBAC") + config, err := server.Start() + must(t, err) + t.Cleanup(func() { + if err := server.Stop(); err != nil { + t.Errorf("stop isolated API server: %v", err) + } + }) + api, err := client.New(config, client.Options{Scheme: scheme}) + must(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + t.Cleanup(cancel) + f := &fixture{ctx: ctx, api: api, scheme: scheme, config: config, server: server, + work: work, binary: binary, helm: helm} + f.kubeconfig = f.writeConfig(t, "admin", config) + // Poison defaults so a missed explicit context/path fails rather than + // accidentally using the user's current cluster. + poison := filepath.Join(work, "default-kubeconfig") + must(t, os.WriteFile(poison, []byte("not a kubeconfig\n"), 0o600)) + f.env = append(os.Environ(), + "KUBECONFIG="+poison, + "PATH="+assets+string(os.PathListSeparator)+filepath.Dir(helm)+string(os.PathListSeparator)+os.Getenv("PATH"), + "HELM_CACHE_HOME="+filepath.Join(work, "helm-cache"), + "HELM_CONFIG_HOME="+filepath.Join(work, "helm-config"), + "HELM_DATA_HOME="+filepath.Join(work, "helm-data"), + ) + for _, name := range []string{"runtime-system", "team", "other-team"} { + must(t, api.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}})) + } + must(t, api.Create(ctx, &nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "brewlet"}, Handler: "brewlet"})) + t.Run("inventory-status-doctor", f.testReadCommands) + t.Run("live-and-dry-run-inventory-updates", f.testInventoryUpdates) + t.Run("helm-owned-profile-and-offline-values", f.testHelmOwnership) + t.Run("rbac-denied-dry-run", f.testRBAC) + t.Run("admission-denied-dry-run", f.testAdmission) + t.Run("concurrent-updates", f.testConflicts) + t.Run("application-inspection", f.testAppInspection) + t.Run("install-refuses-existing-crds", f.testInstallGuard) + unchanged, err := os.ReadFile(poison) + must(t, err) + if string(unchanged) != "not a kubeconfig\n" { + t.Fatal("CLI changed the default kubeconfig") + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +func (f *fixture) writeConfig(t *testing.T, name string, config *rest.Config) string { + t.Helper() + file := filepath.Join(f.work, name+"-kubeconfig") + must(t, clientcmd.WriteToFile(clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{"isolated": { + Server: config.Host, CertificateAuthorityData: config.CAData, CertificateAuthority: config.CAFile, + }}, + AuthInfos: map[string]*clientcmdapi.AuthInfo{"isolated": { + ClientCertificateData: config.CertData, ClientKeyData: config.KeyData, + ClientCertificate: config.CertFile, ClientKey: config.KeyFile, + }}, + Contexts: map[string]*clientcmdapi.Context{ + "selected": {Cluster: "isolated", AuthInfo: "isolated", Namespace: "other-team"}, + "wrong": {Cluster: "missing", AuthInfo: "missing"}, + }, + CurrentContext: "wrong", + }, file)) + return file +} + +func (f *fixture) command(t *testing.T, program string, args ...string) result { + t.Helper() + ctx, cancel := context.WithTimeout(f.ctx, 25*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, program, args...) + cmd.WaitDelay = time.Second + cmd.Env = f.env + cmd.Dir = f.work + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + if ctx.Err() != nil { + t.Fatalf("%s %v timed out: %v\n%s", program, args, ctx.Err(), stderr.String()) + } + code := 0 + if err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + t.Fatalf("start %s: %v", program, err) + } + code = exit.ExitCode() + } + return result{stdout.String(), stderr.String(), code} +} + +func (f *fixture) cli(t *testing.T, args ...string) result { + t.Helper() + return f.cliConfig(t, f.kubeconfig, args...) +} + +func (f *fixture) cliConfig(t *testing.T, config string, args ...string) result { + t.Helper() + base := []string{"k8s", "--kubeconfig", config, "--context", "selected", "--timeout", "10s"} + return f.command(t, f.binary, append(base, args...)...) +} + +func (r result) success(t *testing.T) string { + t.Helper() + if r.code != 0 { + t.Fatalf("CLI failed with exit %d\nstdout: %s\nstderr: %s", r.code, r.stdout, r.stderr) + } + return r.stdout +} + +func (r result) failure(t *testing.T, reason string) { + t.Helper() + if r.code != 1 || r.stdout != "" || !strings.Contains(r.stderr, reason) { + t.Fatalf("expected exit 1, empty stdout, and %q; got %+v", reason, r) + } +} + +func decode[T any](t *testing.T, raw string) T { + t.Helper() + var value T + must(t, json.Unmarshal([]byte(raw), &value)) + return value +} + +func (f *fixture) profile(t *testing.T, name string) *nodeapi.NodeProfile { + t.Helper() + p := &nodeapi.NodeProfile{ + TypeMeta: metav1.TypeMeta{APIVersion: nodeapi.GroupVersion.String(), Kind: "NodeProfile"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"fixture": name}}, + Spec: nodeapi.NodeProfileSpec{ + NodePool: nodeapi.NodePoolRef{Names: []string{name}, Key: "example.com/pool"}, + JDKs: []nodeapi.JDKRef{{Distribution: "temurin", Feature: 21, + Source: nodeapi.JDKSource{Image: sourceImage, JavaHome: "/opt/java/openjdk"}}}, + Launchers: []nodeapi.LauncherRef{{Name: "jaz", + Source: nodeapi.LauncherSource{Image: sourceImage, Path: "/usr/bin/jaz"}}}, + Rollout: nodeapi.RolloutSpec{MaxUnavailable: ptr.To(intstr.FromInt(1)), Validate: ptr.To(true)}, + }, + } + must(t, f.api.Create(f.ctx, p, client.FieldOwner("kubectl"))) + return p +} + +func (f *fixture) getProfile(t *testing.T, name string) *nodeapi.NodeProfile { + t.Helper() + p := &nodeapi.NodeProfile{} + must(t, f.api.Get(f.ctx, client.ObjectKey{Name: name}, p)) + return p +} + +func (f *fixture) assertUnchanged(t *testing.T, before *nodeapi.NodeProfile) { + t.Helper() + after := f.getProfile(t, before.Name) + if after.ResourceVersion != before.ResourceVersion || !reflect.DeepEqual(after.Spec, before.Spec) { + t.Fatalf("dry run or refused update persisted changes: before=%+v after=%+v", before, after) + } +} + +func jdkArgs(profile string, extra ...string) []string { + return append([]string{"jdk", "add", "--profile", profile, "--distribution", "temurin", + "--feature", "25", "--image", sourceImage, "--java-home", "/opt/java/openjdk", "--output", "json"}, extra...) +} + +func launcherArgs(profile string, extra ...string) []string { + return append([]string{"launcher", "add", "--profile", profile, "--name", "custom", + "--image", sourceImage, "--path", "/usr/bin/custom", "--output", "json"}, extra...) +} + +func (f *fixture) node(t *testing.T, p *nodeapi.NodeProfile) *corev1.Node { + t.Helper() + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: p.Name + "-node", Labels: map[string]string{ + "example.com/pool": p.Name, "kubernetes.io/arch": "amd64", + "brewlet.sh/runtime": "ready", "brewlet.sh/owner-uid": string(p.UID), + }, Annotations: map[string]string{ + "brewlet.sh/owner-name": p.Name, "brewlet.sh/profile-generation": fmt.Sprint(p.Generation), + "brewlet.sh/jdks": "temurin-21", "brewlet.sh/launchers": "jaz", + "brewlet.sh/jdks-info": `[{"distribution":"temurin","vendor":"Adoptium","feature":21,"version":"21.0.5","arch":"amd64"}]`, + }}} + must(t, f.api.Create(f.ctx, node)) + node.Labels["brewlet.sh/owner-node-uid"] = string(node.UID) + must(t, f.api.Update(f.ctx, node)) + node.Status.NodeInfo.ContainerRuntimeVersion = "containerd://2.0.0" + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + must(t, f.api.Status().Update(f.ctx, node)) + // No node controller is present to remove the initial NotReady taint. + node.Spec.Taints = nil + must(t, f.api.Update(f.ctx, node)) + return node +} + +func (f *fixture) deployment(t *testing.T, name string) *appsv1.Deployment { + t.Helper() + labels := map[string]string{"app": name} + d := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "runtime-system"}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fixture", Image: sourceImage}}}}, + }} + must(t, f.api.Create(f.ctx, d)) + f.readyDeployment(t, d) + return d +} + +func (f *fixture) readyDeployment(t *testing.T, d *appsv1.Deployment) { + t.Helper() + d.Status = appsv1.DeploymentStatus{ObservedGeneration: d.Generation, Replicas: 1, + UpdatedReplicas: 1, ReadyReplicas: 1, AvailableReplicas: 1} + must(t, f.api.Status().Update(f.ctx, d)) +} + +func (f *fixture) testReadCommands(t *testing.T) { + p := f.profile(t, "inventory") + f.node(t, p) + p.Status = nodeapi.NodeProfileStatus{ObservedGeneration: p.Generation, AssignedNodes: 1, ReadyNodes: 1, + Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "AllNodesProvisioned", + Message: "fixture readiness", ObservedGeneration: p.Generation, LastTransitionTime: metav1.Now()}}} + must(t, f.api.Status().Update(f.ctx, p)) + operator := f.deployment(t, "brewlet-operator") + f.deployment(t, "brewlet-admission") + jdks := f.cli(t, "jdk", "list", "--selector", "example.com/pool=inventory", "--output", "json").success(t) + rows := decode[[]struct { + Distribution, Version, Arch string + Nodes []string + }](t, jdks) + if len(rows) != 1 || rows[0].Distribution != "temurin" || rows[0].Version != "21.0.5" || + rows[0].Arch != "amd64" || !reflect.DeepEqual(rows[0].Nodes, []string{"inventory-node"}) { + t.Fatalf("unexpected advertised inventory: %s", jdks) + } + aliases := f.command(t, f.binary, "jdks", "--kubeconfig", f.kubeconfig, "--context", "selected", + "--selector", "example.com/pool=inventory", "--output", "json").success(t) + if aliases != jdks { + t.Fatal("legacy jdks alias differs from k8s inventory") + } + empty := f.cli(t, "jdk", "list", "--selector", "example.com/pool=absent", "--output", "json").success(t) + if len(decode[[]json.RawMessage](t, empty)) != 0 { + t.Fatalf("node selector was ignored: %s", empty) + } + launchers := f.cli(t, "launcher", "list", "--output", "json").success(t) + if !strings.Contains(launchers, `"jaz"`) || !strings.Contains(launchers, `"inventory-node"`) { + t.Fatal(launchers) + } + for _, command := range []string{"jdk", "launcher"} { + if out := f.cli(t, command, "list", "--output", "wide").success(t); !strings.Contains(out, "inventory-node") { + t.Fatalf("%s per-node inventory missing node: %s", command, out) + } + } + if out := f.cli(t, "profile", "list").success(t); !strings.Contains(out, "temurin-21") { + t.Fatal(out) + } + profile := decode[struct { + Profile struct{ Ready bool } + Nodes []struct{ Name string } + }](t, f.cli(t, "profile", "inspect", p.Name, "--output", "json").success(t)) + if !profile.Profile.Ready || len(profile.Nodes) != 1 || profile.Nodes[0].Name != "inventory-node" { + t.Fatalf("profile inspection: %+v", profile) + } + statusArgs := []string{"status", "--system-namespace", "runtime-system", "--output", "json"} + status := decode[struct{ Healthy bool }](t, f.cli(t, statusArgs...).success(t)) + if !status.Healthy { + t.Fatal("ready fixtures reported unhealthy") + } + f.cli(t, "doctor", "--namespace", "team", "--output", "json").success(t) + f.command(t, f.binary, "doctor", "--kubeconfig", f.kubeconfig, "--context", "selected", + "--namespace", "team", "--output", "json").success(t) + + operator.Status.UpdatedReplicas = 0 + must(t, f.api.Status().Update(f.ctx, operator)) + unready := f.cli(t, statusArgs...) + if unready.code != 1 || decode[struct{ Healthy bool }](t, unready.stdout).Healthy { + t.Fatalf("old available replicas counted as a current rollout: %+v", unready) + } + f.readyDeployment(t, operator) + p.Status.Conditions[0].ObservedGeneration-- + must(t, f.api.Status().Update(f.ctx, p)) + unready = f.cli(t, statusArgs...) + if unready.code != 1 || decode[struct{ Healthy bool }](t, unready.stdout).Healthy { + t.Fatalf("stale profile condition counted as ready: %+v", unready) + } +} + +func (f *fixture) testInventoryUpdates(t *testing.T) { + p := f.profile(t, "updates") + for _, args := range [][]string{jdkArgs(p.Name, "--image", "invalid:source", "--dry-run"), + launcherArgs(p.Name, "--path", "/usr/../bin/custom", "--dry-run")} { + r := f.cli(t, args...) + if r.code != 1 || r.stdout != "" || r.stderr == "" { + t.Fatalf("invalid client dry run should fail with empty output: %+v", r) + } + f.assertUnchanged(t, p) + } + for _, flags := range [][]string{{"--dry-run"}, {"--dry-run=client"}, {"--dry-run=server"}} { + for _, args := range [][]string{jdkArgs(p.Name, flags...), launcherArgs(p.Name, flags...)} { + proposal := decode[nodeapi.NodeProfile](t, f.cli(t, args...).success(t)) + if len(proposal.Spec.JDKs)+len(proposal.Spec.Launchers) != 3 { + t.Fatalf("dry run did not render addition: %+v", proposal.Spec) + } + f.assertUnchanged(t, p) + } + } + f.cli(t, jdkArgs(p.Name)...).success(t) + f.cli(t, launcherArgs(p.Name)...).success(t) + updated := f.getProfile(t, p.Name) + if len(updated.Spec.JDKs) != 2 || len(updated.Spec.Launchers) != 2 || + !reflect.DeepEqual(updated.Spec.Rollout, p.Spec.Rollout) || updated.Labels["fixture"] != p.Name || + updated.Generation <= p.Generation { + t.Fatalf("live updates did not preserve/apply configuration: %+v", updated) + } + f.cli(t, jdkArgs(p.Name)...).success(t) + repeated := f.getProfile(t, p.Name) + if repeated.Generation != updated.Generation || !reflect.DeepEqual(repeated.Spec, updated.Spec) { + t.Fatal("identical add was not idempotent") + } + f.cli(t, jdkArgs(p.Name, "--image", replacementImage)...).failure(t, "--replace") + f.assertUnchanged(t, repeated) + f.cli(t, jdkArgs(p.Name, "--image", replacementImage, "--replace")...).success(t) + if got := f.getProfile(t, p.Name).Spec.JDKs[1].Source.Image; got != replacementImage { + t.Fatalf("explicit replacement was not saved: %s", got) + } + // Exercise the real profile reconciler without starting any host worker. + r := &controller.NodeProfileReconciler{Client: f.api, Recorder: record.NewFakeRecorder(100), + Config: controller.Config{Namespace: "runtime-system", ProvisionerImage: "registry.invalid/never-executed:cli-test"}} + for range 2 { + _, err := r.Reconcile(f.ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: p.Name}}) + must(t, err) + } + ds := &appsv1.DaemonSet{} + must(t, f.api.Get(f.ctx, client.ObjectKey{Namespace: "runtime-system", Name: "brewlet-node-provisioner-" + p.Name}, ds)) + env := map[string]string{} + for _, entry := range ds.Spec.Template.Spec.Containers[0].Env { + env[entry.Name] = entry.Value + } + if env["JDK_SOURCE_COUNT"] != "2" || env["LAUNCHER_SOURCE_COUNT"] != "2" { + t.Fatalf("controller did not consume CLI additions: %+v", env) + } + inspected := f.cli(t, "profile", "inspect", p.Name, "--output", "json").success(t) + if !strings.Contains(inspected, replacementImage) || !strings.Contains(inspected, "custom") { + t.Fatal("profile inspection did not show the reconciled desired inventory", inspected) + } +} + +func (f *fixture) testHelmOwnership(t *testing.T) { + values := filepath.Join(f.work, "values.json") + must(t, os.WriteFile(values, []byte(fmt.Sprintf(`{"provisioner":{"pools":["helm-pool"],"jdks":[ + {"distribution":"temurin","feature":21,"source":{"image":%q,"javaHome":"/opt/java/openjdk"}}]}}`, sourceImage)), 0o600)) + chart, err := filepath.Abs("../../charts/brewlet") + must(t, err) + rendered := f.command(t, f.helm, "template", "brewlet", chart, "--namespace", "runtime-system", + "--values", values, "--show-only", "templates/nodeprofiles.yaml").success(t) + manifest := filepath.Join(f.work, "helm-profile.yaml") + must(t, os.WriteFile(manifest, []byte(rendered), 0o600)) + f.command(t, "kubectl", "--kubeconfig", f.kubeconfig, "--context", "selected", + "apply", "-f", manifest).success(t) + p := f.getProfile(t, "default") + for _, args := range [][]string{jdkArgs("default"), launcherArgs("default"), jdkArgs("default", "--dry-run=server")} { + f.cli(t, args...).failure(t, "managed by") + f.assertUnchanged(t, p) + } + proposal := f.cli(t, jdkArgs("default", "--dry-run", "--values", values)...).success(t) + next := filepath.Join(f.work, "values-next.json") + must(t, os.WriteFile(next, []byte(proposal), 0o600)) + rendered = f.command(t, f.helm, "template", "brewlet", chart, "--namespace", "runtime-system", + "--values", next, "--show-only", "templates/nodeprofiles.yaml").success(t) + if !strings.Contains(rendered, "feature: 25") || !strings.Contains(rendered, "feature: 21") { + t.Fatal("CLI-generated Helm values lost or failed to add inventory", rendered) + } + f.assertUnchanged(t, p) + offline := f.cli(t, launcherArgs("default", "--dry-run", "--file", manifest)...).success(t) + if len(decode[nodeapi.NodeProfile](t, offline).Spec.Launchers) != 1 { + t.Fatal("offline profile proposal omitted launcher") + } + f.assertUnchanged(t, p) +} + +func (f *fixture) testRBAC(t *testing.T) { + p := f.profile(t, "read-only") + user, err := f.server.AddUser(envtest.User{Name: "cli-reader"}, f.config) + must(t, err) + config := f.writeConfig(t, "reader", user.Config()) + must(t, f.api.Create(f.ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "cli-reader"}, + Rules: []rbacv1.PolicyRule{{APIGroups: []string{"node.brewlet.sh"}, Resources: []string{"nodeprofiles"}, + ResourceNames: []string{p.Name}, Verbs: []string{"get"}}}})) + must(t, f.api.Create(f.ctx, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "cli-reader"}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "cli-reader"}, + Subjects: []rbacv1.Subject{{Kind: "User", APIGroup: rbacv1.GroupName, Name: "cli-reader"}}})) + must(t, wait.PollUntilContextTimeout(f.ctx, 100*time.Millisecond, 10*time.Second, true, func(context.Context) (bool, error) { + r := f.command(t, "kubectl", "--kubeconfig", config, "--context", "selected", "auth", "can-i", + "get", "nodeprofiles.node.brewlet.sh/"+p.Name) + return r.code == 0 && strings.TrimSpace(r.stdout) == "yes", nil + })) + f.cliConfig(t, config, jdkArgs(p.Name, "--dry-run")...).success(t) + for _, args := range [][]string{jdkArgs(p.Name), jdkArgs(p.Name, "--dry-run=server")} { + f.cliConfig(t, config, args...).failure(t, "forbidden") + f.assertUnchanged(t, p) + } + f.cliConfig(t, config, "jdk", "list", "--output", "json").failure(t, "forbidden") +} + +func (f *fixture) testAdmission(t *testing.T) { + p := f.profile(t, "policy") + fail := admissionv1.Fail + policy := &admissionv1.ValidatingAdmissionPolicy{ObjectMeta: metav1.ObjectMeta{Name: "cli-test-source-policy"}, + Spec: admissionv1.ValidatingAdmissionPolicySpec{ + FailurePolicy: &fail, + MatchConstraints: &admissionv1.MatchResources{ + ObjectSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"fixture": p.Name}}, + ResourceRules: []admissionv1.NamedRuleWithOperations{{RuleWithOperations: admissionv1.RuleWithOperations{ + Operations: []admissionv1.OperationType{admissionv1.Update}, + Rule: admissionv1.Rule{APIGroups: []string{"node.brewlet.sh"}, APIVersions: []string{"v1alpha1"}, Resources: []string{"nodeprofiles"}}, + }}}, + }, + Validations: []admissionv1.Validation{{Expression: "!object.spec.jdks.exists(j, j.distribution == 'blocked')", + Message: "CLI integration source policy denied this distribution"}}, + }} + must(t, f.api.Create(f.ctx, policy)) + must(t, f.api.Create(f.ctx, &admissionv1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{Name: policy.Name}, + Spec: admissionv1.ValidatingAdmissionPolicyBindingSpec{PolicyName: policy.Name, ValidationActions: []admissionv1.ValidationAction{admissionv1.Deny}}, + })) + // Wait for admission registration using an API dry run; never persist the + // deliberately disallowed inventory while the policy cache is warming. + must(t, wait.PollUntilContextTimeout(f.ctx, 100*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (bool, error) { + candidate := p.DeepCopy() + candidate.Spec.JDKs[0].Distribution = "blocked" + err := f.api.Update(ctx, candidate, client.DryRunAll) + if err == nil { + return false, nil + } + if strings.Contains(err.Error(), "CLI integration source policy denied") { + return true, nil + } + return false, err + })) + for _, format := range []string{"json", "yaml"} { + for _, dryRun := range []bool{false, true} { + args := jdkArgs(p.Name, "--distribution", "blocked", "--output", format) + if dryRun { + args = append(args, "--dry-run=server") + } + f.cli(t, args...).failure(t, "CLI integration source policy denied") + f.assertUnchanged(t, p) + } + } + f.cli(t, jdkArgs(p.Name, "--distribution", "blocked", "--dry-run")...).success(t) + f.assertUnchanged(t, p) +} + +func (f *fixture) testConflicts(t *testing.T) { + for _, replace := range []bool{false, true} { + for _, dryRun := range []bool{false, true} { + t.Run(fmt.Sprintf("recreate=%t/dry-run=%t", replace, dryRun), func(t *testing.T) { + p := f.profile(t, fmt.Sprintf("conflict-%t-%t", replace, dryRun)) + upstream, err := url.Parse(f.config.Host) + must(t, err) + proxy := httputil.NewSingleHostReverseProxy(upstream) + proxy.Transport, err = rest.TransportFor(f.config) + must(t, err) + var once sync.Once + mutations := make(chan error, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPatch && strings.HasSuffix(r.URL.Path, "/nodeprofiles/"+p.Name) { + once.Do(func() { + current := &nodeapi.NodeProfile{} + err := f.api.Get(r.Context(), client.ObjectKey{Name: p.Name}, current) + if err == nil && replace { + err = f.api.Delete(r.Context(), current) + if err == nil { + err = f.api.Create(r.Context(), &nodeapi.NodeProfile{ + ObjectMeta: metav1.ObjectMeta{Name: p.Name}, Spec: current.Spec}) + } + } else if err == nil { + current.Annotations = map[string]string{"concurrent-writer": "preserve-me"} + err = f.api.Update(r.Context(), current) + } + mutations <- err + }) + } + proxy.ServeHTTP(w, r) + })) + defer server.Close() + config := f.writeConfig(t, p.Name, &rest.Config{Host: server.URL}) + args := jdkArgs(p.Name) + if dryRun { + args = append(args, "--dry-run=server") + } + r := f.cliConfig(t, config, args...) + if r.code != 1 || r.stdout != "" || !strings.Contains(r.stderr, "profile update failed") { + t.Fatalf("stale patch was not rejected with empty output: %+v", r) + } + select { + case err := <-mutations: + must(t, err) + default: + t.Fatal("test did not inject a concurrent write") + } + current := f.getProfile(t, p.Name) + if !reflect.DeepEqual(current.Spec, p.Spec) { + t.Fatal("stale patch overwrote the current profile") + } + if replace && current.UID == p.UID { + t.Fatal("replacement did not change the UID") + } + if !replace && current.Annotations["concurrent-writer"] != "preserve-me" { + t.Fatal("concurrent writer's change was lost") + } + }) + } + } +} + +func (f *fixture) testAppInspection(t *testing.T) { + app := &appapi.JavaApplication{ObjectMeta: metav1.ObjectMeta{Name: "orders", Namespace: "team"}, + Spec: appapi.JavaApplicationSpec{Artifact: appapi.ArtifactSpec{Image: sourceImage}, + JVM: appapi.JVMSpec{Version: 21}, Replicas: ptr.To(int32(1)), + Env: []corev1.EnvVar{{Name: "TOKEN", Value: "fixture-secret-must-not-appear"}}}} + must(t, f.api.Create(f.ctx, app)) + reconciler := &controller.JavaApplicationReconciler{Client: f.api, APIReader: f.api, Scheme: f.scheme, + Recorder: record.NewFakeRecorder(100)} + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(app)} + _, err := reconciler.Reconcile(f.ctx, request) + must(t, err) + d := &appsv1.Deployment{} + must(t, f.api.Get(f.ctx, request.NamespacedName, d)) + rs := &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Name: "orders-rs", Namespace: "team", Labels: d.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(d, appsv1.SchemeGroupVersion.WithKind("Deployment"))}}, + Spec: appsv1.ReplicaSetSpec{Replicas: ptr.To(int32(1)), Selector: d.Spec.Selector, Template: d.Spec.Template}} + must(t, f.api.Create(f.ctx, rs)) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orders-pod", Namespace: "team", Labels: d.Spec.Template.Labels, + Annotations: d.Spec.Template.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(rs, appsv1.SchemeGroupVersion.WithKind("ReplicaSet"))}}, + Spec: d.Spec.Template.Spec} + must(t, f.api.Create(f.ctx, pod)) + pod.Status.Phase = corev1.PodPending + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: pod.Spec.Containers[0].Name, + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}} + must(t, f.api.Status().Update(f.ctx, pod)) + foreign := pod.DeepCopy() + foreign.ObjectMeta = metav1.ObjectMeta{Name: "foreign", Namespace: "team", Labels: pod.Labels} + foreign.Status = corev1.PodStatus{} + must(t, f.api.Create(f.ctx, foreign)) + for _, target := range []*corev1.Pod{pod, foreign} { + must(t, f.api.Create(f.ctx, &corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: target.Name + "-event", Namespace: "team"}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: target.Name, Namespace: "team", UID: target.UID}, + Type: "Warning", Reason: "Failed", Message: "fixture image unavailable"})) + } + args := []string{"inspect", "app", "orders", "--namespace", "team", "--output", "json"} + raw := f.cli(t, args...).success(t) + report := decode[struct { + Ready bool + JDKRequest string + Pods []struct{ Name string } + Events []struct{ Object string } + }](t, raw) + if report.Ready || report.JDKRequest != "21" || len(report.Pods) != 1 || len(report.Events) != 1 || + report.Pods[0].Name != pod.Name || strings.Contains(raw, "fixture-secret") || strings.Contains(raw, "foreign") || + !strings.Contains(raw, "ImagePullBackOff") { + t.Fatalf("wrong ownership, readiness, or sensitive data in report: %s", raw) + } + f.readyDeployment(t, d) + _, err = reconciler.Reconcile(f.ctx, request) + must(t, err) + if report := decode[struct{ Ready bool }](t, f.cli(t, args...).success(t)); !report.Ready { + t.Fatal("current completed deployment rollout was not reported ready") + } + f.cli(t, "inspect", "app", "orders", "--output", "json").failure(t, "not found") +} + +func (f *fixture) testInstallGuard(t *testing.T) { + values := filepath.Join(f.work, "no-profiles.json") + must(t, os.WriteFile(values, []byte(`{"defaultProfile":{"enabled":false}}`), 0o600)) + f.cli(t, "install", "--version", "0.0.0", "--values", values). + failure(t, "Brewlet CRDs already exist") +} diff --git a/site/scripts/test_installation_examples.py b/site/scripts/test_installation_examples.py index 769a0d1a..d784f32c 100644 --- a/site/scripts/test_installation_examples.py +++ b/site/scripts/test_installation_examples.py @@ -156,6 +156,40 @@ def test_every_documented_install_preview_and_upgrade_renders(self): self.assertIn(image, result.stdout, "Upgrade lost administrator image choice") self.assertGreaterEqual(command_count, 10, "A documented command disappeared from coverage") + def test_cli_values_example_is_self_contained_and_renders(self): + document = (ROOT / "docs/cli-reference.md").read_text() + installation = document.split("### Installation\n", 1)[1].split( + "### Safe JDK and launcher additions\n", 1 + )[0] + inventory = self.inventory(installation) + self.assertIn(" - java-workers\n", inventory) + self.assertIn("@sha256:<64-lowercase-hex>", inventory) + self.assertIn("configuration.md#helm-chart-values", installation) + self.assertIn("kubernetes/charts/brewlet/values.yaml", installation) + self.assertIn("save the following as `values.yaml`", installation) + self.assertLess(installation.index("```yaml"), installation.index("brewlet k8s install")) + self.assertIn("--values values.yaml --dry-run", installation) + + values = self.work / "cli-values.yaml" + values.write_text(inventory.replace("<64-lowercase-hex>", DIGEST)) + result = self.render(["--values", str(values)]) + self.assertEqual(result.returncode, 0, result.stderr) + profiles = [doc for doc in result.stdout.split("---\n") + if "\nkind: NodeProfile\n" in doc] + self.assertEqual(len(profiles), 1) + profile = profiles[0] + self.assertIn("\n name: default\n", profile) + self.assertIn('\n - "java-workers"\n', profile) + self.assertIn(" - distribution: temurin\n", profile) + self.assertIn(" feature: 21\n", profile) + self.assertIn( + f" image: docker.io/library/eclipse-temurin@sha256:{DIGEST}\n", + profile, + ) + self.assertIn(" javaHome: /opt/java/openjdk\n", profile) + self.assertNotIn("includeControlPlane: true", profile) + self.assertNotIn("\n launchers:", profile) + def test_fresh_install_guides_default_to_latest_chart(self): for filename in ("README.md", "docs/installation.md", "kubernetes/README.md", "kubernetes/charts/brewlet/README.md",