From 7d99a8c05568f019431823cc3aae7b35ad84429d Mon Sep 17 00:00:00 2001 From: Bruno Borges Date: Thu, 24 Sep 2026 15:48:00 -0300 Subject: [PATCH 1/2] Add Kubernetes operations to the Brewlet CLI Add cluster inventory, status, profile and application inspection, fresh Helm installation, and guarded JDK and launcher updates. Consolidate previews under client/server dry-run modes with no rendered output on failure, preserve existing aliases, and document installation values and operational safeguards. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59dd5181-736f-4687-9205-d75210e7035f --- core/README.md | 23 + core/cmd/brewlet/main.go | 108 +-- core/go.mod | 1 + core/go.sum | 3 + core/internal/kube/cli.go | 274 ++++++ core/internal/kube/cli_test.go | 942 +++++++++++++++++++++ core/internal/kube/client.go | 233 +++++ core/internal/kube/install.go | 105 +++ core/internal/kube/read.go | 486 +++++++++++ core/internal/kube/update.go | 507 +++++++++++ docs/cli-reference.md | 311 ++++++- docs/configuration.md | 2 +- docs/installation.md | 6 + docs/jdk-management.md | 9 + docs/launchers.md | 9 + site/scripts/test_installation_examples.py | 34 + 16 files changed, 2941 insertions(+), 112 deletions(-) create mode 100644 core/internal/kube/cli.go create mode 100644 core/internal/kube/cli_test.go create mode 100644 core/internal/kube/client.go create mode 100644 core/internal/kube/install.go create mode 100644 core/internal/kube/read.go create mode 100644 core/internal/kube/update.go diff --git a/core/README.md b/core/README.md index 0a88ddd6..0d26f891 100644 --- a/core/README.md +++ b/core/README.md @@ -34,6 +34,29 @@ go -C core build ./... go -C core test ./... ``` +## Kubernetes operations + +`brewlet k8s` provides JDK/launcher inventory, profile and application inspection, +control-plane/node status, readiness checks, fresh Helm installation, and safe +profile additions. It uses installed `kubectl`/Helm binaries and existing +kubeconfig credentials rather than embedding a Kubernetes client. + +```bash +brewlet k8s --context staging jdk list +brewlet k8s profile list +brewlet k8s status +brewlet k8s inspect app orders --namespace my-team +``` + +`jdk add` and `launcher add` update live profiles by default; use `--dry-run` +for client-validated previews or `--dry-run=server` for server validation. +Proposed YAML/JSON is printed only after validation succeeds; a failed dry run +returns an error without rendering output. Helm and GitOps-owned profiles must +be changed at their source; client dry-run mode's `--values` and `--file` options +support offline generation without overwriting the input. See the +[CLI reference](../docs/cli-reference.md#brewlet-k8s) for installation, source +validation, conditional updates, output formats, and permissions. + ## Managed dependency bundles Platform teams can publish an approved dependency lock and deterministic diff --git a/core/cmd/brewlet/main.go b/core/cmd/brewlet/main.go index 2c087687..7d3b639c 100644 --- a/core/cmd/brewlet/main.go +++ b/core/cmd/brewlet/main.go @@ -13,6 +13,7 @@ package main import ( + "context" "crypto/ecdsa" "encoding/json" "flag" @@ -25,8 +26,7 @@ import ( "time" "github.com/microsoft/brewlet/internal/artifact" - "github.com/microsoft/brewlet/internal/doctor" - "github.com/microsoft/brewlet/internal/inventory" + "github.com/microsoft/brewlet/internal/kube" "github.com/microsoft/brewlet/internal/runtime" ) @@ -55,6 +55,8 @@ func main() { err = cmdJDKs(os.Args[2:]) case "doctor": err = cmdDoctor(os.Args[2:]) + case "k8s": + err = kube.Run(context.Background(), os.Args[2:], os.Stdout, os.Stderr) case "version", "--version": fmt.Println(version) return @@ -84,6 +86,7 @@ USAGE: brewlet bundle [--store DIR] [--cpu N] [--memory M] [--uid UID] [--gid GID] [--jdk-root DIR] [--launcher NAME] [--launcher-root DIR] [--out DIR] brewlet jdks [--output table|wide|json] [--kubeconfig FILE] [--context CTX] [--selector SEL] brewlet doctor [--namespace NS] [--output table|json] [--kubeconfig FILE] [--context CTX] + brewlet k8s inventory, status, inspection, installation and profile updates (see k8s --help) brewlet version is name:tag, e.g. demo/hello:1.0.0 @@ -800,106 +803,11 @@ func parseProcessIDFlag(name, value string) (uint32, error) { return uint32(id), nil } -// cmdJDKs lists the JDKs available across the cluster (vendor, major/minor -// version, architecture) by reading the inventory each Brewlet node advertises. -// It shells out to `kubectl get nodes -o json` and renders the result, so it -// needs a working kubectl/kubeconfig but no in-process Kubernetes client. +// Preserve the original entry points as aliases for the Kubernetes group. func cmdJDKs(args []string) error { - fs := flag.NewFlagSet("jdks", flag.ExitOnError) - output := fs.String("output", "table", "output format: table (distinct JDKs), wide (per node), or json") - kubeconfig := fs.String("kubeconfig", "", "path to kubeconfig (default: kubectl's own resolution)") - context := fs.String("context", "", "kubeconfig context to use") - selector := fs.String("selector", "", "label selector to filter nodes (kubectl -l)") - if err := fs.Parse(args); err != nil { - return err - } - - kubeArgs := []string{"get", "nodes", "-o", "json"} - if *kubeconfig != "" { - kubeArgs = append(kubeArgs, "--kubeconfig", *kubeconfig) - } - if *context != "" { - kubeArgs = append(kubeArgs, "--context", *context) - } - if *selector != "" { - kubeArgs = append(kubeArgs, "-l", *selector) - } - - cmd := exec.Command("kubectl", kubeArgs...) - var stderr strings.Builder - cmd.Stderr = &stderr - out, err := cmd.Output() - if err != nil { - if msg := strings.TrimSpace(stderr.String()); msg != "" { - return fmt.Errorf("kubectl get nodes failed: %s", msg) - } - return fmt.Errorf("running kubectl (is it installed and configured?): %w", err) - } - - nodes, err := inventory.ParseNodes(out) - if err != nil { - return err - } - - switch *output { - case "table": - inventory.RenderTable(os.Stdout, nodes) - case "wide": - inventory.RenderByNode(os.Stdout, nodes) - case "json": - return inventory.RenderJSON(os.Stdout, nodes) - default: - return fmt.Errorf("unknown --output %q (want table, wide, or json)", *output) - } - return nil + return kube.Run(context.Background(), append([]string{"jdk", "list"}, args...), os.Stdout, os.Stderr) } func cmdDoctor(args []string) error { - fs := flag.NewFlagSet("doctor", flag.ExitOnError) - namespace := fs.String("namespace", "default", "namespace where the developer will create JavaApplication resources") - output := fs.String("output", "table", "output format: table or json") - kubeconfig := fs.String("kubeconfig", "", "path to kubeconfig (default: kubectl's own resolution)") - context := fs.String("context", "", "kubeconfig context to use") - if err := fs.Parse(args); err != nil { - return err - } - - executor := func(kubectlArgs ...string) ([]byte, error) { - cmd := exec.Command("kubectl", kubectlArgs...) - out, err := cmd.CombinedOutput() - if err != nil { - if _, lookupErr := exec.LookPath("kubectl"); lookupErr != nil { - return out, fmt.Errorf("kubectl is not installed or not on PATH") - } - } - return out, err - } - report := doctor.Run(executor, doctor.Options{ - Kubeconfig: *kubeconfig, - Context: *context, - Namespace: *namespace, - }) - - switch *output { - case "table": - for _, check := range report.Checks { - fmt.Printf("[%-4s] %-20s %s\n", strings.ToUpper(string(check.Status)), check.Name, check.Detail) - if check.Remediation != "" && check.Status != doctor.Pass { - fmt.Printf(" fix: %s\n", check.Remediation) - } - } - case "json": - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - if err := enc.Encode(report); err != nil { - return err - } - default: - return fmt.Errorf("unknown --output %q (want table or json)", *output) - } - - if !report.OK() { - return fmt.Errorf("doctor found one or more blocking checks") - } - return nil + return kube.Run(context.Background(), append([]string{"doctor"}, args...), os.Stdout, os.Stderr) } diff --git a/core/go.mod b/core/go.mod index 7c061666..9ece9f91 100644 --- a/core/go.mod +++ b/core/go.mod @@ -14,6 +14,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/runtime-spec v1.3.0 github.com/prometheus/client_golang v1.24.1 + go.yaml.in/yaml/v3 v3.0.4 google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.12 ) diff --git a/core/go.sum b/core/go.sum index f11d079b..546621af 100644 --- a/core/go.sum +++ b/core/go.sum @@ -164,6 +164,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -230,6 +232,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/core/internal/kube/cli.go b/core/internal/kube/cli.go new file mode 100644 index 00000000..82208a56 --- /dev/null +++ b/core/internal/kube/cli.go @@ -0,0 +1,274 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "strings" + "time" + + "github.com/microsoft/brewlet/internal/doctor" + "github.com/microsoft/brewlet/internal/inventory" +) + +const help = `Brewlet Kubernetes operations + +USAGE: + brewlet k8s [connection flags] [flags] + +COMMANDS: + jdk list advertised node JDKs (table, wide, json) + launcher list advertised optional launchers (table, wide, json) + profile list desired inventories and profile readiness + profile inspect NAME profile configuration and its claimed nodes + inspect app NAME application, owned workloads, pods and events + status control-plane rollouts, profiles and node failures + doctor developer and cluster readiness checks + install --version X.Y.Z -f FILE install the released Helm chart on a fresh cluster + jdk add --profile NAME --distribution NAME --feature N --image REF --java-home PATH + launcher add --profile NAME --name NAME --image REF --path PATH + +COMMON FLAGS (before the command or after its full name): + --kubeconfig FILE kubeconfig path (otherwise kubectl/Helm defaults) + --context NAME context to use without changing kubeconfig + --namespace NAME application namespace (current context); doctor defaults to default; install to brewlet + --timeout 30s per-kubectl-command deadline + +READ FLAGS: + --output table|json|yaml default table (inventory also supports wide, not yaml) + --selector LABELS node selector for inventory + --system-namespace NAME status control-plane namespace (default brewlet) + +PROFILE ADD FLAGS: + --profile NAME existing target profile + --image REF digest-pinned runtime source + JDK: --distribution NAME --feature N --java-home PATH + Launcher: --name NAME --path PATH + --output yaml|json default yaml; prints the full updated declaration + --replace explicitly replace an existing source with the same identity + +ADD: + Updates the live profile by default, even when stdout is redirected. + --dry-run validate locally and print the proposed declaration (same as client) + --dry-run=client local validation only; no patch is sent to the API server + --dry-run=server validate through the API server and print the result without saving + Failed dry runs return an error and print no rendered YAML/JSON. + Helm/GitOps-owned live profiles cannot be patched. Edit their source of truth. + +OFFLINE INPUT (requires --dry-run or --dry-run=client): + By default add reads the live profile. These options read files instead: + --file FILE read a single NodeProfile YAML/JSON file instead of the cluster + --values FILE read complete Helm values and emit updated values + Input files are never overwritten. + +INSTALL FLAGS: + --values FILE, -f FILE complete Helm values (repeatable, at least one required) + --release NAME Helm release name (default brewlet) + --wait-timeout 5m Helm rollout deadline (node provisioning is separate) + --dry-run render chart manifests locally without installing + +Installation and profile changes never create a cluster or debug a workload. +` + +// Run dispatches the Kubernetes command group. Reads and generated declarations +// go to stdout; mutation notices and diagnostic hints go to stderr. +func Run(ctx context.Context, args []string, out, stderr io.Writer) error { + return run(ctx, args, out, stderr, execute) +} + +func commonFlags(fs *flag.FlagSet, opts *options) { + fs.StringVar(&opts.kubeconfig, "kubeconfig", opts.kubeconfig, "kubeconfig path") + fs.StringVar(&opts.context, "context", opts.context, "Kubernetes context") + fs.StringVar(&opts.namespace, "namespace", opts.namespace, "namespace (defaults to current context)") + fs.DurationVar(&opts.timeout, "timeout", opts.timeout, "per-command kubectl timeout") +} + +func run(ctx context.Context, args []string, out, stderr io.Writer, exec executor) error { + opts := options{timeout: 30 * time.Second, systemNamespace: "brewlet"} + root := flag.NewFlagSet("k8s", flag.ContinueOnError) + root.SetOutput(stderr) + // Keep usage/validation failures off stdout so redirected previews stay empty. + root.Usage = func() {} + commonFlags(root, &opts) + if err := root.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + _, err := fmt.Fprint(out, help) + return err + } + return err + } + args = root.Args() + if len(args) == 0 || args[0] == "help" { + _, err := fmt.Fprint(out, help) + return err + } + command := args[0] + args = args[1:] + if command == "jdk" || command == "launcher" || command == "profile" || command == "inspect" { + if len(args) == 0 || args[0] == "--help" || args[0] == "-h" { + _, err := fmt.Fprint(out, help) + return err + } + command += " " + args[0] + args = args[1:] + } + fs := flag.NewFlagSet("k8s "+command, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = root.Usage + commonFlags(fs, &opts) + var update updateOptions + var install installOptions + switch command { + case "jdk list", "launcher list": + fs.StringVar(&opts.output, "output", "table", "table, wide, or json") + fs.StringVar(&opts.selector, "selector", "", "node label selector") + case "status": + fs.StringVar(&opts.systemNamespace, "system-namespace", "brewlet", "Brewlet control-plane namespace") + fallthrough + case "profile list", "profile inspect", "inspect app", "doctor": + fs.StringVar(&opts.output, "output", "table", "table, json, or yaml") + case "jdk add", "launcher add": + update.flags(fs, command == "jdk add") + fs.StringVar(&opts.output, "output", "yaml", "yaml or json") + case "install": + install.flags(fs) + default: + return fmt.Errorf("unknown Kubernetes command %q; see brewlet k8s --help", command) + } + pos, err := parseFlags(fs, args) + if errors.Is(err, flag.ErrHelp) { + _, err := fmt.Fprint(out, help) + return err + } + if err != nil { + return err + } + expected := 0 + if command == "profile inspect" || command == "inspect app" { + expected = 1 + } + if len(pos) != expected { + return fmt.Errorf("%s expects %d positional arguments; see brewlet k8s --help", command, expected) + } + if opts.timeout <= 0 { + return fmt.Errorf("--timeout must be positive") + } + if expected == 1 { + if err := validateName(pos[0], 253); err != nil { + return err + } + } + allowed := []string{"table", "json", "yaml"} + switch command { + case "jdk list", "launcher list": + allowed = []string{"table", "wide", "json"} + case "jdk add", "launcher add": + allowed = []string{"yaml", "json"} + case "install": + allowed = []string{""} + } + if !oneOf(opts.output, allowed...) { + return fmt.Errorf("invalid --output %q (want %s)", opts.output, strings.Join(allowed, ", ")) + } + c := &client{ctx: ctx, opts: opts, exec: exec, out: out, err: stderr} + switch command { + case "jdk list": + return c.jdks() + case "launcher list": + return c.launchers() + case "profile list": + return c.profiles() + case "profile inspect": + return c.inspectProfile(pos[0]) + case "inspect app": + return c.inspectApp(pos[0]) + case "status": + return c.status() + case "doctor": + return c.doctor() + case "jdk add", "launcher add": + return c.updateProfile(update) + case "install": + return c.install(install) + } + panic("unreachable command") +} + +func parseFlags(fs *flag.FlagSet, args []string) ([]string, error) { + var pos []string + for len(args) > 0 { + if args[0] == "--" { + return append(pos, args[1:]...), nil + } + if err := fs.Parse(args); err != nil { + return nil, err + } + args = fs.Args() + if len(args) > 0 { + pos = append(pos, args[0]) + args = args[1:] + } + } + return pos, nil +} + +func oneOf(value string, choices ...string) bool { + for _, choice := range choices { + if value == choice { + return true + } + } + return false +} + +func (c *client) jdks() error { + objects, err := c.list("nodes", c.nodeArgs()...) + if err != nil { + return err + } + raw, err := json.Marshal(struct { + Items []object `json:"items"` + }{objects}) + if err != nil { + return err + } + nodes, err := inventory.ParseNodes(raw) + if err != nil { + return err + } + switch c.opts.output { + case "table": + inventory.RenderTable(c.out, nodes) + case "wide": + inventory.RenderByNode(c.out, nodes) + case "json": + return inventory.RenderJSON(c.out, nodes) + } + return nil +} + +func (c *client) doctor() error { + report := doctor.Run(func(args ...string) ([]byte, error) { + return c.kubectl(nil, args...) + }, doctor.Options{Context: c.opts.context, Namespace: c.opts.namespace}) + if c.opts.output == "table" { + for _, check := range report.Checks { + fmt.Fprintf(c.out, "[%-4s] %-20s %s\n", strings.ToUpper(string(check.Status)), check.Name, check.Detail) + if check.Remediation != "" && check.Status != doctor.Pass { + fmt.Fprintf(c.out, " fix: %s\n", check.Remediation) + } + } + } else if err := encode(c.out, report, c.opts.output); err != nil { + return err + } + if !report.OK() { + return fmt.Errorf("doctor found one or more blocking checks") + } + return nil +} diff --git a/core/internal/kube/cli_test.go b/core/internal/kube/cli_test.go new file mode 100644 index 00000000..723e4438 --- /dev/null +++ b/core/internal/kube/cli_test.go @@ -0,0 +1,942 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const testImage = "docker.io/library/eclipse-temurin@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const secondImage = "docker.io/library/eclipse-temurin@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +func rawJSON(t *testing.T, value any) []byte { + t.Helper() + raw, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return raw +} + +func objectJSON(t *testing.T, raw string) object { + t.Helper() + var obj object + if err := json.Unmarshal([]byte(raw), &obj); err != nil { + t.Fatal(err) + } + return obj +} + +func listJSON(t *testing.T, items ...object) []byte { + t.Helper() + if items == nil { + items = []object{} + } + return rawJSON(t, struct { + Items []object `json:"items"` + }{items}) +} + +func fixtureProfile() string { + return `{ + "apiVersion":"node.brewlet.sh/v1alpha1","kind":"NodeProfile", + "metadata":{"name":"workers","uid":"profile-uid","resourceVersion":"42","generation":3, + "labels":{"team":"platform"},"finalizers":["node.brewlet.sh/cleanup"]}, + "spec":{"nodePool":{"names":["java"]},"jdks":[{"distribution":"temurin","feature":21, + "source":{"image":"` + testImage + `","javaHome":"/opt/java/openjdk"}}], + "launchers":[{"name":"jaz","source":{"image":"` + testImage + `","path":"/usr/bin/jaz"}}], + "rollout":{"maxUnavailable":1},"appCDS":{"regenerationEnabled":true}}, + "status":{"observedGeneration":3,"assignedNodes":1,"readyNodes":1, + "conditions":[{"type":"Ready","status":"True","observedGeneration":3,"reason":"AllNodesProvisioned"}]} + }` +} + +func addArgs(extra ...string) []string { + return append([]string{"jdk", "add", "--profile", "workers", "--distribution", "temurin", + "--feature", "25", "--image", testImage, "--java-home", "/opt/java/openjdk"}, extra...) +} + +func dryRunArgs(extra ...string) []string { + return addArgs(append([]string{"--dry-run"}, extra...)...) +} + +func runTest(t *testing.T, args []string, execute executor) (string, string, error) { + t.Helper() + var out, stderr bytes.Buffer + err := run(context.Background(), args, &out, &stderr, execute) + return out.String(), stderr.String(), err +} + +func noExecution(t *testing.T) executor { + t.Helper() + return func(_ context.Context, program string, args []string, _ []byte) ([]byte, error) { + t.Fatalf("unexpected execution: %s %v", program, args) + return nil, nil + } +} + +func writeFixture(t *testing.T, value string) string { + t.Helper() + file := filepath.Join(t.TempDir(), "input.yaml") + if err := os.WriteFile(file, []byte(value), 0o600); err != nil { + t.Fatal(err) + } + return file +} + +func hasArgs(args []string, want ...string) bool { + for i := range args { + if i+len(want) <= len(args) && reflect.DeepEqual(args[i:i+len(want)], want) { + return true + } + } + return false +} + +func flagValue(t *testing.T, args []string, flag string) string { + t.Helper() + for i, arg := range args { + if arg == flag && i+1 < len(args) { + return args[i+1] + } + } + t.Fatalf("missing %s in %v", flag, args) + return "" +} + +func TestCommandValidationBeforeExecution(t *testing.T) { + cases := [][]string{ + {"unknown"}, {"jdk", "remove"}, {"debug", "app", "orders"}, + {"jdk", "render"}, {"launcher", "render"}, + {"status", "extra"}, {"inspect", "app"}, {"profile", "inspect", "--bad"}, + {"profile", "inspect", "../bad"}, {"jdk", "list", "--output", "yaml"}, + {"status", "--output", "bogus"}, {"status", "--timeout", "0"}, + {"jdk", "list", "--timeout", "-1s"}, {"launcher", "list", "--oops"}, + addArgs("--apply"), addArgs("--file", "missing"), addArgs("--values", "missing"), + dryRunArgs("--values", "missing", "--file", "missing"), + dryRunArgs("--apply"), addArgs("--output", "table"), dryRunArgs("--output", "table"), + addArgs("--dry-run=invalid"), addArgs("--dry-run=false"), addArgs("--dry-run="), + addArgs("--dry-run=server", "--file", "missing"), addArgs("--dry-run=server", "--values", "missing"), + addArgs("--feature", "0"), addArgs("--feature", "2147483648"), + addArgs("--distribution", "../jdk"), addArgs("--distribution", "com.example"), + addArgs("--image", "temurin:21"), addArgs("--image", "docker.io/library/eclipse-temurin:21@sha256:"+strings.Repeat("a", 64)), + addArgs("--java-home", "/"), addArgs("--java-home", "/opt/../jdk"), + {"launcher", "add", "--profile", "workers", "--name", "java", "--image", testImage, "--path", "/usr/bin/java"}, + {"install", "--version", "latest", "-f", "missing"}, + {"install", "--version", "1.2.3"}, + {"install", "--version", "1.2.3", "-f", "missing", "--wait-timeout", "-1s"}, + } + for _, args := range cases { + t.Run(strings.Join(args, " "), func(t *testing.T) { + if out, _, err := runTest(t, args, noExecution(t)); err == nil || out != "" { + t.Fatalf("expected usage/validation error with empty stdout, got %q, %v", out, err) + } + }) + } + for _, args := range [][]string{nil, {"--help"}, {"jdk", "--help"}, {"jdk", "add", "--help"}, {"launcher", "add", "--help"}} { + out, _, err := runTest(t, args, noExecution(t)) + if err != nil || !strings.Contains(out, "Brewlet Kubernetes operations") { + t.Fatalf("help = %q, %v", out, err) + } + } +} + +func TestInventoryAndConnectionFlags(t *testing.T) { + node := objectJSON(t, `{"kind":"Node","metadata":{"name":"worker-a","annotations":{ + "brewlet.sh/jdks-info":"[{\"distribution\":\"temurin\",\"vendor\":\"Adoptium\",\"feature\":21,\"version\":\"21.0.5\",\"arch\":\"amd64\"}]", + "brewlet.sh/launchers":"jaz,jaz, custom"}}}`) + for _, command := range []string{"jdk", "launcher"} { + t.Run(command, func(t *testing.T) { + out, _, err := runTest(t, []string{"--context", "staging", command, "list", + "--kubeconfig", "/config with spaces", "--selector", "pool=java", "--timeout", "2s", "--output", "json"}, + func(ctx context.Context, program string, args []string, _ []byte) ([]byte, error) { + if program != "kubectl" || !hasArgs(args, "--context", "staging") || + !hasArgs(args, "--kubeconfig", "/config with spaces") || !hasArgs(args, "--selector", "pool=java") || + !hasArgs(args, "--request-timeout", "2s") { + t.Fatalf("connection/selector not forwarded: %s %v", program, args) + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > 2*time.Second { + t.Fatal("missing command deadline") + } + return listJSON(t, node), nil + }) + if err != nil || !json.Valid([]byte(out)) || !strings.Contains(out, "worker-a") { + t.Fatalf("inventory = %s, %v", out, err) + } + if command == "launcher" { + var rows []launcherInventory + if err := json.Unmarshal([]byte(out), &rows); err != nil || len(rows) != 2 || len(rows[1].Nodes) != 1 { + t.Fatalf("launcher deduplication: %s %v", out, err) + } + } + }) + } + for _, format := range []string{"table", "wide"} { + out, _, err := runTest(t, []string{"launcher", "list", "--output", format}, + func(context.Context, string, []string, []byte) ([]byte, error) { return listJSON(t, node), nil }) + if err != nil || !strings.Contains(out, "jaz") || !strings.Contains(out, "LAUNCHER") { + t.Fatalf("launcher %s = %s %v", format, out, err) + } + } +} + +func TestReadErrorsAreNotEmptySuccess(t *testing.T) { + for _, command := range [][]string{{"profile", "list"}, {"jdk", "list"}, {"launcher", "list"}, {"status"}} { + for _, response := range []string{`broken`, `{}`, `{"items":null}`} { + if _, _, err := runTest(t, command, func(context.Context, string, []string, []byte) ([]byte, error) { + return []byte(response), nil + }); err == nil { + t.Fatalf("%v accepted %s", command, response) + } + } + if _, _, err := runTest(t, command, func(context.Context, string, []string, []byte) ([]byte, error) { + return nil, errors.New("Forbidden: RBAC denied") + }); err == nil || !strings.Contains(err.Error(), "Forbidden") { + t.Fatalf("%v hid API failure: %v", command, err) + } + } +} + +func TestProfileInspectionAndStaleReadiness(t *testing.T) { + profile := objectJSON(t, fixtureProfile()) + for _, mutate := range []func(*object){ + func(o *object) { o.Status.ObservedGeneration-- }, + func(o *object) { o.Status.Conditions[0].ObservedGeneration-- }, + func(o *object) { o.Metadata.DeletionTimestamp = "2026-09-22T00:00:00Z" }, + } { + obj := objectJSON(t, fixtureProfile()) + mutate(&obj) + if ready, _ := readyCondition(obj); ready { + t.Fatal("stale/terminating profile reported ready") + } + } + node := objectJSON(t, `{"kind":"Node","metadata":{"name":"worker-a","uid":"node-uid", + "labels":{"brewlet.sh/owner-uid":"profile-uid","brewlet.sh/owner-node-uid":"node-uid","brewlet.sh/runtime":"ready"}, + "annotations":{"brewlet.sh/owner-name":"workers","brewlet.sh/profile-generation":"3","brewlet.sh/jdks":"temurin-21"}}, + "spec":{},"status":{"conditions":[{"type":"Ready","status":"True"}]}}`) + foreign := node + foreign.Metadata.UID = "recreated-node" + out, _, err := runTest(t, []string{"profile", "inspect", "workers", "--output", "json"}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "get", profilesResource) { + return rawJSON(t, profile), nil + } + if !hasArgs(args, "--selector", "brewlet.sh/owner-uid=profile-uid") { + t.Fatalf("profile node query not scoped: %v", args) + } + return listJSON(t, node, foreign), nil + }) + if err != nil { + t.Fatal(err) + } + var report struct { + Profile profileSummary `json:"profile"` + Nodes []nodeSummary `json:"nodes"` + } + if err := json.Unmarshal([]byte(out), &report); err != nil || len(report.Nodes) != 1 || !report.Profile.Ready { + t.Fatalf("profile report = %s %v", out, err) + } +} + +func healthyDeployment(t *testing.T) object { + return objectJSON(t, `{"kind":"Deployment","metadata":{"name":"brewlet-operator","generation":2}, + "spec":{"replicas":1},"status":{"observedGeneration":2,"replicas":1,"updatedReplicas":1,"readyReplicas":1,"availableReplicas":1}}`) +} + +func TestStatusDistinguishesOldReplicasAndMissingComponents(t *testing.T) { + for _, state := range []string{"ready", "old replicas", "operator missing", "admission missing", "node failed", "no nodes", "no profiles"} { + t.Run(state, func(t *testing.T) { + dep := healthyDeployment(t) + if state == "old replicas" { + dep.Status.UpdatedReplicas = 0 + } + node := objectJSON(t, `{"kind":"Node","metadata":{"name":"worker","labels":{"brewlet.sh/runtime":"ready"}}, + "spec":{},"status":{"conditions":[{"type":"Ready","status":"True"}]}}`) + if state == "node failed" { + node.Metadata.Annotations = map[string]string{"brewlet.sh/provision-error": "jdk-copy-failed"} + } + out, _, err := runTest(t, []string{"status", "--output", "json", "--system-namespace", "runtime-system"}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + switch { + case hasArgs(args, "get", "deployments"): + if !hasArgs(args, "--namespace", "runtime-system") { + t.Fatal("wrong component namespace") + } + if state == "operator missing" { + return listJSON(t), nil + } + if state == "admission missing" { + return listJSON(t, dep), nil + } + admission := healthyDeployment(t) + admission.Metadata.Name = "brewlet-admission" + return listJSON(t, dep, admission), nil + case hasArgs(args, "get", profilesResource): + if state == "no profiles" { + return listJSON(t), nil + } + return listJSON(t, objectJSON(t, fixtureProfile())), nil + default: + if state == "no nodes" { + return listJSON(t), nil + } + return listJSON(t, node), nil + } + }) + var report statusReport + if decodeErr := json.Unmarshal([]byte(out), &report); decodeErr != nil { + t.Fatal(decodeErr, out) + } + if report.Healthy != (state == "ready") || (err == nil) != report.Healthy { + t.Fatalf("status %s: %s, %v", state, out, err) + } + if state == "admission missing" && report.Components[1].Present { + t.Fatal("absent admission should be explicit") + } + }) + } +} + +func TestAppInspectionFollowsOwnershipAndOmitsEnvironment(t *testing.T) { + app := objectJSON(t, `{"apiVersion":"apps.brewlet.sh/v1alpha1","kind":"JavaApplication", + "metadata":{"name":"orders","namespace":"team","uid":"app-uid","generation":3}, + "spec":{"artifact":{"image":"registry.example/orders@sha256:abc"},"jvm":{"version":21},"env":[{"name":"TOKEN","value":"secret-value"}]}, + "status":{"observedGeneration":3,"conditions":[{"type":"Ready","status":"True","observedGeneration":3,"reason":"Reconciled"}]}}`) + dep := healthyDeployment(t) + dep.Metadata.Name, dep.Metadata.UID = "orders", "dep-uid" + dep.Metadata.OwnerReferences = []ownerReference{{UID: "app-uid", Controller: true}} + dep.Status.UpdatedReplicas = 0 + rs := objectJSON(t, `{"kind":"ReplicaSet","metadata":{"name":"orders-rs","uid":"rs-uid", + "ownerReferences":[{"uid":"dep-uid","controller":true}]}}`) + pod := objectJSON(t, `{"kind":"Pod","metadata":{"name":"orders-pod","uid":"pod-uid", + "ownerReferences":[{"uid":"rs-uid","controller":true}],"annotations":{"brewlet.sh/jdk":"21"}}, + "spec":{"nodeName":"worker","containers":[{"env":[{"name":"TOKEN","value":"secret-value"}]}]}, + "status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"ImagePullBackOff"}}}]}}`) + foreign := objectJSON(t, `{"kind":"Pod","metadata":{"name":"foreign","uid":"foreign-uid","ownerReferences":[{"uid":"other-app","controller":true}]},"spec":{}}`) + event := objectJSON(t, `{"involvedObject":{"kind":"Pod","name":"orders-pod","uid":"pod-uid"},"reason":"Failed","message":"Cannot pull image","type":"Warning"}`) + foreignEvent := objectJSON(t, `{"involvedObject":{"kind":"Pod","name":"foreign","uid":"foreign-uid"},"message":"unrelated event"}`) + out, _, err := runTest(t, []string{"inspect", "app", "orders", "--namespace", "team", "--output", "json"}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "--namespace", "team") { + t.Fatalf("missing namespace: %v", args) + } + switch { + case hasArgs(args, "get", appsResource): + return rawJSON(t, app), nil + case hasArgs(args, "get", "events"): + return listJSON(t, event, foreignEvent), nil + default: + return listJSON(t, pod, foreign, rs, dep), nil + } + }) + if err != nil { + t.Fatal(err) + } + var report appReport + if err := json.Unmarshal([]byte(out), &report); err != nil { + t.Fatal(err) + } + if report.Ready || report.JDKRequest != "21" || len(report.Pods) != 1 || len(report.Events) != 1 || + len(report.Pods[0].Problems) != 1 || strings.Contains(out, "secret-value") || strings.Contains(out, "foreign") { + t.Fatalf("unsafe or incorrect app inspection: %s", out) + } +} + +func TestOfflineProfileAndValuesUpdates(t *testing.T) { + profile := writeFixture(t, fixtureProfile()) + out, _, err := runTest(t, dryRunArgs("--file", profile, "--output", "json"), noExecution(t)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "resourceVersion") || strings.Contains(out, "finalizers") || + strings.Contains(out, `"status"`) || !strings.Contains(out, `"maxUnavailable"`) || + !strings.Contains(out, `"regenerationEnabled"`) || !strings.Contains(out, `"jaz"`) { + t.Fatalf("manifest did not preserve configuration/strip live metadata: %s", out) + } + doc, err := decodeDocument([]byte(out)) + if err != nil { + t.Fatal(err) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + t.Fatal(err) + } + var jdks []jdkSource + if err := json.Unmarshal(spec["jdks"], &jdks); err != nil || len(jdks) != 2 { + t.Fatalf("inventory: %s %v", spec["jdks"], err) + } + original, err := os.ReadFile(profile) + if err != nil || string(original) != fixtureProfile() { + t.Fatal("source file was overwritten") + } + values := writeFixture(t, `namespace: brewlet +provisioner: + pools: [java] + jdks: [] + rollout: {maxUnavailable: 1} +profiles: + - name: workers + pools: [batch] + jdks: [] + - name: unrelated + pools: [other] + jdks: [] +metrics: {enabled: true} +`) + for _, name := range []string{"default", "workers"} { + out, _, err := runTest(t, dryRunArgs("--values", values, "--profile", name), noExecution(t)) + if err != nil || !strings.Contains(out, "metrics:") || !strings.Contains(out, "name: unrelated") || !strings.Contains(out, "feature: 25") { + t.Fatalf("values update: %s, %v", out, err) + } + } + for _, invalid := range []string{"a: 1\na: 2", "spec: {}\n---\nspec: {}", "null", "provisioner: ["} { + if out, _, err := runTest(t, dryRunArgs("--file", writeFixture(t, invalid)), noExecution(t)); err == nil || out != "" { + t.Fatalf("invalid source %s must fail without output, got %q, %v", invalid, out, err) + } + } +} + +func TestInventoryReplacementAndLimits(t *testing.T) { + file := writeFixture(t, fixtureProfile()) + out, _, err := runTest(t, dryRunArgs("--file", file, "--feature", "21", "--output", "json"), noExecution(t)) + if err != nil || strings.Count(out, `"distribution"`) != 1 { + t.Fatalf("identical add is not idempotent: %s %v", out, err) + } + args := dryRunArgs("--file", file, "--feature", "21", "--image", secondImage) + if _, _, err := runTest(t, args, noExecution(t)); err == nil || !strings.Contains(err.Error(), "--replace") { + t.Fatalf("implicit replacement allowed: %v", err) + } + out, _, err = runTest(t, append(args, "--replace"), noExecution(t)) + if err != nil || !strings.Contains(out, secondImage) { + t.Fatalf("explicit replacement failed: %s %v", out, err) + } + var entries []jdkSource + for n := 1; n <= 32; n++ { + jdk := jdkSource{Distribution: "temurin", Feature: int64(n)} + jdk.Source.Image, jdk.Source.JavaHome = testImage, "/opt/java" + entries = append(entries, jdk) + } + spec := document{"jdks": rawJSON(t, entries)} + u := updateOptions{jdk: true, distribution: "temurin", feature: 99, image: testImage, path: "/opt/java"} + field, entry, err := u.entry() + if err != nil { + t.Fatal(err) + } + if err := updateInventory(spec, field, entry, false); err == nil { + t.Fatal("allowed inventory larger than CRD maxItems") + } + spec["jdks"] = rawJSON(t, []jdkSource{entries[0], entries[0]}) + if err := updateInventory(spec, field, entry, false); err == nil || !strings.Contains(err.Error(), "duplicate") { + t.Fatal("duplicate inventory was not rejected", err) + } +} + +func TestManagedProfilesCannotBePatched(t *testing.T) { + cases := []string{ + `"labels":{"app.kubernetes.io/managed-by":"Helm"}`, + `"annotations":{"meta.helm.sh/release-name":"brewlet"}`, + `"annotations":{"argocd.argoproj.io/tracking-id":"apps:profile/workers"}`, + `"labels":{"kustomize.toolkit.fluxcd.io/name":"runtime"}`, + `"ownerReferences":[{"uid":"controller","controller":true}]`, + `"managedFields":[{"manager":"custom-gitops","fieldsV1":{"f:spec":{}}}]`, + } + for _, ownership := range cases { + t.Run(ownership, func(t *testing.T) { + raw := strings.Replace(fixtureProfile(), `"labels":{"team":"platform"}`, ownership, 1) + executor := func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "get", profilesResource) || !hasArgs(args, "--show-managed-fields=true") { + t.Fatalf("attempted mutation or missed ownership fields: %v", args) + } + return []byte(raw), nil + } + for _, args := range [][]string{addArgs(), addArgs("--dry-run=server")} { + if out, _, err := runTest(t, args, executor); err == nil || !strings.Contains(err.Error(), "managed by") || out != "" { + t.Fatalf("managed profile must fail without output, got %q, %v", out, err) + } + } + out, stderr, err := runTest(t, dryRunArgs(), executor) + if err != nil || !strings.Contains(stderr, "source of truth") || out == "" { + t.Fatalf("managed declaration must remain available: %s %s %v", out, stderr, err) + } + }) + } +} + +func TestAddMutatesByDefaultAndDryRunUsesConditionalPatch(t *testing.T) { + for _, mode := range []string{"apply", "dry-run", "conflict"} { + t.Run(mode, func(t *testing.T) { + var patchPath string + calls := 0 + args := addArgs("--output", "json") + if mode == "dry-run" { + args = append(args, "--dry-run=server") + } + out, stderr, err := runTest(t, args, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + calls++ + if hasArgs(args, "get", profilesResource) { + return []byte(fixtureProfile()), nil + } + if !hasArgs(args, "patch", profilesResource, "workers") || !hasArgs(args, "--type=json") { + t.Fatalf("unexpected update: %v", args) + } + if hasArgs(args, "--dry-run=server") != (mode == "dry-run") { + t.Fatalf("wrong dry-run mode: %v", args) + } + patchPath = flagValue(t, args, "--patch-file") + info, err := os.Stat(patchPath) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("patch file is not private: %v", err) + } + raw, err := os.ReadFile(patchPath) + if err != nil { + t.Fatal(err) + } + var patch []struct { + Op, Path string + Value json.RawMessage + } + if err := json.Unmarshal(raw, &patch); err != nil || len(patch) != 3 { + t.Fatalf("patch: %s %v", raw, err) + } + if patch[0].Op != "test" || patch[0].Path != "/metadata/uid" || string(patch[0].Value) != `"profile-uid"` || + patch[1].Op != "test" || patch[1].Path != "/metadata/resourceVersion" || string(patch[1].Value) != `"42"` || + patch[2].Op != "add" || patch[2].Path != "/spec/jdks" { + t.Fatalf("patch is not identity/version fenced and narrow: %s", raw) + } + if mode == "conflict" { + return nil, errors.New("Conflict: resourceVersion changed") + } + doc, err := decodeDocument([]byte(fixtureProfile())) + if err != nil { + t.Fatal(err) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + t.Fatal(err) + } + spec["jdks"] = patch[2].Value + doc["spec"] = rawJSON(t, spec) + return rawJSON(t, doc), nil + }) + if calls != 2 { + t.Fatalf("unexpected retries: %d", calls) + } + if mode == "conflict" { + if err == nil || !strings.Contains(err.Error(), "Conflict") || out != "" || stderr != "" { + t.Fatalf("conflict concealed: %s %s %v", out, stderr, err) + } + } else { + if err != nil || !json.Valid([]byte(out)) || !strings.Contains(out, `"feature": 25`) { + t.Fatalf("add result: %s %s %v", out, stderr, err) + } + want := "Profile update accepted; provisioning is asynchronous." + if mode == "dry-run" { + want = "Server dry run succeeded; no inventory was changed." + } + if !strings.Contains(stderr, want) { + t.Fatalf("missing mutation/preview distinction: %s", stderr) + } + } + if _, err := os.Stat(patchPath); !os.IsNotExist(err) { + t.Fatal("temporary patch file leaked", err) + } + }) + } +} + +func TestFailedServerDryRunDoesNotRender(t *testing.T) { + for _, kind := range []string{"jdk", "launcher"} { + for _, format := range []string{"yaml", "json"} { + for _, failure := range []string{"read", "validation", "conflict", "decode", "metadata"} { + t.Run(kind+"/"+format+"/"+failure, func(t *testing.T) { + args := addArgs("--dry-run=server", "--output", format) + if kind == "launcher" { + args = []string{"launcher", "add", "--profile", "workers", "--name", "custom", + "--image", testImage, "--path", "/usr/bin/custom", "--dry-run=server", "--output", format} + } + calls := 0 + var patchFile string + out, stderr, err := runTest(t, args, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + calls++ + if hasArgs(args, "get", profilesResource) { + if failure == "read" { + return []byte(fixtureProfile()), errors.New("Forbidden: cannot read profile") + } + return []byte(fixtureProfile()), nil + } + if !hasArgs(args, "patch", profilesResource, "workers") || !hasArgs(args, "--dry-run=server") { + t.Fatalf("dry run attempted a non-preview operation: %v", args) + } + patchFile = flagValue(t, args, "--patch-file") + switch failure { + case "validation": + return []byte(fixtureProfile()), errors.New("admission rejected the source") + case "conflict": + return []byte(fixtureProfile()), errors.New("Conflict: resourceVersion changed") + case "decode": + return []byte("invalid JSON"), nil + case "metadata": + return []byte(`{"metadata":false}`), nil + default: + t.Fatal("unexpected patch after failed read") + return nil, nil + } + }) + if err == nil || out != "" || strings.Contains(stderr, "succeeded") || strings.Contains(stderr, "accepted") { + t.Fatalf("failed dry run rendered output or claimed success: stdout=%q stderr=%q err=%v", out, stderr, err) + } + wantCalls := 2 + if failure == "read" { + wantCalls = 1 + } + if calls != wantCalls { + t.Fatalf("unexpected fallback/retry after failure: got %d calls, want %d", calls, wantCalls) + } + if patchFile != "" { + if _, err := os.Stat(patchFile); !os.IsNotExist(err) { + t.Fatal("failed dry run left a temporary patch file", err) + } + } + }) + } + } + } +} + +func TestClientDryRunModesAndFailures(t *testing.T) { + file := writeFixture(t, fixtureProfile()) + for _, flag := range []string{"--dry-run", "--dry-run=client"} { + for _, format := range []string{"yaml", "json"} { + t.Run(flag+"/"+format, func(t *testing.T) { + out, _, err := runTest(t, addArgs(flag, "--file", file, "--output", format), noExecution(t)) + if err != nil || out == "" || !strings.Contains(out, "25") { + t.Fatalf("client preview failed: %s %v", out, err) + } + for _, invalid := range [][]string{ + {"--image", "temurin:25"}, + {"--feature", "21", "--image", secondImage}, + {"--file", filepath.Join(t.TempDir(), "missing.yaml")}, + } { + args := append(addArgs(flag, "--file", file, "--output", format), invalid...) + out, stderr, err := runTest(t, args, noExecution(t)) + if err == nil || out != "" || strings.Contains(stderr, "succeeded") { + t.Fatalf("failed client dry run rendered output: %q %q %v", out, stderr, err) + } + } + out, _, err = runTest(t, addArgs(flag, "--output", format), + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "get", profilesResource) { + t.Fatalf("client dry run attempted a write: %v", args) + } + return []byte(fixtureProfile()), nil + }) + if err != nil || out == "" { + t.Fatalf("live-profile client preview failed: %q %v", out, err) + } + }) + } + } +} + +func TestFailedValuesDryRunDoesNotRender(t *testing.T) { + values := writeFixture(t, `provisioner: + jdks: [] +profiles: + - name: workers + jdks: [] + - name: workers + jdks: [] +`) + out, stderr, err := runTest(t, dryRunArgs("--values", values), noExecution(t)) + if err == nil || !strings.Contains(err.Error(), "duplicate profile") || out != "" || stderr != "" { + t.Fatalf("partially processed values must not be rendered: %q %q %v", out, stderr, err) + } +} + +func TestKubectlLocalPatchContract(t *testing.T) { + if _, err := exec.LookPath("kubectl"); err != nil { + t.Skip("kubectl not installed") + } + for _, stale := range []bool{false, true} { + t.Run(fmt.Sprint(stale), func(t *testing.T) { + snapshot := fixtureProfile() + if stale { + snapshot = strings.Replace(snapshot, `"resourceVersion":"42"`, `"resourceVersion":"43"`, 1) + } + file := writeFixture(t, snapshot) + out, _, err := runTest(t, addArgs("--output", "json"), + func(ctx context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "get", profilesResource) { + return []byte(fixtureProfile()), nil + } + return execute(ctx, "kubectl", []string{"patch", "--local", "-f", file, "--type=json", + "--patch-file", flagValue(t, args, "--patch-file"), "-o", "json"}, nil) + }) + if stale { + if err == nil { + t.Fatal("kubectl accepted a stale resourceVersion") + } + } else if err != nil || !strings.Contains(out, `"feature": 25`) || !strings.Contains(out, `"maxUnavailable": 1`) { + t.Fatalf("kubectl rejected or lost fields: %s %v", out, err) + } + }) + } +} + +func TestInstallDelegatesToPinnedHelmChart(t *testing.T) { + file := writeFixture(t, "provisioner:\n pools: [workers]\n jdks: []\n") + for _, dryRun := range []bool{false, true} { + args := []string{"--context", "staging", "install", "--kubeconfig", "/kubeconfig", "--version", "0.1.0-rc.1", + "-f", file, "--values", file, "--namespace", "runtime-system", "--wait-timeout", "2m"} + if dryRun { + args = append(args, "--dry-run") + } + out, _, err := runTest(t, args, + func(ctx context.Context, program string, args []string, _ []byte) ([]byte, error) { + if program == "kubectl" { + if dryRun || !hasArgs(args, "get", "customresourcedefinitions") || + !hasArgs(args, profilesResource, appsResource, "--ignore-not-found") { + t.Fatalf("unexpected install preflight: %v", args) + } + return listJSON(t), nil + } + if program != "helm" || !hasArgs(args, "--version", "0.1.0-rc.1") || + !hasArgs(args, "--namespace", "runtime-system") || !hasArgs(args, "--set-string", "namespace=runtime-system") || + !hasArgs(args, "--kube-context", "staging") || !hasArgs(args, "--kubeconfig", "/kubeconfig") { + t.Fatalf("unsafe install command: %s %v", program, args) + } + if dryRun { + if !hasArgs(args, "template", "brewlet", chart) || hasArgs(args, "--wait") { + t.Fatalf("dry-run should only template: %v", args) + } + } else if !hasArgs(args, "install", "brewlet", chart) || !hasArgs(args, "--wait") { + t.Fatalf("install did not use Helm: %v", args) + } + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) < 2*time.Minute { + t.Fatal("Helm deadline does not account for rollout time") + } + return []byte("helm output\n"), nil + }) + if err != nil || out != "helm output\n" { + t.Fatalf("install: %s %v", out, err) + } + } + out, stderr, err := runTest(t, []string{"install", "--version", "1.2.3", "-f", file, "--dry-run"}, + func(context.Context, string, []string, []byte) ([]byte, error) { + return []byte("partially rendered Helm manifests"), errors.New("Helm release failed") + }) + if err == nil || !strings.Contains(err.Error(), "Helm release failed") || out != "" || stderr != "" { + t.Fatalf("Helm dry-run failure must not render partial output: %q %q %v", out, stderr, err) + } + _, _, err = runTest(t, []string{"install", "--version", "1.2.3", "-f", file}, + func(_ context.Context, program string, _ []string, _ []byte) ([]byte, error) { + if program != "kubectl" { + t.Fatal("attempted to install over existing CRDs") + } + return listJSON(t, objectJSON(t, `{"kind":"CustomResourceDefinition","metadata":{"name":"nodeprofiles.node.brewlet.sh"}}`)), nil + }) + if err == nil || !strings.Contains(err.Error(), "CRDs already exist") { + t.Fatal("existing CRDs were not protected", err) + } +} + +func TestExecuteErrorsAndCancellation(t *testing.T) { + if _, err := execute(context.Background(), "brewlet-test-nonexistent-program", nil, nil); err == nil { + t.Fatal("missing binary should fail") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := execute(ctx, "kubectl", []string{"version", "--client"}, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation not propagated: %v", err) + } +} + +func TestDoctorDelegatesChecksAndPropagatesFailure(t *testing.T) { + for _, fail := range []bool{false, true} { + for _, format := range []string{"table", "json", "yaml"} { + out, _, err := runTest(t, []string{"doctor", "--context", "staging", "--namespace", "team", "--output", format}, + func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if !hasArgs(args, "--context", "staging") { + t.Fatalf("context missing: %v", args) + } + switch { + case hasArgs(args, "config", "current-context"): + return []byte("staging"), nil + case hasArgs(args, "get", "nodes"): + return []byte(`{"items":[{"metadata":{"name":"node","labels":{"brewlet.sh/runtime":"ready"}, + "annotations":{"brewlet.sh/jdks":"temurin-21"}},"spec":{}, + "status":{"nodeInfo":{"containerRuntimeVersion":"containerd://2.0.0"}}}]}`), nil + case hasArgs(args, "auth", "can-i"): + if !hasArgs(args, "-n", "team") { + t.Fatal("doctor used the wrong namespace") + } + if fail { + return []byte("no"), nil + } + return []byte("yes"), nil + default: + return []byte("ok"), nil + } + }) + if (err != nil) != fail || !strings.Contains(out, "developer-rbac") { + t.Fatalf("doctor fail=%t format=%s: %s %v", fail, format, out, err) + } + } + } +} + +func TestProfileTableIncludesDesiredInventory(t *testing.T) { + out, _, err := runTest(t, []string{"profile", "list"}, + func(context.Context, string, []string, []byte) ([]byte, error) { + return listJSON(t, objectJSON(t, fixtureProfile())), nil + }) + for _, text := range []string{"DESIRED JDKS", "DESIRED LAUNCHERS", "temurin-21", "jaz", "AllNodesProvisioned"} { + if !strings.Contains(out, text) { + t.Fatalf("profile table lacks %q: %s", text, out) + } + } + if err != nil { + t.Fatal(err) + } +} + +func TestLauncherAdditionAndTerminatingProfile(t *testing.T) { + profile := fixtureProfile() + args := []string{"launcher", "add", "--profile", "workers", "--name", "custom", + "--image", testImage, "--path", "/usr/bin/custom", "--output", "json"} + for _, mode := range []string{"add", "server", "client", "bare"} { + t.Run(mode, func(t *testing.T) { + command := append([]string(nil), args...) + if mode == "bare" { + command = append(command, "--dry-run") + } else if mode != "add" { + command = append(command, "--dry-run="+mode) + } + patches := 0 + out, _, err := runTest(t, command, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "get", profilesResource) { + return []byte(profile), nil + } + if mode == "client" || mode == "bare" || !hasArgs(args, "patch", profilesResource, "workers") { + t.Fatalf("unexpected command: %v", args) + } + patches++ + if hasArgs(args, "--dry-run=server") != (mode == "server") { + t.Fatalf("incorrect launcher mutation/preview mode: %v", args) + } + raw, err := os.ReadFile(flagValue(t, args, "--patch-file")) + if err != nil { + t.Fatal(err) + } + var patch []struct { + Op, Path string + Value json.RawMessage + } + if err := json.Unmarshal(raw, &patch); err != nil || len(patch) != 3 || + patch[0].Path != "/metadata/uid" || patch[1].Path != "/metadata/resourceVersion" || + patch[2].Path != "/spec/launchers" || patch[2].Op != "add" { + t.Fatalf("launcher patch is not conditional and narrow: %s %v", raw, err) + } + doc, err := decodeDocument([]byte(profile)) + if err != nil { + t.Fatal(err) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + t.Fatal(err) + } + spec["launchers"] = patch[2].Value + doc["spec"] = rawJSON(t, spec) + return rawJSON(t, doc), nil + }) + if err != nil || !strings.Contains(out, `"name": "custom"`) || !strings.Contains(out, `"name": "jaz"`) || + !strings.Contains(out, `"feature": 21`) { + t.Fatalf("launcher %s: %s %v", mode, out, err) + } + if (mode == "add" || mode == "server") && patches != 1 { + t.Fatalf("launcher add did not issue exactly one patch: %d", patches) + } + }) + } + for _, field := range []string{`"deletionTimestamp":"2026-09-22T00:00:00Z"`, `"namespace":"forbidden"`} { + invalid := strings.Replace(profile, `"generation":3`, `"generation":3,`+field, 1) + if _, _, err := runTest(t, args, func(_ context.Context, _ string, args []string, _ []byte) ([]byte, error) { + if hasArgs(args, "patch") { + t.Fatal("mutated an invalid/terminating profile") + } + return []byte(invalid), nil + }); err == nil { + t.Fatal("invalid/terminating profile accepted") + } + } +} + +func TestHelmPreviewAndValuesContract(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("Helm not installed") + } + localChart, err := filepath.Abs(filepath.Join("..", "..", "..", "kubernetes", "charts", "brewlet")) + if err != nil { + t.Fatal(err) + } + values := writeFixture(t, `provisioner: + pools: [java-workers] + jdks: + - distribution: temurin + feature: 21 + source: + image: `+testImage+` + javaHome: /opt/java/openjdk + rollout: {maxUnavailable: 1} +profiles: + - name: batch + pools: [batch-workers] + jdks: + - distribution: temurin + feature: 21 + source: + image: `+testImage+` + javaHome: /opt/java/openjdk +`) + updated, _, err := runTest(t, dryRunArgs("--values", values, "--profile", "default"), noExecution(t)) + if err != nil { + t.Fatal(err) + } + nextValues := writeFixture(t, updated) + out, _, err := runTest(t, []string{"install", "--version", "1.2.3", "--values", nextValues, "--namespace", "runtime-system", "--dry-run"}, + func(ctx context.Context, program string, args []string, input []byte) ([]byte, error) { + if program != "helm" || args[0] != "template" { + t.Fatalf("preview attempted cluster access: %s %v", program, args) + } + for i, arg := range args { + if arg == chart { + args[i] = localChart + } + } + return execute(ctx, program, args, input) + }) + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ + "kind: NodeProfile", "kind: CustomResourceDefinition", "runtime-system", + "java-workers", "batch-workers", "feature: 21", "feature: 25", testImage, "maxUnavailable: 1", + } { + if !strings.Contains(out, text) { + t.Fatalf("Helm render missing %q", text) + } + } +} diff --git a/core/internal/kube/client.go b/core/internal/kube/client.go new file mode 100644 index 00000000..509b7cf9 --- /dev/null +++ b/core/internal/kube/client.go @@ -0,0 +1,233 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package kube provides Brewlet-specific Kubernetes operations through kubectl +// and Helm, using the caller's existing credentials and context. +package kube + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "go.yaml.in/yaml/v3" +) + +const profilesResource = "nodeprofiles.node.brewlet.sh" +const appsResource = "javaapplications.apps.brewlet.sh" + +type options struct { + kubeconfig, context, namespace, systemNamespace string + output, selector string + timeout time.Duration +} + +type executor func(context.Context, string, []string, []byte) ([]byte, error) + +type client struct { + ctx context.Context + opts options + exec executor + out io.Writer + err io.Writer +} + +func execute(ctx context.Context, program string, args []string, input []byte) ([]byte, error) { + cmd := exec.CommandContext(ctx, program, args...) + cmd.WaitDelay = time.Second + cmd.Stdin = bytes.NewReader(input) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if ctx.Err() != nil { + return nil, fmt.Errorf("%s: %w", program, ctx.Err()) + } + if err != nil { + return nil, fmt.Errorf("%s: %w: %s", program, err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +func (c *client) kubectl(input []byte, args ...string) ([]byte, error) { + base := []string{"--request-timeout", c.opts.timeout.String()} + if c.opts.kubeconfig != "" { + base = append(base, "--kubeconfig", c.opts.kubeconfig) + } + if c.opts.context != "" { + base = append(base, "--context", c.opts.context) + } + ctx, cancel := context.WithTimeout(c.ctx, c.opts.timeout) + defer cancel() + return c.exec(ctx, "kubectl", append(base, args...), input) +} + +func (c *client) get(resource, name string, extra ...string) (object, error) { + args := []string{"get", resource, name, "-o", "json"} + raw, err := c.kubectl(nil, append(args, extra...)...) + if err != nil { + return object{}, err + } + var obj object + if err := json.Unmarshal(raw, &obj); err != nil { + return obj, fmt.Errorf("decode %s: %w", resource, err) + } + if obj.Metadata.Name != name { + return obj, fmt.Errorf("get %s %q returned unexpected object %q", resource, name, obj.Metadata.Name) + } + obj.raw = raw + return obj, nil +} + +func (c *client) list(resource string, extra ...string) ([]object, error) { + args := append([]string{"get", resource, "-o", "json"}, extra...) + raw, err := c.kubectl(nil, args...) + if err != nil { + return nil, err + } + var list struct { + Items []object `json:"items"` + } + if err := json.Unmarshal(raw, &list); err != nil { + return nil, fmt.Errorf("decode %s list: %w", resource, err) + } + if list.Items == nil { + return nil, fmt.Errorf("%s response is not a Kubernetes list", resource) + } + return list.Items, nil +} + +func (c *client) namespaceArgs() []string { + if c.opts.namespace == "" { + return nil + } + return []string{"--namespace", c.opts.namespace} +} + +func (c *client) nodeArgs() []string { + if c.opts.selector == "" { + return nil + } + return []string{"--selector", c.opts.selector} +} + +func encode(w io.Writer, value any, format string) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + if format == "yaml" { + var document any + if err := yaml.Unmarshal(data, &document); err != nil { + return err + } + enc := yaml.NewEncoder(w) + enc.SetIndent(2) + defer enc.Close() + return enc.Encode(document) + } + _, err = fmt.Fprintln(w, string(data)) + return err +} + +type metadata struct { + Name string `json:"name"` + Namespace string `json:"namespace,omitempty"` + UID string `json:"uid"` + ResourceVersion string `json:"resourceVersion"` + Generation int64 `json:"generation"` + DeletionTimestamp string `json:"deletionTimestamp,omitempty"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + OwnerReferences []ownerReference `json:"ownerReferences"` + ManagedFields []struct { + Manager string `json:"manager"` + Subresource string `json:"subresource"` + Fields json.RawMessage `json:"fieldsV1"` + } `json:"managedFields"` +} + +type ownerReference struct { + UID string `json:"uid"` + Controller bool `json:"controller"` +} + +type condition struct { + Type string `json:"type"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Message string `json:"message,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +type object struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata metadata `json:"metadata"` + Spec json.RawMessage `json:"spec"` + Status struct { + ObservedGeneration int64 `json:"observedGeneration"` + Conditions []condition `json:"conditions"` + AssignedNodes int `json:"assignedNodes"` + ReadyNodes int `json:"readyNodes"` + Replicas int `json:"replicas"` + UpdatedReplicas int `json:"updatedReplicas"` + ReadyReplicas int `json:"readyReplicas"` + AvailableReplicas int `json:"availableReplicas"` + Phase string `json:"phase"` + ContainerStatuses []struct { + Name string `json:"name"` + Ready bool `json:"ready"` + RestartCount int `json:"restartCount"` + State struct { + Waiting *struct { + Reason string `json:"reason"` + } `json:"waiting"` + Terminated *struct { + Reason string `json:"reason"` + ExitCode int `json:"exitCode"` + } `json:"terminated"` + } `json:"state"` + } `json:"containerStatuses"` + } `json:"status"` + InvolvedObject struct { + UID string `json:"uid"` + Name string `json:"name"` + Kind string `json:"kind"` + } `json:"involvedObject"` + Type string `json:"type"` + Reason string `json:"reason"` + Message string `json:"message"` + raw json.RawMessage +} + +func readyCondition(obj object) (bool, string) { + if obj.Metadata.DeletionTimestamp != "" { + return false, "Terminating" + } + if obj.Status.ObservedGeneration != obj.Metadata.Generation { + return false, "NotObserved" + } + for _, cond := range obj.Status.Conditions { + if cond.Type == "Ready" { + if cond.ObservedGeneration != obj.Metadata.Generation { + return false, "NotObserved" + } + return cond.Status == "True", cond.Reason + } + } + return false, "Unknown" +} + +func ownedBy(obj object, uids map[string]bool) bool { + for _, owner := range obj.Metadata.OwnerReferences { + if owner.Controller && owner.UID != "" && uids[owner.UID] { + return true + } + } + return false +} diff --git a/core/internal/kube/install.go b/core/internal/kube/install.go new file mode 100644 index 00000000..9ae40f58 --- /dev/null +++ b/core/internal/kube/install.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "context" + "flag" + "fmt" + "regexp" + "time" +) + +const chart = "oci://ghcr.io/microsoft/charts/brewlet" + +type valuesFiles []string + +func (v *valuesFiles) String() string { return fmt.Sprint([]string(*v)) } +func (v *valuesFiles) Set(value string) error { + *v = append(*v, value) + return nil +} + +type installOptions struct { + version, release string + values valuesFiles + waitTimeout time.Duration + dryRun bool +} + +func (i *installOptions) flags(fs *flag.FlagSet) { + fs.StringVar(&i.version, "version", "", "required exact Helm chart release version") + fs.StringVar(&i.release, "release", "brewlet", "Helm release name") + fs.Var(&i.values, "values", "complete Helm values file (repeatable)") + fs.Var(&i.values, "f", "complete Helm values file (repeatable)") + fs.DurationVar(&i.waitTimeout, "wait-timeout", 5*time.Minute, "Helm rollout timeout") + fs.BoolVar(&i.dryRun, "dry-run", false, "render manifests locally instead of installing") +} + +var chartVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`) + +func (c *client) install(i installOptions) error { + if !chartVersion.MatchString(i.version) { + return fmt.Errorf("--version must be an exact chart version, such as 0.1.0 (not latest or a range)") + } + if err := validateName(i.release, 53); err != nil { + return fmt.Errorf("--release: %w", err) + } + if i.waitTimeout <= 0 { + return fmt.Errorf("--wait-timeout must be positive") + } + if len(i.values) == 0 { + return fmt.Errorf("at least one --values/-f file is required: explicitly configure node pools and digest-pinned runtime sources, or disable defaultProfile") + } + for _, file := range i.values { + if _, err := readDocument(file); err != nil { + return err + } + } + ns := c.opts.namespace + if ns == "" { + ns = "brewlet" + } + if err := validateToken(ns, 63); err != nil { + return fmt.Errorf("--namespace: %w", err) + } + if !i.dryRun { + existing, err := c.list("customresourcedefinitions", profilesResource, appsResource, "--ignore-not-found") + if err != nil { + return fmt.Errorf("check for existing Brewlet CRDs: %w", err) + } + if len(existing) != 0 { + return fmt.Errorf("Brewlet CRDs already exist; install is fresh-install-only. Follow the documented CRD migration and Helm upgrade procedure") + } + } + args := []string{"install", i.release, chart, "--create-namespace", "--wait", "--timeout", i.waitTimeout.String()} + if i.dryRun { + args = []string{"template", i.release, chart, "--include-crds"} + } + args = append(args, "--version", i.version, "--namespace", ns) + if c.opts.kubeconfig != "" { + args = append(args, "--kubeconfig", c.opts.kubeconfig) + } + if c.opts.context != "" { + args = append(args, "--kube-context", c.opts.context) + } + for _, file := range i.values { + args = append(args, "--values", file) + } + // The chart uses Values.namespace for resources, not Release.Namespace. + args = append(args, "--set-string", "namespace="+ns) + ctx, cancel := context.WithTimeout(c.ctx, i.waitTimeout+c.opts.timeout) + defer cancel() + result, err := c.exec(ctx, "helm", args, nil) + if err != nil { + return err + } + if _, err := c.out.Write(result); err != nil { + return err + } + if !i.dryRun { + fmt.Fprintf(c.err, "Helm release is ready. Node provisioning is separate; run brewlet k8s status --system-namespace %s with the same kubeconfig/context and inspect advertised JDK/launcher inventory.\n", ns) + } + return nil +} diff --git a/core/internal/kube/read.go b/core/internal/kube/read.go new file mode 100644 index 00000000..1cd4bd29 --- /dev/null +++ b/core/internal/kube/read.go @@ -0,0 +1,486 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "text/tabwriter" +) + +type launcherInventory struct { + Name string `json:"name"` + Nodes []string `json:"nodes"` +} + +func (c *client) launchers() error { + nodes, err := c.list("nodes", c.nodeArgs()...) + if err != nil { + return err + } + byName := map[string][]string{} + for _, node := range nodes { + seen := map[string]bool{} + for _, name := range strings.Split(node.Metadata.Annotations["brewlet.sh/launchers"], ",") { + name = strings.TrimSpace(name) + if name != "" && !seen[name] { + byName[name] = append(byName[name], node.Metadata.Name) + seen[name] = true + } + } + } + rows := []launcherInventory{} + for name, nodes := range byName { + sort.Strings(nodes) + rows = append(rows, launcherInventory{Name: name, Nodes: nodes}) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Name < rows[j].Name }) + if c.opts.output == "json" { + return encode(c.out, rows, "json") + } + w := tabwriter.NewWriter(c.out, 0, 4, 2, ' ', 0) + if c.opts.output == "wide" { + fmt.Fprintln(w, "LAUNCHER\tNODE") + for _, row := range rows { + for _, node := range row.Nodes { + fmt.Fprintf(w, "%s\t%s\n", row.Name, node) + } + } + } else { + fmt.Fprintln(w, "LAUNCHER\tNODES") + for _, row := range rows { + fmt.Fprintf(w, "%s\t%d\n", row.Name, len(row.Nodes)) + } + } + return w.Flush() +} + +type profileSummary struct { + Name string `json:"name"` + Generation int64 `json:"generation"` + ObservedGeneration int64 `json:"observedGeneration"` + Ready bool `json:"ready"` + Reason string `json:"reason"` + AssignedNodes int `json:"assignedNodes"` + ReadyNodes int `json:"readyNodes"` + ManagedBy string `json:"managedBy,omitempty"` + Spec json.RawMessage `json:"spec"` + Conditions []condition `json:"conditions"` +} + +func summarizeProfile(obj object) profileSummary { + ready, reason := readyCondition(obj) + return profileSummary{ + Name: obj.Metadata.Name, Generation: obj.Metadata.Generation, + ObservedGeneration: obj.Status.ObservedGeneration, + Ready: ready, Reason: reason, AssignedNodes: obj.Status.AssignedNodes, + ReadyNodes: obj.Status.ReadyNodes, ManagedBy: managedBy(obj.Metadata), + Spec: obj.Spec, Conditions: obj.Status.Conditions, + } +} + +func (c *client) profiles() error { + objects, err := c.list(profilesResource) + if err != nil { + return err + } + rows := []profileSummary{} + for _, obj := range objects { + rows = append(rows, summarizeProfile(obj)) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Name < rows[j].Name }) + if c.opts.output != "table" { + return encode(c.out, rows, c.opts.output) + } + return c.profileTable(rows) +} + +func (c *client) profileTable(rows []profileSummary) error { + w := tabwriter.NewWriter(c.out, 0, 4, 2, ' ', 0) + fmt.Fprintln(w, "PROFILE\tPOOLS\tDESIRED JDKS\tDESIRED LAUNCHERS\tREADY\tNODES\tREASON") + for _, row := range rows { + var spec struct { + NodePool struct { + Names []string `json:"names"` + } `json:"nodePool"` + JDKs []jdkSource `json:"jdks"` + Launchers []launcherSource `json:"launchers"` + } + if err := json.Unmarshal(row.Spec, &spec); err != nil { + return fmt.Errorf("decode profile %s spec: %w", row.Name, err) + } + jdks, launchers := []string{}, []string{} + for _, jdk := range spec.JDKs { + jdks = append(jdks, fmt.Sprintf("%s-%d", jdk.Distribution, jdk.Feature)) + } + for _, launcher := range spec.Launchers { + launchers = append(launchers, launcher.Name) + } + pools := strings.Join(spec.NodePool.Names, ",") + if pools == "" { + pools = "(all eligible nodes)" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%t\t%d/%d\t%s\n", + row.Name, pools, strings.Join(jdks, ","), strings.Join(launchers, ","), + row.Ready, row.ReadyNodes, row.AssignedNodes, row.Reason) + } + return w.Flush() +} + +type nodeSummary struct { + Name string `json:"name"` + Profile string `json:"profile,omitempty"` + ProfileGeneration string `json:"profileGeneration,omitempty"` + RuntimeReady bool `json:"runtimeReady"` + NodeReady bool `json:"nodeReady"` + Unschedulable bool `json:"unschedulable"` + ProvisionState string `json:"provisionState,omitempty"` + Error string `json:"error,omitempty"` + Message string `json:"message,omitempty"` + JDKs string `json:"advertisedJdks,omitempty"` + Launchers string `json:"advertisedLaunchers,omitempty"` +} + +func summarizeNodes(objects []object, profileUID string) ([]nodeSummary, error) { + nodes := []nodeSummary{} + for _, obj := range objects { + labels, ann := obj.Metadata.Labels, obj.Metadata.Annotations + if profileUID != "" { + if labels["brewlet.sh/owner-uid"] != profileUID || labels["brewlet.sh/owner-node-uid"] != obj.Metadata.UID { + continue + } + } else if labels["brewlet.sh/runtime"] == "" && labels["brewlet.sh/owner-uid"] == "" && + ann["brewlet.sh/profile"] == "" && ann["brewlet.sh/provision-error"] == "" { + continue + } + var spec struct { + Unschedulable bool `json:"unschedulable"` + } + if err := json.Unmarshal(obj.Spec, &spec); err != nil { + return nil, fmt.Errorf("decode node %s: %w", obj.Metadata.Name, err) + } + nodeReady := false + for _, cond := range obj.Status.Conditions { + if cond.Type == "Ready" { + nodeReady = cond.Status == "True" + } + } + nodes = append(nodes, nodeSummary{ + Name: obj.Metadata.Name, Profile: ann["brewlet.sh/owner-name"], + ProfileGeneration: ann["brewlet.sh/profile-generation"], + RuntimeReady: labels["brewlet.sh/runtime"] == "ready", NodeReady: nodeReady, + Unschedulable: spec.Unschedulable, ProvisionState: labels["brewlet.sh/provision-state"], + Error: ann["brewlet.sh/provision-error"], Message: ann["brewlet.sh/provision-error-message"], + JDKs: ann["brewlet.sh/jdks"], Launchers: ann["brewlet.sh/launchers"], + }) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].Name < nodes[j].Name }) + return nodes, nil +} + +func (c *client) nodeTable(nodes []nodeSummary) error { + w := tabwriter.NewWriter(c.out, 0, 4, 2, ' ', 0) + fmt.Fprintln(w, "NODE\tPROFILE\tGENERATION\tRUNTIME READY\tNODE READY\tCORDONED\tPROVISIONING\tERROR") + for _, node := range nodes { + fmt.Fprintf(w, "%s\t%s\t%s\t%t\t%t\t%t\t%s\t%s\n", node.Name, node.Profile, + node.ProfileGeneration, node.RuntimeReady, node.NodeReady, node.Unschedulable, + node.ProvisionState, node.Error) + } + return w.Flush() +} + +func (c *client) inspectProfile(name string) error { + profile, err := c.get(profilesResource, name) + if err != nil { + return err + } + if profile.Metadata.UID == "" { + return fmt.Errorf("profile %s has no UID", name) + } + objects, err := c.list("nodes", "--selector", "brewlet.sh/owner-uid="+profile.Metadata.UID) + if err != nil { + return err + } + nodes, err := summarizeNodes(objects, profile.Metadata.UID) + if err != nil { + return err + } + report := struct { + Profile profileSummary `json:"profile"` + Nodes []nodeSummary `json:"nodes"` + }{summarizeProfile(profile), nodes} + format := c.opts.output + if format == "table" { + format = "yaml" + } + return encode(c.out, report, format) +} + +type deploymentSummary struct { + Name string `json:"name"` + Present bool `json:"present"` + Ready bool `json:"ready"` + Generation int64 `json:"generation"` + ObservedGeneration int64 `json:"observedGeneration"` + Desired int `json:"desired"` + Updated int `json:"updated"` + Available int `json:"available"` + Conditions []condition `json:"conditions,omitempty"` +} + +func summarizeDeployment(obj object) (deploymentSummary, error) { + var spec struct { + Replicas *int `json:"replicas"` + } + if err := json.Unmarshal(obj.Spec, &spec); err != nil { + return deploymentSummary{}, fmt.Errorf("decode deployment %s: %w", obj.Metadata.Name, err) + } + desired := 1 + if spec.Replicas != nil { + desired = *spec.Replicas + } + s := obj.Status + ready := obj.Metadata.DeletionTimestamp == "" && s.ObservedGeneration == obj.Metadata.Generation && + s.Replicas == desired && s.UpdatedReplicas == desired && s.ReadyReplicas == desired && + s.AvailableReplicas == desired + return deploymentSummary{ + Name: obj.Metadata.Name, Present: true, Ready: ready, + Generation: obj.Metadata.Generation, ObservedGeneration: s.ObservedGeneration, + Desired: desired, Updated: s.UpdatedReplicas, Available: s.AvailableReplicas, Conditions: s.Conditions, + }, nil +} + +type statusReport struct { + Healthy bool `json:"healthy"` + Components []deploymentSummary `json:"components"` + Profiles []profileSummary `json:"profiles"` + Nodes []nodeSummary `json:"nodes"` +} + +func (c *client) status() error { + deployments, err := c.list("deployments", "--namespace", c.opts.systemNamespace) + if err != nil { + return err + } + profiles, err := c.list(profilesResource) + if err != nil { + return err + } + objects, err := c.list("nodes") + if err != nil { + return err + } + nodes, err := summarizeNodes(objects, "") + if err != nil { + return err + } + report := statusReport{Healthy: true, Profiles: []profileSummary{}, Nodes: nodes} + for _, name := range []string{"brewlet-operator", "brewlet-admission"} { + component := deploymentSummary{Name: name} + for _, obj := range deployments { + if obj.Metadata.Name == name { + component, err = summarizeDeployment(obj) + if err != nil { + return err + } + break + } + } + report.Components = append(report.Components, component) + if !component.Ready { + report.Healthy = false + } + } + for _, obj := range profiles { + profile := summarizeProfile(obj) + report.Profiles = append(report.Profiles, profile) + report.Healthy = report.Healthy && profile.Ready + } + sort.Slice(report.Profiles, func(i, j int) bool { return report.Profiles[i].Name < report.Profiles[j].Name }) + readyNodes := 0 + for _, node := range nodes { + if node.Error != "" || !node.RuntimeReady || !node.NodeReady { + report.Healthy = false + } + if node.RuntimeReady && node.NodeReady && !node.Unschedulable { + readyNodes++ + } + } + report.Healthy = report.Healthy && len(profiles) > 0 && readyNodes > 0 + if c.opts.output == "table" { + for _, component := range report.Components { + fmt.Fprintf(c.out, "%s: present=%t ready=%t updated=%d/%d available=%d\n", + component.Name, component.Present, component.Ready, component.Updated, component.Desired, component.Available) + } + if err := c.profileTable(report.Profiles); err != nil { + return err + } + if err := c.nodeTable(report.Nodes); err != nil { + return err + } + } else if err := encode(c.out, report, c.opts.output); err != nil { + return err + } + if !report.Healthy { + return fmt.Errorf("Brewlet is not ready: inspect component rollouts, profile conditions and node failures (an intentionally disabled admission deployment is also reported as not ready)") + } + return nil +} + +type podSummary struct { + Name string `json:"name"` + Node string `json:"node,omitempty"` + Phase string `json:"phase"` + Ready bool `json:"ready"` + Restarts int `json:"restarts"` + Problems []string `json:"problems"` + JDKRequest string `json:"jdkRequest,omitempty"` + Launcher string `json:"launcher,omitempty"` +} + +type eventSummary struct { + Object string `json:"object"` + Type string `json:"type"` + Reason string `json:"reason"` + Message string `json:"message"` +} + +type appReport struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Ready bool `json:"ready"` + Reason string `json:"reason"` + Image string `json:"image"` + JDKRequest string `json:"jdkRequest"` + Launcher string `json:"launcher"` + Arch []string `json:"arch,omitempty"` + Conditions []condition `json:"conditions"` + Deployments []deploymentSummary `json:"deployments"` + Pods []podSummary `json:"pods"` + Events []eventSummary `json:"events"` +} + +func (c *client) inspectApp(name string) error { + app, err := c.get(appsResource, name, c.namespaceArgs()...) + if err != nil { + return err + } + if app.Metadata.UID == "" || app.Metadata.Namespace == "" { + return fmt.Errorf("application %s has no UID or namespace", name) + } + nsArgs := []string{"--namespace", app.Metadata.Namespace} + selector := "app.kubernetes.io/name=" + name + ",app.kubernetes.io/managed-by=brewlet-operator" + workloads, err := c.list("deployments,replicasets,pods", append(nsArgs, "--selector", selector)...) + if err != nil { + return err + } + events, err := c.list("events", nsArgs...) + if err != nil { + return err + } + var spec struct { + Artifact struct { + Image string `json:"image"` + } `json:"artifact"` + JVM struct { + Version int `json:"version"` + Distribution string `json:"distribution"` + Launcher string `json:"launcher"` + } `json:"jvm"` + Arch []string `json:"arch"` + } + if err := json.Unmarshal(app.Spec, &spec); err != nil { + return fmt.Errorf("decode application spec: %w", err) + } + ready, reason := readyCondition(app) + report := appReport{ + Name: name, Namespace: app.Metadata.Namespace, Ready: ready, Reason: reason, + Image: spec.Artifact.Image, Launcher: spec.JVM.Launcher, Arch: spec.Arch, + Conditions: app.Status.Conditions, Deployments: []deploymentSummary{}, Pods: []podSummary{}, Events: []eventSummary{}, + } + if report.Launcher == "" { + report.Launcher = "java" + } + if spec.JVM.Version > 0 { + report.JDKRequest = fmt.Sprint(spec.JVM.Version) + if spec.JVM.Distribution != "" { + report.JDKRequest = spec.JVM.Distribution + "-" + report.JDKRequest + } + } + uids := map[string]bool{app.Metadata.UID: true} + // Traverse controller ownership, not just labels: old ReplicaSets belong, + // but a similarly labelled Pod from another application does not. + for _, kind := range []string{"Deployment", "ReplicaSet", "Pod"} { + for _, obj := range workloads { + if obj.Kind != kind || !ownedBy(obj, uids) || obj.Metadata.UID == "" { + continue + } + uids[obj.Metadata.UID] = true + switch kind { + case "Deployment": + deployment, err := summarizeDeployment(obj) + if err != nil { + return err + } + report.Deployments = append(report.Deployments, deployment) + if report.Ready && !deployment.Ready { + report.Reason = "DeploymentNotReady" + } + report.Ready = report.Ready && deployment.Ready + case "Pod": + var podSpec struct { + NodeName string `json:"nodeName"` + } + if err := json.Unmarshal(obj.Spec, &podSpec); err != nil { + return fmt.Errorf("decode pod %s: %w", obj.Metadata.Name, err) + } + pod := podSummary{ + Name: obj.Metadata.Name, Node: podSpec.NodeName, Phase: obj.Status.Phase, + Problems: []string{}, JDKRequest: obj.Metadata.Annotations["brewlet.sh/jdk"], + Launcher: obj.Metadata.Annotations["brewlet.sh/launcher"], + } + for _, cond := range obj.Status.Conditions { + if cond.Type == "Ready" { + pod.Ready = cond.Status == "True" && obj.Metadata.DeletionTimestamp == "" + } + } + for _, container := range obj.Status.ContainerStatuses { + pod.Restarts += container.RestartCount + if container.State.Waiting != nil { + pod.Problems = append(pod.Problems, container.Name+": "+container.State.Waiting.Reason) + } + if term := container.State.Terminated; term != nil && term.ExitCode != 0 { + pod.Problems = append(pod.Problems, fmt.Sprintf("%s: %s (exit %d)", container.Name, term.Reason, term.ExitCode)) + } + } + report.Pods = append(report.Pods, pod) + } + } + } + report.Ready = report.Ready && len(report.Deployments) > 0 + if len(report.Deployments) == 0 { + report.Reason = "DeploymentMissing" + } + for _, event := range events { + if uids[event.InvolvedObject.UID] { + report.Events = append(report.Events, eventSummary{ + Object: event.InvolvedObject.Kind + "/" + event.InvolvedObject.Name, + Type: event.Type, Reason: event.Reason, Message: event.Message, + }) + } + } + sort.Slice(report.Deployments, func(i, j int) bool { return report.Deployments[i].Name < report.Deployments[j].Name }) + sort.Slice(report.Pods, func(i, j int) bool { return report.Pods[i].Name < report.Pods[j].Name }) + sort.Slice(report.Events, func(i, j int) bool { + a, b := report.Events[i], report.Events[j] + return a.Object+a.Reason+a.Message < b.Object+b.Reason+b.Message + }) + format := c.opts.output + if format == "table" { + format = "yaml" + } + return encode(c.out, report, format) +} diff --git a/core/internal/kube/update.go b/core/internal/kube/update.go new file mode 100644 index 00000000..6fa0fbc5 --- /dev/null +++ b/core/internal/kube/update.go @@ -0,0 +1,507 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package kube + +import ( + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "regexp" + "strings" + + "github.com/microsoft/brewlet/internal/sourcepolicy" + "go.yaml.in/yaml/v3" +) + +type document map[string]json.RawMessage + +type jdkSource struct { + Distribution string `json:"distribution"` + Feature int64 `json:"feature"` + Source struct { + Image string `json:"image"` + JavaHome string `json:"javaHome"` + } `json:"source"` +} + +type launcherSource struct { + Name string `json:"name"` + Source struct { + Image string `json:"image"` + Path string `json:"path"` + } `json:"source"` +} + +type updateOptions struct { + profile, file, values, distribution, name, image, path string + feature int64 + jdk, replace bool + dryRun dryRunMode +} + +type dryRunMode string + +const ( + dryRunClient dryRunMode = "client" + dryRunServer dryRunMode = "server" +) + +func (m *dryRunMode) String() string { return string(*m) } +func (m *dryRunMode) IsBoolFlag() bool { return true } +func (m *dryRunMode) Set(value string) error { + switch value { + case "true", "client": + *m = dryRunClient + case "server": + *m = dryRunServer + default: + return fmt.Errorf("--dry-run accepts client or server; omit the flag to apply changes") + } + return nil +} + +func (u *updateOptions) flags(fs *flag.FlagSet, jdk bool) { + u.jdk = jdk + fs.StringVar(&u.profile, "profile", "", "explicit target NodeProfile name") + fs.StringVar(&u.file, "file", "", "read a NodeProfile YAML/JSON file (requires --dry-run=client)") + fs.StringVar(&u.values, "values", "", "read complete Helm values (requires --dry-run=client)") + fs.Var(&u.dryRun, "dry-run", "validate and print without saving: client (default when bare) or server") + fs.StringVar(&u.image, "image", "", "fully qualified, tagless SHA-256 digest reference") + fs.BoolVar(&u.replace, "replace", false, "replace a source with the same inventory identity") + if jdk { + fs.StringVar(&u.distribution, "distribution", "", "JDK distribution token") + fs.Int64Var(&u.feature, "feature", 0, "JDK feature version") + fs.StringVar(&u.path, "java-home", "", "absolute JDK home inside the image") + } else { + fs.StringVar(&u.name, "name", "", "optional launcher name (not java)") + fs.StringVar(&u.path, "path", "", "absolute launcher binary path inside the image") + } +} + +var dnsLabel = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +func validateName(value string, max int) error { + if value == "" || len(value) > max { + return fmt.Errorf("name %q must contain 1-%d characters", value, max) + } + for _, part := range strings.Split(value, ".") { + if len(part) > 63 || !dnsLabel.MatchString(part) { + return fmt.Errorf("name %q must be a lowercase DNS name", value) + } + } + return nil +} + +func validateToken(value string, max int) error { + if len(value) > max || !dnsLabel.MatchString(value) { + return fmt.Errorf("%q must be a lowercase DNS label of at most %d characters", value, max) + } + return nil +} + +func validateSource(image, path string) error { + if len(image) > 2048 || len(path) > 1024 { + return fmt.Errorf("source image must be at most 2048 characters and source path at most 1024") + } + if err := sourcepolicy.ValidateDigestReference(image); err != nil { + return err + } + return sourcepolicy.ValidateAbsolutePath(path) +} + +func (u updateOptions) entry() (string, json.RawMessage, error) { + if err := validateSource(u.image, u.path); err != nil { + return "", nil, err + } + if u.jdk { + if err := validateToken(u.distribution, 48); err != nil { + return "", nil, err + } + if u.feature < 1 || u.feature > 2147483647 { + return "", nil, fmt.Errorf("--feature must be between 1 and 2147483647") + } + jdk := jdkSource{Distribution: u.distribution, Feature: u.feature} + jdk.Source.Image, jdk.Source.JavaHome = u.image, u.path + raw, err := json.Marshal(jdk) + return "jdks", raw, err + } + if err := validateToken(u.name, 54); err != nil { + return "", nil, err + } + if u.name == "java" { + return "", nil, fmt.Errorf("java is supplied by each JDK and must not be added as a launcher") + } + launcher := launcherSource{Name: u.name} + launcher.Source.Image, launcher.Source.Path = u.image, u.path + raw, err := json.Marshal(launcher) + return "launchers", raw, err +} + +func entryIdentity(field string, raw json.RawMessage) (string, error) { + if field == "jdks" { + var entry jdkSource + if err := json.Unmarshal(raw, &entry); err != nil { + return "", err + } + if err := validateToken(entry.Distribution, 48); err != nil { + return "", err + } + if entry.Feature < 1 || entry.Feature > 2147483647 { + return "", fmt.Errorf("invalid JDK feature %d", entry.Feature) + } + return fmt.Sprintf("%s-%d", entry.Distribution, entry.Feature), nil + } + var entry launcherSource + if err := json.Unmarshal(raw, &entry); err != nil { + return "", err + } + if err := validateToken(entry.Name, 54); err != nil { + return "", err + } + if entry.Name == "java" { + return "", fmt.Errorf("java must not appear in the launcher inventory") + } + return entry.Name, nil +} + +func updateInventory(spec document, field string, entry json.RawMessage, replace bool) error { + entries := []json.RawMessage{} + if raw, exists := spec[field]; exists { + if err := json.Unmarshal(raw, &entries); err != nil { + return fmt.Errorf("decode %s: %w", field, err) + } + } + key, err := entryIdentity(field, entry) + if err != nil { + return err + } + found := false + seen := map[string]bool{} + for i, existing := range entries { + id, err := entryIdentity(field, existing) + if err != nil { + return fmt.Errorf("invalid existing %s entry: %w", field, err) + } + if seen[id] { + return fmt.Errorf("duplicate %s entry %q; repair the source configuration first", field, id) + } + seen[id] = true + if id != key { + continue + } + found = true + var oldValue, newValue any + if err := json.Unmarshal(existing, &oldValue); err != nil { + return err + } + if err := json.Unmarshal(entry, &newValue); err != nil { + return err + } + oldJSON, err := json.Marshal(oldValue) + if err != nil { + return err + } + newJSON, err := json.Marshal(newValue) + if err != nil { + return err + } + if !bytes.Equal(oldJSON, newJSON) && !replace { + return fmt.Errorf("%s %q already exists with a different source; use --replace deliberately", field, key) + } + entries[i] = entry + } + if !found { + entries = append(entries, entry) + } + if len(entries) > 32 { + return fmt.Errorf("%s inventory cannot exceed 32 entries", field) + } + raw, err := json.Marshal(entries) + if err != nil { + return err + } + spec[field] = raw + return nil +} + +func readDocument(file string) (document, error) { + raw, err := os.ReadFile(file) + if err != nil { + return nil, err + } + dec := yaml.NewDecoder(bytes.NewReader(raw)) + var value map[string]any + if err := dec.Decode(&value); err != nil { + return nil, fmt.Errorf("decode %s: %w", file, err) + } + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + if err != nil { + return nil, fmt.Errorf("decode %s: %w", file, err) + } + return nil, fmt.Errorf("%s must contain exactly one YAML/JSON document", file) + } + data, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("convert %s to JSON: %w", file, err) + } + return decodeDocument(data) +} + +func decodeDocument(raw []byte) (document, error) { + var doc document + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, err + } + if doc == nil { + return nil, fmt.Errorf("expected a non-null object") + } + return doc, nil +} + +func managedBy(meta metadata) string { + if meta.Labels["app.kubernetes.io/managed-by"] != "" && + !oneOf(strings.ToLower(meta.Labels["app.kubernetes.io/managed-by"]), "brewlet", "kubectl") { + return meta.Labels["app.kubernetes.io/managed-by"] + } + for _, fields := range []map[string]string{meta.Labels, meta.Annotations} { + for _, key := range []string{ + "meta.helm.sh/release-name", "meta.helm.sh/release-namespace", + "argocd.argoproj.io/instance", "argocd.argoproj.io/tracking-id", + "kustomize.toolkit.fluxcd.io/name", "helm.toolkit.fluxcd.io/name", + } { + if fields[key] != "" { + return key + "=" + fields[key] + } + } + } + if len(meta.OwnerReferences) > 0 { + return "an owning Kubernetes resource" + } + for _, field := range meta.ManagedFields { + if field.Subresource != "" { + continue + } + var fields document + // Unknown ownership is not permission to overwrite a profile. + if err := json.Unmarshal(field.Fields, &fields); err != nil { + return "unrecognized field ownership" + } + if _, ownsSpec := fields["f:spec"]; ownsSpec && + !oneOf(field.Manager, "brewlet", "kubectl", "kubectl-client-side-apply", "kubectl-edit", "kubectl-patch") { + return "field manager " + field.Manager + } + } + return "" +} + +func (c *client) updateProfile(u updateOptions) error { + if err := validateName(u.profile, 253); err != nil { + return fmt.Errorf("--profile: %w", err) + } + if u.file != "" && u.values != "" { + return fmt.Errorf("--file and --values are mutually exclusive") + } + if u.dryRun != dryRunClient && (u.file != "" || u.values != "") { + return fmt.Errorf("--file and --values require --dry-run=client (or bare --dry-run)") + } + field, entry, err := u.entry() + if err != nil { + return err + } + if u.values != "" { + return c.updateValues(u, field, entry) + } + var doc document + var live object + if u.file != "" { + doc, err = readDocument(u.file) + } else { + live, err = c.get(profilesResource, u.profile, "--show-managed-fields=true") + if err == nil { + doc, err = decodeDocument(live.raw) + } + } + if err != nil { + return err + } + raw, err := json.Marshal(doc) + if err != nil { + return err + } + var profile object + if err := json.Unmarshal(raw, &profile); err != nil { + return fmt.Errorf("decode profile: %w", err) + } + if profile.APIVersion != "node.brewlet.sh/v1alpha1" || profile.Kind != "NodeProfile" || + profile.Metadata.Name != u.profile || profile.Metadata.Namespace != "" { + return fmt.Errorf("expected cluster-scoped node.brewlet.sh/v1alpha1 NodeProfile %q", u.profile) + } + if profile.Metadata.DeletionTimestamp != "" { + return fmt.Errorf("profile %q is terminating; refusing to change its inventory", u.profile) + } + owner := managedBy(profile.Metadata) + if u.dryRun != dryRunClient && owner != "" { + return fmt.Errorf("profile %q is managed by %s; edit its source of truth (use add --dry-run --values FILE for Helm), not the live object", u.profile, owner) + } + spec, err := decodeDocument(doc["spec"]) + if err != nil { + return fmt.Errorf("decode profile spec: %w", err) + } + if err := updateInventory(spec, field, entry, u.replace); err != nil { + return err + } + doc["spec"], err = json.Marshal(spec) + if err != nil { + return err + } + if u.dryRun != dryRunClient { + if live.Metadata.UID == "" || live.Metadata.ResourceVersion == "" { + return fmt.Errorf("live profile has no UID/resourceVersion; cannot update safely") + } + patch := []struct { + Op string `json:"op"` + Path string `json:"path"` + Value any `json:"value"` + }{ + {"test", "/metadata/uid", live.Metadata.UID}, + {"test", "/metadata/resourceVersion", live.Metadata.ResourceVersion}, + {"add", "/spec/" + field, spec[field]}, + } + input, err := json.Marshal(patch) + if err != nil { + return err + } + patchFile, err := os.CreateTemp("", "brewlet-profile-patch-*.json") + if err != nil { + return err + } + defer os.Remove(patchFile.Name()) + if _, err := patchFile.Write(input); err != nil { + patchFile.Close() + return err + } + if err := patchFile.Close(); err != nil { + return err + } + args := []string{"patch", profilesResource, u.profile, "--type=json", "--patch-file", patchFile.Name(), "--field-manager=brewlet", "-o", "json"} + if u.dryRun == dryRunServer { + args = append(args, "--dry-run=server") + } + result, err := c.kubectl(nil, args...) + if err != nil { + return fmt.Errorf("profile update failed (concurrent changes require re-reading and reviewing the profile): %w", err) + } + doc, err = decodeDocument(result) + if err != nil { + return fmt.Errorf("decode updated profile: %w", err) + } + } + if err := cleanManifest(doc); err != nil { + return err + } + if err := encode(c.out, doc, c.opts.output); err != nil { + return err + } + switch u.dryRun { + case dryRunServer: + fmt.Fprintln(c.err, "Server dry run succeeded; no inventory was changed.") + case dryRunClient: + fmt.Fprintln(c.err, "Client dry run succeeded; no inventory was changed. Server validation was not performed.") + if owner != "" { + fmt.Fprintf(c.err, "Profile is managed by %s. Update its source of truth; for Helm use add --dry-run --values FILE.\n", owner) + } + default: + fmt.Fprintln(c.err, "Profile update accepted; provisioning is asynchronous. Inspect the profile and advertised node inventory before using the new source.") + } + return nil +} + +func cleanManifest(doc document) error { + meta, err := decodeDocument(doc["metadata"]) + if err != nil { + return err + } + for _, key := range []string{"uid", "resourceVersion", "generation", "creationTimestamp", "deletionTimestamp", "managedFields", "selfLink", "finalizers"} { + delete(meta, key) + } + if raw, exists := meta["annotations"]; exists && string(raw) != "null" { + annotations, err := decodeDocument(raw) + if err != nil { + return err + } + delete(annotations, "kubectl.kubernetes.io/last-applied-configuration") + meta["annotations"], err = json.Marshal(annotations) + if err != nil { + return err + } + } + doc["metadata"], err = json.Marshal(meta) + delete(doc, "status") + return err +} + +func (c *client) updateValues(u updateOptions, field string, entry json.RawMessage) error { + doc, err := readDocument(u.values) + if err != nil { + return err + } + if u.profile == "default" { + var defaultProfile struct { + Enabled *bool `json:"enabled"` + } + if raw, exists := doc["defaultProfile"]; exists { + if err := json.Unmarshal(raw, &defaultProfile); err != nil { + return err + } + if defaultProfile.Enabled != nil && !*defaultProfile.Enabled { + return fmt.Errorf("defaultProfile.enabled is false; edit the enabled profile's source instead") + } + } + spec, err := decodeDocument(doc["provisioner"]) + if err != nil { + return fmt.Errorf("values must contain the complete provisioner configuration: %w", err) + } + if err := updateInventory(spec, field, entry, u.replace); err != nil { + return err + } + doc["provisioner"], err = json.Marshal(spec) + if err != nil { + return err + } + } else { + var profiles []document + if err := json.Unmarshal(doc["profiles"], &profiles); err != nil { + return fmt.Errorf("values must contain profiles: %w", err) + } + found := false + for _, profile := range profiles { + var name string + if err := json.Unmarshal(profile["name"], &name); err != nil { + return err + } + if name == u.profile { + if found { + return fmt.Errorf("duplicate profile %q in values", name) + } + found = true + if err := updateInventory(profile, field, entry, u.replace); err != nil { + return err + } + } + } + if !found { + return fmt.Errorf("profile %q is not present in the values file", u.profile) + } + doc["profiles"], err = json.Marshal(profiles) + if err != nil { + return err + } + } + return encode(c.out, doc, c.opts.output) +} diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 38a2bf44..9b13c984 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -25,6 +25,7 @@ brewlet run [flags] pull + launch java -jar on this node brewlet bundle [flags] emit an OCI runc bundle (the shim path) brewlet jdks [flags] list JDKs available across the cluster brewlet doctor [flags] diagnose cluster and developer readiness +brewlet k8s [flags] manage and inspect Brewlet on Kubernetes brewlet version print the CLI version ``` @@ -50,7 +51,7 @@ brewlet keygen --private FILE --public FILE ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--private` | *(required)* | Output path for the PKCS#8 PEM private key. | | `--public` | *(required)* | Output path for the SubjectPublicKeyInfo PEM public key. | @@ -77,7 +78,7 @@ brewlet dependency-bundle \ ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--name` | *(required)* | Stable bundle name. | | `--version` | *(required)* | Bundle version. | | `--source-bom` | *(required)* | Approved Maven BOM in `groupId:artifactId:version` form. | @@ -126,7 +127,7 @@ brewlet push [--format image|artifact] [--store DIR] [--config FILE] ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--format` | `image` | Delivery format: `image` (standard, kubelet-pullable OCI image — the production `runtimeClassName: brewlet` pod image) or `artifact` (native Brewlet OCI artifact for local OCI-layout / CLI / bundle workflows, not the production Kubernetes pod image path). See [runnable-image delivery](runnable-image.md). | | `--store` | `./oci` | OCI layout directory to write the artifact into. | | `--config` | *(none)* | Path to a `jvm-config.json` to embed verbatim (overrides the generated one). See the [launch config schema](building-and-publishing.md#2-the-launch-config). | @@ -187,7 +188,7 @@ brewlet inspect [--store DIR] ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--store` | `./oci` | OCI layout directory to read from. | | `--trusted-public-key` | *(none)* | ECDSA P-256 public key used to verify managed provenance or attestation; requires `--trusted-signer-identity`. | | `--trusted-signer-identity` | *(none)* | Expected signed identity. For a bundle this is its publisher; for a final image this is its application builder. Requires `--trusted-public-key`. | @@ -211,7 +212,7 @@ brewlet run [--store DIR] [--jdk-root DIR] [--launcher NAME] [--appcds-reg ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--store` | `./oci` | OCI layout directory to read from. | | `--jdk-root` | *(none)* | Node JDK home to launch with. When unset, falls back to `BREWLET_JDK_HOME`, then `JAVA_HOME`, then `java` on `PATH`. | | `--launcher` | `java` | Launcher binary **name** under the selected JDK (or a compatible node-installed launcher name such as `jaz`), resolved on `PATH`. It is a lowercase DNS-1123 token, never a path: separators, `..` and absolute paths are rejected. | @@ -242,7 +243,7 @@ brewlet bundle [--store DIR] [--cpu N] [--memory M] [--uid UID] [--gid GID ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--store` | `./oci` | OCI layout directory to read from. | | `--cpu` | *(unlimited)* | CPU limit, e.g. `2` or `500m` → sandbox `cpu.max`. Minimum `10m` (`0.01` CPU), matching Linux's minimum quota for the 100ms period. | | `--memory` | *(unlimited)* | Memory limit, e.g. `512Mi` or `1Gi` → sandbox `memory.max`. | @@ -281,7 +282,7 @@ brewlet jdks [--output table|wide|json] [--kubeconfig FILE] [--context CTX] [--s ``` | Flag | Default | Meaning | -|---|---|---| +| --- | --- | --- | | `--output` | `table` | `table` = distinct JDKs across the fleet with a node count; `wide` = one row per node; `json` = machine-readable aggregation (e.g. to drive a CI matrix). | | `--kubeconfig` | *(kubectl default)* | Path to a kubeconfig file. | | `--context` | *(current)* | kubeconfig context to use. | @@ -332,14 +333,302 @@ for CI and platform handoff automation. --- -## `brewlet version` +## `brewlet k8s` + +Brewlet-specific Kubernetes operations, using your installed `kubectl` and +existing kubeconfig credentials. Installation also requires Helm. No command +creates a cluster, changes your current context, directly modifies a node, or +opens a debugger. `brewlet jdks` and `brewlet doctor` remain compatibility +aliases for `brewlet k8s jdk list` and `brewlet k8s doctor`. + +```text +brewlet k8s jdk list +brewlet k8s launcher list +brewlet k8s profile list +brewlet k8s profile inspect NAME +brewlet k8s inspect app NAME +brewlet k8s status +brewlet k8s doctor +brewlet k8s install --version X.Y.Z --values FILE +brewlet k8s jdk add --profile NAME --distribution NAME --feature N --image REF --java-home PATH +brewlet k8s launcher add --profile NAME --name NAME --image REF --path PATH +``` + +**Command verbs describe their effects:** `add` updates the live profile and +`install` installs Brewlet. `--dry-run` previews those operations without saving +changes and prints the proposed YAML/JSON only after validation succeeds. +Failed dry runs exit nonzero with an error on stderr and no rendered output on +stdout. `list`, `inspect`, `status`, and `doctor` are read-only. Redirecting +stdout to a file does not change these semantics. + +### Connections and output + +Common flags may precede the command or follow its full name, for example +`brewlet k8s --context staging profile inspect java-workers --output json`. +Flags may also follow positional names. + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--kubeconfig FILE` | Tool default | Kubeconfig passed to every Kubernetes/Helm invocation. | +| `--context NAME` | Current context | Explicit context, without modifying kubeconfig. | +| `--namespace NAME` | Context namespace | Application inspection namespace. `doctor` preserves its original `default` namespace; installation defaults to `brewlet`. Profiles and node inventories are cluster-scoped. | +| `--timeout DURATION` | `30s` | Deadline for each kubectl invocation, including credential helpers. | +| `--output FORMAT` | Command-dependent | Read commands default to `table` and support `json`/`yaml`. JDK/launcher inventory supports `table`/`wide`/`json` instead. `add`, including dry runs, defaults to `yaml` and also supports `json`. Installation prints Helm output after a successful command. | -Print the release version embedded in the binary: +Inspection's default output is a structured YAML report rather than a flattened +table. All commands propagate tool/API failures; RBAC-denied reads are not +reported as empty inventories. Output goes to stdout and mutation notices to +stderr. + +### Inventory, status, and inspection ```bash -brewlet version +brewlet k8s jdk list --output wide +brewlet k8s launcher list --selector brewlet.sh/runtime=ready +brewlet k8s profile list --output json +brewlet k8s profile inspect java-workers +brewlet k8s status --system-namespace brewlet --output json +brewlet k8s inspect app orders --namespace my-team +brewlet k8s doctor --namespace my-team +``` + +JDK inventory reuses the existing [JDK aggregation](#brewlet-jdks), including +legacy annotation fallback. Launcher inventory aggregates optional launcher +names from `brewlet.sh/launchers`; the JDK's implicit `java` launcher is not a +separate entry. Both inventories describe **node-advertised state**, not a +catalog, a live probe of node files, or proof that a particular Pod uses a JDK. +Use `--selector` to restrict nodes, such as to runtime-ready nodes. + +Profile listing shows **desired** JDK/launcher inventories, pool selection, +readiness, and node counts. Profile inspection adds the full spec, conditions, +field-management information, and claimed node summaries. Node identity and +ownership must match the profile's UID. Missing or stale observed generations +never count as a current Ready condition. + +`status` reports the operator and admission **Deployment rollout state** in +`--system-namespace` (default `brewlet`), profile conditions, and node readiness +and provisioning errors. It exits nonzero if either component is absent or +unready, any profile/node is unready, or there are no profiles or no uncordoned, +runtime-ready, Kubernetes-ready nodes. An intentionally disabled admission +Deployment therefore also produces a nonzero result. This is a conservative +rollout snapshot, not a webhook TLS, certificate, endpoint, or scheduling probe. + +Application inspection follows controller ownership from JavaApplication to +Deployment, ReplicaSets, and Pods, and includes their events. Matching labels +alone do not establish ownership. Reports distinguish the **requested JDK** +from the actual per-replica runtime, which Brewlet does not yet report. They +show Pod placement, restarts, and container failure reasons without dumping +environment variables or Secrets. Condition/event messages are operator-provided +text; review them before sharing a report. Inspection succeeds even for an +unready resource; use `status` or `doctor` for readiness exit codes. + +Read permissions are required for the objects each command inspects. +Application inspection lists Deployments, ReplicaSets, Pods, and Events in its +namespace; profile inspection reads the profile and lists Nodes. `status` lists +control-plane Deployments, NodeProfiles, and Nodes. + +### Installation + +`--values FILE` takes a **Helm values file that you create**, not a Brewlet CLI +configuration file. `values.yaml` is an example filename, not a file generated +by the command or automatically loaded from this repository. You may name it +anything and pass its path with `--values` or `-f`. + +#### Draft your values file + +For a first installation, save the following as `values.yaml`. This is a +minimal configuration for one existing worker pool and one JDK; other settings +inherit the selected chart release's defaults. + +```yaml +provisioner: + pools: + - java-workers + jdks: + - distribution: temurin + feature: 21 + source: + image: docker.io/library/eclipse-temurin@sha256:<64-lowercase-hex> + javaHome: /opt/java/openjdk ``` +Replace the example choices before running the command: + +| Field | What to supply | +|---|---| +| `provisioner.pools` | Names of existing node pools Brewlet may provision, not individual node names. The CLI does not create these pools. Replace `java-workers` with a pool you control. | +| `provisioner.jdks[].distribution` | A lowercase inventory name for the JDK, such as `temurin` or `microsoft`. It does not select or download an image automatically. | +| `provisioner.jdks[].feature` | The Java feature version actually supplied by your source image, such as `21` or `25`. | +| `provisioner.jdks[].source.image` | The fully qualified, tagless image reference pinned to the approved manifest or multi-platform index digest. Replace `<64-lowercase-hex>` with all 64 lowercase hexadecimal characters from that digest. | +| `provisioner.jdks[].source.javaHome` | The absolute JDK home **inside the source image**, not your workstation's `JAVA_HOME`. Confirm that `/bin/java` exists in that image. | + +The Temurin entry is an example, not a built-in runtime recommendation. Choose +an image that supports every architecture in the selected pool; see the +[JDK source model](jdk-management.md#source-model) for source requirements. +Pool labels are detected automatically for supported providers. For bare-metal +or nonstandard labels, set `provisioner.poolKey` as described in +[pool configuration](configuration.md#where-the-provisioner-may-run). + +The chart deliberately leaves `provisioner.pools` and `provisioner.jdks` empty: +they are required while `defaultProfile.enabled` is true. Copying the chart's +default file unchanged will therefore fail. You do not need to copy every +setting into your own file; include your pool, complete JDK inventory, and any +other overrides you need. Additional JDKs are entries in the same `jdks` list. +Optional launchers belong in `provisioner.launchers`; omit that list to use the +JDK's implicit `java` launcher. + +The [full Helm values reference](configuration.md#helm-chart-values) and +[annotated chart defaults](https://github.com/microsoft/brewlet/blob/main/kubernetes/charts/brewlet/values.yaml) +describe optional settings, including launchers, mirrors, tolerations, rollout +policy, and additional profiles. Use settings supported by the chart version +you select. For GitOps-managed profiles, you may instead disable +`defaultProfile.enabled` and supply your own NodeProfiles; see +[installation and profile ownership](installation.md#helm-recommended). + +#### Preview and install + +After saving and reviewing `values.yaml`, replace `x.y.z` with the exact chart +release you want and `evaluation` with your intended kubeconfig context: + +```bash +# Set this to the exact approved chart release, not "latest". +RELEASE_VERSION=x.y.z + +# Download and render the chart; does not contact the Kubernetes API. +brewlet k8s install --version "$RELEASE_VERSION" --values values.yaml --dry-run + +# Install into a deliberately selected fresh cluster. +brewlet k8s install --context evaluation --version "$RELEASE_VERSION" \ + --values values.yaml --namespace brewlet +``` + +Preview validates chart rendering, not source image contents or node readiness. +If it fails, fix the reported values error before installing. Installation is +privileged and modifies the selected hosts; review the +[cluster prerequisites](installation.md#prerequisites) first. Control-plane +nodes and tainted pools require deliberate configuration, not an implicit opt-in. + +The CLI invokes `helm install` against +`oci://ghcr.io/microsoft/charts/brewlet`, creates the namespace, and waits for +chart resources. `--release` defaults to `brewlet`; `--wait-timeout` defaults to +`5m`. The Helm process deadline includes that duration plus `--timeout` for +setup. `--values`/`-f` may be repeated in Helm precedence order. The CLI sets the +chart's `namespace` value to the selected release namespace so resources and the +release cannot accidentally land in different namespaces. + +Installation refuses existing Brewlet CRDs and Helm refuses an existing release. +It does **not** upgrade a release or migrate CRDs: follow +[Upgrading](installation.md#upgrading) for those operations, including clusters +retaining CRDs after uninstall. Preview uses `helm template --include-crds`; it +still needs chart registry access, and it is not server-side validation. + +Helm readiness is not node provisioning completion. Follow installation with +`brewlet k8s status` and inventory inspection using the same kubeconfig/context +and the installation namespace as `--system-namespace`. Sources are never chosen for you, +control-plane provisioning is never implicitly enabled, and no cleanup or +uninstall is attempted after a failed installation. + +### Safe JDK and launcher additions + +`jdk add` and `launcher add` **update the live NodeProfile by default**. They +always select an existing profile and return its updated declaration. The +source image must be a fully qualified, tagless SHA-256 digest reference; paths +and inventory names follow the [JDK](jdk-management.md#source-model) and +[launcher](launchers.md#launcher-names-are-tokens-not-paths) source contracts. + +| Command mode | Effect | +| --- | --- | +| `jdk add` / `launcher add` | Persist the inventory change to the live profile. | +| `add --dry-run` | Validate locally and print the proposed declaration; equivalent to `--dry-run=client`. | +| `add --dry-run=client` | Validate the requested source and inventory change locally without sending a patch to the API server. | +| `add --dry-run=server` | Validate the conditional patch through the API server and print the result without saving it. | +| `add --dry-run --file FILE` | Read a source NodeProfile and validate/generate its proposed update offline. | +| `add --dry-run --values FILE` | Read complete Helm values and validate/generate their proposed update offline. | + +Bare `--dry-run` and `--dry-run=client` still read the live profile unless +`--file` or `--values` is supplied. They do not validate registry contents, node +compatibility, or server admission policy. Use `--dry-run=server` when you need +API-server validation. Invalid modes are rejected; omit `--dry-run` entirely +to apply changes. + +**Validation happens before rendering.** A failed source check, file read, +inventory update, cluster read, server dry run, or result decode returns an error +without printing a manifest. There is no fallback from a failed server dry run +to client-generated output. Shell redirection may still create or truncate its +destination file before the command runs; always redirect to a different file +and check the exit status. + +Replace the digest placeholders with approved 64-character lowercase digests: + +```bash +# Add a JDK to an unmanaged live profile. This changes the cluster. +brewlet k8s jdk add --profile java-workers \ + --distribution temurin --feature 25 \ + --image 'docker.io/library/eclipse-temurin@sha256:<64-lowercase-hex>' \ + --java-home /opt/java/openjdk + +# Validate a launcher addition through the API server without saving it. +brewlet k8s launcher add --profile java-workers \ + --name jaz \ + --image 'mcr.microsoft.com/openjdk/jdk@sha256:<64-lowercase-hex>' \ + --path /usr/bin/jaz --dry-run=server + +# Validate and print a GitOps change entirely offline from its source manifest. +brewlet k8s launcher add --dry-run --profile java-workers --file profile.yaml \ + --name jaz \ + --image 'mcr.microsoft.com/openjdk/jdk@sha256:<64-lowercase-hex>' \ + --path /usr/bin/jaz > profile-next.yaml + +# Generate proposed values for a Helm-owned profile; does not deploy them. +brewlet k8s jdk add --dry-run --profile default --values values.yaml \ + --distribution temurin --feature 25 \ + --image 'docker.io/library/eclipse-temurin@sha256:<64-lowercase-hex>' \ + --java-home /opt/java/openjdk > values-next.yaml +``` + +`--file` accepts one NodeProfile YAML/JSON document whose name matches +`--profile`. `--values` accepts one complete Helm values document: `default` +updates `provisioner`, while other names select an existing entry in `profiles`. +Other inventories, profiles, rollout policies, and chart settings are retained. +`--file` and `--values` are mutually exclusive and require client dry-run mode +(bare `--dry-run` or `--dry-run=client`). They cannot be used for a real update +or a server dry run. Input files are never overwritten by the CLI; redirect to +a **different** file, review the diff, and commit the desired source change. +YAML formatting and comments are not retained. For layered Helm values, use the +complete inventory-owning values, not a partial list overlay: Helm replaces +arrays rather than merging entries. + +Adding an identical entry is idempotent. Replacing the source for an existing +distribution/feature or launcher name requires `--replace`. Duplicate identities +and inventories above the CRD's 32-entry limit are rejected. Local generation +validates the supplied entry, not registry contents, node architecture +compatibility, or the cluster's full admission policy. + +For an **unmanaged live profile**, append `--dry-run=server` to `add` to validate a +conditional patch without saving; remove `--dry-run` to persist it. +There is no `--apply` flag. **`add ... > result.yaml` still changes the cluster**; +use `add ... --dry-run > proposal.yaml` for a client-validated preview. + +Live application rejects terminating profiles, Helm/Argo CD/Flux ownership +markers, owner references, and unrecognized spec field managers. It never +forces ownership. The patch changes only the selected inventory and tests both +the profile UID and resource version; a concurrent change fails without a +blind retry. These ownership checks also apply to `add --dry-run=server`. Use +client dry-run mode to prepare Helm/GitOps source changes and deploy them through +the owning system instead of bypassing those guards. Generation strips +server-owned status and lifecycle metadata from the emitted manifest; the actual +API patch leaves them untouched. + +A successful patch means **the declaration was accepted**, not that every node +has installed the source. Inspect the profile, its node generations, and +advertised inventory before deploying workloads that require the new runtime. + +--- + +## `brewlet version` + +Print the release version embedded in the binarybrewlet version + Source builds without release linker flags print `dev`. --- @@ -347,7 +636,7 @@ Source builds without release linker flags print `dev`. ## Environment variables | Variable | Used by | Meaning | -|---|---|---| +| --- | --- | --- | | `JAVA_HOME` | `run`, `push --appcds` | Default node JDK home if `--jdk-root` is unset (`run`); default training JVM if `--appcds-java` is unset (`push --appcds`). | | `BREWLET_JDK_HOME` | `run` | Overrides `JAVA_HOME` for JDK resolution. | | `BREWLET_STORE_ROOT` | shim (`layout` resolver) | OCI layout root used by the local layout resolver. | diff --git a/docs/configuration.md b/docs/configuration.md index 8d065312..60a06b54 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -64,7 +64,7 @@ with `--set key=value` or a values file. | `provisioner.includeControlPlane` | `false` | Allow the default profile onto control-plane nodes. Off by default, and only needed on single-node clusters such as kind or Docker Desktop, which label their node as the control plane without tainting it. | | `provisioner.tolerations` | `[]` | Tolerations for the default profile's provisioner pods. Standard Kubernetes toleration entries; each must name a `key`. Nothing is tolerated implicitly. | | `provisioner.jdks` | `[]` (**required**) | JDK entries with `distribution`, `feature`, digest-pinned `source.image`, and absolute `source.javaHome` ([§JDK management](jdk-management.md#source-model)). Brewlet ships no built-in runtime catalog, so rendering fails when `defaultProfile.enabled` is `true` and this is empty. | -| `provisioner.launchers` | structured `jaz` example | Optional launcher entries with `name`, digest-pinned `source.image`, and absolute `source.path` ([§Launchers](launchers.md#helm-example-jaz)). Empty = vanilla `java` only. | +| `provisioner.launchers` | `[]` | Optional launcher entries with `name`, digest-pinned `source.image`, and absolute `source.path` ([§Launchers](launchers.md#helm-example-jaz)). Empty = vanilla `java` only. | | `provisioner.appCDS.regenerationEnabled` | `false` | Authorize node-side AppCDS regeneration for the chart-managed default `NodeProfile`. | | `provisioner.rollout.maxUnavailable` | `null` | Bounds the default profile's provisioner DaemonSet rolling update. `null` keeps the DaemonSet default. | | `provisioner.rollout.validate` | `true` | Gate node readiness on post-install JDK smoke tests and staged-launcher executable checks. Arbitrary launchers are not executed as a probe. Renders the provisioner `BREWLET_VALIDATE` env. | diff --git a/docs/installation.md b/docs/installation.md index 33e012c3..95e54ac3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -212,6 +212,12 @@ digests. To reproduce a specific release, add `--version x.y.z`, replacing `x.y.z` with that release number. For an existing installation, follow [Upgrading](#upgrading) to update matching CRDs and retain your chosen values. +For a fresh cluster, the CLI also provides +[`brewlet k8s install`](cli-reference.md#installation), a version-pinned Helm +installation wrapper. Put both your pools and runtime sources in a complete +values file; preview with `--dry-run` before installing. This command refuses +existing Brewlet CRDs rather than silently skipping their migration. + `provisioner.poolKey` pins the node label the pool names are matched on. Leave it unset on AKS, EKS, and GKE, where the well-known provider label is auto-detected; set it explicitly on bare metal or kubeadm. diff --git a/docs/jdk-management.md b/docs/jdk-management.md index ebf8ec0d..b6d4061f 100644 --- a/docs/jdk-management.md +++ b/docs/jdk-management.md @@ -47,6 +47,15 @@ Mutable tags are intentionally rejected. A tag may resolve to different bytes between reviews or nodes; a digest identifies the exact OCI manifest or multi-platform index the administrator approved. +The CLI's [`brewlet k8s jdk add`](cli-reference.md#safe-jdk-and-launcher-additions) +updates an unmanaged live NodeProfile by default. `--dry-run` validates locally +and prints the proposed change; `--dry-run=server` validates through the API +server without saving. Failed dry runs print no rendered output. Use client +dry-run mode with `--file` or `--values` for offline source changes. +Live additions and server dry runs reject Helm/GitOps ownership. Use +`brewlet k8s profile list` for desired inventory and `brewlet k8s jdk list` for +node-advertised inventory; they are not interchangeable. + --- ## Helm examples: Temurin and Microsoft diff --git a/docs/launchers.md b/docs/launchers.md index 135c02d5..eefe1990 100644 --- a/docs/launchers.md +++ b/docs/launchers.md @@ -21,6 +21,15 @@ When a workload omits its launcher or requests `java`, Brewlet executes: Do not add `java` to `spec.launchers`. It comes from every declared JDK and is a reserved launcher name. +Use `brewlet k8s launcher list` to aggregate optional launchers advertised by +nodes, or `--output wide` for per-node rows. The +[`launcher add` workflow](cli-reference.md#safe-jdk-and-launcher-additions) +updates the live NodeProfile by default. `--dry-run` validates locally and prints +the proposal; `--dry-run=server` validates through the API server without saving. +If validation fails, no rendered output is printed. Client dry-run mode also +supports offline `--file` and `--values` inputs. The CLI never installs a +launcher directly onto a node; the operator provisions it after a profile update. + ## Launcher names are tokens, not paths A launcher name identifies a node-installed launcher layer directory diff --git a/site/scripts/test_installation_examples.py b/site/scripts/test_installation_examples.py index 769a0d1a..d784f32c 100644 --- a/site/scripts/test_installation_examples.py +++ b/site/scripts/test_installation_examples.py @@ -156,6 +156,40 @@ def test_every_documented_install_preview_and_upgrade_renders(self): self.assertIn(image, result.stdout, "Upgrade lost administrator image choice") self.assertGreaterEqual(command_count, 10, "A documented command disappeared from coverage") + def test_cli_values_example_is_self_contained_and_renders(self): + document = (ROOT / "docs/cli-reference.md").read_text() + installation = document.split("### Installation\n", 1)[1].split( + "### Safe JDK and launcher additions\n", 1 + )[0] + inventory = self.inventory(installation) + self.assertIn(" - java-workers\n", inventory) + self.assertIn("@sha256:<64-lowercase-hex>", inventory) + self.assertIn("configuration.md#helm-chart-values", installation) + self.assertIn("kubernetes/charts/brewlet/values.yaml", installation) + self.assertIn("save the following as `values.yaml`", installation) + self.assertLess(installation.index("```yaml"), installation.index("brewlet k8s install")) + self.assertIn("--values values.yaml --dry-run", installation) + + values = self.work / "cli-values.yaml" + values.write_text(inventory.replace("<64-lowercase-hex>", DIGEST)) + result = self.render(["--values", str(values)]) + self.assertEqual(result.returncode, 0, result.stderr) + profiles = [doc for doc in result.stdout.split("---\n") + if "\nkind: NodeProfile\n" in doc] + self.assertEqual(len(profiles), 1) + profile = profiles[0] + self.assertIn("\n name: default\n", profile) + self.assertIn('\n - "java-workers"\n', profile) + self.assertIn(" - distribution: temurin\n", profile) + self.assertIn(" feature: 21\n", profile) + self.assertIn( + f" image: docker.io/library/eclipse-temurin@sha256:{DIGEST}\n", + profile, + ) + self.assertIn(" javaHome: /opt/java/openjdk\n", profile) + self.assertNotIn("includeControlPlane: true", profile) + self.assertNotIn("\n launchers:", profile) + def test_fresh_install_guides_default_to_latest_chart(self): for filename in ("README.md", "docs/installation.md", "kubernetes/README.md", "kubernetes/charts/brewlet/README.md", From 86aa86f1003acd887781bbd9b767fc6b780fe6c0 Mon Sep 17 00:00:00 2001 From: Bruno Borges Date: Thu, 24 Sep 2026 19:04:47 -0300 Subject: [PATCH 2/2] Add API-backed integration tests for Kubernetes CLI Exercise the built Brewlet CLI through real kubectl and Helm against an isolated envtest API server. Cover inventory and inspection, profile updates and reconciliation, dry-run persistence and failure output, RBAC and admission denials, Helm ownership, concurrency fences, and the install guard. Require the suite in PR CI and document its node-runtime coverage limits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 59dd5181-736f-4687-9205-d75210e7035f --- .github/workflows/ci.yml | 4 +- integration-tests/README.md | 18 + kubernetes/Makefile | 6 +- kubernetes/README.md | 32 + kubernetes/internal/cli/integration_test.go | 684 ++++++++++++++++++++ 5 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 kubernetes/internal/cli/integration_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e709101..bdfbe63e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,10 +89,12 @@ jobs: setup_envtest="$(go env GOPATH)/bin/setup-envtest" echo "KUBEBUILDER_ASSETS=$("$setup_envtest" use 1.31.0 -p path)" >> "$GITHUB_ENV" - - name: Set up Helm for uninstall lifecycle tests + - name: Set up Helm for lifecycle and CLI integration tests uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Test + env: + BREWLET_REQUIRE_CLI_INTEGRATION: "true" run: make test admission: diff --git a/integration-tests/README.md b/integration-tests/README.md index 78eb7db8..565bf70f 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -53,3 +53,21 @@ an exercised capability fails. The suite covers: | 15 | live opt-in Prometheus metrics through Helm, provisioner, shim, and exporter | both + fixtures | See [AGENTS.md](AGENTS.md) for cluster requirements, cleanup, and troubleshooting. + +## Kubernetes CLI API integration + +The process/API integration suite lives in the existing Kubernetes Go module +so it can reuse envtest and the production controllers: + +```bash +make -C kubernetes test-cli-integration +``` + +It builds the CLI and invokes real kubectl/Helm against a fresh API server and +etcd using only private fixture kubeconfigs. No Docker daemon, existing cluster, +or tier reset is used. Missing prerequisites fail this explicit target. The +Kubernetes pull-request CI job runs the same tests. + +See [CLI integration tests](../kubernetes/README.md#cli-integration-tests) for +coverage and limitations. In particular, fixture readiness and node inventory +are simulated; this is not coverage of host provisioning or JVM execution. diff --git a/kubernetes/Makefile b/kubernetes/Makefile index 58f53078..e29a52a3 100644 --- a/kubernetes/Makefile +++ b/kubernetes/Makefile @@ -10,7 +10,7 @@ ENVTEST_K8S_VERSION ?= 1.31.0 SETUP_ENVTEST_VERSION ?= v0.0.0-20250308055145-5fe7bb3edc86 SETUP_ENVTEST ?= $(shell $(GO) env GOPATH)/bin/setup-envtest -.PHONY: all fmt-check vet build test test-envtest ci operator-build admission-build operator-image admission-image helm-lint helm-template helm-digest-check helm-scheduling-check helm-admission-check helm-lifecycle-check helm-check print-setup-envtest-version clean +.PHONY: all fmt-check vet build test test-envtest test-cli-integration ci operator-build admission-build operator-image admission-image helm-lint helm-template helm-digest-check helm-scheduling-check helm-admission-check helm-lifecycle-check helm-check print-setup-envtest-version clean all: build @@ -40,6 +40,10 @@ print-setup-envtest-version: test-envtest: $(SETUP_ENVTEST) KUBEBUILDER_ASSETS="$$($(SETUP_ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GO) test -race ./... +test-cli-integration: $(SETUP_ENVTEST) + KUBEBUILDER_ASSETS="$${KUBEBUILDER_ASSETS:-$$($(SETUP_ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)}" \ + BREWLET_REQUIRE_CLI_INTEGRATION=true $(GO) test -race -count=1 -v ./internal/cli + ci: fmt-check vet build test-envtest helm-check operator-build: diff --git a/kubernetes/README.md b/kubernetes/README.md index 34cd87d0..82530907 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -104,6 +104,38 @@ make -C kubernetes test-envtest make -C kubernetes helm-check ``` +### CLI integration tests + +With Go and Helm installed, run the built CLI against an isolated API server: + +```bash +make -C kubernetes test-cli-integration +``` + +This target obtains the pinned envtest assets (kube-apiserver, etcd, and kubectl) +and runs `internal/cli` without contacting your configured cluster. It builds +`core/cmd/brewlet`, uses real kubectl and Helm subprocesses, and installs the +shipped CRDs in a fresh disposable API server with RBAC enabled. Its private +kubeconfigs deliberately have an invalid default context so explicit connection +and namespace handling are exercised. The existing Kubernetes CI test job runs +the suite with prerequisites required, rather than silently skipping it. + +Coverage includes inventory/status/doctor and compatibility aliases; profile +and application inspection; persistent JDK/launcher additions and replacement; +non-persisting client/server dry runs; failed dry runs with empty stdout; +admission and RBAC rejection; Helm ownership and offline values; stale +resource-version/recreated-UID conflicts; and the existing-CRD install guard. +Real profile and application reconciliation verifies that CLI writes feed the +controller's desired resources. Helm renders generated values using the local +chart, without downloading a released chart. + +These are API/process integration tests, not node-runtime E2E. Fixture node +inventory and readiness statuses are explicitly simulated: envtest has no +kubelet, scheduler, or Deployment controller. The suite does not prove a fresh +`brewlet k8s install` completes, download JDK images, execute privileged +provisioners, or launch a JVM. See the +[E2E runbook](../integration-tests/AGENTS.md) for the separate live-node tiers. + Build the component images from the repository root so the image also receives the shared license and notice-generation inputs: diff --git a/kubernetes/internal/cli/integration_test.go b/kubernetes/internal/cli/integration_test.go new file mode 100644 index 00000000..0b8c1536 --- /dev/null +++ b/kubernetes/internal/cli/integration_test.go @@ -0,0 +1,684 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package cli_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + nodeapi "brewlet-operator/api/nodeprofile/v1alpha1" + appapi "brewlet-operator/api/v1alpha1" + "brewlet-operator/internal/controller" + + admissionv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +const sourceImage = "registry.example.com/jdk@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const replacementImage = "registry.example.com/jdk@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +type fixture struct { + ctx context.Context + api client.Client + scheme *runtime.Scheme + config *rest.Config + server *envtest.Environment + binary, helm string + kubeconfig, work string + env []string +} + +type result struct { + stdout, stderr string + code int +} + +// These are API/process integration tests, not node-runtime E2E: envtest has no +// kubelet or workload controllers. Only fixture readiness/inventory is simulated. +func TestCLIIntegration(t *testing.T) { + assets := os.Getenv("KUBEBUILDER_ASSETS") + if assets == "" { + if os.Getenv("BREWLET_REQUIRE_CLI_INTEGRATION") == "true" { + t.Fatal("KUBEBUILDER_ASSETS is required; run make test-cli-integration") + } + t.Skip("requires envtest assets; run make test-cli-integration") + } + assets, err := filepath.Abs(assets) + must(t, err) + for _, name := range []string{"kube-apiserver", "etcd", "kubectl"} { + info, err := os.Stat(filepath.Join(assets, name)) + if err != nil || info.IsDir() { + t.Fatalf("missing envtest binary %s: %v", name, err) + } + } + helm, err := exec.LookPath("helm") + must(t, err) + work := t.TempDir() + binary := filepath.Join(work, "brewlet") + buildCtx, cancelBuild := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelBuild() + build := exec.CommandContext(buildCtx, "go", "-C", "../../../core", "build", "-o", binary, "./cmd/brewlet") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build CLI: %v\n%s", err, out) + } + scheme := runtime.NewScheme() + must(t, clientgoscheme.AddToScheme(scheme)) + must(t, nodeapi.AddToScheme(scheme)) + must(t, appapi.AddToScheme(scheme)) + server := &envtest.Environment{ + UseExistingCluster: ptr.To(false), + BinaryAssetsDirectory: assets, + CRDDirectoryPaths: []string{"../../charts/brewlet/crds"}, + ErrorIfCRDPathMissing: true, + } + server.ControlPlane.GetAPIServer().Configure().Set("authorization-mode", "RBAC") + config, err := server.Start() + must(t, err) + t.Cleanup(func() { + if err := server.Stop(); err != nil { + t.Errorf("stop isolated API server: %v", err) + } + }) + api, err := client.New(config, client.Options{Scheme: scheme}) + must(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + t.Cleanup(cancel) + f := &fixture{ctx: ctx, api: api, scheme: scheme, config: config, server: server, + work: work, binary: binary, helm: helm} + f.kubeconfig = f.writeConfig(t, "admin", config) + // Poison defaults so a missed explicit context/path fails rather than + // accidentally using the user's current cluster. + poison := filepath.Join(work, "default-kubeconfig") + must(t, os.WriteFile(poison, []byte("not a kubeconfig\n"), 0o600)) + f.env = append(os.Environ(), + "KUBECONFIG="+poison, + "PATH="+assets+string(os.PathListSeparator)+filepath.Dir(helm)+string(os.PathListSeparator)+os.Getenv("PATH"), + "HELM_CACHE_HOME="+filepath.Join(work, "helm-cache"), + "HELM_CONFIG_HOME="+filepath.Join(work, "helm-config"), + "HELM_DATA_HOME="+filepath.Join(work, "helm-data"), + ) + for _, name := range []string{"runtime-system", "team", "other-team"} { + must(t, api.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}})) + } + must(t, api.Create(ctx, &nodev1.RuntimeClass{ + ObjectMeta: metav1.ObjectMeta{Name: "brewlet"}, Handler: "brewlet"})) + t.Run("inventory-status-doctor", f.testReadCommands) + t.Run("live-and-dry-run-inventory-updates", f.testInventoryUpdates) + t.Run("helm-owned-profile-and-offline-values", f.testHelmOwnership) + t.Run("rbac-denied-dry-run", f.testRBAC) + t.Run("admission-denied-dry-run", f.testAdmission) + t.Run("concurrent-updates", f.testConflicts) + t.Run("application-inspection", f.testAppInspection) + t.Run("install-refuses-existing-crds", f.testInstallGuard) + unchanged, err := os.ReadFile(poison) + must(t, err) + if string(unchanged) != "not a kubeconfig\n" { + t.Fatal("CLI changed the default kubeconfig") + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +func (f *fixture) writeConfig(t *testing.T, name string, config *rest.Config) string { + t.Helper() + file := filepath.Join(f.work, name+"-kubeconfig") + must(t, clientcmd.WriteToFile(clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{"isolated": { + Server: config.Host, CertificateAuthorityData: config.CAData, CertificateAuthority: config.CAFile, + }}, + AuthInfos: map[string]*clientcmdapi.AuthInfo{"isolated": { + ClientCertificateData: config.CertData, ClientKeyData: config.KeyData, + ClientCertificate: config.CertFile, ClientKey: config.KeyFile, + }}, + Contexts: map[string]*clientcmdapi.Context{ + "selected": {Cluster: "isolated", AuthInfo: "isolated", Namespace: "other-team"}, + "wrong": {Cluster: "missing", AuthInfo: "missing"}, + }, + CurrentContext: "wrong", + }, file)) + return file +} + +func (f *fixture) command(t *testing.T, program string, args ...string) result { + t.Helper() + ctx, cancel := context.WithTimeout(f.ctx, 25*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, program, args...) + cmd.WaitDelay = time.Second + cmd.Env = f.env + cmd.Dir = f.work + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + if ctx.Err() != nil { + t.Fatalf("%s %v timed out: %v\n%s", program, args, ctx.Err(), stderr.String()) + } + code := 0 + if err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + t.Fatalf("start %s: %v", program, err) + } + code = exit.ExitCode() + } + return result{stdout.String(), stderr.String(), code} +} + +func (f *fixture) cli(t *testing.T, args ...string) result { + t.Helper() + return f.cliConfig(t, f.kubeconfig, args...) +} + +func (f *fixture) cliConfig(t *testing.T, config string, args ...string) result { + t.Helper() + base := []string{"k8s", "--kubeconfig", config, "--context", "selected", "--timeout", "10s"} + return f.command(t, f.binary, append(base, args...)...) +} + +func (r result) success(t *testing.T) string { + t.Helper() + if r.code != 0 { + t.Fatalf("CLI failed with exit %d\nstdout: %s\nstderr: %s", r.code, r.stdout, r.stderr) + } + return r.stdout +} + +func (r result) failure(t *testing.T, reason string) { + t.Helper() + if r.code != 1 || r.stdout != "" || !strings.Contains(r.stderr, reason) { + t.Fatalf("expected exit 1, empty stdout, and %q; got %+v", reason, r) + } +} + +func decode[T any](t *testing.T, raw string) T { + t.Helper() + var value T + must(t, json.Unmarshal([]byte(raw), &value)) + return value +} + +func (f *fixture) profile(t *testing.T, name string) *nodeapi.NodeProfile { + t.Helper() + p := &nodeapi.NodeProfile{ + TypeMeta: metav1.TypeMeta{APIVersion: nodeapi.GroupVersion.String(), Kind: "NodeProfile"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"fixture": name}}, + Spec: nodeapi.NodeProfileSpec{ + NodePool: nodeapi.NodePoolRef{Names: []string{name}, Key: "example.com/pool"}, + JDKs: []nodeapi.JDKRef{{Distribution: "temurin", Feature: 21, + Source: nodeapi.JDKSource{Image: sourceImage, JavaHome: "/opt/java/openjdk"}}}, + Launchers: []nodeapi.LauncherRef{{Name: "jaz", + Source: nodeapi.LauncherSource{Image: sourceImage, Path: "/usr/bin/jaz"}}}, + Rollout: nodeapi.RolloutSpec{MaxUnavailable: ptr.To(intstr.FromInt(1)), Validate: ptr.To(true)}, + }, + } + must(t, f.api.Create(f.ctx, p, client.FieldOwner("kubectl"))) + return p +} + +func (f *fixture) getProfile(t *testing.T, name string) *nodeapi.NodeProfile { + t.Helper() + p := &nodeapi.NodeProfile{} + must(t, f.api.Get(f.ctx, client.ObjectKey{Name: name}, p)) + return p +} + +func (f *fixture) assertUnchanged(t *testing.T, before *nodeapi.NodeProfile) { + t.Helper() + after := f.getProfile(t, before.Name) + if after.ResourceVersion != before.ResourceVersion || !reflect.DeepEqual(after.Spec, before.Spec) { + t.Fatalf("dry run or refused update persisted changes: before=%+v after=%+v", before, after) + } +} + +func jdkArgs(profile string, extra ...string) []string { + return append([]string{"jdk", "add", "--profile", profile, "--distribution", "temurin", + "--feature", "25", "--image", sourceImage, "--java-home", "/opt/java/openjdk", "--output", "json"}, extra...) +} + +func launcherArgs(profile string, extra ...string) []string { + return append([]string{"launcher", "add", "--profile", profile, "--name", "custom", + "--image", sourceImage, "--path", "/usr/bin/custom", "--output", "json"}, extra...) +} + +func (f *fixture) node(t *testing.T, p *nodeapi.NodeProfile) *corev1.Node { + t.Helper() + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: p.Name + "-node", Labels: map[string]string{ + "example.com/pool": p.Name, "kubernetes.io/arch": "amd64", + "brewlet.sh/runtime": "ready", "brewlet.sh/owner-uid": string(p.UID), + }, Annotations: map[string]string{ + "brewlet.sh/owner-name": p.Name, "brewlet.sh/profile-generation": fmt.Sprint(p.Generation), + "brewlet.sh/jdks": "temurin-21", "brewlet.sh/launchers": "jaz", + "brewlet.sh/jdks-info": `[{"distribution":"temurin","vendor":"Adoptium","feature":21,"version":"21.0.5","arch":"amd64"}]`, + }}} + must(t, f.api.Create(f.ctx, node)) + node.Labels["brewlet.sh/owner-node-uid"] = string(node.UID) + must(t, f.api.Update(f.ctx, node)) + node.Status.NodeInfo.ContainerRuntimeVersion = "containerd://2.0.0" + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + must(t, f.api.Status().Update(f.ctx, node)) + // No node controller is present to remove the initial NotReady taint. + node.Spec.Taints = nil + must(t, f.api.Update(f.ctx, node)) + return node +} + +func (f *fixture) deployment(t *testing.T, name string) *appsv1.Deployment { + t.Helper() + labels := map[string]string{"app": name} + d := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "runtime-system"}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "fixture", Image: sourceImage}}}}, + }} + must(t, f.api.Create(f.ctx, d)) + f.readyDeployment(t, d) + return d +} + +func (f *fixture) readyDeployment(t *testing.T, d *appsv1.Deployment) { + t.Helper() + d.Status = appsv1.DeploymentStatus{ObservedGeneration: d.Generation, Replicas: 1, + UpdatedReplicas: 1, ReadyReplicas: 1, AvailableReplicas: 1} + must(t, f.api.Status().Update(f.ctx, d)) +} + +func (f *fixture) testReadCommands(t *testing.T) { + p := f.profile(t, "inventory") + f.node(t, p) + p.Status = nodeapi.NodeProfileStatus{ObservedGeneration: p.Generation, AssignedNodes: 1, ReadyNodes: 1, + Conditions: []metav1.Condition{{Type: "Ready", Status: metav1.ConditionTrue, Reason: "AllNodesProvisioned", + Message: "fixture readiness", ObservedGeneration: p.Generation, LastTransitionTime: metav1.Now()}}} + must(t, f.api.Status().Update(f.ctx, p)) + operator := f.deployment(t, "brewlet-operator") + f.deployment(t, "brewlet-admission") + jdks := f.cli(t, "jdk", "list", "--selector", "example.com/pool=inventory", "--output", "json").success(t) + rows := decode[[]struct { + Distribution, Version, Arch string + Nodes []string + }](t, jdks) + if len(rows) != 1 || rows[0].Distribution != "temurin" || rows[0].Version != "21.0.5" || + rows[0].Arch != "amd64" || !reflect.DeepEqual(rows[0].Nodes, []string{"inventory-node"}) { + t.Fatalf("unexpected advertised inventory: %s", jdks) + } + aliases := f.command(t, f.binary, "jdks", "--kubeconfig", f.kubeconfig, "--context", "selected", + "--selector", "example.com/pool=inventory", "--output", "json").success(t) + if aliases != jdks { + t.Fatal("legacy jdks alias differs from k8s inventory") + } + empty := f.cli(t, "jdk", "list", "--selector", "example.com/pool=absent", "--output", "json").success(t) + if len(decode[[]json.RawMessage](t, empty)) != 0 { + t.Fatalf("node selector was ignored: %s", empty) + } + launchers := f.cli(t, "launcher", "list", "--output", "json").success(t) + if !strings.Contains(launchers, `"jaz"`) || !strings.Contains(launchers, `"inventory-node"`) { + t.Fatal(launchers) + } + for _, command := range []string{"jdk", "launcher"} { + if out := f.cli(t, command, "list", "--output", "wide").success(t); !strings.Contains(out, "inventory-node") { + t.Fatalf("%s per-node inventory missing node: %s", command, out) + } + } + if out := f.cli(t, "profile", "list").success(t); !strings.Contains(out, "temurin-21") { + t.Fatal(out) + } + profile := decode[struct { + Profile struct{ Ready bool } + Nodes []struct{ Name string } + }](t, f.cli(t, "profile", "inspect", p.Name, "--output", "json").success(t)) + if !profile.Profile.Ready || len(profile.Nodes) != 1 || profile.Nodes[0].Name != "inventory-node" { + t.Fatalf("profile inspection: %+v", profile) + } + statusArgs := []string{"status", "--system-namespace", "runtime-system", "--output", "json"} + status := decode[struct{ Healthy bool }](t, f.cli(t, statusArgs...).success(t)) + if !status.Healthy { + t.Fatal("ready fixtures reported unhealthy") + } + f.cli(t, "doctor", "--namespace", "team", "--output", "json").success(t) + f.command(t, f.binary, "doctor", "--kubeconfig", f.kubeconfig, "--context", "selected", + "--namespace", "team", "--output", "json").success(t) + + operator.Status.UpdatedReplicas = 0 + must(t, f.api.Status().Update(f.ctx, operator)) + unready := f.cli(t, statusArgs...) + if unready.code != 1 || decode[struct{ Healthy bool }](t, unready.stdout).Healthy { + t.Fatalf("old available replicas counted as a current rollout: %+v", unready) + } + f.readyDeployment(t, operator) + p.Status.Conditions[0].ObservedGeneration-- + must(t, f.api.Status().Update(f.ctx, p)) + unready = f.cli(t, statusArgs...) + if unready.code != 1 || decode[struct{ Healthy bool }](t, unready.stdout).Healthy { + t.Fatalf("stale profile condition counted as ready: %+v", unready) + } +} + +func (f *fixture) testInventoryUpdates(t *testing.T) { + p := f.profile(t, "updates") + for _, args := range [][]string{jdkArgs(p.Name, "--image", "invalid:source", "--dry-run"), + launcherArgs(p.Name, "--path", "/usr/../bin/custom", "--dry-run")} { + r := f.cli(t, args...) + if r.code != 1 || r.stdout != "" || r.stderr == "" { + t.Fatalf("invalid client dry run should fail with empty output: %+v", r) + } + f.assertUnchanged(t, p) + } + for _, flags := range [][]string{{"--dry-run"}, {"--dry-run=client"}, {"--dry-run=server"}} { + for _, args := range [][]string{jdkArgs(p.Name, flags...), launcherArgs(p.Name, flags...)} { + proposal := decode[nodeapi.NodeProfile](t, f.cli(t, args...).success(t)) + if len(proposal.Spec.JDKs)+len(proposal.Spec.Launchers) != 3 { + t.Fatalf("dry run did not render addition: %+v", proposal.Spec) + } + f.assertUnchanged(t, p) + } + } + f.cli(t, jdkArgs(p.Name)...).success(t) + f.cli(t, launcherArgs(p.Name)...).success(t) + updated := f.getProfile(t, p.Name) + if len(updated.Spec.JDKs) != 2 || len(updated.Spec.Launchers) != 2 || + !reflect.DeepEqual(updated.Spec.Rollout, p.Spec.Rollout) || updated.Labels["fixture"] != p.Name || + updated.Generation <= p.Generation { + t.Fatalf("live updates did not preserve/apply configuration: %+v", updated) + } + f.cli(t, jdkArgs(p.Name)...).success(t) + repeated := f.getProfile(t, p.Name) + if repeated.Generation != updated.Generation || !reflect.DeepEqual(repeated.Spec, updated.Spec) { + t.Fatal("identical add was not idempotent") + } + f.cli(t, jdkArgs(p.Name, "--image", replacementImage)...).failure(t, "--replace") + f.assertUnchanged(t, repeated) + f.cli(t, jdkArgs(p.Name, "--image", replacementImage, "--replace")...).success(t) + if got := f.getProfile(t, p.Name).Spec.JDKs[1].Source.Image; got != replacementImage { + t.Fatalf("explicit replacement was not saved: %s", got) + } + // Exercise the real profile reconciler without starting any host worker. + r := &controller.NodeProfileReconciler{Client: f.api, Recorder: record.NewFakeRecorder(100), + Config: controller.Config{Namespace: "runtime-system", ProvisionerImage: "registry.invalid/never-executed:cli-test"}} + for range 2 { + _, err := r.Reconcile(f.ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: p.Name}}) + must(t, err) + } + ds := &appsv1.DaemonSet{} + must(t, f.api.Get(f.ctx, client.ObjectKey{Namespace: "runtime-system", Name: "brewlet-node-provisioner-" + p.Name}, ds)) + env := map[string]string{} + for _, entry := range ds.Spec.Template.Spec.Containers[0].Env { + env[entry.Name] = entry.Value + } + if env["JDK_SOURCE_COUNT"] != "2" || env["LAUNCHER_SOURCE_COUNT"] != "2" { + t.Fatalf("controller did not consume CLI additions: %+v", env) + } + inspected := f.cli(t, "profile", "inspect", p.Name, "--output", "json").success(t) + if !strings.Contains(inspected, replacementImage) || !strings.Contains(inspected, "custom") { + t.Fatal("profile inspection did not show the reconciled desired inventory", inspected) + } +} + +func (f *fixture) testHelmOwnership(t *testing.T) { + values := filepath.Join(f.work, "values.json") + must(t, os.WriteFile(values, []byte(fmt.Sprintf(`{"provisioner":{"pools":["helm-pool"],"jdks":[ + {"distribution":"temurin","feature":21,"source":{"image":%q,"javaHome":"/opt/java/openjdk"}}]}}`, sourceImage)), 0o600)) + chart, err := filepath.Abs("../../charts/brewlet") + must(t, err) + rendered := f.command(t, f.helm, "template", "brewlet", chart, "--namespace", "runtime-system", + "--values", values, "--show-only", "templates/nodeprofiles.yaml").success(t) + manifest := filepath.Join(f.work, "helm-profile.yaml") + must(t, os.WriteFile(manifest, []byte(rendered), 0o600)) + f.command(t, "kubectl", "--kubeconfig", f.kubeconfig, "--context", "selected", + "apply", "-f", manifest).success(t) + p := f.getProfile(t, "default") + for _, args := range [][]string{jdkArgs("default"), launcherArgs("default"), jdkArgs("default", "--dry-run=server")} { + f.cli(t, args...).failure(t, "managed by") + f.assertUnchanged(t, p) + } + proposal := f.cli(t, jdkArgs("default", "--dry-run", "--values", values)...).success(t) + next := filepath.Join(f.work, "values-next.json") + must(t, os.WriteFile(next, []byte(proposal), 0o600)) + rendered = f.command(t, f.helm, "template", "brewlet", chart, "--namespace", "runtime-system", + "--values", next, "--show-only", "templates/nodeprofiles.yaml").success(t) + if !strings.Contains(rendered, "feature: 25") || !strings.Contains(rendered, "feature: 21") { + t.Fatal("CLI-generated Helm values lost or failed to add inventory", rendered) + } + f.assertUnchanged(t, p) + offline := f.cli(t, launcherArgs("default", "--dry-run", "--file", manifest)...).success(t) + if len(decode[nodeapi.NodeProfile](t, offline).Spec.Launchers) != 1 { + t.Fatal("offline profile proposal omitted launcher") + } + f.assertUnchanged(t, p) +} + +func (f *fixture) testRBAC(t *testing.T) { + p := f.profile(t, "read-only") + user, err := f.server.AddUser(envtest.User{Name: "cli-reader"}, f.config) + must(t, err) + config := f.writeConfig(t, "reader", user.Config()) + must(t, f.api.Create(f.ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "cli-reader"}, + Rules: []rbacv1.PolicyRule{{APIGroups: []string{"node.brewlet.sh"}, Resources: []string{"nodeprofiles"}, + ResourceNames: []string{p.Name}, Verbs: []string{"get"}}}})) + must(t, f.api.Create(f.ctx, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "cli-reader"}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "cli-reader"}, + Subjects: []rbacv1.Subject{{Kind: "User", APIGroup: rbacv1.GroupName, Name: "cli-reader"}}})) + must(t, wait.PollUntilContextTimeout(f.ctx, 100*time.Millisecond, 10*time.Second, true, func(context.Context) (bool, error) { + r := f.command(t, "kubectl", "--kubeconfig", config, "--context", "selected", "auth", "can-i", + "get", "nodeprofiles.node.brewlet.sh/"+p.Name) + return r.code == 0 && strings.TrimSpace(r.stdout) == "yes", nil + })) + f.cliConfig(t, config, jdkArgs(p.Name, "--dry-run")...).success(t) + for _, args := range [][]string{jdkArgs(p.Name), jdkArgs(p.Name, "--dry-run=server")} { + f.cliConfig(t, config, args...).failure(t, "forbidden") + f.assertUnchanged(t, p) + } + f.cliConfig(t, config, "jdk", "list", "--output", "json").failure(t, "forbidden") +} + +func (f *fixture) testAdmission(t *testing.T) { + p := f.profile(t, "policy") + fail := admissionv1.Fail + policy := &admissionv1.ValidatingAdmissionPolicy{ObjectMeta: metav1.ObjectMeta{Name: "cli-test-source-policy"}, + Spec: admissionv1.ValidatingAdmissionPolicySpec{ + FailurePolicy: &fail, + MatchConstraints: &admissionv1.MatchResources{ + ObjectSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"fixture": p.Name}}, + ResourceRules: []admissionv1.NamedRuleWithOperations{{RuleWithOperations: admissionv1.RuleWithOperations{ + Operations: []admissionv1.OperationType{admissionv1.Update}, + Rule: admissionv1.Rule{APIGroups: []string{"node.brewlet.sh"}, APIVersions: []string{"v1alpha1"}, Resources: []string{"nodeprofiles"}}, + }}}, + }, + Validations: []admissionv1.Validation{{Expression: "!object.spec.jdks.exists(j, j.distribution == 'blocked')", + Message: "CLI integration source policy denied this distribution"}}, + }} + must(t, f.api.Create(f.ctx, policy)) + must(t, f.api.Create(f.ctx, &admissionv1.ValidatingAdmissionPolicyBinding{ + ObjectMeta: metav1.ObjectMeta{Name: policy.Name}, + Spec: admissionv1.ValidatingAdmissionPolicyBindingSpec{PolicyName: policy.Name, ValidationActions: []admissionv1.ValidationAction{admissionv1.Deny}}, + })) + // Wait for admission registration using an API dry run; never persist the + // deliberately disallowed inventory while the policy cache is warming. + must(t, wait.PollUntilContextTimeout(f.ctx, 100*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (bool, error) { + candidate := p.DeepCopy() + candidate.Spec.JDKs[0].Distribution = "blocked" + err := f.api.Update(ctx, candidate, client.DryRunAll) + if err == nil { + return false, nil + } + if strings.Contains(err.Error(), "CLI integration source policy denied") { + return true, nil + } + return false, err + })) + for _, format := range []string{"json", "yaml"} { + for _, dryRun := range []bool{false, true} { + args := jdkArgs(p.Name, "--distribution", "blocked", "--output", format) + if dryRun { + args = append(args, "--dry-run=server") + } + f.cli(t, args...).failure(t, "CLI integration source policy denied") + f.assertUnchanged(t, p) + } + } + f.cli(t, jdkArgs(p.Name, "--distribution", "blocked", "--dry-run")...).success(t) + f.assertUnchanged(t, p) +} + +func (f *fixture) testConflicts(t *testing.T) { + for _, replace := range []bool{false, true} { + for _, dryRun := range []bool{false, true} { + t.Run(fmt.Sprintf("recreate=%t/dry-run=%t", replace, dryRun), func(t *testing.T) { + p := f.profile(t, fmt.Sprintf("conflict-%t-%t", replace, dryRun)) + upstream, err := url.Parse(f.config.Host) + must(t, err) + proxy := httputil.NewSingleHostReverseProxy(upstream) + proxy.Transport, err = rest.TransportFor(f.config) + must(t, err) + var once sync.Once + mutations := make(chan error, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPatch && strings.HasSuffix(r.URL.Path, "/nodeprofiles/"+p.Name) { + once.Do(func() { + current := &nodeapi.NodeProfile{} + err := f.api.Get(r.Context(), client.ObjectKey{Name: p.Name}, current) + if err == nil && replace { + err = f.api.Delete(r.Context(), current) + if err == nil { + err = f.api.Create(r.Context(), &nodeapi.NodeProfile{ + ObjectMeta: metav1.ObjectMeta{Name: p.Name}, Spec: current.Spec}) + } + } else if err == nil { + current.Annotations = map[string]string{"concurrent-writer": "preserve-me"} + err = f.api.Update(r.Context(), current) + } + mutations <- err + }) + } + proxy.ServeHTTP(w, r) + })) + defer server.Close() + config := f.writeConfig(t, p.Name, &rest.Config{Host: server.URL}) + args := jdkArgs(p.Name) + if dryRun { + args = append(args, "--dry-run=server") + } + r := f.cliConfig(t, config, args...) + if r.code != 1 || r.stdout != "" || !strings.Contains(r.stderr, "profile update failed") { + t.Fatalf("stale patch was not rejected with empty output: %+v", r) + } + select { + case err := <-mutations: + must(t, err) + default: + t.Fatal("test did not inject a concurrent write") + } + current := f.getProfile(t, p.Name) + if !reflect.DeepEqual(current.Spec, p.Spec) { + t.Fatal("stale patch overwrote the current profile") + } + if replace && current.UID == p.UID { + t.Fatal("replacement did not change the UID") + } + if !replace && current.Annotations["concurrent-writer"] != "preserve-me" { + t.Fatal("concurrent writer's change was lost") + } + }) + } + } +} + +func (f *fixture) testAppInspection(t *testing.T) { + app := &appapi.JavaApplication{ObjectMeta: metav1.ObjectMeta{Name: "orders", Namespace: "team"}, + Spec: appapi.JavaApplicationSpec{Artifact: appapi.ArtifactSpec{Image: sourceImage}, + JVM: appapi.JVMSpec{Version: 21}, Replicas: ptr.To(int32(1)), + Env: []corev1.EnvVar{{Name: "TOKEN", Value: "fixture-secret-must-not-appear"}}}} + must(t, f.api.Create(f.ctx, app)) + reconciler := &controller.JavaApplicationReconciler{Client: f.api, APIReader: f.api, Scheme: f.scheme, + Recorder: record.NewFakeRecorder(100)} + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(app)} + _, err := reconciler.Reconcile(f.ctx, request) + must(t, err) + d := &appsv1.Deployment{} + must(t, f.api.Get(f.ctx, request.NamespacedName, d)) + rs := &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Name: "orders-rs", Namespace: "team", Labels: d.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(d, appsv1.SchemeGroupVersion.WithKind("Deployment"))}}, + Spec: appsv1.ReplicaSetSpec{Replicas: ptr.To(int32(1)), Selector: d.Spec.Selector, Template: d.Spec.Template}} + must(t, f.api.Create(f.ctx, rs)) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orders-pod", Namespace: "team", Labels: d.Spec.Template.Labels, + Annotations: d.Spec.Template.Annotations, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(rs, appsv1.SchemeGroupVersion.WithKind("ReplicaSet"))}}, + Spec: d.Spec.Template.Spec} + must(t, f.api.Create(f.ctx, pod)) + pod.Status.Phase = corev1.PodPending + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: pod.Spec.Containers[0].Name, + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}} + must(t, f.api.Status().Update(f.ctx, pod)) + foreign := pod.DeepCopy() + foreign.ObjectMeta = metav1.ObjectMeta{Name: "foreign", Namespace: "team", Labels: pod.Labels} + foreign.Status = corev1.PodStatus{} + must(t, f.api.Create(f.ctx, foreign)) + for _, target := range []*corev1.Pod{pod, foreign} { + must(t, f.api.Create(f.ctx, &corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: target.Name + "-event", Namespace: "team"}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: target.Name, Namespace: "team", UID: target.UID}, + Type: "Warning", Reason: "Failed", Message: "fixture image unavailable"})) + } + args := []string{"inspect", "app", "orders", "--namespace", "team", "--output", "json"} + raw := f.cli(t, args...).success(t) + report := decode[struct { + Ready bool + JDKRequest string + Pods []struct{ Name string } + Events []struct{ Object string } + }](t, raw) + if report.Ready || report.JDKRequest != "21" || len(report.Pods) != 1 || len(report.Events) != 1 || + report.Pods[0].Name != pod.Name || strings.Contains(raw, "fixture-secret") || strings.Contains(raw, "foreign") || + !strings.Contains(raw, "ImagePullBackOff") { + t.Fatalf("wrong ownership, readiness, or sensitive data in report: %s", raw) + } + f.readyDeployment(t, d) + _, err = reconciler.Reconcile(f.ctx, request) + must(t, err) + if report := decode[struct{ Ready bool }](t, f.cli(t, args...).success(t)); !report.Ready { + t.Fatal("current completed deployment rollout was not reported ready") + } + f.cli(t, "inspect", "app", "orders", "--output", "json").failure(t, "not found") +} + +func (f *fixture) testInstallGuard(t *testing.T) { + values := filepath.Join(f.work, "no-profiles.json") + must(t, os.WriteFile(values, []byte(`{"defaultProfile":{"enabled":false}}`), 0o600)) + f.cli(t, "install", "--version", "0.0.0", "--values", values). + failure(t, "Brewlet CRDs already exist") +}