From e0c2756ce736dace89bef967dd970ceed7865c83 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:28:02 +0000 Subject: [PATCH 1/3] Initial plan From a7da9cce1dc6eba9b4df91b8bbcc19bf53866bce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 28 Sep 2026 23:49:26 +0000 Subject: [PATCH 2/3] Add reference-aware runnable stage cleanup and usage metrics Co-authored-by: brunoborges <129743+brunoborges@users.noreply.github.com> --- core/cmd/brewlet-metrics-exporter/README.md | 15 +- core/cmd/brewlet-metrics-exporter/main.go | 4 + core/cmd/brewlet-metrics-exporter/stages.go | 38 +++ .../brewlet-metrics-exporter/stages_test.go | 56 ++++ core/cmd/brewlet/main.go | 27 +- core/cmd/brewlet/stage_gc.go | 76 +++++ core/cmd/brewlet/stage_gc_test.go | 94 ++++++ core/go.mod | 4 +- core/internal/artifact/blobs.go | 16 +- core/internal/runnablestage/lock_linux.go | 60 ++++ core/internal/runnablestage/mounts_linux.go | 285 +++++++++++++++++ .../runnablestage/mounts_linux_test.go | 180 +++++++++++ core/internal/runnablestage/reap_linux.go | 177 +++++++++++ .../internal/runnablestage/reap_linux_test.go | 294 ++++++++++++++++++ .../runnablestage/references_linux.go | 182 +++++++++++ .../runnablestage/references_linux_test.go | 161 ++++++++++ core/internal/runnablestage/stage.go | 107 +++++++ core/internal/runnablestage/stage_test.go | 72 +++++ core/internal/runnablestage/unsupported.go | 23 ++ core/internal/runtime/bundle.go | 6 + .../service_linux.go | 7 + docs/runnable-image.md | 53 +++- .../controller/nodeprofile_resources.go | 5 +- .../controller/nodeprofile_scheduling_test.go | 15 +- 24 files changed, 1944 insertions(+), 13 deletions(-) create mode 100644 core/cmd/brewlet-metrics-exporter/stages.go create mode 100644 core/cmd/brewlet-metrics-exporter/stages_test.go create mode 100644 core/cmd/brewlet/stage_gc.go create mode 100644 core/cmd/brewlet/stage_gc_test.go create mode 100644 core/internal/runnablestage/lock_linux.go create mode 100644 core/internal/runnablestage/mounts_linux.go create mode 100644 core/internal/runnablestage/mounts_linux_test.go create mode 100644 core/internal/runnablestage/reap_linux.go create mode 100644 core/internal/runnablestage/reap_linux_test.go create mode 100644 core/internal/runnablestage/references_linux.go create mode 100644 core/internal/runnablestage/references_linux_test.go create mode 100644 core/internal/runnablestage/stage.go create mode 100644 core/internal/runnablestage/stage_test.go create mode 100644 core/internal/runnablestage/unsupported.go diff --git a/core/cmd/brewlet-metrics-exporter/README.md b/core/cmd/brewlet-metrics-exporter/README.md index ed630c5f..15cf09f0 100644 --- a/core/cmd/brewlet-metrics-exporter/README.md +++ b/core/cmd/brewlet-metrics-exporter/README.md @@ -3,7 +3,20 @@ The node-local exporter receives bounded, best-effort telemetry from the containerd shim over `/opt/brewlet/metrics/telemetry.sock` and exposes Prometheus metrics for sandbox launches, artifact resolution, AppCDS decisions, -and installed JDK and launcher inventory. +installed JDK and launcher inventory, and runnable-image staging usage. + +`brewlet_runnable_stage_bytes` reports logical regular-file bytes under +`--stage-root` (default: `BREWLET_RUNNABLE_STAGE`, or +`os.TempDir()/brewlet-runnable`). The directory must be the host's actual staging +root; a containerized exporter needs a read-only host mount. The operator +provides that mount for the default location. The gauge includes legacy and +pending stages, does not follow symlinks, and refreshes on every scrape. +Inspection errors fail the scrape instead of reporting zero. + +Use the host command `brewlet stage-gc --dry-run` to inspect reclaimable stages, +then schedule `brewlet stage-gc --min-age 24h` with a node-level timer. +See [stage cleanup](../../../docs/runnable-image.md#reclaiming-unused-stages) +for reference checks and rollout precautions. ## Screenshots diff --git a/core/cmd/brewlet-metrics-exporter/main.go b/core/cmd/brewlet-metrics-exporter/main.go index cbc75a6f..9186a35b 100644 --- a/core/cmd/brewlet-metrics-exporter/main.go +++ b/core/cmd/brewlet-metrics-exporter/main.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "github.com/microsoft/brewlet/internal/runnablestage" "github.com/microsoft/brewlet/internal/telemetry" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -245,10 +246,12 @@ func main() { root string listenAddr string socketPath string + stageRoot string ) flag.StringVar(&root, "root", "/opt/brewlet", "Brewlet host-state root") flag.StringVar(&listenAddr, "listen-address", ":9090", "HTTP metrics listen address") flag.StringVar(&socketPath, "socket-path", "", "shim telemetry Unix datagram socket") + flag.StringVar(&stageRoot, "stage-root", runnablestage.Root(), "runnable staging root (BREWLET_RUNNABLE_STAGE)") flag.Parse() if socketPath == "" { socketPath = filepath.Join(root, "metrics", "telemetry.sock") @@ -261,6 +264,7 @@ func main() { registry.MustRegister(prometheus.NewGoCollector(), prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) events := newEventMetrics(registry) registry.MustRegister(newInventoryCollector(root)) + registry.MustRegister(newStageCollector(stageRoot)) server := &http.Server{ Addr: listenAddr, diff --git a/core/cmd/brewlet-metrics-exporter/stages.go b/core/cmd/brewlet-metrics-exporter/stages.go new file mode 100644 index 00000000..d677c3c9 --- /dev/null +++ b/core/cmd/brewlet-metrics-exporter/stages.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "github.com/microsoft/brewlet/internal/runnablestage" + "github.com/prometheus/client_golang/prometheus" +) + +type stageCollector struct { + root string + bytes *prometheus.Desc +} + +func newStageCollector(root string) *stageCollector { + return &stageCollector{ + root: root, + bytes: prometheus.NewDesc( + "brewlet_runnable_stage_bytes", + "Logical bytes remaining in runnable staging trees, including legacy and pending stages.", + nil, nil, + ), + } +} + +func (c *stageCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.bytes +} + +func (c *stageCollector) Collect(ch chan<- prometheus.Metric) { + size, err := runnablestage.Bytes(c.root) + if err != nil { + ch <- prometheus.NewInvalidMetric(c.bytes, err) + return + } + ch <- prometheus.MustNewConstMetric(c.bytes, prometheus.GaugeValue, float64(size)) +} diff --git a/core/cmd/brewlet-metrics-exporter/stages_test.go b/core/cmd/brewlet-metrics-exporter/stages_test.go new file mode 100644 index 00000000..b0647b33 --- /dev/null +++ b/core/cmd/brewlet-metrics-exporter/stages_test.go @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestStageCollector(t *testing.T) { + root := filepath.Join(t.TempDir(), "stages") + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(newStageCollector(root)) + check := func(value string) { + t.Helper() + expected := ` +# HELP brewlet_runnable_stage_bytes Logical bytes remaining in runnable staging trees, including legacy and pending stages. +# TYPE brewlet_runnable_stage_bytes gauge +brewlet_runnable_stage_bytes ` + value + "\n" + if err := testutil.GatherAndCompare(reg, strings.NewReader(expected)); err != nil { + t.Fatal(err) + } + } + check("0") + dir := filepath.Join(root, "immutable-v2", strings.Repeat("a", 64)) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + payload := filepath.Join(dir, "payload") + if err := os.WriteFile(payload, []byte("12345"), 0o644); err != nil { + t.Fatal(err) + } + check("5") + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + check("0") +} + +func TestStageCollectorReportsReadErrors(t *testing.T) { + root := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(root, nil, 0o644); err != nil { + t.Fatal(err) + } + reg := prometheus.NewRegistry() + reg.MustRegister(newStageCollector(root)) + if _, err := reg.Gather(); err == nil { + t.Fatal("expected a scrape error, not a misleading zero") + } +} diff --git a/core/cmd/brewlet/main.go b/core/cmd/brewlet/main.go index 7d3b639c..cdd3aa07 100644 --- a/core/cmd/brewlet/main.go +++ b/core/cmd/brewlet/main.go @@ -27,6 +27,7 @@ import ( "github.com/microsoft/brewlet/internal/artifact" "github.com/microsoft/brewlet/internal/kube" + "github.com/microsoft/brewlet/internal/runnablestage" "github.com/microsoft/brewlet/internal/runtime" ) @@ -55,6 +56,8 @@ func main() { err = cmdJDKs(os.Args[2:]) case "doctor": err = cmdDoctor(os.Args[2:]) + case "stage-gc": + err = cmdStageGC(os.Args[2:]) case "k8s": err = kube.Run(context.Background(), os.Args[2:], os.Stdout, os.Stderr) case "version", "--version": @@ -86,6 +89,7 @@ USAGE: brewlet bundle [--store DIR] [--cpu N] [--memory M] [--uid UID] [--gid GID] [--jdk-root DIR] [--launcher NAME] [--launcher-root DIR] [--out DIR] brewlet jdks [--output table|wide|json] [--kubeconfig FILE] [--context CTX] [--selector SEL] brewlet doctor [--namespace NS] [--output table|json] [--kubeconfig FILE] [--context CTX] + brewlet stage-gc [--stage-root DIR] [--address SOCKET] [--min-age 24h] [--dry-run] brewlet k8s inventory, status, inspection, installation and profile updates (see k8s --help) brewlet version @@ -674,6 +678,11 @@ func cmdRun(args []string) error { } ref := pos[0] + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return err + } + defer guard.Close() s := artifact.Store{Root: *store} blobs, err := s.ResolveBlobs(ref) if err != nil { @@ -736,6 +745,8 @@ func cmdRun(args []string) error { fmt.Printf("[brewlet] sandbox : %s\n", sandbox) fmt.Printf("[brewlet] launch : %s\n", plan.CommandLine()) fmt.Printf("[brewlet] ----- JVM output below -----\n") + // The sandbox and optional CDS cache now own all payloads. + _ = guard.Close() return plan.Run() } @@ -766,21 +777,35 @@ func cmdBundle(args []string) error { if err != nil { return err } + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return err + } + defer guard.Close() s := artifact.Store{Root: *store} blobs, err := s.ResolveBlobs(pos[0]) if err != nil { return err } + res := runtime.Resources{CPULimit: *cpu, MemoryLimit: *mem} + if err := res.Validate(); err != nil { + return err + } + cleanup, err := retainBundlePayloads(&blobs, *out) + if err != nil { + return err + } cfg := blobs.Config cdsSrc := blobs.CDSHostPath if err := runtime.GenerateBundleWithIdentityAndRegen(cfg, *jdkRoot, *launcherRoot, *launcher, blobs.JarHostPath, blobs.ClasspathHostPaths, blobs.ModulepathHostPaths, cdsSrc, *out, - runtime.Resources{CPULimit: *cpu, MemoryLimit: *mem}, nil, runtime.ProcessIdentity{UID: uid, GID: gid}, + res, nil, runtime.ProcessIdentity{UID: uid, GID: gid}, runtime.CDSRegenOptions{ Regenerate: *appcdsRegen, CacheScope: "local", ArtifactDigest: blobs.ManifestDigest, CacheDir: os.Getenv("BREWLET_CDS_CACHE"), }); err != nil { + cleanup() return err } fmt.Printf("wrote OCI runtime bundle to %s/config.json\n", *out) diff --git a/core/cmd/brewlet/stage_gc.go b/core/cmd/brewlet/stage_gc.go new file mode 100644 index 00000000..d2358f50 --- /dev/null +++ b/core/cmd/brewlet/stage_gc.go @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/microsoft/brewlet/internal/artifact" + "github.com/microsoft/brewlet/internal/runnablestage" +) + +func cmdStageGC(args []string) error { + fs := flag.NewFlagSet("stage-gc", flag.ContinueOnError) + root := fs.String("stage-root", runnablestage.Root(), "runnable staging root (BREWLET_RUNNABLE_STAGE)") + address := fs.String("address", "/run/containerd/containerd.sock", "containerd socket; all namespaces are checked") + age := fs.Duration("min-age", runnablestage.DefaultMinAge, "minimum stage age (must be positive)") + dryRun := fs.Bool("dry-run", false, "report eligible trees without deleting them") + if err := fs.Parse(args); err != nil { + if err == flag.ErrHelp { + return nil + } + return err + } + if fs.NArg() != 0 || *age <= 0 { + return fmt.Errorf("usage: stage-gc [--stage-root DIR] [--address SOCKET] [--min-age positive-duration] [--dry-run]") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + result, err := runnablestage.Reap(ctx, *address, runnablestage.Options{ + Root: *root, MinAge: *age, DryRun: *dryRun, + }) + if err != nil { + return err + } + action := "removed" + if *dryRun { + action = "would remove" + } + fmt.Printf("%s %d stages (%d bytes); remaining stage bytes: %d\n", action, result.Removed, result.ReclaimedBytes, result.RemainingBytes) + return nil +} + +// An exported bundle may be started long after this command exits. Give it +// independent payloads rather than dangling references into the evictable cache. +func retainBundlePayloads(blobs *artifact.ResolvedBlobs, out string) (func(), error) { + if blobs.Format != "image" { + return func() {}, nil + } + out, err := filepath.Abs(out) + if err != nil { + return nil, err + } + if err := os.MkdirAll(out, 0o755); err != nil { + return nil, err + } + dir, err := os.MkdirTemp(out, "brewlet-image-") + if err != nil { + return nil, err + } + cleanup := func() { _ = os.RemoveAll(dir) } + if err := os.CopyFS(dir, os.DirFS(filepath.Dir(blobs.JarHostPath))); err != nil { + cleanup() + return nil, fmt.Errorf("retain bundle payloads: %w", err) + } + blobs.JarHostPath = filepath.Join(dir, filepath.Base(blobs.JarHostPath)) + if blobs.CDSHostPath != "" { + blobs.CDSHostPath = filepath.Join(dir, filepath.Base(blobs.CDSHostPath)) + } + return cleanup, nil +} diff --git a/core/cmd/brewlet/stage_gc_test.go b/core/cmd/brewlet/stage_gc_test.go new file mode 100644 index 00000000..c7bd50ca --- /dev/null +++ b/core/cmd/brewlet/stage_gc_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestStageGCRejectsInvalidArguments(t *testing.T) { + for _, args := range [][]string{ + {"--min-age=0"}, {"--min-age=-1h"}, {"--min-age=invalid"}, {"unexpected"}, + } { + if err := cmdStageGC(args); err == nil { + t.Fatalf("stage-gc %v succeeded", args) + } + if err := cmdStageGC([]string{"--help"}); err != nil { + t.Fatalf("stage-gc help: %v", err) + } + } +} + +func TestRunnableBundleSurvivesStageRemoval(t *testing.T) { + stage := t.TempDir() + t.Setenv("BREWLET_RUNNABLE_STAGE", stage) + dir := t.TempDir() + jar := filepath.Join(dir, "orders.jar") + writeCLIZip(t, jar, "com/example/Orders.class") + archive := filepath.Join(dir, "app.jsa") + if err := os.WriteFile(archive, []byte("archive"), 0o644); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"schemaVersion":1,"entry":{"mode":"jar"}}`), 0o644); err != nil { + t.Fatal(err) + } + for _, cds := range []bool{false, true} { + name := "plain" + if cds { + name = "cds" + } + t.Run(name, func(t *testing.T) { + store := filepath.Join(t.TempDir(), "oci") + args := []string{jar, "apps/orders:1", "--store", store, "--config", config} + if cds { + args = append(args, "--appcds-archive", archive) + } + if err := cmdPush(args); err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "bundle") + if err := cmdBundle([]string{"apps/orders:1", "--store", store, "--out", out}); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(filepath.Join(stage, "immutable-v2")); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(out, "config.json")) + if err != nil { + t.Fatal(err) + } + var spec struct { + Mounts []struct{ Source, Destination string } + } + if err := json.Unmarshal(data, &spec); err != nil { + t.Fatal(err) + } + payloads := 0 + for _, mount := range spec.Mounts { + if !strings.HasPrefix(mount.Destination, "/app/") { + continue + } + payloads++ + if !strings.HasPrefix(mount.Source, out+string(os.PathSeparator)) { + t.Errorf("payload is not bundle-owned: %s", mount.Source) + } + if _, err := os.ReadFile(mount.Source); err != nil { + t.Errorf("bundle payload missing after eviction: %v", err) + } + } + want := 1 + if cds { + want = 2 + } + if payloads != want { + t.Fatalf("payload mounts = %d, want %d", payloads, want) + } + }) + } +} diff --git a/core/go.mod b/core/go.mod index e17c52af..15be0747 100644 --- a/core/go.mod +++ b/core/go.mod @@ -11,10 +11,12 @@ require ( github.com/containerd/ttrpc v1.2.9 github.com/containerd/typeurl/v2 v2.3.0 github.com/distribution/reference v0.6.0 + github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/runtime-spec v1.3.0 github.com/prometheus/client_golang v1.24.1 go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/sys v0.47.0 google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 ) @@ -44,7 +46,6 @@ require ( github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect @@ -54,7 +55,6 @@ require ( golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect ) diff --git a/core/internal/artifact/blobs.go b/core/internal/artifact/blobs.go index 4152cbe8..281122ee 100644 --- a/core/internal/artifact/blobs.go +++ b/core/internal/artifact/blobs.go @@ -13,6 +13,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/microsoft/brewlet/internal/runnablestage" ) // ResolvedBlobs is an artifact after separating the JVM launch config from its @@ -189,8 +191,15 @@ func verifiedBlobPaths(src BlobSource, layers []Descriptor, kind string) ([]stri // reads from — the JAR (and optional CDS archive) as files, and the // classpath/module layers as uncompressed tars the existing staging path // consumes unchanged. Staging is idempotent (keyed on manifestDigest) so -// repeated resolutions of the same image reuse it. +// repeated resolutions of the same image reuse it. Callers must hold a +// runnablestage.Acquire guard until the returned paths have been copied or +// mounted; the guard here protects resolution itself. func ResolveRunnableBlobs(src BlobSource, man Manifest, manifestDigest string) (ResolvedBlobs, error) { + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return ResolvedBlobs{}, err + } + defer guard.Close() cfg, err := man.RunnableConfig() if err != nil { return ResolvedBlobs{}, err @@ -378,10 +387,7 @@ func runnableStageDir(manifestDigest string) (string, error) { if err != nil { return "", fmt.Errorf("runnable image staging: %w", err) } - base := os.Getenv("BREWLET_RUNNABLE_STAGE") - if base == "" { - base = filepath.Join(os.TempDir(), "brewlet-runnable") - } + base := runnablestage.Root() // Previous stages may still be mounted and lack retained verification blobs. // Never rewrite or migrate a live stage during an upgrade. return filepath.Join(base, "immutable-v2", hex), nil diff --git a/core/internal/runnablestage/lock_linux.go b/core/internal/runnablestage/lock_linux.go new file mode 100644 index 00000000..e8605585 --- /dev/null +++ b/core/internal/runnablestage/lock_linux.go @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "errors" + "fmt" + "io" + "os" + + "golang.org/x/sys/unix" +) + +// Acquire holds a shared interprocess directory lock until Close. Callers must +// hold it from stage resolution through establishment of the workload's mounts +// (or for the lifetime of direct host execution). +func Acquire(root string) (io.Closer, error) { + if root == "" { + root = Root() + } + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, err + } + return lock(root, false) +} + +func lock(root string, exclusive bool) (*os.File, error) { + fd, err := unix.Open(root, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), root) + operation := unix.LOCK_SH + if exclusive { + operation = unix.LOCK_EX | unix.LOCK_NB + } + for { + err = unix.Flock(fd, operation) + if !errors.Is(err, unix.EINTR) { + break + } + } + if err != nil { + file.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, ErrBusy + } + return nil, err + } + current, err := os.Lstat(root) + opened, statErr := file.Stat() + if err != nil || statErr != nil || !current.IsDir() || !os.SameFile(current, opened) { + file.Close() + return nil, fmt.Errorf("staging root changed while locking") + } + return file, nil +} diff --git a/core/internal/runnablestage/mounts_linux.go b/core/internal/runnablestage/mounts_linux.go new file mode 100644 index 00000000..0b2696b2 --- /dev/null +++ b/core/internal/runnablestage/mounts_linux.go @@ -0,0 +1,285 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "bufio" + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" +) + +type mount struct { + device string + root string + point string + filesystem string + options string +} + +type mountSnapshot struct { + current []mount + all []mount +} + +type procReader interface { + ReadFile(string) ([]byte, error) + ReadDir(string) ([]fs.DirEntry, error) + Readlink(string) (string, error) +} + +type procFS string + +func (p procFS) ReadFile(name string) ([]byte, error) { + return os.ReadFile(filepath.Join(string(p), name)) +} +func (p procFS) ReadDir(name string) ([]fs.DirEntry, error) { + return os.ReadDir(filepath.Join(string(p), name)) +} +func (p procFS) Readlink(name string) (string, error) { + return os.Readlink(filepath.Join(string(p), name)) +} + +func hostMounts(ctx context.Context) (mountSnapshot, error) { + return readMounts(ctx, procFS("/proc")) +} + +func readMounts(ctx context.Context, proc procReader) (mountSnapshot, error) { + // Linux reserves these inode numbers for the initial PID/user namespaces. + // Merely comparing self with PID 1 would accept a container's private proc. + for name, expected := range map[string]string{ + "self/ns/pid": "pid:[4026531836]", + "1/ns/pid": "pid:[4026531836]", + "self/ns/user": "user:[4026531837]", + } { + value, err := proc.Readlink(name) + if err != nil { + return mountSnapshot{}, fmt.Errorf("verify host namespace %s: %w", name, err) + } + if value != expected { + return mountSnapshot{}, fmt.Errorf("reaper requires the initial host PID and user namespaces") + } + } + selfNS, err := proc.Readlink("self/ns/mnt") + if err != nil { + return mountSnapshot{}, err + } + initNS, err := proc.Readlink("1/ns/mnt") + if err != nil { + return mountSnapshot{}, err + } + if selfNS != initNS { + return mountSnapshot{}, fmt.Errorf("reaper requires the host mount namespace") + } + raw, err := proc.ReadFile("self/mountinfo") + if err != nil { + return mountSnapshot{}, err + } + current, err := parseMounts(raw) + if err != nil { + return mountSnapshot{}, err + } + procMount, ok := containingMount(current, "/proc") + if !ok || procMount.filesystem != "proc" || procMount.point != "/proc" || procMount.root != "/" { + return mountSnapshot{}, fmt.Errorf("cannot verify complete host proc visibility") + } + for _, option := range strings.Split(procMount.options, ",") { + if strings.HasPrefix(option, "hidepid=") && option != "hidepid=0" && option != "hidepid=off" { + return mountSnapshot{}, fmt.Errorf("refusing permission-limited proc mount (%s)", option) + } + } + entries, err := proc.ReadDir(".") + if err != nil { + return mountSnapshot{}, err + } + snapshot := mountSnapshot{current: current, all: append([]mount(nil), current...)} + pids := 0 + for _, entry := range entries { + if !entry.IsDir() || !decimal(entry.Name()) { + continue + } + pids++ + if err := ctx.Err(); err != nil { + return mountSnapshot{}, err + } + raw, err := proc.ReadFile(entry.Name() + "/mountinfo") + if os.IsNotExist(err) { + // A vanished PID is harmless; an existing process with a hidden + // mountinfo is not. Zombies can have an empty mountinfo. + _, aliveErr := proc.ReadFile(entry.Name() + "/stat") + if os.IsNotExist(aliveErr) { + continue + } + } + if err != nil { + return mountSnapshot{}, fmt.Errorf("read PID %s mounts: %w", entry.Name(), err) + } + if len(strings.TrimSpace(string(raw))) == 0 { + stat, statErr := proc.ReadFile(entry.Name() + "/stat") + if os.IsNotExist(statErr) { + continue + } + if statErr == nil { + if end := strings.LastIndex(string(stat), ") "); end >= 0 { + state := string(stat)[end+2:] + if strings.HasPrefix(state, "Z ") || strings.HasPrefix(state, "X ") { + continue + } + } + } + return mountSnapshot{}, fmt.Errorf("empty mountinfo for live or unreadable PID %s", entry.Name()) + } + mounts, err := parseMounts(raw) + if err != nil { + return mountSnapshot{}, fmt.Errorf("parse PID %s mounts: %w", entry.Name(), err) + } + snapshot.all = append(snapshot.all, mounts...) + } + if pids == 0 { + return mountSnapshot{}, fmt.Errorf("no visible processes in host proc") + } + return snapshot, nil +} + +func decimal(value string) bool { + if value == "" { + return false + } + for _, c := range value { + if c < '0' || c > '9' { + return false + } + } + return true +} + +func parseMounts(raw []byte) ([]mount, error) { + var result []mount + scanner := bufio.NewScanner(strings.NewReader(string(raw))) + scanner.Buffer(make([]byte, 4096), 1024*1024) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + separator := -1 + for i, field := range fields { + if field == "-" { + separator = i + break + } + } + if separator < 6 || len(fields) != separator+4 { + return nil, fmt.Errorf("malformed mountinfo record") + } + for _, id := range fields[:2] { + if _, err := strconv.ParseUint(id, 10, 64); err != nil { + return nil, fmt.Errorf("malformed mountinfo mount ID") + } + } + device := strings.Split(fields[2], ":") + if len(device) != 2 { + return nil, fmt.Errorf("malformed mountinfo device") + } + for _, n := range device { + if _, err := strconv.ParseUint(n, 10, 32); err != nil { + return nil, fmt.Errorf("malformed mountinfo device") + } + } + root, err := mountPath(fields[3]) + if err != nil { + return nil, err + } + point, err := mountPath(fields[4]) + if err != nil { + return nil, err + } + result = append(result, mount{ + device: fields[2], root: root, point: point, + filesystem: fields[separator+1], options: fields[5] + "," + fields[separator+3], + }) + } + if err := scanner.Err(); err != nil { + return nil, err + } + if len(result) == 0 { + return nil, fmt.Errorf("empty mountinfo") + } + return result, nil +} + +func mountPath(value string) (string, error) { + var out strings.Builder + for i := 0; i < len(value); i++ { + if value[i] != '\\' { + out.WriteByte(value[i]) + continue + } + if i+3 >= len(value) { + return "", fmt.Errorf("malformed mountinfo escape") + } + switch value[i+1 : i+4] { + case "040": + out.WriteByte(' ') + case "011": + out.WriteByte('\t') + case "012": + out.WriteByte('\n') + case "134": + out.WriteByte('\\') + default: + return "", fmt.Errorf("malformed mountinfo escape") + } + i += 3 + } + path := out.String() + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return "", fmt.Errorf("noncanonical mountinfo path %q", path) + } + return path, nil +} + +func beneath(parent, path string) bool { + return path == parent || strings.HasPrefix(path, strings.TrimSuffix(parent, "/")+"/") +} + +func containingMount(mounts []mount, path string) (mount, bool) { + var best mount + found := false + for _, m := range mounts { + if beneath(m.point, path) && (!found || len(m.point) >= len(best.point)) { + best, found = m, true + } + } + return best, found +} + +func (s mountSnapshot) protects(path, device string) (bool, error) { + base, ok := containingMount(s.current, path) + if !ok || base.device != device { + return false, fmt.Errorf("cannot resolve staging filesystem for %q", path) + } + relative, err := filepath.Rel(base.point, path) + if err != nil { + return false, err + } + source := filepath.Join(base.root, relative) + for _, m := range s.current { + if beneath(path, m.point) { + // Includes same-device bind mounts; RemoveAll must not cross + // any mountpoint, not merely a change in st_dev. + return true, nil + } + } + for _, m := range s.all { + if m.device == device && beneath(source, m.root) { + // Bind sources live in mountinfo's root field, NOT source. + return true, nil + } + } + return false, nil +} diff --git a/core/internal/runnablestage/mounts_linux_test.go b/core/internal/runnablestage/mounts_linux_test.go new file mode 100644 index 00000000..7d90a416 --- /dev/null +++ b/core/internal/runnablestage/mounts_linux_test.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "fmt" + "io/fs" + "os" + "strings" + "testing" + "testing/fstest" +) + +type fakeProc struct { + files fstest.MapFS + links map[string]string + fail string +} + +func (p fakeProc) ReadFile(name string) ([]byte, error) { + if name == p.fail { + return nil, os.ErrPermission + } + return fs.ReadFile(p.files, name) +} +func (p fakeProc) ReadDir(name string) ([]fs.DirEntry, error) { + if name == p.fail { + return nil, os.ErrPermission + } + return fs.ReadDir(p.files, name) +} +func (p fakeProc) Readlink(name string) (string, error) { + if name == p.fail { + return "", os.ErrPermission + } + if value, ok := p.links[name]; ok { + return value, nil + } + return "", os.ErrNotExist +} + +const hostMountinfo = "1 0 8:1 / / rw - ext4 /dev/root rw\n2 1 0:2 / /proc rw - proc proc rw\n" + +func newProc() fakeProc { + return fakeProc{ + files: fstest.MapFS{ + "self/mountinfo": {Data: []byte(hostMountinfo)}, + "1/mountinfo": {Data: []byte(hostMountinfo)}, + "42/mountinfo": {Data: []byte("3 0 8:1 /stage/immutable-v2/hash/app/app.jar /app/app.jar ro - ext4 /dev/root rw\n")}, + }, + links: map[string]string{ + "self/ns/pid": "pid:[4026531836]", "1/ns/pid": "pid:[4026531836]", + "self/ns/user": "user:[4026531837]", "self/ns/mnt": "mnt:[1]", "1/ns/mnt": "mnt:[1]", + }, + } +} + +func TestMountsAcrossProcesses(t *testing.T) { + proc := newProc() + snapshot, err := readMounts(context.Background(), proc) + if err != nil { + t.Fatal(err) + } + got, err := snapshot.protects("/stage/immutable-v2/hash", "8:1") + if err != nil || !got { + t.Fatalf("missed bind source in another PID namespace: %t %v", got, err) + } + if got, err := snapshot.protects("/stage/immutable-v2/has", "8:1"); err != nil || got { + t.Fatalf("non-component prefix matched: %t %v", got, err) + } +} + +func TestMountMapping(t *testing.T) { + for _, tt := range []struct { + name string + mounts []mount + want bool + }{ + {"file source", []mount{{device: "8:1", root: "/subvolume/cache/hash/app.jar", point: "/app.jar"}}, true}, + {"directory source", []mount{{device: "8:1", root: "/subvolume/cache/hash", point: "/app"}}, true}, + {"exporter ancestor source", []mount{{device: "8:1", root: "/subvolume/cache", point: "/cache"}}, false}, + {"filesystem root source", []mount{{device: "8:1", root: "/", point: "/host-root"}}, false}, + {"other device", []mount{{device: "8:2", root: "/subvolume/cache/hash", point: "/app"}}, false}, + {"prefix only", []mount{{device: "8:1", root: "/subvolume/cache/hash-other", point: "/app"}}, false}, + {"unrelated", []mount{{device: "8:1", root: "/other/hash", point: "/app"}}, false}, + } { + t.Run(tt.name, func(t *testing.T) { + base := mount{device: "8:1", root: "/subvolume", point: "/host"} + snapshot := mountSnapshot{current: []mount{base}, all: append([]mount{base}, tt.mounts...)} + got, err := snapshot.protects("/host/cache/hash", "8:1") + if err != nil || got != tt.want { + t.Fatalf("protected=%t error=%v want=%t", got, err, tt.want) + } + }) + } + for _, device := range []string{"8:1", "8:2"} { + base := mount{device: "8:1", root: "/", point: "/"} + inside := mount{device: device, root: "/", point: "/stage/hash/mount"} + snapshot := mountSnapshot{current: []mount{base, inside}, all: []mount{base}} + if protected, err := snapshot.protects("/stage/hash", "8:1"); err != nil || !protected { + t.Fatalf("mount inside stage on %s not protected: %t %v", device, protected, err) + } + } +} + +func TestProcFailClosed(t *testing.T) { + for _, failure := range []string{"hidepid", "pidns", "userns", "mountns", "missing-proc", "permission", "parse", "empty-live", "missing-live", "enumeration", "namespace-permission"} { + t.Run(failure, func(t *testing.T) { + proc := newProc() + switch failure { + case "hidepid": + proc.files["self/mountinfo"].Data = []byte(strings.ReplaceAll(hostMountinfo, "proc proc rw", "proc proc rw,hidepid=2")) + case "pidns": + proc.links["self/ns/pid"] = "pid:[999]" + case "userns": + proc.links["self/ns/user"] = "user:[999]" + case "mountns": + proc.links["self/ns/mnt"] = "mnt:[999]" + case "missing-proc": + proc.files["self/mountinfo"].Data = []byte("1 0 8:1 / / rw - ext4 /dev/root rw\n") + case "permission": + proc.fail = "42/mountinfo" + case "parse": + proc.files["42/mountinfo"].Data = []byte("broken\n") + case "empty-live": + proc.files["42/mountinfo"].Data = nil + proc.files["42/stat"] = &fstest.MapFile{Data: []byte("42 (jvm) S 1 2")} + case "missing-live": + delete(proc.files, "42/mountinfo") + proc.files["42/stat"] = &fstest.MapFile{Data: []byte("42 (jvm) S 1 2")} + case "enumeration": + proc.fail = "." + case "namespace-permission": + proc.fail = "1/ns/mnt" + } + if _, err := readMounts(context.Background(), proc); err == nil { + t.Fatal("accepted incomplete mount snapshot") + } + }) + } +} + +func TestExitedProcesses(t *testing.T) { + for _, zombie := range []bool{false, true} { + t.Run(fmt.Sprint(zombie), func(t *testing.T) { + proc := newProc() + if zombie { + proc.files["42/mountinfo"].Data = nil + proc.files["42/stat"] = &fstest.MapFile{Data: []byte("42 (name with ) parenthesis) Z 1 2")} + } else { + delete(proc.files, "42/mountinfo") + proc.files["42"] = &fstest.MapFile{Mode: fs.ModeDir} + } + if _, err := readMounts(context.Background(), proc); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestMountinfoParsing(t *testing.T) { + mounts, err := parseMounts([]byte(`1 0 8:1 /stage\040root/app\134name /app\011jar ro shared:3 - ext4 /dev/root rw` + "\n")) + if err != nil || mounts[0].root != "/stage root/app\\name" || mounts[0].point != "/app\tjar" { + t.Fatalf("escaped mount paths: %+v %v", mounts, err) + } + for _, line := range []string{ + "", "broken", "1 0 8:1 / / rw - ext4 source", "x 0 8:1 / / rw - ext4 source rw", + "1 0 bad / / rw - ext4 source rw", "1 0 8:x / / rw - ext4 source rw", + "1 0 8:1 relative / rw - ext4 source rw", `1 0 8:1 /bad\777 / rw - ext4 source rw`, + "1 0 8:1 /bad/../path / rw - ext4 source rw", + } { + if _, err := parseMounts([]byte(line)); err == nil { + t.Errorf("accepted malformed mountinfo %q", line) + } + } +} diff --git a/core/internal/runnablestage/reap_linux.go b/core/internal/runnablestage/reap_linux.go new file mode 100644 index 00000000..05ec2fff --- /dev/null +++ b/core/internal/runnablestage/reap_linux.go @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +type dependencies struct { + references func(context.Context, string) (map[string]bool, error) + mounts func(context.Context) (mountSnapshot, error) + now func() time.Time +} + +// Reap removes only old, unreferenced immutable-v2 digest directories. MinAge +// must be strictly positive. Metadata and every visible process's mounts +// must be readable before any deletion. Run in the host namespaces with a +// complete proc mount; concurrent legacy launchers must be quiesced. +func Reap(ctx context.Context, address string, opts Options) (Result, error) { + return reap(ctx, address, opts, dependencies{containerReferences, hostMounts, time.Now}) +} + +func reap(ctx context.Context, address string, opts Options, deps dependencies) (Result, error) { + var result Result + if opts.MinAge <= 0 { + return result, fmt.Errorf("minimum stage age must be positive") + } + if opts.Root == "" { + opts.Root = Root() + } + absolute, err := filepath.Abs(opts.Root) + if err != nil { + return result, err + } + // Resolve ancestor aliases for mount matching, but never accept a symlink + // as the staging root itself. + before, err := os.Lstat(absolute) + if os.IsNotExist(err) { + return result, nil + } + if err != nil { + return result, err + } + if !before.IsDir() { + return result, fmt.Errorf("staging root is not a real directory") + } + absolute, err = filepath.EvalSymlinks(absolute) + if err != nil { + return result, err + } + guard, err := lock(absolute, true) + if err != nil { + return result, err + } + defer guard.Close() + root, err := openRoot(absolute) + if err != nil { + return result, err + } + defer root.Close() + lockedInfo, err := guard.Stat() + if err != nil { + return result, err + } + rootInfo, err := root.Stat(".") + if err != nil || !os.SameFile(lockedInfo, rootInfo) || !os.SameFile(before, rootInfo) { + return result, fmt.Errorf("staging root changed while opening") + } + versionInfo, err := root.Lstat("immutable-v2") + if os.IsNotExist(err) { + result.RemainingBytes, err = treeBytes(root, ".") + return result, err + } + if err != nil { + return result, err + } + if !versionInfo.IsDir() { + return result, fmt.Errorf("immutable-v2 is not a real directory") + } + version, err := root.OpenRoot("immutable-v2") + if err != nil { + return result, err + } + defer version.Close() + openedInfo, err := version.Stat(".") + if err != nil || !os.SameFile(versionInfo, openedInfo) { + return result, fmt.Errorf("immutable-v2 changed while opening") + } + entries, err := fs.ReadDir(version.FS(), ".") + if err != nil { + return result, err + } + refs, err := deps.references(ctx, address) + if err != nil { + return result, err + } + mounts, err := deps.mounts(ctx) + if err != nil { + return result, err + } + type candidate struct { + name string + info fs.FileInfo + size int64 + } + var candidates []candidate + cutoff := deps.now().Add(-opts.MinAge) + for _, entry := range entries { + if err := ctx.Err(); err != nil { + return result, err + } + if !canonical(entry.Name()) || !entry.IsDir() || refs[entry.Name()] { + continue + } + info, err := version.Lstat(entry.Name()) + if err != nil { + return result, err + } + if !info.IsDir() || !info.ModTime().Before(cutoff) { + continue + } + var stat unix.Stat_t + path := filepath.Join(absolute, "immutable-v2", entry.Name()) + if err := unix.Lstat(path, &stat); err != nil { + return result, err + } + device := fmt.Sprintf("%d:%d", unix.Major(uint64(stat.Dev)), unix.Minor(uint64(stat.Dev))) + protected, err := mounts.protects(path, device) + if err != nil { + return result, err + } + if protected { + continue + } + size, err := treeBytes(version, entry.Name()) + if err != nil { + return result, err + } + candidates = append(candidates, candidate{entry.Name(), info, size}) + } + // Finish all accounting and reference validation before the first removal. + result.RemainingBytes, err = treeBytes(root, ".") + if err != nil { + return result, err + } + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return result, err + } + current, err := version.Lstat(candidate.name) + if err != nil || !current.IsDir() || !os.SameFile(current, candidate.info) || + !current.ModTime().Equal(candidate.info.ModTime()) { + return result, fmt.Errorf("stage %s changed during reaping", candidate.name) + } + if !opts.DryRun { + if err := version.RemoveAll(candidate.name); err != nil { + // A filesystem failure can leave a partially removed tree. + result.RemainingBytes, _ = treeBytes(root, ".") + return result, err + } + result.RemainingBytes -= candidate.size + } + result.Removed++ + result.ReclaimedBytes += candidate.size + } + return result, nil +} diff --git a/core/internal/runnablestage/reap_linux_test.go b/core/internal/runnablestage/reap_linux_test.go new file mode 100644 index 00000000..6b8c95a0 --- /dev/null +++ b/core/internal/runnablestage/reap_linux_test.go @@ -0,0 +1,294 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func createStage(t *testing.T, root, name string, age time.Duration) string { + t.Helper() + path := filepath.Join(root, "immutable-v2", name) + writeFile(t, filepath.Join(path, "app", "app.jar"), "jar") + old := time.Now().Add(-age) + if err := os.Chtimes(path, old, old); err != nil { + t.Fatal(err) + } + return path +} + +func fakeDependencies(t *testing.T, root string) dependencies { + t.Helper() + var stat unix.Stat_t + if err := unix.Stat(root, &stat); err != nil { + t.Fatal(err) + } + base := mount{device: fmt.Sprintf("%d:%d", unix.Major(uint64(stat.Dev)), unix.Minor(uint64(stat.Dev))), root: "/", point: "/"} + return dependencies{ + references: func(context.Context, string) (map[string]bool, error) { return map[string]bool{}, nil }, + mounts: func(context.Context) (mountSnapshot, error) { + return mountSnapshot{current: []mount{base}, all: []mount{base}}, nil + }, + now: time.Now, + } +} + +func TestReapFiltersAndDryRun(t *testing.T) { + for _, dry := range []bool{true, false} { + t.Run(fmt.Sprintf("dry=%t", dry), func(t *testing.T) { + root := t.TempDir() + deps := fakeDependencies(t, root) + names := make([]string, 7) + for i := range names { + names[i] = strings.Repeat(fmt.Sprintf("%x", i), 64) + } + old := createStage(t, root, names[0], 48*time.Hour) + image := createStage(t, root, names[1], 48*time.Hour) + content := createStage(t, root, names[2], 48*time.Hour) + mounted := createStage(t, root, names[3], 48*time.Hour) + young := createStage(t, root, names[4], time.Hour) + inside := createStage(t, root, names[5], 48*time.Hour) + unknown := createStage(t, root, "unknown", 48*time.Hour) + pending := createStage(t, root, "."+names[0]+"-pending", 48*time.Hour) + uppercase := createStage(t, root, strings.Repeat("A", 64), 48*time.Hour) + writeFile(t, filepath.Join(root, "legacy", "app.jar"), "jar") + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "app.jar"), "untouched") + link := filepath.Join(root, "immutable-v2", names[6]) + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + deps.references = func(context.Context, string) (map[string]bool, error) { + return map[string]bool{names[1]: true, names[2]: true}, nil + } + snapshot, _ := deps.mounts(context.Background()) + snapshot.all = append(snapshot.all, mount{device: snapshot.current[0].device, root: filepath.Join(mounted, "app", "app.jar"), point: "/app/app.jar"}) + snapshot.all = append(snapshot.all, mount{device: snapshot.current[0].device, root: root, point: "/exporter-stage"}) + snapshot.current = append(snapshot.current, mount{device: "99:9", root: "/", point: filepath.Join(inside, "nested")}) + deps.mounts = func(context.Context) (mountSnapshot, error) { return snapshot, nil } + result, err := reap(context.Background(), "fake", Options{Root: root, MinAge: DefaultMinAge, DryRun: dry}, deps) + if err != nil { + t.Fatal(err) + } + wantRemaining := int64(30) + if !dry { + wantRemaining -= 3 + } + if result.Removed != 1 || result.ReclaimedBytes != 3 || result.RemainingBytes != wantRemaining { + t.Fatalf("unexpected result: %+v, want remaining %d", result, wantRemaining) + } + if _, err := os.Lstat(old); dry && err != nil || !dry && !os.IsNotExist(err) { + t.Fatalf("old stage: %v", err) + } + for _, path := range []string{image, content, mounted, young, inside, unknown, pending, uppercase, link, outside, filepath.Join(root, "legacy")} { + if _, err := os.Lstat(path); err != nil { + t.Fatalf("removed protected path %s: %v", path, err) + } + } + }) + } +} + +func TestReapFailClosed(t *testing.T) { + for _, failure := range []string{"references", "mounts", "mapping", "cancelled"} { + t.Run(failure, func(t *testing.T) { + root := t.TempDir() + path := createStage(t, root, strings.Repeat("a", 64), 48*time.Hour) + deps := fakeDependencies(t, root) + ctx := context.Background() + switch failure { + case "references": + deps.references = func(context.Context, string) (map[string]bool, error) { + return nil, errors.New("containerd unavailable") + } + case "mounts": + deps.mounts = func(context.Context) (mountSnapshot, error) { + return mountSnapshot{}, os.ErrPermission + } + case "mapping": + deps.mounts = func(context.Context) (mountSnapshot, error) { + return mountSnapshot{}, nil + } + case "cancelled": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + if _, err := reap(ctx, "", Options{Root: root, MinAge: DefaultMinAge}, deps); err == nil { + t.Fatal("expected failure") + } + if _, err := os.Stat(filepath.Join(path, "app", "app.jar")); err != nil { + t.Fatalf("deletion before complete validation: %v", err) + } + }) + } +} + +func TestReapSymlinksAndMissing(t *testing.T) { + root := t.TempDir() + deps := fakeDependencies(t, root) + missing := filepath.Join(root, "missing") + if got, err := reap(context.Background(), "", Options{Root: missing, MinAge: DefaultMinAge}, dependencies{}); got != (Result{}) || err != nil { + t.Fatalf("missing root: %+v, %v", got, err) + } + if _, err := os.Stat(missing); !os.IsNotExist(err) { + t.Fatal("reaping created missing root") + } + target := t.TempDir() + createStage(t, target, strings.Repeat("a", 64), 48*time.Hour) + for _, name := range []string{"link", "immutable-v2"} { + path := filepath.Join(root, name) + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } + reapRoot := root + if name == "link" { + reapRoot = path + } + if _, err := reap(context.Background(), "", Options{Root: reapRoot, MinAge: DefaultMinAge}, deps); err == nil { + t.Fatalf("accepted symlink %s", name) + } + } + for _, age := range []time.Duration{-1, 0} { + if _, err := reap(context.Background(), "", Options{Root: root, MinAge: age}, deps); err == nil { + t.Fatalf("accepted nonpositive age %s", age) + } + } +} + +func TestAgeBoundary(t *testing.T) { + root := t.TempDir() + deps := fakeDependencies(t, root) + now := time.Now().Truncate(time.Second) + deps.now = func() time.Time { return now } + path := createStage(t, root, strings.Repeat("a", 64), time.Hour) + boundary := now.Add(-time.Hour) + if err := os.Chtimes(path, boundary, boundary); err != nil { + t.Fatal(err) + } + result, err := reap(context.Background(), "", Options{Root: root, MinAge: time.Hour}, deps) + if err != nil || result.Removed != 0 { + t.Fatalf("age boundary: %+v, %v", result, err) + } + deps.now = func() time.Time { return now.Add(time.Second) } + result, err = reap(context.Background(), "", Options{Root: root, MinAge: time.Hour}, deps) + if err != nil || result.Removed != 1 { + t.Fatalf("past age boundary: %+v, %v", result, err) + } +} + +func TestLocks(t *testing.T) { + root := filepath.Join(t.TempDir(), "created") + a, err := Acquire(root) + if err != nil { + t.Fatal(err) + } + b, err := Acquire(root) + if err != nil { + t.Fatal(err) + } + defer b.Close() + if _, err := Reap(context.Background(), "", Options{Root: root, MinAge: DefaultMinAge}); !errors.Is(err, ErrBusy) { + t.Fatalf("reaper with active readers: %v", err) + } + a.Close() + if _, err := lock(root, true); !errors.Is(err, ErrBusy) { + t.Fatalf("second reader failed to retain lock: %v", err) + } + b.Close() + exclusive, err := lock(root, true) + if err != nil { + t.Fatal(err) + } + acquired := make(chan io.Closer, 1) + failures := make(chan error, 1) + go func() { + reader, err := Acquire(root) + if err != nil { + failures <- err + return + } + acquired <- reader + }() + select { + case reader := <-acquired: + reader.Close() + t.Fatal("reader bypassed exclusive lock") + case err := <-failures: + t.Fatal(err) + case <-time.After(25 * time.Millisecond): + } + exclusive.Close() + select { + case reader := <-acquired: + reader.Close() + case err := <-failures: + t.Fatal(err) + case <-time.After(5 * time.Second): + t.Fatal("reader did not unblock") + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 0 { + t.Fatalf("locking added staging entries: %v, %v", entries, err) + } +} + +func TestLockInterprocess(t *testing.T) { + if root := os.Getenv("BREWLET_STAGE_LOCK_TEST_CHILD"); root != "" { + guard, err := Acquire(root) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + fmt.Println("locked") + _, _ = io.Copy(io.Discard, os.Stdin) + return + } + root := t.TempDir() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(executable, "-test.run=^TestLockInterprocess$") + cmd.Env = append(os.Environ(), "BREWLET_STAGE_LOCK_TEST_CHILD="+root) + input, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + output, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + input.Close() + if err := cmd.Wait(); err != nil { + t.Error(err) + } + }() + line, err := bufio.NewReader(output).ReadString('\n') + if err != nil || line != "locked\n" { + t.Fatalf("child readiness: %q %v", line, err) + } + if _, err := Reap(context.Background(), "", Options{Root: root, MinAge: DefaultMinAge}); !errors.Is(err, ErrBusy) { + t.Fatalf("cross-process reaper lock: %v", err) + } +} diff --git a/core/internal/runnablestage/references_linux.go b/core/internal/runnablestage/references_linux.go new file mode 100644 index 00000000..d06d2592 --- /dev/null +++ b/core/internal/runnablestage/references_linux.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + + contentapi "github.com/containerd/containerd/api/services/content/v1" + imagesapi "github.com/containerd/containerd/api/services/images/v1" + namespacesapi "github.com/containerd/containerd/api/services/namespaces/v1" + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/content/proxy" + "github.com/containerd/containerd/v2/pkg/namespaces" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type metadata interface { + namespaces(context.Context) ([]string, error) + images(context.Context, string) ([]ocispec.Descriptor, error) + contents(context.Context, string, func(content.Info) error) error + read(context.Context, string, ocispec.Descriptor) ([]byte, error) +} + +type containerMetadata struct { + client *grpc.ClientConn + store content.Store +} + +func (m containerMetadata) namespaces(ctx context.Context) ([]string, error) { + response, err := namespacesapi.NewNamespacesClient(m.client).List(ctx, &namespacesapi.ListNamespacesRequest{}) + if err != nil { + return nil, err + } + var names []string + for _, namespace := range response.Namespaces { + if namespace == nil || namespace.Name == "" { + return nil, fmt.Errorf("invalid containerd namespace") + } + names = append(names, namespace.Name) + } + return names, nil +} + +func (m containerMetadata) images(ctx context.Context, ns string) ([]ocispec.Descriptor, error) { + response, err := imagesapi.NewImagesClient(m.client).List(namespaces.WithNamespace(ctx, ns), &imagesapi.ListImagesRequest{}) + if err != nil { + return nil, err + } + targets := make([]ocispec.Descriptor, 0, len(response.Images)) + for _, image := range response.Images { + if image == nil || image.Target == nil { + return nil, fmt.Errorf("invalid containerd image") + } + targets = append(targets, ocispec.Descriptor{ + MediaType: image.Target.MediaType, Digest: digest.Digest(image.Target.Digest), Size: image.Target.Size, + }) + } + return targets, nil +} + +func (m containerMetadata) contents(ctx context.Context, ns string, fn func(content.Info) error) error { + return m.store.Walk(namespaces.WithNamespace(ctx, ns), fn) +} + +func (m containerMetadata) read(ctx context.Context, ns string, d ocispec.Descriptor) ([]byte, error) { + return content.ReadBlob(namespaces.WithNamespace(ctx, ns), m.store, d) +} + +func containerReferences(ctx context.Context, address string) (map[string]bool, error) { + if address == "" { + return nil, fmt.Errorf("containerd address is required") + } + if filepath.IsAbs(address) { + address = "unix://" + address + } + client, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, err + } + defer client.Close() + store := proxy.NewContentStore(contentapi.NewContentClient(client)) + return references(ctx, containerMetadata{client, store}) +} + +func references(ctx context.Context, store metadata) (map[string]bool, error) { + refs := make(map[string]bool) + add := func(d digest.Digest) error { + if err := d.Validate(); err != nil { + return fmt.Errorf("invalid reference digest %q: %w", d, err) + } + if d.Algorithm() == digest.SHA256 { + refs[d.Encoded()] = true + } + return nil + } + nss, err := store.namespaces(ctx) + if err != nil { + return nil, fmt.Errorf("list containerd namespaces: %w", err) + } + for _, ns := range nss { + if err := store.contents(ctx, ns, func(info content.Info) error { + if err := add(info.Digest); err != nil { + return err + } + for key, value := range info.Labels { + if strings.HasPrefix(key, "containerd.io/gc.ref.content.") { + if err := add(digest.Digest(value)); err != nil { + return err + } + } + } + return nil + }); err != nil { + return nil, fmt.Errorf("list content in namespace %q: %w", ns, err) + } + targets, err := store.images(ctx, ns) + if err != nil { + return nil, fmt.Errorf("list images in namespace %q: %w", ns, err) + } + seen := make(map[digest.Digest]bool) + var visit func(ocispec.Descriptor, int) error + visit = func(d ocispec.Descriptor, depth int) error { + if err := ctx.Err(); err != nil { + return err + } + if err := add(d.Digest); err != nil { + return err + } + if d.MediaType == ocispec.MediaTypeImageManifest || d.MediaType == "application/vnd.docker.distribution.manifest.v2+json" { + // A target/child descriptor protects a manifest even if its + // content has already disappeared. + return nil + } + if d.MediaType != ocispec.MediaTypeImageIndex && d.MediaType != "application/vnd.docker.distribution.manifest.list.v2+json" { + return fmt.Errorf("unknown image target media type %q", d.MediaType) + } + if seen[d.Digest] { + return nil + } + if depth > 128 { + return fmt.Errorf("image index nesting limit exceeded") + } + seen[d.Digest] = true + raw, err := store.read(ctx, ns, d) + if err != nil { + return fmt.Errorf("read image index %s: %w", d.Digest, err) + } + if d.Digest.Algorithm().FromBytes(raw) != d.Digest { + return fmt.Errorf("image index %s failed digest verification", d.Digest) + } + var index ocispec.Index + if err := json.Unmarshal(raw, &index); err != nil { + return fmt.Errorf("parse image index %s: %w", d.Digest, err) + } + if index.SchemaVersion != 2 || index.Manifests == nil { + return fmt.Errorf("invalid image index %s", d.Digest) + } + for _, child := range index.Manifests { + if err := visit(child, depth+1); err != nil { + return err + } + } + return nil + } + for _, target := range targets { + if err := visit(target, 0); err != nil { + return nil, fmt.Errorf("image references in namespace %q: %w", ns, err) + } + } + } + return refs, nil +} diff --git a/core/internal/runnablestage/references_linux_test.go b/core/internal/runnablestage/references_linux_test.go new file mode 100644 index 00000000..978e6032 --- /dev/null +++ b/core/internal/runnablestage/references_linux_test.go @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/containerd/containerd/v2/core/content" + digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type fakeMetadata struct { + nss []string + targets map[string][]ocispec.Descriptor + infos map[string][]content.Info + blobs map[string]map[digest.Digest][]byte + fail string + visited []string +} + +func (m *fakeMetadata) namespaces(context.Context) ([]string, error) { + if m.fail == "namespaces" { + return nil, os.ErrPermission + } + return m.nss, nil +} +func (m *fakeMetadata) images(_ context.Context, ns string) ([]ocispec.Descriptor, error) { + m.visited = append(m.visited, "images:"+ns) + if m.fail == "images:"+ns { + return nil, os.ErrPermission + } + return m.targets[ns], nil +} +func (m *fakeMetadata) contents(_ context.Context, ns string, fn func(content.Info) error) error { + m.visited = append(m.visited, "content:"+ns) + if m.fail == "content:"+ns { + return os.ErrPermission + } + for _, info := range m.infos[ns] { + if err := fn(info); err != nil { + return err + } + } + return nil +} +func (m *fakeMetadata) read(_ context.Context, ns string, d ocispec.Descriptor) ([]byte, error) { + if raw, ok := m.blobs[ns][d.Digest]; ok { + return raw, nil + } + return nil, os.ErrNotExist +} + +func descriptor(char string) ocispec.Descriptor { + return ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.Digest("sha256:" + strings.Repeat(char, 64))} +} + +func indexDescriptor(t *testing.T, children ...ocispec.Descriptor) (ocispec.Descriptor, []byte) { + t.Helper() + raw, err := json.Marshal(ocispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, MediaType: ocispec.MediaTypeImageIndex, Manifests: children, + }) + if err != nil { + t.Fatal(err) + } + return ocispec.Descriptor{MediaType: ocispec.MediaTypeImageIndex, Digest: digest.FromBytes(raw), Size: int64(len(raw))}, raw +} + +func TestReferencesAllNamespacesAndIndexes(t *testing.T) { + child := descriptor("a") + nested, nestedRaw := indexDescriptor(t, child) + index, raw := indexDescriptor(t, nested, descriptor("b")) + store := &fakeMetadata{ + nss: []string{"default", "k8s.io", "other"}, + targets: map[string][]ocispec.Descriptor{ + "default": {descriptor("c")}, + "k8s.io": {index}, + }, + infos: map[string][]content.Info{ + "other": {{Digest: descriptor("d").Digest, Labels: map[string]string{"containerd.io/gc.ref.content.child": descriptor("e").Digest.String()}}}, + }, + blobs: map[string]map[digest.Digest][]byte{"k8s.io": {index.Digest: raw, nested.Digest: nestedRaw}}, + } + refs, err := references(context.Background(), store) + if err != nil { + t.Fatal(err) + } + for _, d := range []ocispec.Descriptor{child, descriptor("b"), descriptor("c"), descriptor("d"), descriptor("e"), index, nested} { + if !refs[d.Digest.Encoded()] { + t.Errorf("missing reference %s", d.Digest) + } + } + if len(store.visited) != 6 { + t.Fatalf("did not query both services in every namespace: %v", store.visited) + } +} + +func TestReferencesFailClosed(t *testing.T) { + for _, failure := range []string{"namespaces", "images:two", "content:two", "index-missing", "index-malformed", "index-corrupt", "child-index-missing", "unknown-type", "invalid-digest", "invalid-label"} { + t.Run(failure, func(t *testing.T) { + index, raw := indexDescriptor(t, descriptor("a")) + store := &fakeMetadata{ + nss: []string{"one", "two"}, + targets: map[string][]ocispec.Descriptor{"one": {descriptor("b")}, "two": {index}}, + blobs: map[string]map[digest.Digest][]byte{"two": {index.Digest: raw}}, + } + switch failure { + case "index-missing": + store.blobs = nil + case "index-malformed": + raw = []byte(`{"schemaVersion":2}`) + index.Digest = digest.FromBytes(raw) + store.targets["two"] = []ocispec.Descriptor{index} + store.blobs["two"][index.Digest] = raw + case "index-corrupt": + store.blobs["two"][index.Digest] = []byte(`{}`) + case "child-index-missing": + child := descriptor("a") + child.MediaType = ocispec.MediaTypeImageIndex + index, raw = indexDescriptor(t, child) + store.targets["two"] = []ocispec.Descriptor{index} + store.blobs["two"][index.Digest] = raw + case "unknown-type": + store.targets["two"][0].MediaType = "application/unknown" + case "invalid-digest": + store.targets["two"][0].Digest = "sha256:../bad" + case "invalid-label": + store.infos = map[string][]content.Info{"two": {{Digest: descriptor("c").Digest, Labels: map[string]string{"containerd.io/gc.ref.content.child": "broken"}}}} + default: + store.fail = failure + } + if refs, err := references(context.Background(), store); err == nil || refs != nil { + t.Fatalf("returned partial references on error: %v, %v", refs, err) + } + }) + } +} + +func TestLateNamespaceFailurePreventsAllDeletion(t *testing.T) { + root := t.TempDir() + path := createStage(t, root, strings.Repeat("a", 64), 2*DefaultMinAge) + store := &fakeMetadata{nss: []string{"one", "two"}, fail: "content:two"} + deps := fakeDependencies(t, root) + deps.references = func(ctx context.Context, _ string) (map[string]bool, error) { return references(ctx, store) } + if _, err := reap(context.Background(), "", Options{Root: root, MinAge: DefaultMinAge}, deps); !errors.Is(err, os.ErrPermission) { + t.Fatalf("expected namespace failure, got %v", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatal(fmt.Errorf("deleted before completing namespace scan: %w", err)) + } +} diff --git a/core/internal/runnablestage/stage.go b/core/internal/runnablestage/stage.go new file mode 100644 index 00000000..f335232c --- /dev/null +++ b/core/internal/runnablestage/stage.go @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package runnablestage accounts for and conservatively reclaims immutable +// runnable-image stages. The root must be administrator-owned and must not be +// renamed or replaced while launchers or the reaper are running. +package runnablestage + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" +) + +const DefaultMinAge = 24 * time.Hour + +type Options struct { + Root string + MinAge time.Duration + DryRun bool +} + +type Result struct { + Removed int + ReclaimedBytes int64 + RemainingBytes int64 +} + +// ErrBusy means a launcher or another reaper holds the staging root lock. +var ErrBusy = errors.New("runnable staging root is busy") + +func Root() string { + if root := os.Getenv("BREWLET_RUNNABLE_STAGE"); root != "" { + return root + } + return filepath.Join(os.TempDir(), "brewlet-runnable") +} + +// Bytes sums logical regular-file sizes, including legacy and pending stages, +// without following symlinks. An absent root has size zero. +func Bytes(root string) (int64, error) { + if root == "" { + root = Root() + } + r, err := openRoot(root) + if os.IsNotExist(err) { + return 0, nil + } + if err != nil { + return 0, err + } + defer r.Close() + return treeBytes(r, ".") +} + +func openRoot(path string) (*os.Root, error) { + before, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !before.IsDir() { + return nil, fmt.Errorf("staging root %q is not a real directory", path) + } + r, err := os.OpenRoot(path) + if err != nil { + return nil, err + } + after, err := r.Stat(".") + if err != nil || !os.SameFile(before, after) { + r.Close() + return nil, fmt.Errorf("staging root %q changed while opening", path) + } + return r, nil +} + +func treeBytes(root *os.Root, name string) (int64, error) { + var size int64 + err := fs.WalkDir(root.FS(), name, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + info, err := root.Lstat(path) + if err != nil { + return err + } + if info.Mode().IsRegular() { + size += info.Size() + } + return nil + }) + return size, err +} + +func canonical(name string) bool { + if len(name) != 64 { + return false + } + for _, c := range name { + if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') { + return false + } + } + return true +} diff --git a/core/internal/runnablestage/stage_test.go b/core/internal/runnablestage/stage_test.go new file mode 100644 index 00000000..db833fe2 --- /dev/null +++ b/core/internal/runnablestage/stage_test.go @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package runnablestage + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRootAndBytes(t *testing.T) { + root := t.TempDir() + t.Setenv("BREWLET_RUNNABLE_STAGE", root) + if Root() != root { + t.Fatal("environment override not used") + } + t.Setenv("BREWLET_RUNNABLE_STAGE", "") + if Root() != filepath.Join(os.TempDir(), "brewlet-runnable") { + t.Fatal("incorrect default root") + } + for name, data := range map[string]string{ + "legacy/app.jar": "123", + "immutable-v2/.pending/app.jar": "12345", + "immutable-v2/" + strings.Repeat("a", 64) + "/app.jar": "1234567", + } { + writeFile(t, filepath.Join(root, name), data) + } + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "large.jar"), strings.Repeat("x", 1024)) + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(outside, "large.jar"), filepath.Join(root, "linked.jar")); err != nil { + t.Fatal(err) + } + got, err := Bytes(root) + if err != nil || got != 15 { + t.Fatalf("Bytes = %d, %v; want 15", got, err) + } + if got, err := Bytes(filepath.Join(root, "missing")); got != 0 || err != nil { + t.Fatalf("missing Bytes = %d, %v", got, err) + } + if _, err := Bytes(filepath.Join(root, "linked")); err == nil { + t.Fatal("accepted symlink root") + } + if _, err := Bytes(filepath.Join(outside, "large.jar")); err == nil { + t.Fatal("accepted regular-file root") + } +} + +func writeFile(t *testing.T, path, data string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(data), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestCanonical(t *testing.T) { + for _, name := range []string{"", ".pending", strings.Repeat("A", 64), strings.Repeat("g", 64), strings.Repeat("a", 63), strings.Repeat("a", 65)} { + if canonical(name) { + t.Errorf("accepted %q", name) + } + } + if !canonical(strings.Repeat("0123456789abcdef", 4)) { + t.Fatal("rejected lowercase digest") + } +} diff --git a/core/internal/runnablestage/unsupported.go b/core/internal/runnablestage/unsupported.go new file mode 100644 index 00000000..16fd9e2e --- /dev/null +++ b/core/internal/runnablestage/unsupported.go @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build !linux + +package runnablestage + +import ( + "context" + "fmt" + "io" +) + +type noopLock struct{} + +func (noopLock) Close() error { return nil } + +// Acquire is a no-op outside Linux, where Reap is unsupported. +func Acquire(root string) (io.Closer, error) { return noopLock{}, nil } + +func Reap(context.Context, string, Options) (Result, error) { + return Result{}, fmt.Errorf("runnable stage reaping is supported only on Linux") +} diff --git a/core/internal/runtime/bundle.go b/core/internal/runtime/bundle.go index b765c74b..2187dd59 100644 --- a/core/internal/runtime/bundle.go +++ b/core/internal/runtime/bundle.go @@ -396,6 +396,12 @@ func GenerateBundleWithIdentityAndRegen(cfg artifact.JVMConfig, jdkRoot, launche return os.WriteFile(filepath.Join(outDir, "config.json"), b, 0o644) } +// Validate checks resource limits before a caller creates bundle-owned files. +func (res Resources) Validate() error { + _, err := buildResources(res) + return err +} + func buildResources(res Resources) (ociResources, error) { out := ociResources{} if res.MemoryLimit != "" { diff --git a/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go b/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go index 5c34e93f..3c60e3e8 100644 --- a/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go +++ b/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go @@ -36,6 +36,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/microsoft/brewlet/internal/artifact" + "github.com/microsoft/brewlet/internal/runnablestage" kcruntime "github.com/microsoft/brewlet/internal/runtime" "github.com/microsoft/brewlet/internal/telemetry" ) @@ -175,6 +176,12 @@ func (s *brewletTaskService) Create(ctx context.Context, r *taskAPI.CreateTaskRe return nil, err } if !sandbox { + // Hold the stage guard until runc has installed the workload's mounts. + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return nil, err + } + defer guard.Close() info, err := assembleBrewletBundle(ctx, r, s.imageIdentity) if err != nil { emitLaunch(info, err) diff --git a/docs/runnable-image.md b/docs/runnable-image.md index 8f874081..d3642008 100644 --- a/docs/runnable-image.md +++ b/docs/runnable-image.md @@ -118,7 +118,7 @@ New shims publish stages under an `immutable-v2` subdirectory of `BREWLET_RUNNABLE_STAGE` (or the default temporary staging root). They leave legacy stages untouched because running workloads may still mount those files. Allow extra disk capacity during rollout: legacy stages and staging directories left -by abruptly terminated processes are not automatically garbage-collected. +by abruptly terminated processes are not removed by the stage reaper. Do not remove staging trees while workloads still reference them. The stage retains descriptor-verified packed layers as well as extracted @@ -131,6 +131,57 @@ layers are already missing, or an upgrade with only a legacy stage, requires a verified image re-pull; this does not repair missing content from an unpacked snapshot. Allow disk capacity for the retained compressed bytes per image. +### Reclaiming unused stages + +Run the Linux node command as root, in the host PID and mount namespaces, with +access to the host's `/proc` and containerd socket: + +```sh +sudo brewlet stage-gc --dry-run +sudo brewlet stage-gc --min-age 24h +# Non-default locations (use the same stage root as the shim): +sudo brewlet stage-gc --stage-root /var/lib/brewlet/runnable \ + --address /run/containerd/containerd.sock --min-age 48h +``` + +`--stage-root` defaults to `BREWLET_RUNNABLE_STAGE`, then +`os.TempDir()/brewlet-runnable`. `--min-age` must be positive and defaults to +24 hours. Keep the stage root administrator-owned and do not rename or replace +it while launchers or cleanup are running. Only published +`immutable-v2/` directories older than +that floor are eligible. The command preserves stages whose manifest remains +in containerd image or content metadata in **any namespace**, including platform +manifests referenced by image indexes. Removing an image alone may not free its +stage until containerd also garbage-collects the manifest content. + +The reaper checks live references across process mount namespaces, including +read-only bind mounts of individual staged files. It fails closed if it cannot +establish that reference information is complete. A shared staging guard covers +resolution and workload mount creation; cleanup takes the exclusive guard and +fails if a launch is in progress. Retry on the next scheduled run rather than +removing trees manually. Running workloads do not need to stop. + +Use an operator-managed timer or root cron job to bound accumulation, for +example an hourly invocation of `/usr/local/bin/brewlet stage-gc --min-age 24h`. +Cleanup is not automatically enabled by installing the shim. Before first +enabling cleanup during an upgrade, finish or quiesce launches from older shims +that do not participate in the guard. Legacy trees and abandoned pending trees +remain outside automated eviction. Previously exported OCI bundles may also +name stage paths: retire or regenerate those bundles before enabling cleanup. +New `brewlet bundle` outputs retain their own runnable payloads. +External consumers that read stages without mounts must participate in the +staging guard or be stopped during cleanup; open file descriptors alone are +not tracked. + +The metrics exporter exposes `brewlet_runnable_stage_bytes`, the logical size +of regular files remaining under its stage root (including legacy and pending +trees, without following symlinks). This is not filesystem-allocated space or +free disk space. It is refreshed on each scrape; inspection errors fail the +scrape rather than reporting a misleading zero. The operator's exporter mounts +the default host stage root read-only. For a custom staging location, configure +the exporter's `--stage-root` and its read-only host mount to point to that same +location. Alert on sustained growth and monitor node filesystem free space too. + ## 5. Operator & webhook - The `JavaApplication` controller sets the Deployment's container `image:` to diff --git a/kubernetes/internal/controller/nodeprofile_resources.go b/kubernetes/internal/controller/nodeprofile_resources.go index 6b85e55f..f6351afd 100644 --- a/kubernetes/internal/controller/nodeprofile_resources.go +++ b/kubernetes/internal/controller/nodeprofile_resources.go @@ -346,6 +346,7 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv Command: []string{"/opt/brewlet-dist/brewlet-metrics-exporter"}, Args: []string{ "--root=/opt/brewlet", + "--stage-root=/tmp/brewlet-runnable", fmt.Sprintf("--listen-address=:%d", metricsPort), }, Ports: []corev1.ContainerPort{{ @@ -368,6 +369,7 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv // nested bind mount lands on an existing path. {Name: "host-opt", MountPath: "/opt/brewlet", ReadOnly: true}, {Name: metricsSocketVolume, MountPath: "/opt/brewlet/metrics"}, + {Name: "runnable-stage", MountPath: "/tmp/brewlet-runnable", ReadOnly: true}, }, }, }, @@ -381,6 +383,7 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv // writable, and only when the sidecar is scheduled; // dropMetricsExporter removes it otherwise. hostPathVolume(metricsSocketVolume, "/opt/brewlet/metrics", &hostPathDirOrCreate), + hostPathVolume("runnable-stage", "/tmp/brewlet-runnable", &hostPathDirOrCreate), hostPathVolume("containerd-conf", "/etc/containerd", &hostPathDir), hostPathVolume("host-bin", "/usr/local/bin", &hostPathDir), hostPathVolume("containerd-sock", "/run/containerd/containerd.sock", &hostPathSocket), @@ -454,7 +457,7 @@ func dropMetricsExporter(ds *appsv1.DaemonSet) { spec.Containers = spec.Containers[:1] kept := spec.Volumes[:0] for _, vol := range spec.Volumes { - if vol.Name == metricsSocketVolume { + if vol.Name == metricsSocketVolume || vol.Name == "runnable-stage" { continue } kept = append(kept, vol) diff --git a/kubernetes/internal/controller/nodeprofile_scheduling_test.go b/kubernetes/internal/controller/nodeprofile_scheduling_test.go index 7009a383..0fa5a0b0 100644 --- a/kubernetes/internal/controller/nodeprofile_scheduling_test.go +++ b/kubernetes/internal/controller/nodeprofile_scheduling_test.go @@ -166,8 +166,8 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { // The metrics exporter only reads what the provisioner wrote, except for // the one directory holding the telemetry socket it must bind. exporter := spec.Containers[1] - if len(exporter.VolumeMounts) != 2 { - t.Fatalf("metrics-exporter mounts = %+v, want the read-only tree plus the socket directory", exporter.VolumeMounts) + if len(exporter.VolumeMounts) != 3 { + t.Fatalf("metrics-exporter mounts = %+v, want read-only inventory and stages plus the socket directory", exporter.VolumeMounts) } // The read-only parent must be listed first so the nested socket mount // lands on a path that already exists inside it. @@ -177,6 +177,9 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { if exporter.VolumeMounts[1].MountPath != "/opt/brewlet/metrics" || exporter.VolumeMounts[1].ReadOnly { t.Errorf("second exporter mount = %+v, want a writable /opt/brewlet/metrics", exporter.VolumeMounts[1]) } + if exporter.VolumeMounts[2].MountPath != "/tmp/brewlet-runnable" || !exporter.VolumeMounts[2].ReadOnly { + t.Errorf("third exporter mount = %+v, want read-only runnable staging", exporter.VolumeMounts[2]) + } if exporter.SecurityContext == nil || exporter.SecurityContext.ReadOnlyRootFilesystem == nil || !*exporter.SecurityContext.ReadOnlyRootFilesystem { @@ -193,6 +196,7 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { wantTypes := map[string]corev1.HostPathType{ "/opt/brewlet": corev1.HostPathDirectoryOrCreate, "/opt/brewlet/metrics": corev1.HostPathDirectoryOrCreate, + "/tmp/brewlet-runnable": corev1.HostPathDirectoryOrCreate, "/etc/containerd": corev1.HostPathDirectory, "/usr/local/bin": corev1.HostPathDirectory, "/run/containerd/containerd.sock": corev1.HostPathSocket, @@ -217,6 +221,13 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { t.Error("provisioner must not carry the exporter's socket mount") } } + cfg.MetricsEnabled = false + withoutMetrics := buildProfileDaemonSet(cfg, &p, "agentpool", nil).Spec.Template.Spec + for _, volume := range withoutMetrics.Volumes { + if volume.Name == "runnable-stage" { + t.Error("disabled metrics must not mount the staging tree") + } + } } func TestMetricsSocketVolumeOnlyWithExporter(t *testing.T) { From 523d6458e1e8a864d3e803888a9be5c531f67d01 Mon Sep 17 00:00:00 2001 From: Bruno Borges Date: Tue, 29 Sep 2026 08:05:37 -0400 Subject: [PATCH 3/3] Enable runnable-stage GC by default through the provisioner Run `brewlet stage-gc` periodically from the operator-managed provisioner DaemonSet, configured through new Helm stageGC values (enabled, 5m interval, 24h minimum age). Existing nodes stay blocked until stageGC.upgradeAcknowledged is set, and each compatible node persists a compatibility record. - Ship the brewlet CLI in the provisioner image as brewlet-stage-gc - Supervise each sweep in its own process group; cancel on TERM/INT - Skip sweeps on inconclusive API reads instead of crashing the worker - Use a fixed /tmp/brewlet-runnable default stage root on Linux - Add tier 17 e2e coverage and update docs and specification Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/e2e.yml | 5 + core/cmd/brewlet-metrics-exporter/README.md | 22 +- core/cmd/brewlet/stage_gc.go | 6 +- core/cmd/brewlet/stage_gc_linux_test.go | 87 +++ core/internal/runnablestage/stage.go | 8 + core/internal/runnablestage/stage_test.go | 10 +- docs/cli-reference.md | 51 ++ docs/configuration.md | 40 +- docs/installation.md | 67 ++ docs/runnable-image.md | 99 ++- docs/runtime-metrics.md | 19 +- docs/third-party-notices.md | 2 +- integration-tests/AGENTS.md | 23 +- integration-tests/README.md | 6 +- integration-tests/e2e/reset.sh | 2 +- integration-tests/e2e/run.sh | 8 +- integration-tests/e2e/tier17-stage-gc.sh | 580 ++++++++++++++++++ kubernetes/charts/brewlet/README.md | 27 + .../charts/brewlet/templates/operator.yaml | 4 + kubernetes/charts/brewlet/values.yaml | 11 + kubernetes/cmd/manager/main.go | 9 + kubernetes/internal/chart/stage_gc_test.go | 49 ++ .../controller/nodeprofile_resources.go | 5 + kubernetes/internal/controller/resources.go | 24 +- .../internal/controller/stage_gc_test.go | 70 +++ provisioner/Dockerfile | 7 +- provisioner/README.md | 31 +- provisioner/entrypoint.sh | 193 +++++- provisioner/entrypoint_test.sh | 19 +- provisioner/stage_gc_test.sh | 315 ++++++++++ specs/SPECIFICATION.md | 121 +++- 31 files changed, 1866 insertions(+), 54 deletions(-) create mode 100644 core/cmd/brewlet/stage_gc_linux_test.go create mode 100644 integration-tests/e2e/tier17-stage-gc.sh create mode 100644 kubernetes/internal/chart/stage_gc_test.go create mode 100644 kubernetes/internal/controller/stage_gc_test.go create mode 100644 provisioner/stage_gc_test.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1e87c536..5ea85554 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -102,6 +102,11 @@ jobs: java: true cluster: true helm: false + - name: "Tier 17: Runnable-stage GC" + tiers: --tier 17 + java: true + cluster: true + helm: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/core/cmd/brewlet-metrics-exporter/README.md b/core/cmd/brewlet-metrics-exporter/README.md index 15cf09f0..e1e59ede 100644 --- a/core/cmd/brewlet-metrics-exporter/README.md +++ b/core/cmd/brewlet-metrics-exporter/README.md @@ -6,15 +6,29 @@ Prometheus metrics for sandbox launches, artifact resolution, AppCDS decisions, installed JDK and launcher inventory, and runnable-image staging usage. `brewlet_runnable_stage_bytes` reports logical regular-file bytes under -`--stage-root` (default: `BREWLET_RUNNABLE_STAGE`, or -`os.TempDir()/brewlet-runnable`). The directory must be the host's actual staging +`--stage-root` (default: `BREWLET_RUNNABLE_STAGE`, or `/tmp/brewlet-runnable` +on Linux regardless of `TMPDIR`). The directory must be the host's actual staging root; a containerized exporter needs a read-only host mount. The operator provides that mount for the default location. The gauge includes legacy and pending stages, does not follow symlinks, and refreshes on every scrape. Inspection errors fail the scrape instead of reporting zero. -Use the host command `brewlet stage-gc --dry-run` to inspect reclaimable stages, -then schedule `brewlet stage-gc --min-age 24h` with a node-level timer. +On Helm-managed nodes, the provisioner already schedules stage GC by default, +independently of whether this exporter is enabled. The exporter never deletes +stages. Do not add a second timer: it would run outside the operator's +`stageGC.enabled` setting and migration gate. + +To inspect eligibility on a managed Linux node, run the installed helper as +root in the host namespaces: + +```sh +sudo /usr/local/bin/brewlet-stage-gc stage-gc --dry-run +``` + +Manual invocations retain the reaper's reference and mount checks but do not +consult the provisioner's compatibility record or upgrade acknowledgment. +Outside the provisioner, install the standalone `brewlet` CLI and arrange a +host timer only after completing the same migration prerequisites. See [stage cleanup](../../../docs/runnable-image.md#reclaiming-unused-stages) for reference checks and rollout precautions. diff --git a/core/cmd/brewlet/stage_gc.go b/core/cmd/brewlet/stage_gc.go index d2358f50..1ac27a11 100644 --- a/core/cmd/brewlet/stage_gc.go +++ b/core/cmd/brewlet/stage_gc.go @@ -8,7 +8,9 @@ import ( "flag" "fmt" "os" + "os/signal" "path/filepath" + "syscall" "time" "github.com/microsoft/brewlet/internal/artifact" @@ -30,7 +32,9 @@ func cmdStageGC(args []string) error { if fs.NArg() != 0 || *age <= 0 { return fmt.Errorf("usage: stage-gc [--stage-root DIR] [--address SOCKET] [--min-age positive-duration] [--dry-run]") } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() result, err := runnablestage.Reap(ctx, *address, runnablestage.Options{ Root: *root, MinAge: *age, DryRun: *dryRun, diff --git a/core/cmd/brewlet/stage_gc_linux_test.go b/core/cmd/brewlet/stage_gc_linux_test.go new file mode 100644 index 00000000..e257c783 --- /dev/null +++ b/core/cmd/brewlet/stage_gc_linux_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package main + +import ( + "bytes" + "context" + "net" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestStageGCCancelsOnSIGTERM(t *testing.T) { + if os.Getenv("BREWLET_TEST_GC_CHILD") == "true" { + err := cmdStageGC([]string{ + "--stage-root", os.Getenv("BREWLET_TEST_GC_ROOT"), + "--address", os.Getenv("BREWLET_TEST_GC_SOCKET"), + }) + if status.Code(err) != codes.Canceled { + t.Fatalf("expected canceled metadata inspection, got %v", err) + } + return + } + + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "immutable-v2"), 0o755); err != nil { + t.Fatal(err) + } + socket := filepath.Join(root, "containerd.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + connected := make(chan error, 1) + go func() { + conn, err := listener.Accept() + connected <- err + if err == nil { + defer conn.Close() + // Leave the gRPC handshake pending until the child is signaled. + <-ctx.Done() + } + }() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestStageGCCancelsOnSIGTERM$") + cmd.Env = append(os.Environ(), "BREWLET_TEST_GC_CHILD=true", + "BREWLET_TEST_GC_ROOT="+root, "BREWLET_TEST_GC_SOCKET="+socket) + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + var waitErr error + go func() { + waitErr = cmd.Wait() + close(done) + }() + defer func() { cancel(); <-done }() + select { + case err := <-connected: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal("stage GC did not connect to containerd") + } + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + <-done + if waitErr != nil { + t.Fatalf("stage GC did not cancel cleanly: %v\n%s", waitErr, output.String()) + } +} diff --git a/core/internal/runnablestage/stage.go b/core/internal/runnablestage/stage.go index f335232c..16a021f8 100644 --- a/core/internal/runnablestage/stage.go +++ b/core/internal/runnablestage/stage.go @@ -12,6 +12,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "time" ) @@ -32,10 +33,17 @@ type Result struct { // ErrBusy means a launcher or another reaper holds the staging root lock. var ErrBusy = errors.New("runnable staging root is busy") +// LinuxDefaultRoot is fixed rather than derived from TMPDIR so the shim, which +// inherits containerd's environment, and the provisioner's reaper agree. +const LinuxDefaultRoot = "/tmp/brewlet-runnable" + func Root() string { if root := os.Getenv("BREWLET_RUNNABLE_STAGE"); root != "" { return root } + if runtime.GOOS == "linux" { + return LinuxDefaultRoot + } return filepath.Join(os.TempDir(), "brewlet-runnable") } diff --git a/core/internal/runnablestage/stage_test.go b/core/internal/runnablestage/stage_test.go index db833fe2..42e006b9 100644 --- a/core/internal/runnablestage/stage_test.go +++ b/core/internal/runnablestage/stage_test.go @@ -6,6 +6,7 @@ package runnablestage import ( "os" "path/filepath" + "runtime" "strings" "testing" ) @@ -17,8 +18,13 @@ func TestRootAndBytes(t *testing.T) { t.Fatal("environment override not used") } t.Setenv("BREWLET_RUNNABLE_STAGE", "") - if Root() != filepath.Join(os.TempDir(), "brewlet-runnable") { - t.Fatal("incorrect default root") + t.Setenv("TMPDIR", t.TempDir()) + want := filepath.Join(os.TempDir(), "brewlet-runnable") + if runtime.GOOS == "linux" { + want = "/tmp/brewlet-runnable" + } + if Root() != want { + t.Fatalf("default root = %q, want %q", Root(), want) } for name, data := range map[string]string{ "legacy/app.jar": "123", diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 9b13c984..784cf4ee 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -23,6 +23,7 @@ brewlet keygen [flags] generate an ECDSA P-256 key pair brewlet inspect [flags] show the artifact manifest + config brewlet run [flags] pull + launch java -jar on this node brewlet bundle [flags] emit an OCI runc bundle (the shim path) +brewlet stage-gc [flags] reclaim orphaned runnable-image stages (Linux) brewlet jdks [flags] list JDKs available across the cluster brewlet doctor [flags] diagnose cluster and developer readiness brewlet k8s [flags] manage and inspect Brewlet on Kubernetes @@ -269,6 +270,56 @@ cat ./bundle/config.json --- +## `brewlet stage-gc` + +Perform one reference-aware cleanup sweep on a Linux node. Run as root in the +host PID, mount, and initial user namespaces, with complete host `/proc` +visibility and access to containerd. + +```text +brewlet stage-gc [--stage-root DIR] [--address SOCKET] [--min-age DURATION] [--dry-run] +``` + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--stage-root` | `BREWLET_RUNNABLE_STAGE`, otherwise `/tmp/brewlet-runnable` on Linux (ignores `TMPDIR`) or `os.TempDir()/brewlet-runnable` elsewhere | Administrator-owned staging root; must match the shim. | +| `--address` | `/run/containerd/containerd.sock` | containerd endpoint; references are checked across all namespaces. | +| `--min-age` | `24h` | Positive Go duration. Only published stages with directory modification times older than this floor are eligible; this is not time since last use. | +| `--dry-run` | `false` | Apply the same eligibility checks and report candidates without deleting them. | + +On a node installed by the provisioner, the CLI is packaged under a helper name: + +```bash +sudo /usr/local/bin/brewlet-stage-gc stage-gc --dry-run +``` + +With the standalone `brewlet` CLI installed on the host: + +```bash +sudo brewlet stage-gc --dry-run +# Only after completing the migration prerequisites: +sudo brewlet stage-gc --min-age 24h +``` + +Only canonical `immutable-v2/` directories are considered. Image +and content references, live mounts, or an active staging guard prevent +reclamation. Incomplete reference inspection fails closed; legacy and abandoned +pending directories are not removed. The command reports stage counts, logical +bytes reclaimed (or eligible in dry-run), and remaining logical stage bytes. +In dry-run, remaining bytes describe the current tree, not projected usage. +Errors, including lock contention, exit nonzero. A five-minute context deadline +and TERM/INT cancellation bound cooperative work; filesystem operations already +in progress may take longer to return. + +**Manual invocation does not check `stageGC.enabled`, migration acknowledgment, +node ownership, or the provisioner's compatibility record.** Complete the +[migration prerequisites](runnable-image.md#upgrading-existing-nodes) before +deleting anything. Helm-managed nodes already have a periodic loop; do not add +a second timer. For standalone installations only, arrange a host timer after +those checks. See [Runnable stage cleanup](runnable-image.md#reclaiming-unused-stages). + +--- + ## `brewlet jdks` List the JDKs available across the cluster — **vendor, major version, minor diff --git a/docs/configuration.md b/docs/configuration.md index 60a06b54..8946aa4a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,8 @@ and autoscaling, see [Capability labels and autoscaling](capability-labels-and-autoscaling.md). For the operational workflow behind the metrics values, see [Runtime metrics and Grafana dashboards](runtime-metrics.md). +For runnable-image cache reclamation and migration prerequisites, see +[Runnable stage cleanup](runnable-image.md#reclaiming-unused-stages). --- @@ -75,6 +77,10 @@ with `--set key=value` or a values file. | `operator.replicas` | `1` | Operator replica count. | | `operator.leaderElect` | `true` | Enable leader election for HA. | | `operator.resources` | requests `50m/64Mi`, limits `200m/128Mi` | Operator pod resources. | +| `stageGC.enabled` | `true` | Run periodic orphaned runnable-image stage cleanup in every managed provisioner, independently of metrics. Fresh nodes activate automatically; existing installations require a matching compatibility record or migration acknowledgment. | +| `stageGC.interval` | `5m` | Delay after each attempt, plus up to 10% jitter. Sweeps do not overlap. | +| `stageGC.minAge` | `24h` | Minimum stage directory age, not time since last use or since becoming unreferenced. | +| `stageGC.upgradeAcknowledged` | `false` | Confirm older unguarded shims and stage-dependent exported bundles have been retired across all managed nodes. Reset after the rollout; compatible nodes retain their approval. | | `uninstall.timeoutSeconds` | `240` | Pre-delete cleanup coordinator timeout, 1-86400 whole seconds. Configure before uninstalling; Helm's `--timeout` must exceed this plus 30 seconds. Failure retains the control plane. See [Uninstall](installation.md#uninstall). | | `uninstall.imagePullSecrets` | `[]` | Namespaced registry Secret references for the cleanup Job, which uses the operator image and its own service account. | | `metrics.enabled` | `false` | Opt in to Brewlet runtime and control-plane Prometheus endpoints. Enables the operator and admission listeners plus the node exporter sidecar and Services. | @@ -125,6 +131,27 @@ component-image overrides match it. For an existing installation, follow > tagless SHA-256 digest references. For air-gapped clusters, mirror the exact > digest and approve the destination host explicitly. +### Runnable-image stage cleanup + +`stageGC.*` is operator-wide policy, not a `NodeProfile.spec` field. It applies +to chart-rendered and externally managed profiles alike. Configure it through +Helm values or operator flags, not by patching a reconciled DaemonSet. +`interval` and `minAge` accept Go durations that resolve to positive whole +seconds, at most `2147483647s`; the operator rejects invalid values. + +Kubelet and containerd still own image retention and content reclamation. +Brewlet removes only old, unreferenced, unmounted published stages; it does not +copy kubelet's disk thresholds or change kubelet configuration. Containerd +references in any namespace can delay reclamation. Legacy and abandoned pending +trees are not automatically removed. + +For existing nodes, follow the [GC activation procedure](installation.md#activating-runnable-stage-gc) +before setting `stageGC.upgradeAcknowledged=true`. Readiness does not imply +that GC is active. Setting `stageGC.enabled=false` stops automatic sweeps, not +separately scheduled or manually invoked cleanup commands. + +### AppCDS regeneration + AppCDS regeneration is default-deny. Enable it for the default profile with `--set provisioner.appCDS.regenerationEnabled=true`, or on a named profile with: @@ -220,6 +247,10 @@ When you install via Helm, the chart populates them for you. | `--health-probe-bind-address` | `:8081` | Health/readiness endpoint. | | `--node-metrics-enabled` | `false` | Run the node-local exporter sidecar in managed provisioner pods. | | `--node-metrics-port` | `9090` | Exporter port when node metrics are enabled. | +| `--stage-gc-enabled` | `true` | Enable periodic stage cleanup after successful provisioning and compatibility checks. | +| `--stage-gc-interval` | `5m` | Delay between GC attempts, plus jitter; positive whole seconds in Go duration syntax. | +| `--stage-gc-min-age` | `24h` | Minimum stage directory age; positive whole seconds in Go duration syntax. | +| `--stage-gc-upgrade-acknowledged` | `false` | Acknowledge retirement of unguarded stage consumers across the managed fleet. Reset after rollout. | For a local operator built from the same release as the installed CLI, use that CLI's version to select the matching provisioner image. For custom source builds, @@ -261,6 +292,11 @@ only touch them directly if you hand-wire the DaemonSet. | `LAUNCHER_SOURCE__PATH` | required per entry | Clean absolute launcher path inside the image. | | `JDKS` / `LAUNCHERS` | derived | Internal comma-separated inventories derived from the indexed entries. They are not accepted as independent inputs. | | `BREWLET_APP_CDS_REGENERATION_ENABLED` | `false` | Internal operator-to-provisioner policy transport. When true, atomically creates the root-owned AppCDS authorization sentinel and publishes `brewlet.sh/appcds-regeneration=true`; when false or during cleanup/failure, removes both. Configure `spec.appCDS.regenerationEnabled`, not this variable directly. | +| `BREWLET_STAGE_GC_ENABLED` | `true` | Enable the periodic GC loop after provisioning, subject to compatibility checks. Rendered from operator-wide `stageGC.enabled`. | +| `BREWLET_STAGE_GC_INTERVAL_SECONDS` | `300` | Delay between GC attempts, plus up to 10% jitter; integer seconds in `1..2147483647`. | +| `BREWLET_STAGE_GC_MIN_AGE_SECONDS` | `86400` | Minimum stage directory age; integer seconds in `1..2147483647`. | +| `BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED` | `false` | Confirm older unguarded shims and stage-dependent exported bundles have been retired; rendered from operator-wide policy. | +| `BREWLET_RUNNABLE_STAGE` | `/tmp/brewlet-runnable` (provisioner) | Host stage root used by automatic GC. A custom location must also be configured separately for the shim and exporter's read-only mount; this variable does not reconfigure those processes. | | `NODE_NAME` | (downward API) | The node to label; injected from `spec.nodeName`. | | `BREWLET_PROFILE_NAME` | `default` | Profile name paired with its UID in managed worker authority checks. | | `BREWLET_PROFILE_UID` | *(empty)* | Operator-managed profile UID paired with node ownership and persisted writer authority. | @@ -270,8 +306,8 @@ only touch them directly if you hand-wire the DaemonSet. | `CONTAINERD_CONFIG` | `/etc/containerd/config.toml` | Primary containerd configuration. Validated mode uses an imported drop-in when supported and otherwise patches this file with a backup. | | `CONTAINERD_DROPIN_DIR` | `/etc/containerd/config.toml.d` | Drop-in directory used when the primary config imports `*.toml` from it. | | `CONTAINERD_DROPIN_FILE` | `/etc/containerd/config.toml.d/99-brewlet.toml` | Brewlet-managed runtime drop-in. | -| `CONTAINERD_ADDRESS` | `/run/containerd/containerd.sock` | Host containerd socket (used for copy-from-image). | -| `CONTAINERD_NAMESPACE` | `k8s.io` | containerd namespace for image pulls. | +| `CONTAINERD_ADDRESS` | `/run/containerd/containerd.sock` | Host containerd socket for copy-from-image and stage GC reference inspection. | +| `CONTAINERD_NAMESPACE` | `k8s.io` | containerd namespace for image pulls; does not restrict GC, which checks all namespaces. | | `BREWLET_MODE` | `provision` | `provision` installs the runtime; `cleanup` reverses recorded host state for a retiring target or deleted profile. Managed cleanup uses the frozen per-node containerd policy and runs in `brewlet-cleanup-` (§5.6). | | `BREWLET_CONTAINERD_RESTART` | `validated` | `validated` smoke-tests the runtime inventory, validates the effective config with `containerd config dump`, restarts the host service only when needed, checks containerd and the live `brewlet` handler, and restores known-good config on activation failure. `sighup` preserves the legacy in-place render/reload path without the config-dump gate. `none` neither mutates nor signals containerd. Rendered from `spec.rollout.containerdRestart`. | | `BREWLET_VALIDATE` | `true` | Run `java -version` for every JDK and verify every staged launcher is executable before publishing runtime or capability labels. Arbitrary launchers are not executed. `false` skips both sets of checks. Rendered from `spec.rollout.validate`. | diff --git a/docs/installation.md b/docs/installation.md index 95e54ac3..e84c1b16 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -359,6 +359,11 @@ For source builds, use the local chart instead and retain your digest-pinned component overrides. For incompatible legacy profiles, use the maintenance sequence below instead of an ordinary upgrade. +An upgrade that introduces automatic runnable-stage GC can leave existing nodes +Ready while cleanup remains blocked pending migration acknowledgment. +`helm upgrade --wait` does not prove GC is active. After the runtime rollout, +follow [Activating runnable-stage GC](#activating-runnable-stage-gc). + An older CRD prunes unsupported fields when a resource is saved. Updating the CRD cannot restore those values; reapply the original JavaApplication manifests afterward. @@ -419,6 +424,68 @@ helm upgrade brewlet oci://ghcr.io/microsoft/charts/brewlet \ --wait ``` +#### Activating runnable-stage GC + +Fresh nodes enable periodic cleanup automatically. On an existing installation, +the default `stageGC.upgradeAcknowledged=false` keeps cleanup blocked unless +the node has a matching compatibility record. A blocked node continues serving +workloads and logs `stage GC blocked`. + +First roll out the matching operator and provisioner, then retire older +unguarded shims and finish or quiesce their launches. Merely replacing the shim +binary does not retire already running shim processes; use your workload +restart/drain procedure where needed. Retire or regenerate exported OCI bundles +that point into the stage cache, and stop or upgrade other unguarded stage +consumers. Complete these prerequisites across **all operator-managed nodes**, +including externally managed NodeProfiles, before acknowledging migration: + +```bash +helm upgrade brewlet oci://ghcr.io/microsoft/charts/brewlet \ + --version "$RELEASE_VERSION" \ + --namespace brewlet \ + --values values.yaml \ + --values my-jdks.yaml \ + --set stageGC.upgradeAcknowledged=true \ + --wait +``` + +Keep `stageGC.enabled=true` for activation. For source builds, use the local chart +and retain the matching component overrides as above. Check the provisioner +logs for each managed node, not just pod readiness: + +```bash +kubectl get pods -n brewlet -l app=brewlet-node-provisioner -o wide +kubectl logs -n brewlet -c provisioner --tail=100 +``` + +Look for `stage GC: successful_sweeps=...` with a recent `last_success` timestamp; +zero removed stages is a valid successful sweep. `last_success=never` with +failed attempts means cleanup has not succeeded. Investigate inspection, +namespace, socket-access, and lock errors; do not bypass them by deleting stage +directories. Neither GC failure counters nor compatibility blocking are part of +the pod readiness gate. + +After the rollout, reset the fleet-wide acknowledgment: + +```bash +helm upgrade brewlet oci://ghcr.io/microsoft/charts/brewlet \ + --version "$RELEASE_VERSION" \ + --namespace brewlet \ + --values values.yaml \ + --values my-jdks.yaml \ + --set stageGC.upgradeAcknowledged=false \ + --wait +``` + +Compatible nodes retain their approval, tied to the installed shims and stage +path. Disable automatic GC with `stageGC.enabled=false` before manually rolling +back shims or introducing unguarded consumers. Do not add a separate timer or +invoke destructive manual cleanup to bypass blocked activation: manual commands +do not enforce the provisioner's migration gate. See +[Runnable stage cleanup](runnable-image.md#reclaiming-unused-stages) for the +reclamation rules and [Configuration](configuration.md#runnable-image-stage-cleanup) +for all settings. + ### What the chart deploys vs. what the operator creates | Deployed by the chart | Created/reconciled by the operator at runtime | diff --git a/docs/runnable-image.md b/docs/runnable-image.md index d3642008..ab5760de 100644 --- a/docs/runnable-image.md +++ b/docs/runnable-image.md @@ -133,19 +133,35 @@ snapshot. Allow disk capacity for the retained compressed bytes per image. ### Reclaiming unused stages -Run the Linux node command as root, in the host PID and mount namespaces, with -access to the host's `/proc` and containerd socket: +Run the Linux node command as root, in the host PID, mount, and initial user +namespaces, with complete visibility of the host's `/proc` and access to the +containerd socket. On nodes installed by the provisioner, use its host helper: + +```sh +sudo /usr/local/bin/brewlet-stage-gc stage-gc --dry-run +``` + +If you installed the standalone `brewlet` CLI on the host, use: ```sh sudo brewlet stage-gc --dry-run +# Only after completing the migration prerequisites below: sudo brewlet stage-gc --min-age 24h # Non-default locations (use the same stage root as the shim): sudo brewlet stage-gc --stage-root /var/lib/brewlet/runnable \ --address /run/containerd/containerd.sock --min-age 48h ``` +Manual commands do not enforce `stageGC.enabled`, migration acknowledgment, +node ownership, or the provisioner's compatibility record. The reaper's +reference, mount, and locking checks still apply, but they cannot make +unguarded legacy consumers safe. Do not use a manual deletion or an additional +timer to bypass blocked automatic activation. + `--stage-root` defaults to `BREWLET_RUNNABLE_STAGE`, then -`os.TempDir()/brewlet-runnable`. `--min-age` must be positive and defaults to +`/tmp/brewlet-runnable` on Linux (independent of `TMPDIR`, so it matches the +shim even when containerd sets `TMPDIR`) or `os.TempDir()/brewlet-runnable` +elsewhere. `--min-age` must be positive and defaults to 24 hours. Keep the stage root administrator-owned and do not rename or replace it while launchers or cleanup are running. Only published `immutable-v2/` directories older than @@ -161,18 +177,76 @@ resolution and workload mount creation; cleanup takes the exclusive guard and fails if a launch is in progress. Retry on the next scheduled run rather than removing trees manually. Running workloads do not need to stop. -Use an operator-managed timer or root cron job to bound accumulation, for -example an hourly invocation of `/usr/local/bin/brewlet stage-gc --min-age 24h`. -Cleanup is not automatically enabled by installing the shim. Before first -enabling cleanup during an upgrade, finish or quiesce launches from older shims -that do not participate in the guard. Legacy trees and abandoned pending trees -remain outside automated eviction. Previously exported OCI bundles may also -name stage paths: retire or regenerate those bundles before enabling cleanup. -New `brewlet bundle` outputs retain their own runnable payloads. +The Helm chart enables periodic cleanup by default on fresh nodes through the +existing provisioner, independently of `metrics.enabled`. The global settings +apply to every operator-managed NodeProfile, including externally managed CRs: + +```yaml +stageGC: + enabled: true + interval: 5m + minAge: 24h + upgradeAcknowledged: false +``` + +The provisioner enters the host PID and mount namespaces for each sweep and +rechecks node/profile ownership before it runs. If the Kubernetes API cannot be +read, that sweep is skipped and retried; the worker exits only when a successful +read shows it no longer owns the node. Sweeps start after successful +provisioning, do not overlap, and wait the configured interval plus up to 10% +jitter between attempts. Each attempt has a five-minute context deadline; +filesystem operations already in progress may take longer to return. Lock contention +or inspection failures are logged and retried on the next interval; shutdown +signals cancel the active reaper. Logs include removed stage counts, logical +bytes, process-local success/failure counters, and the last successful sweep. +GC never runs in the provisioner's teardown mode. Disable it with +`--set stageGC.enabled=false`. + +This follows kubelet's image-retention decisions rather than competing with +them: kubelet removes unused image records via CRI, containerd eventually +releases unreferenced content, and Brewlet reclaims the orphaned stages. Brewlet +does not delete containerd images, read or modify kubelet configuration, or apply +another set of disk-pressure thresholds. The `minAge` floor measures stage +directory age, **not** time since last use or since becoming unreferenced, and +is not kubelet's `imageMinimumGCAge` or `imageMaximumGCAge`. This is eventual +orphan reclamation, not guaranteed immediate disk-pressure relief. + +### Upgrading existing nodes + +Existing installations without a matching compatibility record stay provisioned +but log `stage GC blocked` instead of deleting stages. Before acknowledging the +migration, retire older shims that do not participate in the guard, finish or +quiesce their launches, and retire or regenerate previously exported OCI bundles +that name stage paths. Merely installing a new shim does not retire already +running older shims. New `brewlet bundle` outputs retain their own payloads. +Then upgrade with `stageGC.upgradeAcknowledged=true` in addition to your normal +release values. This acknowledges those prerequisites for **all managed nodes**; +do not set it before the whole selected fleet is ready. + +After the rollout, reset `stageGC.upgradeAcknowledged=false`. Each compatible +node retains a root-owned `/opt/brewlet/.stage-gc-compatible` record tied to both +installed shim copies and the staging path. A changed shim identity or path +without a matching record blocks activation again at provisioner startup. An interrupted installation +can also require acknowledgment again. Disable GC before manually rolling back +shims or introducing consumers that do not participate in the guard. +Earlier Linux shims derived their default stage root from `TMPDIR`. If +containerd set a non-default `TMPDIR`, stages created before the upgrade remain +under that old location; new launches use `/tmp/brewlet-runnable`. Automatic GC +does not sweep the old location. Remove it manually only after the migration +checks above confirm nothing still references it. +For complete Helm commands and per-node activation checks, see +[Activating runnable-stage GC](installation.md#activating-runnable-stage-gc). + +Standalone shim installation still does not schedule GC. Outside the provisioner, +use a host timer or root cron job, such as an hourly invocation of +`/usr/local/bin/brewlet stage-gc --min-age 24h`, after the same migration checks. +Legacy trees and abandoned pending trees remain outside automated eviction. External consumers that read stages without mounts must participate in the staging guard or be stopped during cleanup; open file descriptors alone are not tracked. +### Observing stage usage and cleanup + The metrics exporter exposes `brewlet_runnable_stage_bytes`, the logical size of regular files remaining under its stage root (including legacy and pending trees, without following symlinks). This is not filesystem-allocated space or @@ -181,6 +255,9 @@ scrape rather than reporting a misleading zero. The operator's exporter mounts the default host stage root read-only. For a custom staging location, configure the exporter's `--stage-root` and its read-only host mount to point to that same location. Alert on sustained growth and monitor node filesystem free space too. +Sweep success/failure counters and last-success timestamps are currently +provisioner logs, not Prometheus metrics. See the +[metric catalog](runtime-metrics.md#runtime-and-node-metrics). ## 5. Operator & webhook diff --git a/docs/runtime-metrics.md b/docs/runtime-metrics.md index 2d9919d7..2b48edf4 100644 --- a/docs/runtime-metrics.md +++ b/docs/runtime-metrics.md @@ -105,7 +105,8 @@ observability must never block or fail workload launch. The `metrics-exporter` sidecar runs in every profile-managed provisioner DaemonSet. It receives shim events, exposes them as Prometheus counters and histograms, and derives inventory gauges from the node's active JDK and launcher -state under `/opt/brewlet`. +state under `/opt/brewlet`. It also measures runnable-image stage bytes through +a read-only host mount of `/tmp/brewlet-runnable`. The sidecar is deliberately narrow: it runs with a read-only root filesystem, all capabilities dropped, and no privilege escalation, and it mounts @@ -113,6 +114,9 @@ all capabilities dropped, and no privilege escalation, and it mounts `DirectoryOrCreate` mount of `/opt/brewlet/metrics`, which exists solely so the exporter can bind the telemetry socket. That volume is present only when the sidecar itself is — a metrics-disabled or cleanup DaemonSet carries neither. +The read-only stage volume is also omitted without the exporter. Automatic +stage GC runs separately in the provisioner through the host namespaces and +does not depend on exporter mounts or `metrics.enabled`. The operator and admission webhook use controller-runtime's Prometheus registry. Their endpoints include standard controller-runtime/process metrics alongside @@ -205,6 +209,19 @@ If you changed `metrics.nodePort`, replace `9090` with that value. | `brewlet_jdk_info` | Gauge | `distribution`, `feature`, `version`, `vendor`, `arch`, `source` | Value `1` identifies each active JDK build on a node. | | `brewlet_jdk_installed_timestamp_seconds` | Gauge | `distribution`, `feature`, `version` | Unix timestamp when that JDK root was installed on the node. It is **not** the upstream JDK patch release date. | | `brewlet_launcher_info` | Gauge | `launcher` | Value `1` identifies an available launcher. Vanilla `java` is always emitted; additional installed launchers such as `jaz` are also reported. | +| `brewlet_runnable_stage_bytes` | Gauge | None | Logical regular-file bytes under the host staging root, including legacy and pending stages. Refreshed on each scrape without following symlinks. This is not allocated disk space, free space, or reclaimable bytes. | + +Stage inspection errors fail the scrape instead of reporting a misleading zero. +Monitor node filesystem free space alongside stage usage; protected, legacy, +and pending trees can keep the gauge high even when GC is working correctly. +The exporter's `--stage-root` and read-only host mount must match the shim for a +custom stage location. + +Sweep success/failure counters and last-success timestamps are currently +reported in **provisioner logs**, not Prometheus metrics. The exporter only +observes usage; it never deletes stages. See +[Runnable stage cleanup](runnable-image.md#observing-stage-usage-and-cleanup) +and the [activation checks](installation.md#activating-runnable-stage-gc). `brewlet_sandbox_launches_total` uses these bounded failure reasons: `NoCompatibleJDK`, `NoCompatibleLauncher`, `NoCompatibleArch`, diff --git a/docs/third-party-notices.md b/docs/third-party-notices.md index a065125a..5f30c943 100644 --- a/docs/third-party-notices.md +++ b/docs/third-party-notices.md @@ -27,7 +27,7 @@ only inside a built artifact and cannot be browsed in the source tree. | `brewlet` CLI archive | `NOTICE.txt` beside the executable | `./scripts/generate-go-notice.sh core /tmp/cli-notice.txt ./cmd/brewlet` | | Operator image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh kubernetes /tmp/operator-notice.txt ./cmd/manager` | | Admission image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh kubernetes /tmp/admission-notice.txt ./cmd/admission` | -| Node provisioner image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh core /tmp/provisioner-notice.txt ./shim/cmd/containerd-shim-brewlet-v2 ./cmd/brewlet-metrics-exporter ./cmd/brewlet-source-policy` | +| Node provisioner image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh core /tmp/provisioner-notice.txt ./shim/cmd/containerd-shim-brewlet-v2 ./cmd/brewlet-metrics-exporter ./cmd/brewlet-source-policy ./cmd/brewlet` | | Helm chart | No third-party code is embedded in the chart package | — | The provisioner image build appends the `kubectl`, `ctr`, and `crictl` licenses diff --git a/integration-tests/AGENTS.md b/integration-tests/AGENTS.md index cedad591..eb6ccb1d 100644 --- a/integration-tests/AGENTS.md +++ b/integration-tests/AGENTS.md @@ -40,15 +40,15 @@ The harness does not switch branches or modify component sources. It uses | Tool | Tiers | |---|---| | Go | all | -| Python 3 | 2, 4, 12, 13, 16 | -| JDK 21+ | 2, 3, 7, 8, 9, 12, 14, 15, 16 | -| Docker | 3, 6, 7, 8-12, 14, 15, 16 | -| kubectl and a reachable cluster | 4-16 | -| Helm | 4 (optional), 10, 15 | +| Python 3 | 2, 4, 12, 13, 16, 17 | +| JDK 21+ | 2, 3, 7, 8, 9, 12, 14, 15, 16, 17 | +| Docker | 3, 6, 7, 8-12, 14-17 | +| kubectl and a reachable cluster | 4-17 | +| Helm | 4 (optional), 10, 15, 17 | | OpenSSL | 4, 5, 6, 10, 11 | Host-only tiers 1-3 need no cluster. Tiers 4-7 and 13 exercise API-server -behavior. Tiers 6, 8-12, 14, 15, and 16 require local containerd nodes that +behavior. Tiers 6, 8-12, and 14-17 require local containerd nodes that Docker can enter, such as kind. Managed clusters skip those node-side paths. Tier 13 also proves the control-plane guard: a NodeProfile that does not set `nodePool.includeControlPlane` never counts or schedules onto a control-plane @@ -69,6 +69,15 @@ a silently non-running pod. It moves the node's shim aside for the last case and it both inline and from its cleanup trap. §14's remaining row, the cgroup-v1 refusal, cannot be produced on a cgroup-v2 CI node and is covered deterministically by `provisioner/entrypoint_test.sh` over `require_cgroup_v2`. +Tier 17 installs the chart with default `stageGC` values, verifies they reach +the provisioner DaemonSet without the metrics exporter, and proves the upgrade +gate blocks GC on a node whose stage root already holds data. It then +acknowledges the upgrade with `interval=5s` and `minAge=1s`, resets the +acknowledgment to prove the per-node compatibility record persists, and shows +that a runnable-image stage survives while its pod runs and while containerd +still holds the image, then is reclaimed after the image is removed. Once GC is +active, it may also reclaim unreferenced stages left on the node by earlier +tiers. **Architecture coverage.** Tiers pick up the node's architecture automatically, but hosted runners are amd64, so `.github/workflows/e2e.yml` runs the host-only @@ -106,7 +115,7 @@ Common environment-specific skips: - Tier 5 skips when a cluster cannot reach a host-bound webhook; tier 6 covers the same assertions in-cluster. -- Tiers 8, 9, 12, 14, 15, and 16 skip if no schedulable local containerd node +- Tiers 8, 9, 12, and 14-17 skip if no schedulable local containerd node can be provisioned. - Tier 12 skips when the node's `ctr` supports neither `images unpack` nor import-time unpack; tier 16 applies the same rule. diff --git a/integration-tests/README.md b/integration-tests/README.md index 565bf70f..aad34627 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -7,7 +7,7 @@ manifests, and specifications remain in their owning monorepo directories. ## Layout ```text -e2e/ runner, reset helper, shared library, and tiers 1-15 +e2e/ runner, reset helper, shared library, and tiers 1-17 fixtures/ harness-owned Java demo applications and PetClinic build fixture ``` @@ -26,7 +26,7 @@ testing external checkouts. ## Running ```bash -integration-tests/e2e/run.sh # all 15 tiers +integration-tests/e2e/run.sh # all 17 tiers integration-tests/e2e/run.sh --list # tier catalog integration-tests/e2e/run.sh --tier 4 # one tier integration-tests/e2e/run.sh --reset # remove Brewlet test state @@ -51,6 +51,8 @@ an exercised capability fails. The suite covers: | 13 | NodeProfile lifecycle | Kubernetes | | 14 | custom JDK + jaz NodeProfile, live workload, and broken-launcher readiness failure | both + fixtures | | 15 | live opt-in Prometheus metrics through Helm, provisioner, shim, and exporter | both + fixtures | +| 16 | SPECIFICATION §14 failure modes on a live node | core + fixtures | +| 17 | default-enabled runnable-stage GC: chart defaults, upgrade gate, reference protection, and reclamation | both + fixtures | See [AGENTS.md](AGENTS.md) for cluster requirements, cleanup, and troubleshooting. diff --git a/integration-tests/e2e/reset.sh b/integration-tests/e2e/reset.sh index 3e360018..ad09fe55 100755 --- a/integration-tests/e2e/reset.sh +++ b/integration-tests/e2e/reset.sh @@ -31,7 +31,7 @@ fi # Fixed namespaces the tiers use for their fixtures. Throwaway per-run # namespaces (tiers 5/6/10/11) are random and self-cleaned by their RETURN # traps; we only need to sweep the deterministic ones here. -E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e) +E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e brewlet-stagegc-e2e) # scrub_node_labels: remove every brewlet.sh/* label and annotation from every # node. These are advertised by the provisioner (or simulated by tiers) and are diff --git a/integration-tests/e2e/run.sh b/integration-tests/e2e/run.sh index 75941800..9f981fd3 100755 --- a/integration-tests/e2e/run.sh +++ b/integration-tests/e2e/run.sh @@ -32,7 +32,7 @@ mkdir -p "$WORK" for arg in "$@"; do case "$arg" in --list) - printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster)\n' + printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster)\n' exit 0 ;; esac @@ -97,6 +97,7 @@ source "$E2E_DIR/tier13-nodeprofile.sh" source "$E2E_DIR/tier14-custom-jdk.sh" source "$E2E_DIR/tier15-metrics-incluster.sh" source "$E2E_DIR/tier16-failure-modes.sh" +source "$E2E_DIR/tier17-stage-gc.sh" declare -a TIERS=() DO_RESET=0 @@ -106,7 +107,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --tier) TIERS+=("$2"); shift 2 ;; --reset) DO_RESET=1; shift ;; - --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster)\n'; exit 0 ;; + --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster)\n'; exit 0 ;; -h|--help) usage; exit 0 ;; *) warn "unknown arg: $1"; usage; exit 2 ;; esac @@ -119,7 +120,7 @@ if [[ "$DO_RESET" -eq 1 ]]; then [[ ${#TIERS[@]} -eq 0 ]] && exit 0 fi -[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16) +[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17) section "Brewlet E2E — environment" info "harness : $REPO_DIR" @@ -177,6 +178,7 @@ for t in "${TIERS[@]}"; do 14) tier14_custom_jdk ;; 15) tier15_metrics_incluster ;; 16) tier16_failure_modes ;; + 17) tier17_stage_gc ;; *) warn "no such tier: $t" ;; esac done diff --git a/integration-tests/e2e/tier17-stage-gc.sh b/integration-tests/e2e/tier17-stage-gc.sh new file mode 100644 index 00000000..526d8bdd --- /dev/null +++ b/integration-tests/e2e/tier17-stage-gc.sh @@ -0,0 +1,580 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Tier 17 — default-enabled runnable-stage GC through the real Helm chart, +# operator, provisioner DaemonSet, shim, kubelet, and containerd. +# +# What this tier proves on a local containerd node (kind/CI): +# (1) the chart's defaults enable GC and propagate interval/min-age/ack to the +# provisioner DaemonSet without the metrics exporter; +# (2) the upgrade gate blocks GC on a node whose stage root already holds data +# and no compatibility record exists, and never deletes anything; +# (3) stageGC.upgradeAcknowledged=true activates sweeps and writes the per-node +# compatibility record, which keeps GC active after the acknowledgment is +# reset; +# (4) a stage stays while its image is referenced by containerd and mounted by +# a running pod, and still stays after the pod is gone but the image +# remains; +# (5) once the image is removed and containerd collects its content, a sweep +# deletes the orphaned stage and leaves non-canonical trees alone. +# +# The tier uses minAge=1s and interval=5s to finish quickly. With GC activated, +# the reaper may also remove other unreferenced canonical stages left on the +# test node by earlier tiers; that is exactly its production behavior. + +T17_RELEASE="brewlet-stagegc-e2e" +T17_RELEASE_NS="default" +T17_NS="brewlet" +T17_APP_NS="brewlet-stagegc-e2e" +T17_PROFILE="stagegc" +T17_POOL_KEY="brewlet.sh/e2e-pool" +T17_POOL="stagegc" +T17_JDK="temurin-21" +T17_REF="demo/hello:stagegc-e2e-$$" +T17_APP="stagegc-orders" +T17_OP_IMG="brewlet.local/brewlet-operator:stagegc-e2e" +T17_ADM_IMG="brewlet.local/brewlet-admission:stagegc-e2e" +T17_PROV_IMG="brewlet.local/brewlet-node-provisioner:stagegc-e2e" +T17_STAGE_ROOT="/tmp/brewlet-runnable" +T17_RECORD="/opt/brewlet/.stage-gc-compatible" +T17_SENTINEL="$T17_STAGE_ROOT/t17-legacy-sentinel" +T17_NODE="" +T17_ARCH="" +T17_HELM_INSTALLED="" +T17_PROFILE_CREATED="" +T17_APP_NS_CREATED="" +T17_NODE_TOUCHED="" +T17_JDK_PREEXISTING="" +T17_JDK_ACTIVE_PREEXISTING="" +T17_RECORD_PREEXISTING="" +T17_SENTINEL_CREATED="" +T17_IMAGE_DIGEST="" +declare -a T17_LOADED_NODES=() +declare -a T17_BUILT_IMAGES=() + +# Snapshot/restore host files the provisioner may mutate. An empty snapshot +# plus a missing ".present" marker means the file did not exist. +_t17_snapshot_file() { + local path="$1" out="$2" + rm -f "$out" "$out.present" + if docker exec "$T17_NODE" test -f "$path" >/dev/null 2>&1; then + docker exec "$T17_NODE" cat "$path" >"$out" || return 1 + : >"$out.present" + else + : >"$out" + fi +} + +_t17_restore_file() { + local path="$1" snapshot="$2" + [[ -f "$snapshot" ]] || return 0 + if [[ -f "$snapshot.present" ]]; then + docker exec "$T17_NODE" mkdir -p "$(dirname "$path")" >/dev/null 2>&1 && + docker exec -i "$T17_NODE" sh -c 'cat > "$1"' sh "$path" <"$snapshot" >/dev/null 2>&1 + else + docker exec "$T17_NODE" rm -f "$path" >/dev/null 2>&1 + fi +} + +_t17_restore_containerd() { + local before="$WORK/t17-containerd-state-before" after="$WORK/t17-containerd-state-after" tries=30 + _t17_restore_file /etc/containerd/config.toml "$WORK/t17-containerd.toml" || return 1 + _t17_restore_file /etc/containerd/config.toml.brewlet.bak "$WORK/t17-containerd-bak.toml" || return 1 + _t17_restore_file /etc/containerd/config.toml.d/99-brewlet.toml "$WORK/t17-containerd-dropin.toml" || return 1 + docker exec "$T17_NODE" sh -c \ + 'cat /etc/containerd/config.toml /etc/containerd/config.toml.d/99-brewlet.toml 2>/dev/null' \ + >"$after" 2>/dev/null || true + cmp -s "$before" "$after" && return 0 + docker exec "$T17_NODE" systemctl restart containerd >/dev/null 2>&1 || return 1 + while (( tries-- > 0 )); do + docker exec "$T17_NODE" ctr version >/dev/null 2>&1 && return 0 + sleep 1 + done + return 1 +} + +_t17_image_refs() { + docker exec "$T17_NODE" ctr -n k8s.io images ls 2>/dev/null | + awk -v d="$1" 'NR > 1 && $3 == d { print $1 }' +} + +_t17_remove_image() { + local refs + refs="$(_t17_image_refs "$1")" + [[ -z "$refs" ]] && return 0 + # shellcheck disable=SC2086 + docker exec "$T17_NODE" ctr -n k8s.io images rm --sync $refs >>"$WORK/t17-app.log" 2>&1 +} + +_t17_cleanup() { + info "tier17: cleaning up" + if [[ -n "$T17_APP_NS_CREATED" ]]; then + kubectl delete ns "$T17_APP_NS" --ignore-not-found --wait=true --timeout=120s \ + >/dev/null 2>&1 || true + fi + if [[ -n "$T17_PROFILE_CREATED" ]] && + kubectl get nodeprofile "$T17_PROFILE" >/dev/null 2>&1; then + kubectl delete nodeprofile "$T17_PROFILE" --wait=false >/dev/null 2>&1 || true + if ! wait_for_seconds 120 bash -c "! kubectl get nodeprofile '$T17_PROFILE'"; then + warn "tier17: profile cleanup did not finish; removing the test finalizer" + docker exec "$T17_NODE" rm -f /opt/brewlet/bin/containerd-shim-brewlet-v2 \ + /usr/local/bin/containerd-shim-brewlet-v2 /usr/local/bin/brewlet-ctr \ + /usr/local/bin/brewlet-stage-gc "$T17_RECORD" >/dev/null 2>&1 || true + kubectl patch nodeprofile "$T17_PROFILE" --type=merge \ + -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || true + fi + fi + if [[ -n "$T17_HELM_INSTALLED" ]]; then + helm uninstall "$T17_RELEASE" -n "$T17_RELEASE_NS" >/dev/null 2>&1 || true + kubectl delete runtimeclass brewlet --ignore-not-found >/dev/null 2>&1 || true + kubectl delete crd javaapplications.apps.brewlet.sh nodeprofiles.node.brewlet.sh \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + kubectl delete mutatingwebhookconfiguration brewlet-admission \ + --ignore-not-found >/dev/null 2>&1 || true + kubectl delete validatingwebhookconfiguration brewlet-nodeprofiles \ + --ignore-not-found >/dev/null 2>&1 || true + kubectl delete ns "$T17_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true + wait_for_seconds 120 bash -c "! kubectl get namespace '$T17_NS'" || true + fi + + if [[ -n "$T17_NODE_TOUCHED" ]]; then + _t17_restore_containerd || + fail "tier17: restore the node's original containerd configuration" + if [[ -n "$T17_RECORD_PREEXISTING" ]]; then + _t17_restore_file "$T17_RECORD" "$WORK/t17-record" || true + fi + [[ -n "$T17_SENTINEL_CREATED" ]] && + docker exec "$T17_NODE" rm -rf "$T17_SENTINEL" >/dev/null 2>&1 || true + label_node "$T17_NODE" "$T17_POOL_KEY-" brewlet.sh/provision- \ + brewlet.sh/runtime- "brewlet.sh/jdk.$T17_JDK-" \ + "brewlet.sh/jdk-feature.${T17_JDK##*-}-" brewlet.sh/launcher.java- \ + >/dev/null 2>&1 || true + annotate_node "$T17_NODE" brewlet.sh/jdks- brewlet.sh/jdks-info- \ + brewlet.sh/launchers- brewlet.sh/profile- brewlet.sh/profile-generation- \ + brewlet.sh/provision-state- brewlet.sh/provision-error- >/dev/null 2>&1 || true + if [[ -z "$T17_JDK_PREEXISTING" ]]; then + docker exec "$T17_NODE" sh -c \ + 'chmod -R u+w "$1" 2>/dev/null || true; rm -rf "$1"' \ + sh "/opt/brewlet/jdks/$T17_JDK" >/dev/null 2>&1 || true + fi + _t17_restore_file /opt/brewlet/jdks/.brewlet-active "$WORK/t17-jdk-active" || true + [[ -n "$T17_IMAGE_DIGEST" ]] && _t17_remove_image "$T17_IMAGE_DIGEST" || true + fi + + local n + for n in ${T17_LOADED_NODES[@]+"${T17_LOADED_NODES[@]}"}; do + docker exec "$n" ctr -n k8s.io images rm \ + "$T17_OP_IMG" "$T17_ADM_IMG" "$T17_PROV_IMG" >/dev/null 2>&1 || true + done + if [[ -n "${T17_BUILT_IMAGES[*]-}" ]]; then + docker rmi ${T17_BUILT_IMAGES[@]+"${T17_BUILT_IMAGES[@]}"} >/dev/null 2>&1 || true + fi +} + +_t17_build_load() { + local image="$1" dockerfile="$2" nodes="$3" name="$4"; shift 4 + local tarball="$WORK/t17-$name.tar" n + if ! docker build --provenance=false "$@" -t "$image" -f "$dockerfile" "$MONOREPO_DIR" \ + >>"$WORK/t17-build.log" 2>&1; then + docker build "$@" -t "$image" -f "$dockerfile" "$MONOREPO_DIR" \ + >>"$WORK/t17-build.log" 2>&1 || return 1 + fi + T17_BUILT_IMAGES+=("$image") + docker save "$image" -o "$tarball" >>"$WORK/t17-load.log" 2>&1 || return 1 + for n in $nodes; do + docker exec -i "$n" ctr -n k8s.io images import - <"$tarball" \ + >>"$WORK/t17-load.log" 2>&1 || return 1 + done +} + +# Newest non-terminating provisioner pod on the test node. +_t17_provisioner_pod() { + kubectl get pods -n "$T17_NS" -l "brewlet.sh/nodeprofile=$T17_PROFILE" \ + --field-selector "spec.nodeName=$T17_NODE" \ + --sort-by=.metadata.creationTimestamp \ + -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' \ + 2>/dev/null | awk -F'|' '$1 != "" && $2 == "" { name = $1 } END { print name }' +} + +# _t17_platform_stage STORE REF ARCH: the stage path for the platform manifest +# the shim selects (stages are keyed by manifest digest, not index digest). +_t17_platform_stage() { + local hex + hex="$(python3 - "$1" "$2" "$3" <<'PY' +import json, sys +root, ref, arch = sys.argv[1:4] +tag = ref.rsplit(":", 1)[-1] +def blob(digest): + with open(f"{root}/blobs/{digest.replace(':', '/')}", encoding="utf-8") as stream: + return json.load(stream) +with open(f"{root}/index.json", encoding="utf-8") as stream: + index = json.load(stream) +top = next((m for m in index["manifests"] + if m.get("annotations", {}).get("org.opencontainers.image.ref.name") in (ref, tag)), + index["manifests"][0]) +if top["mediaType"].endswith("image.index.v1+json"): + top = next(m for m in blob(top["digest"])["manifests"] + if m.get("platform", {}).get("os") == "linux" + and m.get("platform", {}).get("architecture") == arch) +print(top["digest"].split(":", 1)[1]) +PY +)" || return 1 + [[ -n "$hex" ]] && printf '%s/immutable-v2/%s' "$T17_STAGE_ROOT" "$hex" +} + +_t17_logs() { + kubectl logs "$1" -n "$T17_NS" -c provisioner 2>/dev/null +} + +_t17_last_sweeps() { + _t17_logs "$1" | sed -n 's/.*stage GC: successful_sweeps=\([0-9]*\).*/\1/p' | tail -1 +} + +# _t17_wait_sweeps POD N: wait until POD has logged at least N successful sweeps. +_t17_wait_sweeps() { + local pod="$1" want="$2" deadline=$(( $(date +%s) + 120 )) got + while (( $(date +%s) < deadline )); do + got="$(_t17_last_sweeps "$pod")" + [[ -n "$got" ]] && (( got >= want )) && return 0 + sleep 2 + done + return 1 +} + +_t17_ds_env() { + kubectl get ds "brewlet-node-provisioner-$T17_PROFILE" -n "$T17_NS" \ + -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}" 2>/dev/null +} + +# _t17_wait_rollout ACK: after a Helm change, wait for the operator to render ACK +# into the DaemonSet, the rollout to finish, and the new pod to provision. +_t17_wait_rollout() { + local ack="$1" ds="brewlet-node-provisioner-$T17_PROFILE" pod + wait_for_seconds 180 bash -c \ + "[[ \"\$(kubectl get ds '$ds' -n '$T17_NS' -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED\")].value}')\" == '$ack' ]]" || + return 1 + kubectl rollout status ds/"$ds" -n "$T17_NS" --timeout=300s >>"$WORK/t17-profile.log" 2>&1 || + return 1 + local deadline=$(( $(date +%s) + 180 )) + while (( $(date +%s) < deadline )); do + pod="$(_t17_provisioner_pod)" + if [[ -n "$pod" ]] && _t17_logs "$pod" | grep -Fq "node ${T17_NODE} provisioned successfully"; then + printf '%s' "$pod" + return 0 + fi + sleep 2 + done + return 1 +} + +tier17_stage_gc() { + section "Tier 17 — default-enabled runnable-stage GC in-cluster" + if ! have kubectl || ! k8s_reachable; then skip "tier17: stage GC" "no reachable cluster"; return 0; fi + if ! have docker || ! docker info >/dev/null 2>&1; then + skip "tier17: stage GC" "docker daemon not available"; return 0 + fi + if ! have helm; then skip "tier17: stage GC" "helm not installed"; return 0; fi + if ! have go; then skip "tier17: stage GC" "go not installed"; return 0; fi + if ! have java; then skip "tier17: stage GC" "JDK not installed"; return 0; fi + if ! have python3; then skip "tier17: stage GC" "python3 not installed"; return 0; fi + + local nodes n + nodes="$(kubectl get nodes -o name 2>/dev/null | sed 's#node/##')" + for n in $nodes; do + if ! node_provisionable "$n"; then + skip "tier17: stage GC" "node '$n' is not a local containerd docker container"; return 0 + fi + done + T17_NODE="$(pick_provisionable_node)" + if [[ -z "$T17_NODE" ]] || ! node_schedulable "$T17_NODE"; then + skip "tier17: stage GC" "no schedulable local containerd node"; return 0 + fi + T17_ARCH="$(_t15_node_arch "$T17_NODE")" + [[ -n "$T17_ARCH" ]] || { skip "tier17: stage GC" "unknown node architecture"; return 0; } + if [[ -n "$(detect_leftovers)" ]] || + helm status "$T17_RELEASE" -n "$T17_RELEASE_NS" >/dev/null 2>&1 || + kubectl get ns "$T17_NS" >/dev/null 2>&1 || + kubectl get ns "$T17_APP_NS" >/dev/null 2>&1; then + fail "tier17: clean Brewlet cluster state" "run ./run.sh --reset before tier 17" + return 0 + fi + + # --- snapshot and prepare host state -------------------------------------- + docker exec "$T17_NODE" test -e "/opt/brewlet/jdks/$T17_JDK" >/dev/null 2>&1 && + T17_JDK_PREEXISTING=1 + if ! _t17_snapshot_file /etc/containerd/config.toml "$WORK/t17-containerd.toml" || + ! _t17_snapshot_file /etc/containerd/config.toml.brewlet.bak "$WORK/t17-containerd-bak.toml" || + ! _t17_snapshot_file /etc/containerd/config.toml.d/99-brewlet.toml "$WORK/t17-containerd-dropin.toml" || + ! _t17_snapshot_file /opt/brewlet/jdks/.brewlet-active "$WORK/t17-jdk-active" || + ! _t17_snapshot_file "$T17_RECORD" "$WORK/t17-record"; then + fail "tier17: snapshot node state"; return 0 + fi + docker exec "$T17_NODE" sh -c \ + 'cat /etc/containerd/config.toml /etc/containerd/config.toml.d/99-brewlet.toml 2>/dev/null' \ + >"$WORK/t17-containerd-state-before" 2>/dev/null || true + [[ -f "$WORK/t17-record.present" ]] && T17_RECORD_PREEXISTING=1 + T17_NODE_TOUCHED=1 + trap _t17_cleanup RETURN + + # An existing, record-less stage root makes this an "upgraded" node. The + # sentinel is a non-canonical tree, which the reaper must never delete. + docker exec "$T17_NODE" rm -f "$T17_RECORD" >/dev/null 2>&1 || true + if ! docker exec "$T17_NODE" test -e "$T17_SENTINEL" >/dev/null 2>&1; then + docker exec "$T17_NODE" sh -c 'mkdir -p "$1" && echo keep >"$1/app.jar"' sh "$T17_SENTINEL" \ + >/dev/null 2>&1 || { fail "tier17: plant pre-existing stage data"; return 0; } + T17_SENTINEL_CREATED=1 + fi + + # --- build and load images ------------------------------------------------ + for n in $nodes; do T17_LOADED_NODES+=("$n"); done + : >"$WORK/t17-build.log"; : >"$WORK/t17-load.log" + info "tier17: building and side-loading operator, admission, and provisioner images" + if ! _t17_build_load "$T17_OP_IMG" "$BREWLET_KUBERNETES_DIR/Dockerfile" "$nodes" manager --build-arg CMD=manager || + ! _t17_build_load "$T17_ADM_IMG" "$BREWLET_KUBERNETES_DIR/Dockerfile" "$nodes" admission --build-arg CMD=admission || + ! _t17_build_load "$T17_PROV_IMG" "$MONOREPO_DIR/provisioner/Dockerfile" "$nodes" provisioner --platform "linux/$T17_ARCH"; then + fail "tier17: build and load images" "see t17-build.log and t17-load.log"; return 0 + fi + if docker run --rm --platform "linux/$T17_ARCH" --entrypoint /bin/test "$T17_PROV_IMG" \ + -x /opt/brewlet-dist/brewlet; then + pass "tier17: provisioner image ships the stage-GC CLI" + else + fail "tier17: provisioner image ships the stage-GC CLI"; return 0 + fi + + local jar="$FIXTURES_DIR/demo-app/target/app.jar" store="$WORK/t17-oci" jh image_ref + if [[ ! -f "$jar" ]]; then + jh="$(resolve_java_home)" + env JAVA_HOME="$jh" PATH="$jh/bin:$PATH" "$FIXTURES_DIR/demo-app/build.sh" \ + >>"$WORK/t17-app.log" 2>&1 || { fail "tier17: build demo JAR" "see $WORK/t17-app.log"; return 0; } + fi + rm -rf "$store" + if ! (cd "$BREWLET_CORE_DIR" && go build -o "$WORK/t17-brewlet" ./cmd/brewlet) >>"$WORK/t17-app.log" 2>&1 || + ! "$WORK/t17-brewlet" push "$jar" "$T17_REF" --store "$store" --format=image >>"$WORK/t17-app.log" 2>&1; then + fail "tier17: build runnable image" "see $WORK/t17-app.log"; return 0 + fi + local digest + digest="$(oci_layout_digest "$store" "$T17_REF")" + if [[ -z "$digest" ]] || ! import_oci_layout "$T17_NODE" "$store" "$WORK/t17-app.log" || + ! image_ref="$(pin_image_for_cri "$T17_NODE" "$T17_REF" "$digest" "$WORK/t17-app.log")"; then + fail "tier17: import runnable image into node" "see $WORK/t17-app.log"; return 0 + fi + T17_IMAGE_DIGEST="$digest" + + # --- (1)+(2): chart defaults, blocked on an upgraded node ----------------- + info "tier17: installing the chart with default stageGC values" + T17_HELM_INSTALLED=1 + if ! helm install "$T17_RELEASE" "$BREWLET_KUBERNETES_DIR/charts/brewlet" \ + --namespace "$T17_RELEASE_NS" \ + --set images.operator="$T17_OP_IMG" \ + --set images.admission="$T17_ADM_IMG" \ + --set images.provisioner="$T17_PROV_IMG" \ + --set images.pullPolicy=IfNotPresent \ + --set defaultProfile.enabled=false \ + --set operator.leaderElect=false \ + --wait --timeout 180s >"$WORK/t17-install.log" 2>&1; then + save_pod_diag t17-install "$T17_NS" >>"$WORK/t17-install.log" 2>&1 || true + fail "tier17: install chart" "see $WORK/t17-install.log"; return 0 + fi + pass "tier17: chart installed with default stageGC values" + + label_node "$T17_NODE" --overwrite "$T17_POOL_KEY=$T17_POOL" brewlet.sh/provision=true \ + brewlet.sh/runtime- >>"$WORK/t17-profile.log" 2>&1 + annotate_node "$T17_NODE" brewlet.sh/provision-state- brewlet.sh/provision-error- \ + >>"$WORK/t17-profile.log" 2>&1 || true + T17_PROFILE_CREATED=1 + if ! kubectl apply -f - >>"$WORK/t17-profile.log" 2>&1 <>"$WORK/t17-install.log" 2>&1 || + ! pod="$(_t17_wait_rollout true)"; then + fail "tier17: roll out upgradeAcknowledged=true" \ + "diag: $(save_pod_diag t17-ack "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + if _t17_wait_sweeps "$pod" 1; then + pass "tier17: acknowledgment activates periodic sweeps" + else + fail "tier17: acknowledgment activates periodic sweeps" \ + "diag: $(save_pod_diag t17-ack-sweep "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + check "tier17: compatible node persists its compatibility record" \ + docker exec "$T17_NODE" test -f "$T17_RECORD" + + if ! helm upgrade "$T17_RELEASE" "$BREWLET_KUBERNETES_DIR/charts/brewlet" \ + --namespace "$T17_RELEASE_NS" --reuse-values \ + --set stageGC.upgradeAcknowledged=false \ + --wait --timeout 180s >>"$WORK/t17-install.log" 2>&1 || + ! pod="$(_t17_wait_rollout false)"; then + fail "tier17: roll out upgradeAcknowledged=false" \ + "diag: $(save_pod_diag t17-reset "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + if _t17_wait_sweeps "$pod" 1; then + pass "tier17: compatibility record keeps GC active after resetting the acknowledgment" + else + fail "tier17: compatibility record keeps GC active after resetting the acknowledgment" \ + "diag: $(save_pod_diag t17-reset-sweep "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + + # --- (4): referenced and mounted stages survive --------------------------- + local stage sweeps + if ! stage="$(_t17_platform_stage "$store" "$T17_REF" "$T17_ARCH")"; then + fail "tier17: resolve the platform manifest stage path"; return 0 + fi + T17_APP_NS_CREATED=1 + kubectl create namespace "$T17_APP_NS" >/dev/null 2>&1 || true + if ! kubectl apply -n "$T17_APP_NS" -f - >>"$WORK/t17-app.log" 2>&1 <>"$WORK/t17-app.log" 2>&1; then + fail "tier17: runnable-image workload launched" \ + "diag: $(save_pod_diag t17-workload "$T17_APP_NS" "app=$T17_APP")" + return 0 + fi + if ! docker exec "$T17_NODE" test -d "$stage"; then + fail "tier17: workload published its runnable stage" "missing $stage" + return 0 + fi + info "tier17: workload stage $stage" + + sweeps="$(_t17_last_sweeps "$pod")" + if _t17_wait_sweeps "$pod" $(( ${sweeps:-0} + 2 )) && + docker exec "$T17_NODE" test -d "$stage"; then + pass "tier17: stage of a running, referenced image survives sweeps" + else + fail "tier17: stage of a running, referenced image survives sweeps"; return 0 + fi + + kubectl delete deploy "$T17_APP" -n "$T17_APP_NS" --wait=true --timeout=60s \ + >>"$WORK/t17-app.log" 2>&1 || true + wait_for_seconds 60 bash -c "[[ -z \"\$(kubectl get pods -n '$T17_APP_NS' -o name)\" ]]" || true + local containers + containers="$(docker exec "$T17_NODE" crictl ps -a -q \ + --label "io.kubernetes.pod.namespace=$T17_APP_NS" 2>/dev/null || true)" + if [[ -n "$containers" ]]; then + # shellcheck disable=SC2086 + docker exec "$T17_NODE" crictl rm $containers >>"$WORK/t17-app.log" 2>&1 || true + fi + sweeps="$(_t17_last_sweeps "$pod")" + if _t17_wait_sweeps "$pod" $(( ${sweeps:-0} + 2 )) && + docker exec "$T17_NODE" test -d "$stage"; then + pass "tier17: stage survives while containerd still holds its image" + else + fail "tier17: stage survives while containerd still holds its image"; return 0 + fi + + # --- (5): image removal makes the stage an orphan ------------------------- + if ! _t17_remove_image "$T17_IMAGE_DIGEST" || [[ -n "$(_t17_image_refs "$T17_IMAGE_DIGEST")" ]]; then + fail "tier17: remove the workload image from containerd" "see $WORK/t17-app.log"; return 0 + fi + if wait_for_seconds 120 docker exec "$T17_NODE" test ! -e "$stage"; then + pass "tier17: sweep reclaims the stage after its image is removed" + else + fail "tier17: sweep reclaims the stage after its image is removed" \ + "diag: $(save_pod_diag t17-reclaim "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + if wait_for_seconds 30 bash -c \ + "kubectl logs '$pod' -n '$T17_NS' -c provisioner | grep -Eq 'removed [1-9][0-9]* stages'"; then + pass "tier17: provisioner logs the reclaimed stage" + else + fail "tier17: provisioner logs the reclaimed stage" + fi + check "tier17: non-canonical stage data is never reclaimed" \ + docker exec "$T17_NODE" test -f "$T17_SENTINEL/app.jar" +} diff --git a/kubernetes/charts/brewlet/README.md b/kubernetes/charts/brewlet/README.md index fe2b6a10..363b5d74 100644 --- a/kubernetes/charts/brewlet/README.md +++ b/kubernetes/charts/brewlet/README.md @@ -104,6 +104,10 @@ for upgrades; pass them again rather than substituting example defaults. | `provisioner.launchers` | `[]` | Optional entries with `name`, digest-pinned `source.image`, and absolute `source.path` (§5.4). Empty = vanilla `java` only. | | `provisioner.registry.mirrors` | `{}` | Upstream host → approved mirror host/path map. Rewrites preserve the source digest. | | `provisioner.appCDS.regenerationEnabled` | `false` | Authorize node-side AppCDS regeneration for the default profile. | +| `stageGC.enabled` | `true` | Automatically reclaim orphaned runnable-image stages on managed nodes, independently of metrics. | +| `stageGC.interval` | `5m` | Time between sweeps, plus up to 10% jitter. | +| `stageGC.minAge` | `24h` | Minimum stage directory age, not time since last use or loss of references. | +| `stageGC.upgradeAcknowledged` | `false` | Confirm older unguarded shims and stage-dependent bundles have been retired across the managed fleet. Reset after rollout. | | `operator.leaderElect` | `true` | Enable operator leader election. | | `profiles` | `[]` | Additional profiles, each with a unique name, nonempty named `pools`, and explicit JDK sources. | | `uninstall.timeoutSeconds` | `240` | Cleanup coordinator timeout, 1-86400 whole seconds. Configure before uninstall; Helm's `--timeout` must exceed this plus 30 seconds. | @@ -128,6 +132,29 @@ for upgrades; pass them again rather than substituting example defaults. | `admission.certManager.renewBefore` | `720h` | How early cert-manager renews the serving certificate. | | `admission.certManager.caRenewBefore` | `2160h` | How early a chart-managed CA renews; must exceed the serving certificate renewal interval. | +### Automatic stage cleanup + +Stage GC settings are operator-wide defaults for **all** managed NodeProfiles, +including profiles created outside Helm. Fresh nodes activate after successful +provisioning; no extra DaemonSet, CronJob, kubelet access, or RBAC is needed. +The provisioner cleans only eligible unreferenced, unmounted stages; kubelet and +containerd remain responsible for image retention and content reclamation. +Durations must be positive whole seconds (Go duration syntax, at most +`2147483647s`); invalid settings are rejected by the operator. + +Existing nodes without a matching compatibility record log `stage GC blocked` +and continue serving workloads. First retire older unguarded shims and exported +bundles that reference the stage cache across the selected fleet. Then include +`--set stageGC.upgradeAcknowledged=true` in your normal Helm upgrade. Reset it +to `false` after the rollout; compatible nodes remember activation. Installing +the new shim alone is not proof that older consumers have stopped. +To opt out, set `stageGC.enabled=false`. + +See [runnable stage cleanup](../../../docs/runnable-image.md#reclaiming-unused-stages) +for safety constraints, migration details, and observability. These settings do +not copy or alter kubelet image GC thresholds, and reclamation may lag +containerd content GC. + All JDK distributions use the structured inventory form: ```yaml diff --git a/kubernetes/charts/brewlet/templates/operator.yaml b/kubernetes/charts/brewlet/templates/operator.yaml index ba0d18c6..5ff59b67 100644 --- a/kubernetes/charts/brewlet/templates/operator.yaml +++ b/kubernetes/charts/brewlet/templates/operator.yaml @@ -142,6 +142,10 @@ spec: - --provisioner-image={{ include "brewlet.image" (dict "root" . "component" "provisioner" "name" "brewlet-node-provisioner") }} - --node-metrics-port={{ .Values.metrics.nodePort }} - --node-metrics-enabled={{ .Values.metrics.enabled }} + - {{ printf "--stage-gc-enabled=%v" .Values.stageGC.enabled | quote }} + - {{ printf "--stage-gc-interval=%v" .Values.stageGC.interval | quote }} + - {{ printf "--stage-gc-min-age=%v" .Values.stageGC.minAge | quote }} + - {{ printf "--stage-gc-upgrade-acknowledged=%v" .Values.stageGC.upgradeAcknowledged | quote }} - {{ printf "--allowed-source-mirror-hosts=%s" (join "," .Values.security.allowedSourceMirrorHosts) | quote }} {{- if .Values.metrics.enabled }} - --metrics-bind-address=:8080 diff --git a/kubernetes/charts/brewlet/values.yaml b/kubernetes/charts/brewlet/values.yaml index fa883472..43c9832a 100644 --- a/kubernetes/charts/brewlet/values.yaml +++ b/kubernetes/charts/brewlet/values.yaml @@ -131,6 +131,17 @@ defaultProfile: # registry: { mirrors: { "mcr.microsoft.com": "mirror.internal/mcr" } } profiles: [] +# Applies to every operator-managed NodeProfile, independently of metrics. +stageGC: + enabled: true + interval: 5m + # Stage directory age, not time since last use or since becoming unreferenced. + minAge: 24h + # On existing installations, first retire older unguarded shims and exported + # bundles that point into the stage cache. Fresh nodes activate automatically. + # Reset to false after the rollout; compatible nodes remember activation. + upgradeAcknowledged: false + operator: replicas: 1 leaderElect: true diff --git a/kubernetes/cmd/manager/main.go b/kubernetes/cmd/manager/main.go index 9a3df467..87382d97 100644 --- a/kubernetes/cmd/manager/main.go +++ b/kubernetes/cmd/manager/main.go @@ -12,6 +12,7 @@ import ( "flag" "fmt" "os" + "time" nodev1alpha1 "brewlet-operator/api/nodeprofile/v1alpha1" appsv1alpha1 "brewlet-operator/api/v1alpha1" @@ -57,6 +58,10 @@ func main() { flag.StringVar(&cfg.ProvisionerImage, "provisioner-image", "ghcr.io/microsoft/brewlet-node-provisioner:0.1.0", "brewlet-node-provisioner image to run") flag.IntVar(&cfg.MetricsPort, "node-metrics-port", 9090, "node provisioner metrics exporter port") flag.BoolVar(&cfg.MetricsEnabled, "node-metrics-enabled", false, "run the node-local metrics exporter in provisioner pods") + flag.BoolVar(&cfg.StageGCEnabled, "stage-gc-enabled", true, "periodically reclaim unreferenced runnable-image stages") + flag.DurationVar(&cfg.StageGCInterval, "stage-gc-interval", 5*time.Minute, "stage GC interval (positive whole seconds)") + flag.DurationVar(&cfg.StageGCMinAge, "stage-gc-min-age", 24*time.Hour, "minimum stage age (positive whole seconds)") + flag.BoolVar(&cfg.StageGCUpgradeAcknowledged, "stage-gc-upgrade-acknowledged", false, "confirm older unguarded shims and stage-dependent exported bundles have been retired") flag.StringVar(&allowedMirrors, "allowed-source-mirror-hosts", "", "comma-separated exact registry hosts approved as runtime source mirror destinations") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "address the metric endpoint binds to; 0 disables metrics") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "address the health probe endpoint binds to") @@ -84,6 +89,10 @@ func main() { fmt.Fprintf(os.Stderr, "invalid --allowed-source-mirror-hosts: %v\n", err) os.Exit(1) } + if err := cfg.ValidateStageGC(); err != nil { + fmt.Fprintf(os.Stderr, "invalid stage GC configuration: %v\n", err) + os.Exit(1) + } ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog := ctrl.Log.WithName("setup") diff --git a/kubernetes/internal/chart/stage_gc_test.go b/kubernetes/internal/chart/stage_gc_test.go new file mode 100644 index 00000000..5c43322b --- /dev/null +++ b/kubernetes/internal/chart/stage_gc_test.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package chart_test + +import ( + "slices" + "testing" + + appsv1 "k8s.io/api/apps/v1" +) + +func TestStageGCOperatorArguments(t *testing.T) { + for _, tc := range []struct { + name string + args []string + want []string + }{ + { + name: "defaults without metrics or chart-managed profiles", + want: []string{"--stage-gc-enabled=true", "--stage-gc-interval=5m", "--stage-gc-min-age=24h", "--stage-gc-upgrade-acknowledged=false", "--node-metrics-enabled=false"}, + }, + { + name: "overrides", + args: []string{"--set", "stageGC.enabled=false", "--set", "stageGC.interval=10m", "--set", "stageGC.minAge=48h", "--set", "stageGC.upgradeAcknowledged=true"}, + want: []string{"--stage-gc-enabled=false", "--stage-gc-interval=10m", "--stage-gc-min-age=48h", "--stage-gc-upgrade-acknowledged=true"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + found := false + for _, object := range render(t, tc.args...) { + if object.GetKind() != "Deployment" || object.GetName() != "brewlet-operator" { + continue + } + found = true + operator := convert[appsv1.Deployment](t, object) + args := operator.Spec.Template.Spec.Containers[0].Args + for _, want := range tc.want { + if !slices.Contains(args, want) { + t.Errorf("missing %q in %v", want, args) + } + } + } + if !found { + t.Fatal("operator Deployment missing") + } + }) + } +} diff --git a/kubernetes/internal/controller/nodeprofile_resources.go b/kubernetes/internal/controller/nodeprofile_resources.go index f6351afd..9e8f5e9b 100644 --- a/kubernetes/internal/controller/nodeprofile_resources.go +++ b/kubernetes/internal/controller/nodeprofile_resources.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "time" nodev1alpha1 "brewlet-operator/api/nodeprofile/v1alpha1" "brewlet-operator/internal/brewlet" @@ -285,6 +286,10 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv {Name: "BREWLET_PROFILE_UID", Value: string(profile.UID)}, {Name: "BREWLET_PROFILE_GENERATION", Value: strconv.FormatInt(profile.Generation, 10)}, {Name: "SOURCE_ALLOWED_MIRROR_HOSTS", Value: strings.Join(cfg.AllowedSourceMirrorHosts, ",")}, + {Name: "BREWLET_STAGE_GC_ENABLED", Value: strconv.FormatBool(cfg.StageGCEnabled)}, + {Name: "BREWLET_STAGE_GC_INTERVAL_SECONDS", Value: strconv.FormatInt(int64(cfg.StageGCInterval/time.Second), 10)}, + {Name: "BREWLET_STAGE_GC_MIN_AGE_SECONDS", Value: strconv.FormatInt(int64(cfg.StageGCMinAge/time.Second), 10)}, + {Name: "BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED", Value: strconv.FormatBool(cfg.StageGCUpgradeAcknowledged)}, } if profile.UID != "" { env = append(env, corev1.EnvVar{Name: "BREWLET_REQUIRE_NODE_CLAIM", Value: "true"}) diff --git a/kubernetes/internal/controller/resources.go b/kubernetes/internal/controller/resources.go index d48417e4..c2b73b86 100644 --- a/kubernetes/internal/controller/resources.go +++ b/kubernetes/internal/controller/resources.go @@ -4,6 +4,9 @@ package controller import ( + "fmt" + "time" + "brewlet-operator/internal/brewlet" corev1 "k8s.io/api/core/v1" @@ -24,12 +27,31 @@ type Config struct { MetricsPort int // MetricsEnabled controls whether managed provisioner pods run the node-local // metrics exporter sidecar. - MetricsEnabled bool + MetricsEnabled bool + StageGCEnabled bool + StageGCInterval time.Duration + StageGCMinAge time.Duration + // StageGCUpgradeAcknowledged authorizes activation on previously provisioned + // nodes after the administrator has retired unguarded stage consumers. + StageGCUpgradeAcknowledged bool // AllowedSourceMirrorHosts is the exact operator-level allowlist for registry // mirror destination hosts, including any explicit port. AllowedSourceMirrorHosts []string } +// ValidateStageGC keeps the duration-to-shell-seconds conversion exact and bounded. +func (c Config) ValidateStageGC() error { + for name, value := range map[string]time.Duration{ + "stage-gc-interval": c.StageGCInterval, + "stage-gc-min-age": c.StageGCMinAge, + } { + if value < time.Second || value%time.Second != 0 || value/time.Second > 2147483647 { + return fmt.Errorf("%s must be a positive whole number of seconds, at most 2147483647s", name) + } + } + return nil +} + // buildRuntimeClass returns the desired brewlet RuntimeClass: it schedules only // onto provisioned nodes (LabelRuntimeReady=ready) and reserves JVM overhead so // the scheduler accounts for the runtime baseline (§7). diff --git a/kubernetes/internal/controller/stage_gc_test.go b/kubernetes/internal/controller/stage_gc_test.go new file mode 100644 index 00000000..3f9bac56 --- /dev/null +++ b/kubernetes/internal/controller/stage_gc_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package controller + +import ( + "testing" + "time" +) + +func TestStageGCValidation(t *testing.T) { + for _, duration := range []time.Duration{0, -time.Second, time.Millisecond, time.Second + time.Nanosecond, 2147483648 * time.Second} { + for _, field := range []string{"interval", "minAge"} { + cfg := Config{StageGCInterval: 5 * time.Minute, StageGCMinAge: 24 * time.Hour} + if field == "interval" { + cfg.StageGCInterval = duration + } else { + cfg.StageGCMinAge = duration + } + if err := cfg.ValidateStageGC(); err == nil { + t.Errorf("%s accepted %s", field, duration) + } + } + } + for _, duration := range []time.Duration{time.Second, 5 * time.Minute, 24 * time.Hour, 2147483647 * time.Second} { + cfg := Config{StageGCInterval: duration, StageGCMinAge: duration} + if err := cfg.ValidateStageGC(); err != nil { + t.Errorf("rejected %s: %v", duration, err) + } + } +} + +func TestStageGCProvisionerConfiguration(t *testing.T) { + for _, metrics := range []bool{false, true} { + for _, enabled := range []bool{false, true} { + cfg := testConfig() + cfg.MetricsEnabled = metrics + cfg.StageGCEnabled = enabled + cfg.StageGCInterval = 7 * time.Minute + cfg.StageGCMinAge = 48 * time.Hour + cfg.StageGCUpgradeAcknowledged = true + p := profileNamed("external", []string{"java"}, jdk("temurin", 21)) + ds := buildProfileDaemonSet(cfg, &p, "agentpool", nil) + env := map[string]string{} + for _, item := range ds.Spec.Template.Spec.Containers[0].Env { + env[item.Name] = item.Value + } + wantEnabled := "false" + if enabled { + wantEnabled = "true" + } + for key, want := range map[string]string{ + "BREWLET_STAGE_GC_ENABLED": wantEnabled, + "BREWLET_STAGE_GC_INTERVAL_SECONDS": "420", + "BREWLET_STAGE_GC_MIN_AGE_SECONDS": "172800", + "BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED": "true", + } { + if env[key] != want { + t.Errorf("metrics=%t GC=%t: %s=%q, want %q", metrics, enabled, key, env[key], want) + } + } + if !ds.Spec.Template.Spec.HostPID { + t.Fatal("GC requires host PID visibility") + } + if !metrics && len(ds.Spec.Template.Spec.Containers) != 1 { + t.Fatal("GC must not depend on an exporter sidecar") + } + } + } +} diff --git a/provisioner/Dockerfile b/provisioner/Dockerfile index 4a4a5fd2..67da903d 100644 --- a/provisioner/Dockerfile +++ b/provisioner/Dockerfile @@ -46,11 +46,15 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ && CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ go build -trimpath -ldflags='-s -w' \ -o /out/brewlet-source-policy ./cmd/brewlet-source-policy \ + && CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags='-s -w' \ + -o /out/brewlet ./cmd/brewlet \ && GOOS=linux GOARCH=${TARGETARCH} \ /usr/local/bin/generate-go-notice /src /out/NOTICE.txt \ ./shim/cmd/containerd-shim-brewlet-v2 \ ./cmd/brewlet-metrics-exporter \ - ./cmd/brewlet-source-policy + ./cmd/brewlet-source-policy \ + ./cmd/brewlet # ---- stage 2: download and verify remote assets ---------------------------- FROM --platform=$BUILDPLATFORM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 AS downloaded-assets @@ -163,6 +167,7 @@ COPY --from=verified-assets /verified/NOTICE.txt /NOTICE.txt COPY --from=shim /out/containerd-shim-brewlet-v2 /opt/brewlet-dist/containerd-shim-brewlet-v2 COPY --from=shim /out/brewlet-metrics-exporter /opt/brewlet-dist/brewlet-metrics-exporter COPY --from=shim /out/brewlet-source-policy /opt/brewlet-dist/brewlet-source-policy +COPY --from=shim /out/brewlet /opt/brewlet-dist/brewlet COPY LICENSE.txt /LICENSE.txt COPY legal/CONTAINER-LEGAL-NOTICE.txt /CONTAINER-LEGAL-NOTICE.txt COPY provisioner/entrypoint.sh /usr/local/bin/brewlet-provision diff --git a/provisioner/README.md b/provisioner/README.md index 9697ad48..f39ff7c6 100644 --- a/provisioner/README.md +++ b/provisioner/README.md @@ -32,7 +32,7 @@ make provisioner-image PROVISIONER_IMAGE=ghcr.io/acme/brewlet-provisioner:1.2.3 ``` The multi-stage Docker build compiles `containerd-shim-brewlet-v2`, -`brewlet-source-policy`, and `brewlet-metrics-exporter` for the target Linux +`brewlet`, `brewlet-source-policy`, and `brewlet-metrics-exporter` for the target Linux architecture and packages them with the entrypoint. ## Runtime source model @@ -94,6 +94,11 @@ the checksum gate to reject each build. | `LAUNCHER_SOURCE__PATH` | required per entry | Clean absolute binary path inside that image | | `JDKS` / `LAUNCHERS` | derived | Internal inventories; never accepted as independent input | | `BREWLET_APP_CDS_REGENERATION_ENABLED` | `false` | Authorize node-side AppCDS regeneration for the active profile | +| `BREWLET_STAGE_GC_ENABLED` | `true` | Periodically reclaim unreferenced runnable-image stages after successful provisioning | +| `BREWLET_STAGE_GC_INTERVAL_SECONDS` | `300` | Positive whole seconds between sweeps, plus up to 10% jitter | +| `BREWLET_STAGE_GC_MIN_AGE_SECONDS` | `86400` | Minimum stage directory age, not unused age; positive whole seconds | +| `BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED` | `false` | Confirm all older unguarded shims and stage-dependent exported bundles have been retired | +| `BREWLET_RUNNABLE_STAGE` | `/tmp/brewlet-runnable` | Host staging path for GC; must match the shim's staging root | | `NODE_NAME` | downward API | Kubernetes node to label | | `BREWLET_PROFILE_NAME` | `default` | Profile name paired with its UID for managed writer authority | | `BREWLET_PROFILE_UID` | empty | Operator-managed profile UID bound to durable writer authority | @@ -130,6 +135,30 @@ root-owned, mode `0444` provisioning failure removes both. The node label is a scheduling hint; the shim checks the sentinel authoritatively. +After readiness, the provisioner runs `brewlet stage-gc` through the installed +`/usr/local/bin/brewlet-stage-gc` helper in the host PID/mount namespaces. It +rechecks managed node/profile ownership before every sweep and stops when a +successful read shows loss of authority; if the API is unreachable, it skips +that sweep and retries. Cleanup mode never starts this loop. Failures, including lock +contention, are logged and retried; successful runs report logical bytes +reclaimed, and every attempt reports process-local counters and last success. +TERM/INT stops the active sweep's process group, or the timer, before the +provisioner exits. +This is independent of the optional metrics exporter. + +Fresh nodes activate automatically. Existing shim installations or nonempty +staging roots require migration acknowledgment unless a saved +`$PREFIX/.stage-gc-compatible` record matches both installed shims and the stage +path. Compatibility is recorded even when GC is disabled so later enabling it +on a fresh, guarded installation needs no migration acknowledgment. The record +is invalidated before replacing binaries and removed on teardown. Never +acknowledge an upgrade until older unguarded shims and stage-dependent exported +bundles have been retired; reset the acknowledgment after rollout. +See [runnable stage cleanup](../docs/runnable-image.md#reclaiming-unused-stages). +The shim's environment and metrics exporter's stage mount must also be aligned +when using a nondefault staging location; setting the provisioner variable alone +does not configure those processes. + Mirror destinations are approved outside `NodeProfile` by the manager/admission `--allowed-source-mirror-hosts` setting and passed into each provisioner pod. Destination hosts, including explicit ports, must match exactly. Schemes, diff --git a/provisioner/entrypoint.sh b/provisioner/entrypoint.sh index 138f8921..98f2e6a5 100755 --- a/provisioner/entrypoint.sh +++ b/provisioner/entrypoint.sh @@ -18,8 +18,8 @@ # JDKs/launchers via annotations. # # The script is idempotent: it is safe to re-run, and only does work that is -# still missing. It publishes a container-local completion marker, then sleeps -# forever so the DaemonSet readiness probe can observe successful completion. +# still missing. It publishes a container-local completion marker, then runs +# reference-aware stage GC while the DaemonSet stays Ready. # # It also runs in a reversal mode (BREWLET_MODE=cleanup): a short-lived # brewlet-cleanup DaemonSet the operator launches for a deleted NodeProfile. @@ -57,6 +57,20 @@ JDK_ACTIVE_INVENTORY=".brewlet-active" LAUNCHER_SOURCE_METADATA=".brewlet-source" LAUNCHER_ACTIVE_INVENTORY=".brewlet-active" SOURCE_POLICY_BIN="${SOURCE_POLICY_BIN:-/opt/brewlet-dist/brewlet-source-policy}" +STAGE_GC_SRC="${STAGE_GC_SRC:-/opt/brewlet-dist/brewlet}" +HOST_STAGE_GC_PATH="${HOST_STAGE_GC_PATH:-/usr/local/bin/brewlet-stage-gc}" +STAGE_GC_ENABLED="${BREWLET_STAGE_GC_ENABLED:-true}" +STAGE_GC_INTERVAL_SECONDS="${BREWLET_STAGE_GC_INTERVAL_SECONDS:-300}" +STAGE_GC_MIN_AGE_SECONDS="${BREWLET_STAGE_GC_MIN_AGE_SECONDS:-86400}" +STAGE_GC_UPGRADE_ACKNOWLEDGED="${BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED:-false}" +STAGE_GC_ROOT="${BREWLET_RUNNABLE_STAGE:-/tmp/brewlet-runnable}" +STAGE_GC_COMPATIBLE=false +STAGE_GC_CHILD="" +STAGE_GC_GROUP="" +# Never add --no-fork: setns(CLONE_NEWPID) only affects children. The pod is +# hostPID today, but the reaper must not depend on that to join the host PID +# namespace it verifies. +STAGE_GC_NSENTER_FLAGS=(--mount --pid) # How long a rotated-out JDK/launcher root must age before it may be reclaimed. # Belt-and-braces alongside the live-mount check in reclaim_retired_roots. RETIRED_GRACE_SECONDS="${BREWLET_RETIRED_GRACE_SECONDS:-3600}" @@ -539,6 +553,151 @@ install_shim() { log "installed shim and host containerd/CRI helpers" } +stage_gc_identity() { + printf '%s\n' "$STAGE_GC_ROOT" + sha256sum "$HOST_BIN/$SHIM_NAME" "$PREFIX/bin/$SHIM_NAME" +} + +prepare_stage_gc() { + local value contents identity + for value in "$STAGE_GC_ENABLED" "$STAGE_GC_UPGRADE_ACKNOWLEDGED"; do + case "$value" in + true|false) ;; + *) die invalid-stage-gc-config "stage GC booleans must be true or false" ;; + esac + done + for value in "$STAGE_GC_INTERVAL_SECONDS" "$STAGE_GC_MIN_AGE_SECONDS"; do + [[ "$value" =~ ^[1-9][0-9]{0,9}$ ]] && (( value <= 2147483647 )) \ + || die invalid-stage-gc-config "stage GC durations must be 1..2147483647 whole seconds" + done + [[ "$STAGE_GC_ROOT" == /* && "$STAGE_GC_ROOT" != *$'\n'* ]] \ + || die invalid-stage-gc-config "stage root must be an absolute single-line host path" + STAGE_GC_COMPATIBLE=false + if [[ "$STAGE_GC_UPGRADE_ACKNOWLEDGED" == true ]]; then + STAGE_GC_COMPATIBLE=true + elif [[ -f "$PREFIX/.stage-gc-compatible" && ! -L "$PREFIX/.stage-gc-compatible" && + -f "$HOST_BIN/$SHIM_NAME" && -f "$PREFIX/bin/$SHIM_NAME" ]]; then + identity="$(stage_gc_identity)" \ + || die stage-gc-state-failed "could not verify installed shim identities" + contents="$(cat "$PREFIX/.stage-gc-compatible")" \ + || die stage-gc-state-failed "could not read stage GC compatibility record" + [[ "$contents" != "$identity" ]] || STAGE_GC_COMPATIBLE=true + elif [[ ! -e "$HOST_BIN/$SHIM_NAME" && ! -L "$HOST_BIN/$SHIM_NAME" && + ! -e "$PREFIX/bin/$SHIM_NAME" && ! -L "$PREFIX/bin/$SHIM_NAME" && + ! -e "$PREFIX/.stage-gc-compatible" && ! -L "$PREFIX/.stage-gc-compatible" ]]; then + contents="$(host_exec sh -c ' + if [ -L "$1" ]; then exit 1 + elif [ -d "$1" ]; then find "$1" -mindepth 1 -maxdepth 1 -print -quit + elif [ -e "$1" ]; then exit 1 + fi + ' sh "$STAGE_GC_ROOT")" \ + || die stage-gc-state-failed "could not inspect the host stage root before installation" + [[ -n "$contents" ]] || STAGE_GC_COMPATIBLE=true + fi + # Invalidate before replacing binaries: an interrupted upgrade must not leave + # an approval associated with a different shim installation. + rm -f "$PREFIX/.stage-gc-compatible" \ + || die stage-gc-state-failed "could not invalidate stage GC compatibility record" +} + +install_stage_gc() { + local tmp + [[ -x "$STAGE_GC_SRC" ]] \ + || die shim-image-incomplete "stage GC CLI not found in image at $STAGE_GC_SRC" + tmp="$(mktemp "$HOST_BIN/.brewlet-stage-gc.XXXXXX")" \ + || die stage-gc-state-failed "could not stage GC executable" + if ! install -m 0755 "$STAGE_GC_SRC" "$tmp" || + ! mv -f "$tmp" "$HOST_BIN/brewlet-stage-gc"; then + rm -f "$tmp" + die stage-gc-state-failed "could not install GC executable" + fi + if [[ "$STAGE_GC_COMPATIBLE" == true ]]; then + tmp="$(mktemp "$PREFIX/.stage-gc-compatible.XXXXXX")" \ + || die stage-gc-state-failed "could not stage compatibility record" + if ! stage_gc_identity >"$tmp" || ! mv -f "$tmp" "$PREFIX/.stage-gc-compatible"; then + rm -f "$tmp" + die stage-gc-state-failed "could not record stage GC compatibility" + fi + fi +} + +# Each sweep runs as the leader of its own session so shutdown can signal the +# whole tree, including the reaper that nsenter forks. +stop_stage_gc_child() { + local pid="$STAGE_GC_CHILD" group="$STAGE_GC_GROUP" attempt + STAGE_GC_CHILD="" + STAGE_GC_GROUP="" + [[ -n "$pid" ]] || return 0 + if [[ -n "$group" ]]; then + kill -TERM -- "-$group" 2>/dev/null || true + else + kill -TERM "$pid" 2>/dev/null || true + fi + wait "$pid" 2>/dev/null || true + [[ -n "$group" ]] || return 0 + for attempt in {1..50}; do + kill -0 -- "-$group" 2>/dev/null || return 0 + command sleep 0.1 + done + kill -KILL -- "-$group" 2>/dev/null || true +} + +stop_stage_gc() { + stop_stage_gc_child + exit "$1" +} + +run_stage_gc_sweep() { + setsid nsenter --target 1 "${STAGE_GC_NSENTER_FLAGS[@]}" -- "$HOST_STAGE_GC_PATH" stage-gc \ + --stage-root "$STAGE_GC_ROOT" --address "$CONTAINERD_ADDRESS" \ + --min-age "${STAGE_GC_MIN_AGE_SECONDS}s" & + STAGE_GC_CHILD=$! + # Without job control, a background child is never a group leader, so + # setsid execs in place and the child PID is also the new group ID. + STAGE_GC_GROUP="$STAGE_GC_CHILD" + local status=0 + wait "$STAGE_GC_CHILD" || status=$? + # Reap anything the sweep left behind before the next interval. + stop_stage_gc_child + return "$status" +} + +run_stage_gc_loop() { + if [[ "$STAGE_GC_ENABLED" != true ]]; then + log "stage GC disabled" + exec sleep infinity + fi + local successes=0 failures=0 last_success=never delay ownership + trap 'stop_stage_gc 143' TERM + trap 'stop_stage_gc 130' INT + while true; do + # Recheck authority after every wait and before any sweep. The operator + # drains this worker before allowing a replacement owner or cleanup worker. + # A failed API read skips this sweep; only a proven loss of authority exits. + ownership=0 + verify_node_ownership --allow-inconclusive || ownership=$? + if (( ownership != 0 )); then + log "WARN: stage GC skipped: could not verify node ownership; retrying next interval" + elif [[ "$STAGE_GC_COMPATIBLE" != true ]]; then + log "WARN: stage GC blocked: retire older unguarded shims and stage-dependent bundles, then set stageGC.upgradeAcknowledged=true" + else + if run_stage_gc_sweep; then + successes=$((successes + 1)) + last_success="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + else + failures=$((failures + 1)) + log "WARN: stage GC attempt failed (including lock contention); retrying next interval" + fi + log "stage GC: successful_sweeps=$successes failed_attempts=$failures last_success=$last_success" + fi + delay=$((STAGE_GC_INTERVAL_SECONDS + RANDOM % (STAGE_GC_INTERVAL_SECONDS / 10 + 1))) + sleep "$delay" & + STAGE_GC_CHILD=$! + wait "$STAGE_GC_CHILD" + STAGE_GC_CHILD="" + done +} + # --------------------------------------------------------------------------- # Step 2 — install JDK runtime roots. Each inventory directory is a complete # image rootfs plus metadata pointing at the JDK or jlink runtime within it. @@ -1479,23 +1638,38 @@ verify_profile_identity() { || die profile-changed "profile ${BREWLET_PROFILE_NAME} identity changed before readiness publication" } +# With --allow-inconclusive, a failed API read restores the previous authority +# and returns 75 instead of dying. Only a successful read can revoke authority, +# so a transient control-plane outage cannot crash a provisioned worker. verify_node_ownership() { [[ "$BREWLET_REQUIRE_NODE_CLAIM" == "true" ]] || return 0 + local allow_inconclusive=false previous_authorization="$NODE_WRITE_AUTHORIZED" + [[ "${1:-}" != "--allow-inconclusive" ]] || allow_inconclusive=true NODE_WRITE_AUTHORIZED=false [[ -n "$BREWLET_PROFILE_UID" && -n "$NODE_NAME" ]] \ || die ownership-fence-failed "managed writers require profile and node identities" local node_identity node_uid owner_uid owner_node_uid owner_name node_identity="$(kubectl get node "$NODE_NAME" -o \ - 'jsonpath={.metadata.uid}|{.metadata.labels.brewlet\.sh/owner-uid}|{.metadata.labels.brewlet\.sh/owner-node-uid}|{.metadata.annotations.brewlet\.sh/owner-name}')" \ - || die ownership-fence-failed "could not read the node ownership claim" + 'jsonpath={.metadata.uid}|{.metadata.labels.brewlet\.sh/owner-uid}|{.metadata.labels.brewlet\.sh/owner-node-uid}|{.metadata.annotations.brewlet\.sh/owner-name}' --request-timeout=10s)" || { + if [[ "$allow_inconclusive" == true ]]; then + NODE_WRITE_AUTHORIZED="$previous_authorization" + return 75 + fi + die ownership-fence-failed "could not read the node ownership claim" + } IFS='|' read -r node_uid owner_uid owner_node_uid owner_name <<<"$node_identity" [[ -n "$node_uid" && "$owner_uid" == "$BREWLET_PROFILE_UID" && "$owner_node_uid" == "$node_uid" && "$owner_name" == "$BREWLET_PROFILE_NAME" ]] \ || die ownership-fence-failed "node identity or ownership no longer authorizes this container" local identity uid generation deleting target_uid claimed retirement_generation retirement_phase retiring_uid retiring_claimed target_restart retiring_restart local query='{.metadata.uid}|{.metadata.generation}|{.metadata.deletionTimestamp}|{.status.targets[?(@.name=="'"$NODE_NAME"'")].uid}|{.status.targets[?(@.name=="'"$NODE_NAME"'")].claimed}|{.status.retirement.generation}|{.status.retirement.phase}|{.status.retirement.targets[?(@.name=="'"$NODE_NAME"'")].uid}|{.status.retirement.targets[?(@.name=="'"$NODE_NAME"'")].claimed}|{.status.targets[?(@.name=="'"$NODE_NAME"'")].containerdRestart}|{.status.retirement.targets[?(@.name=="'"$NODE_NAME"'")].containerdRestart}' - identity="$(kubectl get nodeprofile "$BREWLET_PROFILE_NAME" -o "jsonpath=$query")" \ - || die ownership-fence-failed "could not read the durable target ledger" + identity="$(kubectl get nodeprofile "$BREWLET_PROFILE_NAME" -o "jsonpath=$query" --request-timeout=10s)" || { + if [[ "$allow_inconclusive" == true ]]; then + NODE_WRITE_AUTHORIZED="$previous_authorization" + return 75 + fi + die ownership-fence-failed "could not read the durable target ledger" + } IFS='|' read -r uid generation deleting target_uid claimed retirement_generation retirement_phase retiring_uid retiring_claimed target_restart retiring_restart <<<"$identity" [[ "$uid" == "$BREWLET_PROFILE_UID" && "$target_uid" == "$node_uid" && "$claimed" == "true" ]] \ || die ownership-fence-failed "node target was not durably authorized by this profile incarnation" @@ -1595,7 +1769,7 @@ unpatch_containerd() { remove_shim() { rm -f "$PREFIX/bin/$SHIM_NAME" "$HOST_BIN/$SHIM_NAME" \ - "$HOST_CTR" "$HOST_CRICTL" + "$HOST_CTR" "$HOST_CRICTL" "$HOST_BIN/brewlet-stage-gc" "$PREFIX/.stage-gc-compatible" log "removed shim binaries and host containerd/CRI helpers" } @@ -1673,7 +1847,9 @@ main() { parse_runtime_sources require_cgroup_v2 preflight_sources + prepare_stage_gc install_shim + install_stage_gc install_source_mount_traps cleanup_stale_source_mounts require_containerd_image_identity @@ -1696,8 +1872,7 @@ main() { # Keep provisioning independent from observability. The exporter runs as a # sidecar in the same pod, so an exporter failure cannot reprovision the node. - log "entering idle loop; the pod stays Ready to keep the node provisioned" - exec sleep infinity + run_stage_gc_loop } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then diff --git a/provisioner/entrypoint_test.sh b/provisioner/entrypoint_test.sh index 1f11febe..eaf7ff00 100644 --- a/provisioner/entrypoint_test.sh +++ b/provisioner/entrypoint_test.sh @@ -465,6 +465,8 @@ if ( parse_runtime_sources() { return 0; } require_cgroup_v2() { return 0; } preflight_sources() { return 0; } + prepare_stage_gc() { return 0; } + install_stage_gc() { return 0; } install_shim() { printf 'install-shim\n' >>"$calls"; } install_source_mount_traps() { printf 'install-mount-traps\n' >>"$calls"; } cleanup_stale_source_mounts() { printf 'cleanup-stale-mounts\n' >>"$calls"; } @@ -1499,6 +1501,8 @@ completion_case() ( parse_runtime_sources() { completion_step parse-sources; } require_cgroup_v2() { completion_step require-cgroups; } preflight_sources() { completion_step preflight-sources; } + prepare_stage_gc() { completion_step prepare-stage-gc; } + install_stage_gc() { completion_step install-stage-gc; } install_shim() { completion_step install-shim; } install_source_mount_traps() { completion_step install-traps; } cleanup_stale_source_mounts() { completion_step cleanup-mounts; } @@ -1513,6 +1517,11 @@ completion_case() ( configure_appcds_regeneration_policy() { completion_step configure-policy; } label_node() { completion_step label-node; } cleanup_host() { completion_step cleanup-host; } + run_stage_gc_loop() { + [[ -f "$COMPLETION_FILE" ]] || exit 1 + printf 'gc-ready\n' >>"$calls" + exec sleep infinity + } main ) @@ -1524,9 +1533,15 @@ for mode in provision cleanup; do } [[ -f "$COMPLETION_FILE" ]] grep -Fxq "idle-ready" "$calls" + if [[ "$mode" == provision ]]; then + grep -Fxq "gc-ready" "$calls" + elif grep -Fq "stage-gc" "$calls" || grep -Fxq "gc-ready" "$calls"; then + echo "cleanup mode must not install or start stage GC" >&2 + exit 1 + fi done -for scenario in provision:parse-sources provision:install-sources \ +for scenario in provision:parse-sources provision:prepare-stage-gc provision:install-stage-gc provision:install-sources \ provision:activate-containerd provision:label-node \ cleanup:cleanup-mounts cleanup:cleanup-host; do if completion_case "${scenario%%:*}" "${scenario#*:}" >/dev/null 2>&1; then @@ -1636,3 +1651,5 @@ if output="$( exit 1 fi grep -Fq "ERROR: completion-state-failed" <<<"$output" + +bash "$repo_root/provisioner/stage_gc_test.sh" diff --git a/provisioner/stage_gc_test.sh b/provisioner/stage_gc_test.sh new file mode 100644 index 00000000..91225971 --- /dev/null +++ b/provisioner/stage_gc_test.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -euo pipefail +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +source "$repo_root/provisioner/entrypoint.sh" +work="$(mktemp -d)" +loop_pid="" +trap 'if [[ -n "$loop_pid" ]]; then kill -TERM "$loop_pid" 2>/dev/null || true; wait "$loop_pid" 2>/dev/null || true; fi; rm -rf "$work"' EXIT + +fixture() { + local dir + dir="$(mktemp -d "$work/case.XXXXXX")" + PREFIX="$dir/prefix" + HOST_BIN="$dir/bin" + STAGE_GC_ROOT="$dir/stages" + STAGE_GC_SRC="$dir/brewlet" + SHIM_SRC="$dir/shim" + STAGE_GC_ENABLED=true + STAGE_GC_INTERVAL_SECONDS=1 + STAGE_GC_MIN_AGE_SECONDS=86400 + STAGE_GC_UPGRADE_ACKNOWLEDGED=false + STAGE_GC_COMPATIBLE=false + STAGE_GC_CHILD="" + STAGE_GC_GROUP="" + NODE_NAME="" + setsid() { "$@"; } + mkdir -p "$PREFIX/bin" "$HOST_BIN" + printf '#!/bin/sh\nexit 0\n' >"$STAGE_GC_SRC" + chmod +x "$STAGE_GC_SRC" + printf 'guarded-shim\n' >"$SHIM_SRC" + calls="$dir/calls" + : >"$calls" + host_exec() { "$@"; } + die() { printf '%s: %s\n' "$1" "$*" >&2; exit 1; } +} + +install_fixture() { + cp "$SHIM_SRC" "$HOST_BIN/$SHIM_NAME" + cp "$SHIM_SRC" "$PREFIX/bin/$SHIM_NAME" + install_stage_gc +} + +# Fresh nodes activate even when the optional exporter created an empty root. +for root in absent empty; do + ( + fixture + [[ "$root" != empty ]] || mkdir "$STAGE_GC_ROOT" + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + install_fixture + [[ -x "$HOST_BIN/brewlet-stage-gc" ]] + [[ -s "$PREFIX/.stage-gc-compatible" ]] + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + printf 'next-guarded-shim\n' >"$SHIM_SRC" + install_fixture + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + ) +done + +# Existing shims or any retained staging tree require explicit acknowledgment. +for existing in shim stage; do + ( + fixture + if [[ "$existing" == shim ]]; then + printf 'old-shim\n' >"$HOST_BIN/$SHIM_NAME" + else + mkdir -p "$STAGE_GC_ROOT/immutable-v2/retained" + fi + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == false ]] + install_fixture + [[ ! -e "$PREFIX/.stage-gc-compatible" ]] + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == false ]] + STAGE_GC_UPGRADE_ACKNOWLEDGED=true + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + install_fixture + STAGE_GC_UPGRADE_ACKNOWLEDGED=false + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + ) +done + +# A rollback or changed staging location invalidates previously saved approval. +for changed in host-shim prefix-shim root; do + ( + fixture + prepare_stage_gc + install_fixture + case "$changed" in + host-shim) printf 'old-shim\n' >"$HOST_BIN/$SHIM_NAME" ;; + prefix-shim) printf 'old-shim\n' >"$PREFIX/bin/$SHIM_NAME" ;; + root) STAGE_GC_ROOT="$STAGE_GC_ROOT/other" ;; + esac + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == false ]] + [[ ! -e "$PREFIX/.stage-gc-compatible" ]] + ) +done + +for invalid in zero negative fractional overflow boolean age inspection symlink; do + if ( + fixture + case "$invalid" in + zero) STAGE_GC_INTERVAL_SECONDS=0 ;; + negative) STAGE_GC_INTERVAL_SECONDS=-1 ;; + fractional) STAGE_GC_INTERVAL_SECONDS=1.5 ;; + overflow) STAGE_GC_INTERVAL_SECONDS=2147483648 ;; + boolean) STAGE_GC_ENABLED=yes ;; + age) STAGE_GC_MIN_AGE_SECONDS=0 ;; + inspection) host_exec() { return 1; } ;; + symlink) ln -s "$PREFIX" "$STAGE_GC_ROOT" ;; + esac + prepare_stage_gc + ) >"$work/invalid.log" 2>&1; then + echo "stage GC accepted invalid configuration or inspection: $invalid" >&2 + exit 1 + fi +done + +# Retry failures, recheck authority on every tick, and preserve all host flags. +( + fixture + STAGE_GC_COMPATIBLE=true + checks=0 + verify_node_ownership() { + checks=$((checks + 1)) + if (( checks == 3 )); then + [[ "$(grep -c '^sweep ' "$calls")" == 2 ]] + grep -Fq -- '--target 1 --mount --pid -- /usr/local/bin/brewlet-stage-gc stage-gc' "$calls" + ! grep -Fq -- '--no-fork' "$calls" + grep -Fq -- "--stage-root $STAGE_GC_ROOT --address $CONTAINERD_ADDRESS --min-age 86400s" "$calls" + exit 0 + fi + } + nsenter() { + printf 'sweep %s\n' "$*" >>"$calls" + [[ "$(grep -c '^sweep ' "$calls")" != 1 ]] + } + sleep() { [[ "$1" == 1 ]]; } + run_stage_gc_loop +) >"$work/loop.log" +grep -Eq 'successful_sweeps=1 failed_attempts=1 last_success=[0-9]{4}-' "$work/loop.log" + +( + fixture + STAGE_GC_COMPATIBLE=true + STAGE_GC_INTERVAL_SECONDS=300 + checks=0 + verify_node_ownership() { + checks=$((checks + 1)) + [[ "$checks" != 2 ]] || exit 0 + } + nsenter() { return 0; } + sleep() { (( $1 >= 300 && $1 <= 330 )); } + run_stage_gc_loop +) >"$work/jitter.log" + +for state in blocked lost-ownership; do + ( + fixture + [[ "$state" != lost-ownership ]] || STAGE_GC_COMPATIBLE=true + checks=0 + verify_node_ownership() { + checks=$((checks + 1)) + [[ "$state" != lost-ownership && "$checks" != 2 ]] || exit 42 + } + nsenter() { echo 'unexpected-sweep' >"$calls"; } + sleep() { return 0; } + run_stage_gc_loop + ) >"$work/$state.log" 2>&1 && { + echo "expected worker to stop when ownership verification exits" >&2 + exit 1 + } +done +# An inconclusive API read skips the sweep but keeps the worker running. +( + fixture + STAGE_GC_COMPATIBLE=true + checks=0 + verify_node_ownership() { + [[ "$1" == --allow-inconclusive ]] + checks=$((checks + 1)) + (( checks != 1 )) || return 75 + [[ "$(grep -c '^sweep' "$calls")" == 0 ]] + exit 0 + } + nsenter() { echo 'sweep' >>"$calls"; } + sleep() { return 0; } + run_stage_gc_loop +) >"$work/inconclusive.log" 2>&1 +grep -Fq 'stage GC skipped: could not verify node ownership' "$work/inconclusive.log" + +# The real fence restores prior authority on failed reads in background mode +# and still dies at startup. +( + fixture + BREWLET_REQUIRE_NODE_CLAIM=true + BREWLET_PROFILE_UID=profile-uid + BREWLET_PROFILE_NAME=profile + NODE_NAME=node + NODE_WRITE_AUTHORIZED=true + kubectl() { return 1; } + status=0 + verify_node_ownership --allow-inconclusive || status=$? + [[ "$status" == 75 && "$NODE_WRITE_AUTHORIZED" == true ]] + if (verify_node_ownership) 2>/dev/null; then exit 1; fi +) + +if grep -Rq 'unexpected-sweep' "$work"; then + echo "blocked or unauthorized worker invoked GC" >&2 + exit 1 +fi +grep -Fq 'stage GC blocked' "$work/blocked.log" + +( + fixture + STAGE_GC_ENABLED=false + prepare_stage_gc + install_fixture + STAGE_GC_ENABLED=true + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + STAGE_GC_ENABLED=false + verify_node_ownership() { exit 1; } + nsenter() { exit 1; } + exec() { [[ "$*" == "sleep infinity" ]]; exit 0; } + run_stage_gc_loop +) + +# Signal the supervisor during a sweep and while sleeping; no descendant may +# outlive it. The fake nsenter forks like the real one and does not forward +# TERM, so only the process-group shutdown reaches the grandchild. +test_child_script="$work/child.sh" +fake_bin="$work/fake-bin" +mkdir -p "$fake_bin" +export test_child_script +cat >"$test_child_script" <<'EOF' +printf '%s\n' "$$" >"$child_file" +exec /bin/sleep 60 +EOF +cat >"$fake_bin/nsenter" <<'EOF' +#!/usr/bin/env bash +trap 'exit 143' TERM +/bin/sh "$test_child_script" & +wait +EOF +chmod +x "$fake_bin/nsenter" +for phase in sweep sleep; do + child_file="$work/$phase.pid" + export phase child_file + PATH="$fake_bin:$PATH" bash -c ' + source "$1" + STAGE_GC_COMPATIBLE=true + STAGE_GC_ENABLED=true + verify_node_ownership() { :; } + if [[ "$phase" == sleep ]]; then + nsenter() { return 0; } + setsid() { "$@"; } + fi + sleep() { + exec /bin/sh "$test_child_script" + } + run_stage_gc_loop + ' bash "$repo_root/provisioner/entrypoint.sh" >"$work/signal.log" 2>&1 & + loop_pid=$! + for attempt in {1..100}; do + [[ ! -s "$child_file" ]] || break + /bin/sleep 0.05 + done + [[ -s "$child_file" ]] || { cat "$work/signal.log" >&2; exit 1; } + child_pid="$(cat "$child_file")" + kill -TERM "$loop_pid" + status=0 + wait "$loop_pid" || status=$? + loop_pid="" + [[ "$status" == 143 ]] || { echo "unexpected shutdown status $status" >&2; exit 1; } + if kill -0 "$child_pid" 2>/dev/null; then + echo "GC supervisor left child $child_pid running" >&2 + exit 1 + fi +done + +# Prove the configured nsenter flags place the helper itself, not just its +# children, in the target PID namespace. setns(2) can only enter a descendant +# PID namespace, so target a nested one. Requires root and util-linux. +if [[ "$(id -u)" == 0 ]] && command -v unshare >/dev/null && command -v nsenter >/dev/null \ + && unshare --pid --fork true 2>/dev/null; then + unshare --pid --fork /bin/sleep 30 & + unshare_pid=$! + target="" + for attempt in {1..100}; do + target="$(pgrep -P "$unshare_pid" || true)" + [[ -z "$target" ]] || break + /bin/sleep 0.05 + done + [[ -n "$target" ]] || { echo "nested PID namespace did not start" >&2; exit 1; } + target_ns="$(readlink "/proc/$target/ns/pid")" + entered="$(nsenter --target "$target" "${STAGE_GC_NSENTER_FLAGS[@]}" -- readlink /proc/self/ns/pid)" + pkill -P "$unshare_pid" || true + wait "$unshare_pid" 2>/dev/null || true + [[ "$target_ns" != "$(readlink /proc/self/ns/pid)" && "$entered" == "$target_ns" ]] || { + echo "nsenter flags did not place the helper in the target PID namespace" >&2 + exit 1 + } +else + echo "skipping PID namespace test (requires root, unshare, and nsenter)" +fi + +echo "Stage GC provisioner tests passed." diff --git a/specs/SPECIFICATION.md b/specs/SPECIFICATION.md index 3b169dd7..bbdbfd10 100644 --- a/specs/SPECIFICATION.md +++ b/specs/SPECIFICATION.md @@ -787,16 +787,73 @@ or readiness advertisement. Brewlet has no built-in runtime catalog. Their exact keys, token grammar, presence semantics, compatibility guarantees, and autoscaler integration are defined by the public [capability-label contract](CAPABILITY_LABELS.md). -9. Publishes the container-local `/tmp/brewlet-complete` marker and idles. +9. Publishes the container-local `/tmp/brewlet-complete` marker, then runs + periodic runnable-stage cleanup when enabled and compatible (§5.2.1). + Disabled provisioners and completed cleanup-mode workers idle instead. Provisioner and cleanup containers have an exec readiness probe for that marker. Each entrypoint invocation removes stale completion state before doing any work; failures do not publish completion. This gate is independent of the optional runtime smoke checks and has no startup deadline that could kill a legitimately slow installation. + Readiness certifies provisioning, not GC activation or successful sweeps. The `brewlet.sh/runtime=ready` label is used by the `RuntimeClass` `nodeSelector` so workloads only schedule onto provisioned nodes. +#### 5.2.1 Runnable-stage cleanup lifecycle + +The existing provisioner owns the automatic stage-GC loop; no additional +DaemonSet, CronJob, or kubelet API access is required. Policy is operator-wide, +including externally managed NodeProfiles, and is independent of +`metrics.enabled`: + +| Helm setting | Default | Contract | +|---|---|---| +| `stageGC.enabled` | `true` | Enable automatic sweeps after successful provisioning and compatibility checks | +| `stageGC.interval` | `5m` | Delay after each attempt, plus up to 10% jitter; sweeps do not overlap | +| `stageGC.minAge` | `24h` | Minimum published stage directory age, not unused age or time since references disappeared | +| `stageGC.upgradeAcknowledged` | `false` | Confirm retirement of older unguarded shims, stage-dependent exported bundles, and other unguarded consumers across the managed fleet | + +Durations must resolve to positive whole seconds, at most `2147483647s`. +The operator validates and passes them to the provisioner; this is not a +per-profile `spec` field. The complete flags and environment mapping is in +[Configuration](../docs/configuration.md#runnable-image-stage-cleanup). + +Fresh installations with no installed shim copies and an absent or empty stage +root activate automatically. Existing installations need a matching saved +compatibility record or explicit migration acknowledgment. The root-owned +record (§14.5) binds approval to both installed shim copies and the stage path, +is invalidated before replacement, and is renewed only for a compatible +installation. It is maintained even when sweeps are disabled. A detected +identity/path change or interrupted installation can require acknowledgment +again at startup. Reset the fleet-wide acknowledgment after rollout; compatible +nodes retain approval. Installing new binaries alone does not prove that older +running shim processes or exported bundles have been retired. + +Each managed sweep rechecks node/profile ownership before invoking the reaper +in the host PID and mount namespaces; `nsenter` MUST fork so the reaper itself +joins the host PID namespace, and each sweep runs in its own process group. A +successful read showing loss of authority stops the worker; an unreadable node +or NodeProfile (API timeout or outage) skips that sweep without exiting or +changing node advertisements; +cleanup-mode workers never start this loop. Reference-inspection errors and +lock contention are logged and retried at the next interval, without treating a +failed sweep as a provisioning failure. Blocked compatibility also leaves the +node provisioned but logs `stage GC blocked`. TERM/INT signals the active +sweep's whole process group, or the timer, before supervisor exit. Each reaper uses a five-minute context deadline; +filesystem operations already in progress may take longer to return. + +Kubelet and containerd remain responsible for image eviction and content +reclamation. Brewlet neither deletes their images nor reads, copies, or changes +kubelet GC configuration. It follows released references through eventual +orphan reclamation (§6.3), not a second disk-pressure eviction policy. +There is no guarantee of immediate disk relief or a bounded total stage size. + +Manual `stage-gc` invocations apply the same reaper safety checks but do not +enforce the provisioner's enablement, ownership, or migration gate. Do not +schedule a second timer on managed nodes or use manual deletion to bypass +blocked activation. See the [upgrade procedure](../docs/installation.md#activating-runnable-stage-gc). + ### 5.3 Installing JDK runtime roots on nodes Which JDKs a node offers is **declarative**: the platform team lists them in the @@ -948,11 +1005,15 @@ directory (`Dockerfile` + `entrypoint.sh`) and deployed by [`deploy/node-provisioner.yaml`](../kubernetes/deploy/node-provisioner.yaml): - **Image** — a multi-stage build that compiles - `containerd-shim-brewlet-v2` and `brewlet-source-policy` from the + `containerd-shim-brewlet-v2`, `brewlet`, `brewlet-source-policy`, and + `brewlet-metrics-exporter` from the [core runtime](https://github.com/microsoft/brewlet) for the target architecture (so installed binaries always match the node arch), then assembles a small - Debian-based runtime carrying the entrypoint, source-policy validator, and - `bash`/`curl`/`kubectl`. Build with `make provisioner-image` + Debian-based runtime carrying the entrypoint, compiled binaries, and host + tools including `bash`, `kubectl`, `ctr`, `crictl`, and `nsenter`. + The CLI is installed on the host as `/usr/local/bin/brewlet-stage-gc` for + one-shot sweeps; it is invoked with the `stage-gc` subcommand. + Build with `make provisioner-image` (single arch) or `make provisioner-image-push` (multi-arch via buildx). - **Entrypoint** — an idempotent script that performs all §5.2 steps: validates all indexed JDK/launcher sources before host mutation; installs @@ -969,7 +1030,8 @@ directory (`Dockerfile` + `entrypoint.sh`) and deployed by annotates the installed JDKs/launchers, and emits the per-capability scheduling labels the admission webhook uses (`brewlet.sh/jdk.*`, `brewlet.sh/jdk-feature.*`, `brewlet.sh/launcher.*`). The manifest ships the `ServiceAccount` + `ClusterRole`/binding - (`get`/`patch` on nodes) the labelling step needs. + (`get`/`patch` on nodes) the labelling step needs. After publishing completion, + the provisioner continues with the cleanup lifecycle in §5.2.1. Node provisioning is driven by the **operator** (§8.1) and admission is handled by the **pod webhook** (§8.3). The whole set is packaged by @@ -1320,6 +1382,25 @@ and builds/runs on Linux: source bytes. This warm-reuse protection does not guarantee cold startup after source-layer GC. Re-pull affected digest-pinned images with packed-layer retention enabled, and allow extra disk capacity for retained layer bytes. +- Runnable-stage reclamation is reference-aware and Linux-only. Only canonical + published `immutable-v2/` directories with modification times + older than the positive age floor are eligible. Containerd image targets, + index-referenced platform manifests, content records and content-reference + labels in **any namespace** protect their stages. Removing an image record + alone may not release a stage until containerd also collects its content. + Live mounts across process mount namespaces, including individual file bind + mounts, protect the corresponding stage. + Launchers hold a shared staging guard through resolution and mount creation + (or direct execution); cleanup requires the exclusive guard. Reclamation + requires complete reference inspection in the initial host PID/user and host + mount namespaces with a complete `/proc`; ambiguous or unreadable evidence + prevents deletion. The root is administrator-owned and must not be renamed + or replaced while consumers or cleanup run. Legacy and abandoned pending + trees remain outside automated eviction. Open file descriptors alone are + not tracked: direct readers must hold the guard or be stopped. New exported + runnable-image bundles retain their own payload copies rather than depend on + the evictable stage; legacy bundles require migration before GC activation. + These rules govern both automatic and manual sweeps. The workload image reference and manifest digest hints are managed **cluster-side, not in the shim**: the `brewlet-admission` webhook (§8.3) overwrites the @@ -1820,6 +1901,14 @@ other, so each shape behaves as plain Kubernetes does: NodeProfile readiness/provisioning metrics and admission outcomes (including `NoCompatibleJDK`, `NoCompatibleLauncher`, and `NoCompatibleArch`) to their controller-runtime endpoints. +- **Stage cleanup observability:** the optional exporter exposes + `brewlet_runnable_stage_bytes` from a read-only host stage mount. It measures + logical regular-file bytes, including legacy and pending trees, not allocated + space, free space, or reclaimable bytes. It does not follow symlinks; inspection + errors fail the scrape rather than report zero. Automatic cleanup runs + independently of metrics. Removed stage counts/bytes, process-local + success/failure counters, and last-success timestamps are provisioner logs, + not additional Prometheus GC metrics. Monitor filesystem free space as well. - **Metric discovery:** with `metrics.enabled=true`, the Helm chart creates Services for node, operator, and admission metrics. `metrics.serviceMonitor.enabled` and @@ -1961,6 +2050,8 @@ repurposed. | `invalid-jdk-source` | A JDK source entry is missing fields, has a malformed `-` token, or is duplicated | | `invalid-launcher-source` | A launcher source entry is missing fields, has a malformed or reserved name, or is duplicated | | `invalid-restart-mode` | `rollout.containerdRestart` is not `validated` / `sighup` / `none` | +| `invalid-stage-gc-config` | Provisioner GC booleans, integer-second durations, or the host staging path are invalid | +| `stage-gc-state-failed` | GC compatibility inspection/recording or helper installation failed; distinct from a periodic sweep failure, which is logged and retried | | `unsupported-architecture` | The node's architecture is not supported | | `host-tooling-missing` | A required host tool (`nsenter`, the host `ctr` helper) is unavailable | | `completion-state-failed` | The container-local readiness marker could not be reset or published; the operation exits unsuccessfully | @@ -1969,7 +2060,7 @@ repurposed. | `containerd-version-invalid` | The reported containerd server version could not be parsed | | `unsupported-containerd-version` | containerd is older than 2.0 (protected CRI requested-image metadata is required) | | `cgroup-v2-required` | The node is cgroup-v1 only | -| `shim-image-incomplete` | The provisioner image is missing the shim / `ctr` / `crictl` binary | +| `shim-image-incomplete` | The provisioner image is missing the shim / `ctr` / `crictl` binary or stage-GC CLI | | `shim-install-failed` | The shim was not present after installation | | `jdk-source-missing` | No configured source exists for a requested JDK token (there is no built-in catalog — §5.2) | | `jdk-install-failed` | Mounting, copying, activating, or retaining a JDK root failed | @@ -2024,6 +2115,7 @@ All are opt-in (`metrics.enabled=true`) and bounded-cardinality. | `brewlet_artifact_resolution_duration_seconds` | Shim → node exporter | | `brewlet_cds_regeneration_decisions_total`, `brewlet_cds_archive_mapped` | Shim → node exporter | | `brewlet_jdk_info`, `brewlet_jdk_installed_timestamp_seconds`, `brewlet_launcher_info` | Node exporter | +| `brewlet_runnable_stage_bytes` | Node exporter; logical regular-file bytes under the host stage root, refreshed per scrape, with no metric-specific labels | | `brewlet_telemetry_events_invalid_total` | Node exporter | | `brewlet_node_provision_transitions_total` | Operator | | `brewlet_nodeprofile_condition`, `brewlet_nodeprofile_nodes` | Operator | @@ -2047,6 +2139,23 @@ read-only mounts §6.1 describes. | `metrics/telemetry.sock` | Unix datagram socket the shim writes best-effort telemetry to | | `.retired../` | A rotated-out JDK/launcher root, retained while live mounts may reference it (§12) | | `.image-mount-*` | Transient source-image mount points; reclaimed on the next pass | +| `.stage-gc-compatible` | Root-owned record binding GC compatibility to the installed shim copies and stage path; checked at provisioner startup, invalidated before replacing binaries, and removed during teardown | + +Related host paths outside this prefix: + +| Path | Contents | +|---|---| +| `/usr/local/bin/brewlet-stage-gc` | Provisioner-installed CLI helper, invoked as `brewlet-stage-gc stage-gc`; removed during teardown | +| `/tmp/brewlet-runnable/immutable-v2//` | Published immutable runnable-image stages eligible for reference-aware reclamation (§6.3) | + +The provisioner's default host stage root is `/tmp/brewlet-runnable`, overridden +by `BREWLET_RUNNABLE_STAGE`. When that variable is unset, the shim and the +standalone CLI on Linux use the same fixed `/tmp/brewlet-runnable`, ignoring +`TMPDIR` so that a containerd service `TMPDIR` cannot diverge the shim's root +from the reaper's; on other platforms the CLI uses +`os.TempDir()/brewlet-runnable`. A custom root must +be configured consistently for the shim, GC, and the exporter's read-only host +mount; configuring the provisioner alone does not reconfigure other processes. ---