diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1e87c536..5ea85554 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -102,6 +102,11 @@ jobs: java: true cluster: true helm: false + - name: "Tier 17: Runnable-stage GC" + tiers: --tier 17 + java: true + cluster: true + helm: true steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/core/cmd/brewlet-metrics-exporter/README.md b/core/cmd/brewlet-metrics-exporter/README.md index ed630c5f..e1e59ede 100644 --- a/core/cmd/brewlet-metrics-exporter/README.md +++ b/core/cmd/brewlet-metrics-exporter/README.md @@ -3,7 +3,34 @@ The node-local exporter receives bounded, best-effort telemetry from the containerd shim over `/opt/brewlet/metrics/telemetry.sock` and exposes Prometheus metrics for sandbox launches, artifact resolution, AppCDS decisions, -and installed JDK and launcher inventory. +installed JDK and launcher inventory, and runnable-image staging usage. + +`brewlet_runnable_stage_bytes` reports logical regular-file bytes under +`--stage-root` (default: `BREWLET_RUNNABLE_STAGE`, or `/tmp/brewlet-runnable` +on Linux regardless of `TMPDIR`). The directory must be the host's actual staging +root; a containerized exporter needs a read-only host mount. The operator +provides that mount for the default location. The gauge includes legacy and +pending stages, does not follow symlinks, and refreshes on every scrape. +Inspection errors fail the scrape instead of reporting zero. + +On Helm-managed nodes, the provisioner already schedules stage GC by default, +independently of whether this exporter is enabled. The exporter never deletes +stages. Do not add a second timer: it would run outside the operator's +`stageGC.enabled` setting and migration gate. + +To inspect eligibility on a managed Linux node, run the installed helper as +root in the host namespaces: + +```sh +sudo /usr/local/bin/brewlet-stage-gc stage-gc --dry-run +``` + +Manual invocations retain the reaper's reference and mount checks but do not +consult the provisioner's compatibility record or upgrade acknowledgment. +Outside the provisioner, install the standalone `brewlet` CLI and arrange a +host timer only after completing the same migration prerequisites. +See [stage cleanup](../../../docs/runnable-image.md#reclaiming-unused-stages) +for reference checks and rollout precautions. ## Screenshots diff --git a/core/cmd/brewlet-metrics-exporter/main.go b/core/cmd/brewlet-metrics-exporter/main.go index cbc75a6f..9186a35b 100644 --- a/core/cmd/brewlet-metrics-exporter/main.go +++ b/core/cmd/brewlet-metrics-exporter/main.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "github.com/microsoft/brewlet/internal/runnablestage" "github.com/microsoft/brewlet/internal/telemetry" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -245,10 +246,12 @@ func main() { root string listenAddr string socketPath string + stageRoot string ) flag.StringVar(&root, "root", "/opt/brewlet", "Brewlet host-state root") flag.StringVar(&listenAddr, "listen-address", ":9090", "HTTP metrics listen address") flag.StringVar(&socketPath, "socket-path", "", "shim telemetry Unix datagram socket") + flag.StringVar(&stageRoot, "stage-root", runnablestage.Root(), "runnable staging root (BREWLET_RUNNABLE_STAGE)") flag.Parse() if socketPath == "" { socketPath = filepath.Join(root, "metrics", "telemetry.sock") @@ -261,6 +264,7 @@ func main() { registry.MustRegister(prometheus.NewGoCollector(), prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) events := newEventMetrics(registry) registry.MustRegister(newInventoryCollector(root)) + registry.MustRegister(newStageCollector(stageRoot)) server := &http.Server{ Addr: listenAddr, diff --git a/core/cmd/brewlet-metrics-exporter/stages.go b/core/cmd/brewlet-metrics-exporter/stages.go new file mode 100644 index 00000000..d677c3c9 --- /dev/null +++ b/core/cmd/brewlet-metrics-exporter/stages.go @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "github.com/microsoft/brewlet/internal/runnablestage" + "github.com/prometheus/client_golang/prometheus" +) + +type stageCollector struct { + root string + bytes *prometheus.Desc +} + +func newStageCollector(root string) *stageCollector { + return &stageCollector{ + root: root, + bytes: prometheus.NewDesc( + "brewlet_runnable_stage_bytes", + "Logical bytes remaining in runnable staging trees, including legacy and pending stages.", + nil, nil, + ), + } +} + +func (c *stageCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.bytes +} + +func (c *stageCollector) Collect(ch chan<- prometheus.Metric) { + size, err := runnablestage.Bytes(c.root) + if err != nil { + ch <- prometheus.NewInvalidMetric(c.bytes, err) + return + } + ch <- prometheus.MustNewConstMetric(c.bytes, prometheus.GaugeValue, float64(size)) +} diff --git a/core/cmd/brewlet-metrics-exporter/stages_test.go b/core/cmd/brewlet-metrics-exporter/stages_test.go new file mode 100644 index 00000000..b0647b33 --- /dev/null +++ b/core/cmd/brewlet-metrics-exporter/stages_test.go @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestStageCollector(t *testing.T) { + root := filepath.Join(t.TempDir(), "stages") + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(newStageCollector(root)) + check := func(value string) { + t.Helper() + expected := ` +# HELP brewlet_runnable_stage_bytes Logical bytes remaining in runnable staging trees, including legacy and pending stages. +# TYPE brewlet_runnable_stage_bytes gauge +brewlet_runnable_stage_bytes ` + value + "\n" + if err := testutil.GatherAndCompare(reg, strings.NewReader(expected)); err != nil { + t.Fatal(err) + } + } + check("0") + dir := filepath.Join(root, "immutable-v2", strings.Repeat("a", 64)) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + payload := filepath.Join(dir, "payload") + if err := os.WriteFile(payload, []byte("12345"), 0o644); err != nil { + t.Fatal(err) + } + check("5") + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } + check("0") +} + +func TestStageCollectorReportsReadErrors(t *testing.T) { + root := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(root, nil, 0o644); err != nil { + t.Fatal(err) + } + reg := prometheus.NewRegistry() + reg.MustRegister(newStageCollector(root)) + if _, err := reg.Gather(); err == nil { + t.Fatal("expected a scrape error, not a misleading zero") + } +} diff --git a/core/cmd/brewlet/main.go b/core/cmd/brewlet/main.go index 7d3b639c..cdd3aa07 100644 --- a/core/cmd/brewlet/main.go +++ b/core/cmd/brewlet/main.go @@ -27,6 +27,7 @@ import ( "github.com/microsoft/brewlet/internal/artifact" "github.com/microsoft/brewlet/internal/kube" + "github.com/microsoft/brewlet/internal/runnablestage" "github.com/microsoft/brewlet/internal/runtime" ) @@ -55,6 +56,8 @@ func main() { err = cmdJDKs(os.Args[2:]) case "doctor": err = cmdDoctor(os.Args[2:]) + case "stage-gc": + err = cmdStageGC(os.Args[2:]) case "k8s": err = kube.Run(context.Background(), os.Args[2:], os.Stdout, os.Stderr) case "version", "--version": @@ -86,6 +89,7 @@ USAGE: brewlet bundle [--store DIR] [--cpu N] [--memory M] [--uid UID] [--gid GID] [--jdk-root DIR] [--launcher NAME] [--launcher-root DIR] [--out DIR] brewlet jdks [--output table|wide|json] [--kubeconfig FILE] [--context CTX] [--selector SEL] brewlet doctor [--namespace NS] [--output table|json] [--kubeconfig FILE] [--context CTX] + brewlet stage-gc [--stage-root DIR] [--address SOCKET] [--min-age 24h] [--dry-run] brewlet k8s inventory, status, inspection, installation and profile updates (see k8s --help) brewlet version @@ -674,6 +678,11 @@ func cmdRun(args []string) error { } ref := pos[0] + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return err + } + defer guard.Close() s := artifact.Store{Root: *store} blobs, err := s.ResolveBlobs(ref) if err != nil { @@ -736,6 +745,8 @@ func cmdRun(args []string) error { fmt.Printf("[brewlet] sandbox : %s\n", sandbox) fmt.Printf("[brewlet] launch : %s\n", plan.CommandLine()) fmt.Printf("[brewlet] ----- JVM output below -----\n") + // The sandbox and optional CDS cache now own all payloads. + _ = guard.Close() return plan.Run() } @@ -766,21 +777,35 @@ func cmdBundle(args []string) error { if err != nil { return err } + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return err + } + defer guard.Close() s := artifact.Store{Root: *store} blobs, err := s.ResolveBlobs(pos[0]) if err != nil { return err } + res := runtime.Resources{CPULimit: *cpu, MemoryLimit: *mem} + if err := res.Validate(); err != nil { + return err + } + cleanup, err := retainBundlePayloads(&blobs, *out) + if err != nil { + return err + } cfg := blobs.Config cdsSrc := blobs.CDSHostPath if err := runtime.GenerateBundleWithIdentityAndRegen(cfg, *jdkRoot, *launcherRoot, *launcher, blobs.JarHostPath, blobs.ClasspathHostPaths, blobs.ModulepathHostPaths, cdsSrc, *out, - runtime.Resources{CPULimit: *cpu, MemoryLimit: *mem}, nil, runtime.ProcessIdentity{UID: uid, GID: gid}, + res, nil, runtime.ProcessIdentity{UID: uid, GID: gid}, runtime.CDSRegenOptions{ Regenerate: *appcdsRegen, CacheScope: "local", ArtifactDigest: blobs.ManifestDigest, CacheDir: os.Getenv("BREWLET_CDS_CACHE"), }); err != nil { + cleanup() return err } fmt.Printf("wrote OCI runtime bundle to %s/config.json\n", *out) diff --git a/core/cmd/brewlet/stage_gc.go b/core/cmd/brewlet/stage_gc.go new file mode 100644 index 00000000..1ac27a11 --- /dev/null +++ b/core/cmd/brewlet/stage_gc.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/microsoft/brewlet/internal/artifact" + "github.com/microsoft/brewlet/internal/runnablestage" +) + +func cmdStageGC(args []string) error { + fs := flag.NewFlagSet("stage-gc", flag.ContinueOnError) + root := fs.String("stage-root", runnablestage.Root(), "runnable staging root (BREWLET_RUNNABLE_STAGE)") + address := fs.String("address", "/run/containerd/containerd.sock", "containerd socket; all namespaces are checked") + age := fs.Duration("min-age", runnablestage.DefaultMinAge, "minimum stage age (must be positive)") + dryRun := fs.Bool("dry-run", false, "report eligible trees without deleting them") + if err := fs.Parse(args); err != nil { + if err == flag.ErrHelp { + return nil + } + return err + } + if fs.NArg() != 0 || *age <= 0 { + return fmt.Errorf("usage: stage-gc [--stage-root DIR] [--address SOCKET] [--min-age positive-duration] [--dry-run]") + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + result, err := runnablestage.Reap(ctx, *address, runnablestage.Options{ + Root: *root, MinAge: *age, DryRun: *dryRun, + }) + if err != nil { + return err + } + action := "removed" + if *dryRun { + action = "would remove" + } + fmt.Printf("%s %d stages (%d bytes); remaining stage bytes: %d\n", action, result.Removed, result.ReclaimedBytes, result.RemainingBytes) + return nil +} + +// An exported bundle may be started long after this command exits. Give it +// independent payloads rather than dangling references into the evictable cache. +func retainBundlePayloads(blobs *artifact.ResolvedBlobs, out string) (func(), error) { + if blobs.Format != "image" { + return func() {}, nil + } + out, err := filepath.Abs(out) + if err != nil { + return nil, err + } + if err := os.MkdirAll(out, 0o755); err != nil { + return nil, err + } + dir, err := os.MkdirTemp(out, "brewlet-image-") + if err != nil { + return nil, err + } + cleanup := func() { _ = os.RemoveAll(dir) } + if err := os.CopyFS(dir, os.DirFS(filepath.Dir(blobs.JarHostPath))); err != nil { + cleanup() + return nil, fmt.Errorf("retain bundle payloads: %w", err) + } + blobs.JarHostPath = filepath.Join(dir, filepath.Base(blobs.JarHostPath)) + if blobs.CDSHostPath != "" { + blobs.CDSHostPath = filepath.Join(dir, filepath.Base(blobs.CDSHostPath)) + } + return cleanup, nil +} diff --git a/core/cmd/brewlet/stage_gc_linux_test.go b/core/cmd/brewlet/stage_gc_linux_test.go new file mode 100644 index 00000000..e257c783 --- /dev/null +++ b/core/cmd/brewlet/stage_gc_linux_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package main + +import ( + "bytes" + "context" + "net" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestStageGCCancelsOnSIGTERM(t *testing.T) { + if os.Getenv("BREWLET_TEST_GC_CHILD") == "true" { + err := cmdStageGC([]string{ + "--stage-root", os.Getenv("BREWLET_TEST_GC_ROOT"), + "--address", os.Getenv("BREWLET_TEST_GC_SOCKET"), + }) + if status.Code(err) != codes.Canceled { + t.Fatalf("expected canceled metadata inspection, got %v", err) + } + return + } + + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "immutable-v2"), 0o755); err != nil { + t.Fatal(err) + } + socket := filepath.Join(root, "containerd.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + connected := make(chan error, 1) + go func() { + conn, err := listener.Accept() + connected <- err + if err == nil { + defer conn.Close() + // Leave the gRPC handshake pending until the child is signaled. + <-ctx.Done() + } + }() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestStageGCCancelsOnSIGTERM$") + cmd.Env = append(os.Environ(), "BREWLET_TEST_GC_CHILD=true", + "BREWLET_TEST_GC_ROOT="+root, "BREWLET_TEST_GC_SOCKET="+socket) + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + var waitErr error + go func() { + waitErr = cmd.Wait() + close(done) + }() + defer func() { cancel(); <-done }() + select { + case err := <-connected: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal("stage GC did not connect to containerd") + } + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + <-done + if waitErr != nil { + t.Fatalf("stage GC did not cancel cleanly: %v\n%s", waitErr, output.String()) + } +} diff --git a/core/cmd/brewlet/stage_gc_test.go b/core/cmd/brewlet/stage_gc_test.go new file mode 100644 index 00000000..c7bd50ca --- /dev/null +++ b/core/cmd/brewlet/stage_gc_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestStageGCRejectsInvalidArguments(t *testing.T) { + for _, args := range [][]string{ + {"--min-age=0"}, {"--min-age=-1h"}, {"--min-age=invalid"}, {"unexpected"}, + } { + if err := cmdStageGC(args); err == nil { + t.Fatalf("stage-gc %v succeeded", args) + } + if err := cmdStageGC([]string{"--help"}); err != nil { + t.Fatalf("stage-gc help: %v", err) + } + } +} + +func TestRunnableBundleSurvivesStageRemoval(t *testing.T) { + stage := t.TempDir() + t.Setenv("BREWLET_RUNNABLE_STAGE", stage) + dir := t.TempDir() + jar := filepath.Join(dir, "orders.jar") + writeCLIZip(t, jar, "com/example/Orders.class") + archive := filepath.Join(dir, "app.jsa") + if err := os.WriteFile(archive, []byte("archive"), 0o644); err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, "config.json") + if err := os.WriteFile(config, []byte(`{"schemaVersion":1,"entry":{"mode":"jar"}}`), 0o644); err != nil { + t.Fatal(err) + } + for _, cds := range []bool{false, true} { + name := "plain" + if cds { + name = "cds" + } + t.Run(name, func(t *testing.T) { + store := filepath.Join(t.TempDir(), "oci") + args := []string{jar, "apps/orders:1", "--store", store, "--config", config} + if cds { + args = append(args, "--appcds-archive", archive) + } + if err := cmdPush(args); err != nil { + t.Fatal(err) + } + out := filepath.Join(t.TempDir(), "bundle") + if err := cmdBundle([]string{"apps/orders:1", "--store", store, "--out", out}); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(filepath.Join(stage, "immutable-v2")); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(out, "config.json")) + if err != nil { + t.Fatal(err) + } + var spec struct { + Mounts []struct{ Source, Destination string } + } + if err := json.Unmarshal(data, &spec); err != nil { + t.Fatal(err) + } + payloads := 0 + for _, mount := range spec.Mounts { + if !strings.HasPrefix(mount.Destination, "/app/") { + continue + } + payloads++ + if !strings.HasPrefix(mount.Source, out+string(os.PathSeparator)) { + t.Errorf("payload is not bundle-owned: %s", mount.Source) + } + if _, err := os.ReadFile(mount.Source); err != nil { + t.Errorf("bundle payload missing after eviction: %v", err) + } + } + want := 1 + if cds { + want = 2 + } + if payloads != want { + t.Fatalf("payload mounts = %d, want %d", payloads, want) + } + }) + } +} diff --git a/core/go.mod b/core/go.mod index e17c52af..15be0747 100644 --- a/core/go.mod +++ b/core/go.mod @@ -11,10 +11,12 @@ require ( github.com/containerd/ttrpc v1.2.9 github.com/containerd/typeurl/v2 v2.3.0 github.com/distribution/reference v0.6.0 + github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 github.com/opencontainers/runtime-spec v1.3.0 github.com/prometheus/client_golang v1.24.1 go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/sys v0.47.0 google.golang.org/grpc v1.84.0 google.golang.org/protobuf v1.36.12 ) @@ -44,7 +46,6 @@ require ( github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect @@ -54,7 +55,6 @@ require ( golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect ) diff --git a/core/internal/artifact/blobs.go b/core/internal/artifact/blobs.go index 4152cbe8..281122ee 100644 --- a/core/internal/artifact/blobs.go +++ b/core/internal/artifact/blobs.go @@ -13,6 +13,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/microsoft/brewlet/internal/runnablestage" ) // ResolvedBlobs is an artifact after separating the JVM launch config from its @@ -189,8 +191,15 @@ func verifiedBlobPaths(src BlobSource, layers []Descriptor, kind string) ([]stri // reads from — the JAR (and optional CDS archive) as files, and the // classpath/module layers as uncompressed tars the existing staging path // consumes unchanged. Staging is idempotent (keyed on manifestDigest) so -// repeated resolutions of the same image reuse it. +// repeated resolutions of the same image reuse it. Callers must hold a +// runnablestage.Acquire guard until the returned paths have been copied or +// mounted; the guard here protects resolution itself. func ResolveRunnableBlobs(src BlobSource, man Manifest, manifestDigest string) (ResolvedBlobs, error) { + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return ResolvedBlobs{}, err + } + defer guard.Close() cfg, err := man.RunnableConfig() if err != nil { return ResolvedBlobs{}, err @@ -378,10 +387,7 @@ func runnableStageDir(manifestDigest string) (string, error) { if err != nil { return "", fmt.Errorf("runnable image staging: %w", err) } - base := os.Getenv("BREWLET_RUNNABLE_STAGE") - if base == "" { - base = filepath.Join(os.TempDir(), "brewlet-runnable") - } + base := runnablestage.Root() // Previous stages may still be mounted and lack retained verification blobs. // Never rewrite or migrate a live stage during an upgrade. return filepath.Join(base, "immutable-v2", hex), nil diff --git a/core/internal/runnablestage/lock_linux.go b/core/internal/runnablestage/lock_linux.go new file mode 100644 index 00000000..e8605585 --- /dev/null +++ b/core/internal/runnablestage/lock_linux.go @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "errors" + "fmt" + "io" + "os" + + "golang.org/x/sys/unix" +) + +// Acquire holds a shared interprocess directory lock until Close. Callers must +// hold it from stage resolution through establishment of the workload's mounts +// (or for the lifetime of direct host execution). +func Acquire(root string) (io.Closer, error) { + if root == "" { + root = Root() + } + if err := os.MkdirAll(root, 0o755); err != nil { + return nil, err + } + return lock(root, false) +} + +func lock(root string, exclusive bool) (*os.File, error) { + fd, err := unix.Open(root, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), root) + operation := unix.LOCK_SH + if exclusive { + operation = unix.LOCK_EX | unix.LOCK_NB + } + for { + err = unix.Flock(fd, operation) + if !errors.Is(err, unix.EINTR) { + break + } + } + if err != nil { + file.Close() + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, ErrBusy + } + return nil, err + } + current, err := os.Lstat(root) + opened, statErr := file.Stat() + if err != nil || statErr != nil || !current.IsDir() || !os.SameFile(current, opened) { + file.Close() + return nil, fmt.Errorf("staging root changed while locking") + } + return file, nil +} diff --git a/core/internal/runnablestage/mounts_linux.go b/core/internal/runnablestage/mounts_linux.go new file mode 100644 index 00000000..0b2696b2 --- /dev/null +++ b/core/internal/runnablestage/mounts_linux.go @@ -0,0 +1,285 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "bufio" + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" +) + +type mount struct { + device string + root string + point string + filesystem string + options string +} + +type mountSnapshot struct { + current []mount + all []mount +} + +type procReader interface { + ReadFile(string) ([]byte, error) + ReadDir(string) ([]fs.DirEntry, error) + Readlink(string) (string, error) +} + +type procFS string + +func (p procFS) ReadFile(name string) ([]byte, error) { + return os.ReadFile(filepath.Join(string(p), name)) +} +func (p procFS) ReadDir(name string) ([]fs.DirEntry, error) { + return os.ReadDir(filepath.Join(string(p), name)) +} +func (p procFS) Readlink(name string) (string, error) { + return os.Readlink(filepath.Join(string(p), name)) +} + +func hostMounts(ctx context.Context) (mountSnapshot, error) { + return readMounts(ctx, procFS("/proc")) +} + +func readMounts(ctx context.Context, proc procReader) (mountSnapshot, error) { + // Linux reserves these inode numbers for the initial PID/user namespaces. + // Merely comparing self with PID 1 would accept a container's private proc. + for name, expected := range map[string]string{ + "self/ns/pid": "pid:[4026531836]", + "1/ns/pid": "pid:[4026531836]", + "self/ns/user": "user:[4026531837]", + } { + value, err := proc.Readlink(name) + if err != nil { + return mountSnapshot{}, fmt.Errorf("verify host namespace %s: %w", name, err) + } + if value != expected { + return mountSnapshot{}, fmt.Errorf("reaper requires the initial host PID and user namespaces") + } + } + selfNS, err := proc.Readlink("self/ns/mnt") + if err != nil { + return mountSnapshot{}, err + } + initNS, err := proc.Readlink("1/ns/mnt") + if err != nil { + return mountSnapshot{}, err + } + if selfNS != initNS { + return mountSnapshot{}, fmt.Errorf("reaper requires the host mount namespace") + } + raw, err := proc.ReadFile("self/mountinfo") + if err != nil { + return mountSnapshot{}, err + } + current, err := parseMounts(raw) + if err != nil { + return mountSnapshot{}, err + } + procMount, ok := containingMount(current, "/proc") + if !ok || procMount.filesystem != "proc" || procMount.point != "/proc" || procMount.root != "/" { + return mountSnapshot{}, fmt.Errorf("cannot verify complete host proc visibility") + } + for _, option := range strings.Split(procMount.options, ",") { + if strings.HasPrefix(option, "hidepid=") && option != "hidepid=0" && option != "hidepid=off" { + return mountSnapshot{}, fmt.Errorf("refusing permission-limited proc mount (%s)", option) + } + } + entries, err := proc.ReadDir(".") + if err != nil { + return mountSnapshot{}, err + } + snapshot := mountSnapshot{current: current, all: append([]mount(nil), current...)} + pids := 0 + for _, entry := range entries { + if !entry.IsDir() || !decimal(entry.Name()) { + continue + } + pids++ + if err := ctx.Err(); err != nil { + return mountSnapshot{}, err + } + raw, err := proc.ReadFile(entry.Name() + "/mountinfo") + if os.IsNotExist(err) { + // A vanished PID is harmless; an existing process with a hidden + // mountinfo is not. Zombies can have an empty mountinfo. + _, aliveErr := proc.ReadFile(entry.Name() + "/stat") + if os.IsNotExist(aliveErr) { + continue + } + } + if err != nil { + return mountSnapshot{}, fmt.Errorf("read PID %s mounts: %w", entry.Name(), err) + } + if len(strings.TrimSpace(string(raw))) == 0 { + stat, statErr := proc.ReadFile(entry.Name() + "/stat") + if os.IsNotExist(statErr) { + continue + } + if statErr == nil { + if end := strings.LastIndex(string(stat), ") "); end >= 0 { + state := string(stat)[end+2:] + if strings.HasPrefix(state, "Z ") || strings.HasPrefix(state, "X ") { + continue + } + } + } + return mountSnapshot{}, fmt.Errorf("empty mountinfo for live or unreadable PID %s", entry.Name()) + } + mounts, err := parseMounts(raw) + if err != nil { + return mountSnapshot{}, fmt.Errorf("parse PID %s mounts: %w", entry.Name(), err) + } + snapshot.all = append(snapshot.all, mounts...) + } + if pids == 0 { + return mountSnapshot{}, fmt.Errorf("no visible processes in host proc") + } + return snapshot, nil +} + +func decimal(value string) bool { + if value == "" { + return false + } + for _, c := range value { + if c < '0' || c > '9' { + return false + } + } + return true +} + +func parseMounts(raw []byte) ([]mount, error) { + var result []mount + scanner := bufio.NewScanner(strings.NewReader(string(raw))) + scanner.Buffer(make([]byte, 4096), 1024*1024) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + separator := -1 + for i, field := range fields { + if field == "-" { + separator = i + break + } + } + if separator < 6 || len(fields) != separator+4 { + return nil, fmt.Errorf("malformed mountinfo record") + } + for _, id := range fields[:2] { + if _, err := strconv.ParseUint(id, 10, 64); err != nil { + return nil, fmt.Errorf("malformed mountinfo mount ID") + } + } + device := strings.Split(fields[2], ":") + if len(device) != 2 { + return nil, fmt.Errorf("malformed mountinfo device") + } + for _, n := range device { + if _, err := strconv.ParseUint(n, 10, 32); err != nil { + return nil, fmt.Errorf("malformed mountinfo device") + } + } + root, err := mountPath(fields[3]) + if err != nil { + return nil, err + } + point, err := mountPath(fields[4]) + if err != nil { + return nil, err + } + result = append(result, mount{ + device: fields[2], root: root, point: point, + filesystem: fields[separator+1], options: fields[5] + "," + fields[separator+3], + }) + } + if err := scanner.Err(); err != nil { + return nil, err + } + if len(result) == 0 { + return nil, fmt.Errorf("empty mountinfo") + } + return result, nil +} + +func mountPath(value string) (string, error) { + var out strings.Builder + for i := 0; i < len(value); i++ { + if value[i] != '\\' { + out.WriteByte(value[i]) + continue + } + if i+3 >= len(value) { + return "", fmt.Errorf("malformed mountinfo escape") + } + switch value[i+1 : i+4] { + case "040": + out.WriteByte(' ') + case "011": + out.WriteByte('\t') + case "012": + out.WriteByte('\n') + case "134": + out.WriteByte('\\') + default: + return "", fmt.Errorf("malformed mountinfo escape") + } + i += 3 + } + path := out.String() + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return "", fmt.Errorf("noncanonical mountinfo path %q", path) + } + return path, nil +} + +func beneath(parent, path string) bool { + return path == parent || strings.HasPrefix(path, strings.TrimSuffix(parent, "/")+"/") +} + +func containingMount(mounts []mount, path string) (mount, bool) { + var best mount + found := false + for _, m := range mounts { + if beneath(m.point, path) && (!found || len(m.point) >= len(best.point)) { + best, found = m, true + } + } + return best, found +} + +func (s mountSnapshot) protects(path, device string) (bool, error) { + base, ok := containingMount(s.current, path) + if !ok || base.device != device { + return false, fmt.Errorf("cannot resolve staging filesystem for %q", path) + } + relative, err := filepath.Rel(base.point, path) + if err != nil { + return false, err + } + source := filepath.Join(base.root, relative) + for _, m := range s.current { + if beneath(path, m.point) { + // Includes same-device bind mounts; RemoveAll must not cross + // any mountpoint, not merely a change in st_dev. + return true, nil + } + } + for _, m := range s.all { + if m.device == device && beneath(source, m.root) { + // Bind sources live in mountinfo's root field, NOT source. + return true, nil + } + } + return false, nil +} diff --git a/core/internal/runnablestage/mounts_linux_test.go b/core/internal/runnablestage/mounts_linux_test.go new file mode 100644 index 00000000..7d90a416 --- /dev/null +++ b/core/internal/runnablestage/mounts_linux_test.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "fmt" + "io/fs" + "os" + "strings" + "testing" + "testing/fstest" +) + +type fakeProc struct { + files fstest.MapFS + links map[string]string + fail string +} + +func (p fakeProc) ReadFile(name string) ([]byte, error) { + if name == p.fail { + return nil, os.ErrPermission + } + return fs.ReadFile(p.files, name) +} +func (p fakeProc) ReadDir(name string) ([]fs.DirEntry, error) { + if name == p.fail { + return nil, os.ErrPermission + } + return fs.ReadDir(p.files, name) +} +func (p fakeProc) Readlink(name string) (string, error) { + if name == p.fail { + return "", os.ErrPermission + } + if value, ok := p.links[name]; ok { + return value, nil + } + return "", os.ErrNotExist +} + +const hostMountinfo = "1 0 8:1 / / rw - ext4 /dev/root rw\n2 1 0:2 / /proc rw - proc proc rw\n" + +func newProc() fakeProc { + return fakeProc{ + files: fstest.MapFS{ + "self/mountinfo": {Data: []byte(hostMountinfo)}, + "1/mountinfo": {Data: []byte(hostMountinfo)}, + "42/mountinfo": {Data: []byte("3 0 8:1 /stage/immutable-v2/hash/app/app.jar /app/app.jar ro - ext4 /dev/root rw\n")}, + }, + links: map[string]string{ + "self/ns/pid": "pid:[4026531836]", "1/ns/pid": "pid:[4026531836]", + "self/ns/user": "user:[4026531837]", "self/ns/mnt": "mnt:[1]", "1/ns/mnt": "mnt:[1]", + }, + } +} + +func TestMountsAcrossProcesses(t *testing.T) { + proc := newProc() + snapshot, err := readMounts(context.Background(), proc) + if err != nil { + t.Fatal(err) + } + got, err := snapshot.protects("/stage/immutable-v2/hash", "8:1") + if err != nil || !got { + t.Fatalf("missed bind source in another PID namespace: %t %v", got, err) + } + if got, err := snapshot.protects("/stage/immutable-v2/has", "8:1"); err != nil || got { + t.Fatalf("non-component prefix matched: %t %v", got, err) + } +} + +func TestMountMapping(t *testing.T) { + for _, tt := range []struct { + name string + mounts []mount + want bool + }{ + {"file source", []mount{{device: "8:1", root: "/subvolume/cache/hash/app.jar", point: "/app.jar"}}, true}, + {"directory source", []mount{{device: "8:1", root: "/subvolume/cache/hash", point: "/app"}}, true}, + {"exporter ancestor source", []mount{{device: "8:1", root: "/subvolume/cache", point: "/cache"}}, false}, + {"filesystem root source", []mount{{device: "8:1", root: "/", point: "/host-root"}}, false}, + {"other device", []mount{{device: "8:2", root: "/subvolume/cache/hash", point: "/app"}}, false}, + {"prefix only", []mount{{device: "8:1", root: "/subvolume/cache/hash-other", point: "/app"}}, false}, + {"unrelated", []mount{{device: "8:1", root: "/other/hash", point: "/app"}}, false}, + } { + t.Run(tt.name, func(t *testing.T) { + base := mount{device: "8:1", root: "/subvolume", point: "/host"} + snapshot := mountSnapshot{current: []mount{base}, all: append([]mount{base}, tt.mounts...)} + got, err := snapshot.protects("/host/cache/hash", "8:1") + if err != nil || got != tt.want { + t.Fatalf("protected=%t error=%v want=%t", got, err, tt.want) + } + }) + } + for _, device := range []string{"8:1", "8:2"} { + base := mount{device: "8:1", root: "/", point: "/"} + inside := mount{device: device, root: "/", point: "/stage/hash/mount"} + snapshot := mountSnapshot{current: []mount{base, inside}, all: []mount{base}} + if protected, err := snapshot.protects("/stage/hash", "8:1"); err != nil || !protected { + t.Fatalf("mount inside stage on %s not protected: %t %v", device, protected, err) + } + } +} + +func TestProcFailClosed(t *testing.T) { + for _, failure := range []string{"hidepid", "pidns", "userns", "mountns", "missing-proc", "permission", "parse", "empty-live", "missing-live", "enumeration", "namespace-permission"} { + t.Run(failure, func(t *testing.T) { + proc := newProc() + switch failure { + case "hidepid": + proc.files["self/mountinfo"].Data = []byte(strings.ReplaceAll(hostMountinfo, "proc proc rw", "proc proc rw,hidepid=2")) + case "pidns": + proc.links["self/ns/pid"] = "pid:[999]" + case "userns": + proc.links["self/ns/user"] = "user:[999]" + case "mountns": + proc.links["self/ns/mnt"] = "mnt:[999]" + case "missing-proc": + proc.files["self/mountinfo"].Data = []byte("1 0 8:1 / / rw - ext4 /dev/root rw\n") + case "permission": + proc.fail = "42/mountinfo" + case "parse": + proc.files["42/mountinfo"].Data = []byte("broken\n") + case "empty-live": + proc.files["42/mountinfo"].Data = nil + proc.files["42/stat"] = &fstest.MapFile{Data: []byte("42 (jvm) S 1 2")} + case "missing-live": + delete(proc.files, "42/mountinfo") + proc.files["42/stat"] = &fstest.MapFile{Data: []byte("42 (jvm) S 1 2")} + case "enumeration": + proc.fail = "." + case "namespace-permission": + proc.fail = "1/ns/mnt" + } + if _, err := readMounts(context.Background(), proc); err == nil { + t.Fatal("accepted incomplete mount snapshot") + } + }) + } +} + +func TestExitedProcesses(t *testing.T) { + for _, zombie := range []bool{false, true} { + t.Run(fmt.Sprint(zombie), func(t *testing.T) { + proc := newProc() + if zombie { + proc.files["42/mountinfo"].Data = nil + proc.files["42/stat"] = &fstest.MapFile{Data: []byte("42 (name with ) parenthesis) Z 1 2")} + } else { + delete(proc.files, "42/mountinfo") + proc.files["42"] = &fstest.MapFile{Mode: fs.ModeDir} + } + if _, err := readMounts(context.Background(), proc); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestMountinfoParsing(t *testing.T) { + mounts, err := parseMounts([]byte(`1 0 8:1 /stage\040root/app\134name /app\011jar ro shared:3 - ext4 /dev/root rw` + "\n")) + if err != nil || mounts[0].root != "/stage root/app\\name" || mounts[0].point != "/app\tjar" { + t.Fatalf("escaped mount paths: %+v %v", mounts, err) + } + for _, line := range []string{ + "", "broken", "1 0 8:1 / / rw - ext4 source", "x 0 8:1 / / rw - ext4 source rw", + "1 0 bad / / rw - ext4 source rw", "1 0 8:x / / rw - ext4 source rw", + "1 0 8:1 relative / rw - ext4 source rw", `1 0 8:1 /bad\777 / rw - ext4 source rw`, + "1 0 8:1 /bad/../path / rw - ext4 source rw", + } { + if _, err := parseMounts([]byte(line)); err == nil { + t.Errorf("accepted malformed mountinfo %q", line) + } + } +} diff --git a/core/internal/runnablestage/reap_linux.go b/core/internal/runnablestage/reap_linux.go new file mode 100644 index 00000000..05ec2fff --- /dev/null +++ b/core/internal/runnablestage/reap_linux.go @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "time" + + "golang.org/x/sys/unix" +) + +type dependencies struct { + references func(context.Context, string) (map[string]bool, error) + mounts func(context.Context) (mountSnapshot, error) + now func() time.Time +} + +// Reap removes only old, unreferenced immutable-v2 digest directories. MinAge +// must be strictly positive. Metadata and every visible process's mounts +// must be readable before any deletion. Run in the host namespaces with a +// complete proc mount; concurrent legacy launchers must be quiesced. +func Reap(ctx context.Context, address string, opts Options) (Result, error) { + return reap(ctx, address, opts, dependencies{containerReferences, hostMounts, time.Now}) +} + +func reap(ctx context.Context, address string, opts Options, deps dependencies) (Result, error) { + var result Result + if opts.MinAge <= 0 { + return result, fmt.Errorf("minimum stage age must be positive") + } + if opts.Root == "" { + opts.Root = Root() + } + absolute, err := filepath.Abs(opts.Root) + if err != nil { + return result, err + } + // Resolve ancestor aliases for mount matching, but never accept a symlink + // as the staging root itself. + before, err := os.Lstat(absolute) + if os.IsNotExist(err) { + return result, nil + } + if err != nil { + return result, err + } + if !before.IsDir() { + return result, fmt.Errorf("staging root is not a real directory") + } + absolute, err = filepath.EvalSymlinks(absolute) + if err != nil { + return result, err + } + guard, err := lock(absolute, true) + if err != nil { + return result, err + } + defer guard.Close() + root, err := openRoot(absolute) + if err != nil { + return result, err + } + defer root.Close() + lockedInfo, err := guard.Stat() + if err != nil { + return result, err + } + rootInfo, err := root.Stat(".") + if err != nil || !os.SameFile(lockedInfo, rootInfo) || !os.SameFile(before, rootInfo) { + return result, fmt.Errorf("staging root changed while opening") + } + versionInfo, err := root.Lstat("immutable-v2") + if os.IsNotExist(err) { + result.RemainingBytes, err = treeBytes(root, ".") + return result, err + } + if err != nil { + return result, err + } + if !versionInfo.IsDir() { + return result, fmt.Errorf("immutable-v2 is not a real directory") + } + version, err := root.OpenRoot("immutable-v2") + if err != nil { + return result, err + } + defer version.Close() + openedInfo, err := version.Stat(".") + if err != nil || !os.SameFile(versionInfo, openedInfo) { + return result, fmt.Errorf("immutable-v2 changed while opening") + } + entries, err := fs.ReadDir(version.FS(), ".") + if err != nil { + return result, err + } + refs, err := deps.references(ctx, address) + if err != nil { + return result, err + } + mounts, err := deps.mounts(ctx) + if err != nil { + return result, err + } + type candidate struct { + name string + info fs.FileInfo + size int64 + } + var candidates []candidate + cutoff := deps.now().Add(-opts.MinAge) + for _, entry := range entries { + if err := ctx.Err(); err != nil { + return result, err + } + if !canonical(entry.Name()) || !entry.IsDir() || refs[entry.Name()] { + continue + } + info, err := version.Lstat(entry.Name()) + if err != nil { + return result, err + } + if !info.IsDir() || !info.ModTime().Before(cutoff) { + continue + } + var stat unix.Stat_t + path := filepath.Join(absolute, "immutable-v2", entry.Name()) + if err := unix.Lstat(path, &stat); err != nil { + return result, err + } + device := fmt.Sprintf("%d:%d", unix.Major(uint64(stat.Dev)), unix.Minor(uint64(stat.Dev))) + protected, err := mounts.protects(path, device) + if err != nil { + return result, err + } + if protected { + continue + } + size, err := treeBytes(version, entry.Name()) + if err != nil { + return result, err + } + candidates = append(candidates, candidate{entry.Name(), info, size}) + } + // Finish all accounting and reference validation before the first removal. + result.RemainingBytes, err = treeBytes(root, ".") + if err != nil { + return result, err + } + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return result, err + } + current, err := version.Lstat(candidate.name) + if err != nil || !current.IsDir() || !os.SameFile(current, candidate.info) || + !current.ModTime().Equal(candidate.info.ModTime()) { + return result, fmt.Errorf("stage %s changed during reaping", candidate.name) + } + if !opts.DryRun { + if err := version.RemoveAll(candidate.name); err != nil { + // A filesystem failure can leave a partially removed tree. + result.RemainingBytes, _ = treeBytes(root, ".") + return result, err + } + result.RemainingBytes -= candidate.size + } + result.Removed++ + result.ReclaimedBytes += candidate.size + } + return result, nil +} diff --git a/core/internal/runnablestage/reap_linux_test.go b/core/internal/runnablestage/reap_linux_test.go new file mode 100644 index 00000000..6b8c95a0 --- /dev/null +++ b/core/internal/runnablestage/reap_linux_test.go @@ -0,0 +1,294 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func createStage(t *testing.T, root, name string, age time.Duration) string { + t.Helper() + path := filepath.Join(root, "immutable-v2", name) + writeFile(t, filepath.Join(path, "app", "app.jar"), "jar") + old := time.Now().Add(-age) + if err := os.Chtimes(path, old, old); err != nil { + t.Fatal(err) + } + return path +} + +func fakeDependencies(t *testing.T, root string) dependencies { + t.Helper() + var stat unix.Stat_t + if err := unix.Stat(root, &stat); err != nil { + t.Fatal(err) + } + base := mount{device: fmt.Sprintf("%d:%d", unix.Major(uint64(stat.Dev)), unix.Minor(uint64(stat.Dev))), root: "/", point: "/"} + return dependencies{ + references: func(context.Context, string) (map[string]bool, error) { return map[string]bool{}, nil }, + mounts: func(context.Context) (mountSnapshot, error) { + return mountSnapshot{current: []mount{base}, all: []mount{base}}, nil + }, + now: time.Now, + } +} + +func TestReapFiltersAndDryRun(t *testing.T) { + for _, dry := range []bool{true, false} { + t.Run(fmt.Sprintf("dry=%t", dry), func(t *testing.T) { + root := t.TempDir() + deps := fakeDependencies(t, root) + names := make([]string, 7) + for i := range names { + names[i] = strings.Repeat(fmt.Sprintf("%x", i), 64) + } + old := createStage(t, root, names[0], 48*time.Hour) + image := createStage(t, root, names[1], 48*time.Hour) + content := createStage(t, root, names[2], 48*time.Hour) + mounted := createStage(t, root, names[3], 48*time.Hour) + young := createStage(t, root, names[4], time.Hour) + inside := createStage(t, root, names[5], 48*time.Hour) + unknown := createStage(t, root, "unknown", 48*time.Hour) + pending := createStage(t, root, "."+names[0]+"-pending", 48*time.Hour) + uppercase := createStage(t, root, strings.Repeat("A", 64), 48*time.Hour) + writeFile(t, filepath.Join(root, "legacy", "app.jar"), "jar") + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "app.jar"), "untouched") + link := filepath.Join(root, "immutable-v2", names[6]) + if err := os.Symlink(outside, link); err != nil { + t.Fatal(err) + } + deps.references = func(context.Context, string) (map[string]bool, error) { + return map[string]bool{names[1]: true, names[2]: true}, nil + } + snapshot, _ := deps.mounts(context.Background()) + snapshot.all = append(snapshot.all, mount{device: snapshot.current[0].device, root: filepath.Join(mounted, "app", "app.jar"), point: "/app/app.jar"}) + snapshot.all = append(snapshot.all, mount{device: snapshot.current[0].device, root: root, point: "/exporter-stage"}) + snapshot.current = append(snapshot.current, mount{device: "99:9", root: "/", point: filepath.Join(inside, "nested")}) + deps.mounts = func(context.Context) (mountSnapshot, error) { return snapshot, nil } + result, err := reap(context.Background(), "fake", Options{Root: root, MinAge: DefaultMinAge, DryRun: dry}, deps) + if err != nil { + t.Fatal(err) + } + wantRemaining := int64(30) + if !dry { + wantRemaining -= 3 + } + if result.Removed != 1 || result.ReclaimedBytes != 3 || result.RemainingBytes != wantRemaining { + t.Fatalf("unexpected result: %+v, want remaining %d", result, wantRemaining) + } + if _, err := os.Lstat(old); dry && err != nil || !dry && !os.IsNotExist(err) { + t.Fatalf("old stage: %v", err) + } + for _, path := range []string{image, content, mounted, young, inside, unknown, pending, uppercase, link, outside, filepath.Join(root, "legacy")} { + if _, err := os.Lstat(path); err != nil { + t.Fatalf("removed protected path %s: %v", path, err) + } + } + }) + } +} + +func TestReapFailClosed(t *testing.T) { + for _, failure := range []string{"references", "mounts", "mapping", "cancelled"} { + t.Run(failure, func(t *testing.T) { + root := t.TempDir() + path := createStage(t, root, strings.Repeat("a", 64), 48*time.Hour) + deps := fakeDependencies(t, root) + ctx := context.Background() + switch failure { + case "references": + deps.references = func(context.Context, string) (map[string]bool, error) { + return nil, errors.New("containerd unavailable") + } + case "mounts": + deps.mounts = func(context.Context) (mountSnapshot, error) { + return mountSnapshot{}, os.ErrPermission + } + case "mapping": + deps.mounts = func(context.Context) (mountSnapshot, error) { + return mountSnapshot{}, nil + } + case "cancelled": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + if _, err := reap(ctx, "", Options{Root: root, MinAge: DefaultMinAge}, deps); err == nil { + t.Fatal("expected failure") + } + if _, err := os.Stat(filepath.Join(path, "app", "app.jar")); err != nil { + t.Fatalf("deletion before complete validation: %v", err) + } + }) + } +} + +func TestReapSymlinksAndMissing(t *testing.T) { + root := t.TempDir() + deps := fakeDependencies(t, root) + missing := filepath.Join(root, "missing") + if got, err := reap(context.Background(), "", Options{Root: missing, MinAge: DefaultMinAge}, dependencies{}); got != (Result{}) || err != nil { + t.Fatalf("missing root: %+v, %v", got, err) + } + if _, err := os.Stat(missing); !os.IsNotExist(err) { + t.Fatal("reaping created missing root") + } + target := t.TempDir() + createStage(t, target, strings.Repeat("a", 64), 48*time.Hour) + for _, name := range []string{"link", "immutable-v2"} { + path := filepath.Join(root, name) + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } + reapRoot := root + if name == "link" { + reapRoot = path + } + if _, err := reap(context.Background(), "", Options{Root: reapRoot, MinAge: DefaultMinAge}, deps); err == nil { + t.Fatalf("accepted symlink %s", name) + } + } + for _, age := range []time.Duration{-1, 0} { + if _, err := reap(context.Background(), "", Options{Root: root, MinAge: age}, deps); err == nil { + t.Fatalf("accepted nonpositive age %s", age) + } + } +} + +func TestAgeBoundary(t *testing.T) { + root := t.TempDir() + deps := fakeDependencies(t, root) + now := time.Now().Truncate(time.Second) + deps.now = func() time.Time { return now } + path := createStage(t, root, strings.Repeat("a", 64), time.Hour) + boundary := now.Add(-time.Hour) + if err := os.Chtimes(path, boundary, boundary); err != nil { + t.Fatal(err) + } + result, err := reap(context.Background(), "", Options{Root: root, MinAge: time.Hour}, deps) + if err != nil || result.Removed != 0 { + t.Fatalf("age boundary: %+v, %v", result, err) + } + deps.now = func() time.Time { return now.Add(time.Second) } + result, err = reap(context.Background(), "", Options{Root: root, MinAge: time.Hour}, deps) + if err != nil || result.Removed != 1 { + t.Fatalf("past age boundary: %+v, %v", result, err) + } +} + +func TestLocks(t *testing.T) { + root := filepath.Join(t.TempDir(), "created") + a, err := Acquire(root) + if err != nil { + t.Fatal(err) + } + b, err := Acquire(root) + if err != nil { + t.Fatal(err) + } + defer b.Close() + if _, err := Reap(context.Background(), "", Options{Root: root, MinAge: DefaultMinAge}); !errors.Is(err, ErrBusy) { + t.Fatalf("reaper with active readers: %v", err) + } + a.Close() + if _, err := lock(root, true); !errors.Is(err, ErrBusy) { + t.Fatalf("second reader failed to retain lock: %v", err) + } + b.Close() + exclusive, err := lock(root, true) + if err != nil { + t.Fatal(err) + } + acquired := make(chan io.Closer, 1) + failures := make(chan error, 1) + go func() { + reader, err := Acquire(root) + if err != nil { + failures <- err + return + } + acquired <- reader + }() + select { + case reader := <-acquired: + reader.Close() + t.Fatal("reader bypassed exclusive lock") + case err := <-failures: + t.Fatal(err) + case <-time.After(25 * time.Millisecond): + } + exclusive.Close() + select { + case reader := <-acquired: + reader.Close() + case err := <-failures: + t.Fatal(err) + case <-time.After(5 * time.Second): + t.Fatal("reader did not unblock") + } + entries, err := os.ReadDir(root) + if err != nil || len(entries) != 0 { + t.Fatalf("locking added staging entries: %v, %v", entries, err) + } +} + +func TestLockInterprocess(t *testing.T) { + if root := os.Getenv("BREWLET_STAGE_LOCK_TEST_CHILD"); root != "" { + guard, err := Acquire(root) + if err != nil { + t.Fatal(err) + } + defer guard.Close() + fmt.Println("locked") + _, _ = io.Copy(io.Discard, os.Stdin) + return + } + root := t.TempDir() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(executable, "-test.run=^TestLockInterprocess$") + cmd.Env = append(os.Environ(), "BREWLET_STAGE_LOCK_TEST_CHILD="+root) + input, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + output, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + input.Close() + if err := cmd.Wait(); err != nil { + t.Error(err) + } + }() + line, err := bufio.NewReader(output).ReadString('\n') + if err != nil || line != "locked\n" { + t.Fatalf("child readiness: %q %v", line, err) + } + if _, err := Reap(context.Background(), "", Options{Root: root, MinAge: DefaultMinAge}); !errors.Is(err, ErrBusy) { + t.Fatalf("cross-process reaper lock: %v", err) + } +} diff --git a/core/internal/runnablestage/references_linux.go b/core/internal/runnablestage/references_linux.go new file mode 100644 index 00000000..d06d2592 --- /dev/null +++ b/core/internal/runnablestage/references_linux.go @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + + contentapi "github.com/containerd/containerd/api/services/content/v1" + imagesapi "github.com/containerd/containerd/api/services/images/v1" + namespacesapi "github.com/containerd/containerd/api/services/namespaces/v1" + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/content/proxy" + "github.com/containerd/containerd/v2/pkg/namespaces" + digest "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type metadata interface { + namespaces(context.Context) ([]string, error) + images(context.Context, string) ([]ocispec.Descriptor, error) + contents(context.Context, string, func(content.Info) error) error + read(context.Context, string, ocispec.Descriptor) ([]byte, error) +} + +type containerMetadata struct { + client *grpc.ClientConn + store content.Store +} + +func (m containerMetadata) namespaces(ctx context.Context) ([]string, error) { + response, err := namespacesapi.NewNamespacesClient(m.client).List(ctx, &namespacesapi.ListNamespacesRequest{}) + if err != nil { + return nil, err + } + var names []string + for _, namespace := range response.Namespaces { + if namespace == nil || namespace.Name == "" { + return nil, fmt.Errorf("invalid containerd namespace") + } + names = append(names, namespace.Name) + } + return names, nil +} + +func (m containerMetadata) images(ctx context.Context, ns string) ([]ocispec.Descriptor, error) { + response, err := imagesapi.NewImagesClient(m.client).List(namespaces.WithNamespace(ctx, ns), &imagesapi.ListImagesRequest{}) + if err != nil { + return nil, err + } + targets := make([]ocispec.Descriptor, 0, len(response.Images)) + for _, image := range response.Images { + if image == nil || image.Target == nil { + return nil, fmt.Errorf("invalid containerd image") + } + targets = append(targets, ocispec.Descriptor{ + MediaType: image.Target.MediaType, Digest: digest.Digest(image.Target.Digest), Size: image.Target.Size, + }) + } + return targets, nil +} + +func (m containerMetadata) contents(ctx context.Context, ns string, fn func(content.Info) error) error { + return m.store.Walk(namespaces.WithNamespace(ctx, ns), fn) +} + +func (m containerMetadata) read(ctx context.Context, ns string, d ocispec.Descriptor) ([]byte, error) { + return content.ReadBlob(namespaces.WithNamespace(ctx, ns), m.store, d) +} + +func containerReferences(ctx context.Context, address string) (map[string]bool, error) { + if address == "" { + return nil, fmt.Errorf("containerd address is required") + } + if filepath.IsAbs(address) { + address = "unix://" + address + } + client, err := grpc.NewClient(address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, err + } + defer client.Close() + store := proxy.NewContentStore(contentapi.NewContentClient(client)) + return references(ctx, containerMetadata{client, store}) +} + +func references(ctx context.Context, store metadata) (map[string]bool, error) { + refs := make(map[string]bool) + add := func(d digest.Digest) error { + if err := d.Validate(); err != nil { + return fmt.Errorf("invalid reference digest %q: %w", d, err) + } + if d.Algorithm() == digest.SHA256 { + refs[d.Encoded()] = true + } + return nil + } + nss, err := store.namespaces(ctx) + if err != nil { + return nil, fmt.Errorf("list containerd namespaces: %w", err) + } + for _, ns := range nss { + if err := store.contents(ctx, ns, func(info content.Info) error { + if err := add(info.Digest); err != nil { + return err + } + for key, value := range info.Labels { + if strings.HasPrefix(key, "containerd.io/gc.ref.content.") { + if err := add(digest.Digest(value)); err != nil { + return err + } + } + } + return nil + }); err != nil { + return nil, fmt.Errorf("list content in namespace %q: %w", ns, err) + } + targets, err := store.images(ctx, ns) + if err != nil { + return nil, fmt.Errorf("list images in namespace %q: %w", ns, err) + } + seen := make(map[digest.Digest]bool) + var visit func(ocispec.Descriptor, int) error + visit = func(d ocispec.Descriptor, depth int) error { + if err := ctx.Err(); err != nil { + return err + } + if err := add(d.Digest); err != nil { + return err + } + if d.MediaType == ocispec.MediaTypeImageManifest || d.MediaType == "application/vnd.docker.distribution.manifest.v2+json" { + // A target/child descriptor protects a manifest even if its + // content has already disappeared. + return nil + } + if d.MediaType != ocispec.MediaTypeImageIndex && d.MediaType != "application/vnd.docker.distribution.manifest.list.v2+json" { + return fmt.Errorf("unknown image target media type %q", d.MediaType) + } + if seen[d.Digest] { + return nil + } + if depth > 128 { + return fmt.Errorf("image index nesting limit exceeded") + } + seen[d.Digest] = true + raw, err := store.read(ctx, ns, d) + if err != nil { + return fmt.Errorf("read image index %s: %w", d.Digest, err) + } + if d.Digest.Algorithm().FromBytes(raw) != d.Digest { + return fmt.Errorf("image index %s failed digest verification", d.Digest) + } + var index ocispec.Index + if err := json.Unmarshal(raw, &index); err != nil { + return fmt.Errorf("parse image index %s: %w", d.Digest, err) + } + if index.SchemaVersion != 2 || index.Manifests == nil { + return fmt.Errorf("invalid image index %s", d.Digest) + } + for _, child := range index.Manifests { + if err := visit(child, depth+1); err != nil { + return err + } + } + return nil + } + for _, target := range targets { + if err := visit(target, 0); err != nil { + return nil, fmt.Errorf("image references in namespace %q: %w", ns, err) + } + } + } + return refs, nil +} diff --git a/core/internal/runnablestage/references_linux_test.go b/core/internal/runnablestage/references_linux_test.go new file mode 100644 index 00000000..978e6032 --- /dev/null +++ b/core/internal/runnablestage/references_linux_test.go @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build linux + +package runnablestage + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/containerd/containerd/v2/core/content" + digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type fakeMetadata struct { + nss []string + targets map[string][]ocispec.Descriptor + infos map[string][]content.Info + blobs map[string]map[digest.Digest][]byte + fail string + visited []string +} + +func (m *fakeMetadata) namespaces(context.Context) ([]string, error) { + if m.fail == "namespaces" { + return nil, os.ErrPermission + } + return m.nss, nil +} +func (m *fakeMetadata) images(_ context.Context, ns string) ([]ocispec.Descriptor, error) { + m.visited = append(m.visited, "images:"+ns) + if m.fail == "images:"+ns { + return nil, os.ErrPermission + } + return m.targets[ns], nil +} +func (m *fakeMetadata) contents(_ context.Context, ns string, fn func(content.Info) error) error { + m.visited = append(m.visited, "content:"+ns) + if m.fail == "content:"+ns { + return os.ErrPermission + } + for _, info := range m.infos[ns] { + if err := fn(info); err != nil { + return err + } + } + return nil +} +func (m *fakeMetadata) read(_ context.Context, ns string, d ocispec.Descriptor) ([]byte, error) { + if raw, ok := m.blobs[ns][d.Digest]; ok { + return raw, nil + } + return nil, os.ErrNotExist +} + +func descriptor(char string) ocispec.Descriptor { + return ocispec.Descriptor{MediaType: ocispec.MediaTypeImageManifest, Digest: digest.Digest("sha256:" + strings.Repeat(char, 64))} +} + +func indexDescriptor(t *testing.T, children ...ocispec.Descriptor) (ocispec.Descriptor, []byte) { + t.Helper() + raw, err := json.Marshal(ocispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, MediaType: ocispec.MediaTypeImageIndex, Manifests: children, + }) + if err != nil { + t.Fatal(err) + } + return ocispec.Descriptor{MediaType: ocispec.MediaTypeImageIndex, Digest: digest.FromBytes(raw), Size: int64(len(raw))}, raw +} + +func TestReferencesAllNamespacesAndIndexes(t *testing.T) { + child := descriptor("a") + nested, nestedRaw := indexDescriptor(t, child) + index, raw := indexDescriptor(t, nested, descriptor("b")) + store := &fakeMetadata{ + nss: []string{"default", "k8s.io", "other"}, + targets: map[string][]ocispec.Descriptor{ + "default": {descriptor("c")}, + "k8s.io": {index}, + }, + infos: map[string][]content.Info{ + "other": {{Digest: descriptor("d").Digest, Labels: map[string]string{"containerd.io/gc.ref.content.child": descriptor("e").Digest.String()}}}, + }, + blobs: map[string]map[digest.Digest][]byte{"k8s.io": {index.Digest: raw, nested.Digest: nestedRaw}}, + } + refs, err := references(context.Background(), store) + if err != nil { + t.Fatal(err) + } + for _, d := range []ocispec.Descriptor{child, descriptor("b"), descriptor("c"), descriptor("d"), descriptor("e"), index, nested} { + if !refs[d.Digest.Encoded()] { + t.Errorf("missing reference %s", d.Digest) + } + } + if len(store.visited) != 6 { + t.Fatalf("did not query both services in every namespace: %v", store.visited) + } +} + +func TestReferencesFailClosed(t *testing.T) { + for _, failure := range []string{"namespaces", "images:two", "content:two", "index-missing", "index-malformed", "index-corrupt", "child-index-missing", "unknown-type", "invalid-digest", "invalid-label"} { + t.Run(failure, func(t *testing.T) { + index, raw := indexDescriptor(t, descriptor("a")) + store := &fakeMetadata{ + nss: []string{"one", "two"}, + targets: map[string][]ocispec.Descriptor{"one": {descriptor("b")}, "two": {index}}, + blobs: map[string]map[digest.Digest][]byte{"two": {index.Digest: raw}}, + } + switch failure { + case "index-missing": + store.blobs = nil + case "index-malformed": + raw = []byte(`{"schemaVersion":2}`) + index.Digest = digest.FromBytes(raw) + store.targets["two"] = []ocispec.Descriptor{index} + store.blobs["two"][index.Digest] = raw + case "index-corrupt": + store.blobs["two"][index.Digest] = []byte(`{}`) + case "child-index-missing": + child := descriptor("a") + child.MediaType = ocispec.MediaTypeImageIndex + index, raw = indexDescriptor(t, child) + store.targets["two"] = []ocispec.Descriptor{index} + store.blobs["two"][index.Digest] = raw + case "unknown-type": + store.targets["two"][0].MediaType = "application/unknown" + case "invalid-digest": + store.targets["two"][0].Digest = "sha256:../bad" + case "invalid-label": + store.infos = map[string][]content.Info{"two": {{Digest: descriptor("c").Digest, Labels: map[string]string{"containerd.io/gc.ref.content.child": "broken"}}}} + default: + store.fail = failure + } + if refs, err := references(context.Background(), store); err == nil || refs != nil { + t.Fatalf("returned partial references on error: %v, %v", refs, err) + } + }) + } +} + +func TestLateNamespaceFailurePreventsAllDeletion(t *testing.T) { + root := t.TempDir() + path := createStage(t, root, strings.Repeat("a", 64), 2*DefaultMinAge) + store := &fakeMetadata{nss: []string{"one", "two"}, fail: "content:two"} + deps := fakeDependencies(t, root) + deps.references = func(ctx context.Context, _ string) (map[string]bool, error) { return references(ctx, store) } + if _, err := reap(context.Background(), "", Options{Root: root, MinAge: DefaultMinAge}, deps); !errors.Is(err, os.ErrPermission) { + t.Fatalf("expected namespace failure, got %v", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatal(fmt.Errorf("deleted before completing namespace scan: %w", err)) + } +} diff --git a/core/internal/runnablestage/stage.go b/core/internal/runnablestage/stage.go new file mode 100644 index 00000000..16a021f8 --- /dev/null +++ b/core/internal/runnablestage/stage.go @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package runnablestage accounts for and conservatively reclaims immutable +// runnable-image stages. The root must be administrator-owned and must not be +// renamed or replaced while launchers or the reaper are running. +package runnablestage + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "time" +) + +const DefaultMinAge = 24 * time.Hour + +type Options struct { + Root string + MinAge time.Duration + DryRun bool +} + +type Result struct { + Removed int + ReclaimedBytes int64 + RemainingBytes int64 +} + +// ErrBusy means a launcher or another reaper holds the staging root lock. +var ErrBusy = errors.New("runnable staging root is busy") + +// LinuxDefaultRoot is fixed rather than derived from TMPDIR so the shim, which +// inherits containerd's environment, and the provisioner's reaper agree. +const LinuxDefaultRoot = "/tmp/brewlet-runnable" + +func Root() string { + if root := os.Getenv("BREWLET_RUNNABLE_STAGE"); root != "" { + return root + } + if runtime.GOOS == "linux" { + return LinuxDefaultRoot + } + return filepath.Join(os.TempDir(), "brewlet-runnable") +} + +// Bytes sums logical regular-file sizes, including legacy and pending stages, +// without following symlinks. An absent root has size zero. +func Bytes(root string) (int64, error) { + if root == "" { + root = Root() + } + r, err := openRoot(root) + if os.IsNotExist(err) { + return 0, nil + } + if err != nil { + return 0, err + } + defer r.Close() + return treeBytes(r, ".") +} + +func openRoot(path string) (*os.Root, error) { + before, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !before.IsDir() { + return nil, fmt.Errorf("staging root %q is not a real directory", path) + } + r, err := os.OpenRoot(path) + if err != nil { + return nil, err + } + after, err := r.Stat(".") + if err != nil || !os.SameFile(before, after) { + r.Close() + return nil, fmt.Errorf("staging root %q changed while opening", path) + } + return r, nil +} + +func treeBytes(root *os.Root, name string) (int64, error) { + var size int64 + err := fs.WalkDir(root.FS(), name, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + info, err := root.Lstat(path) + if err != nil { + return err + } + if info.Mode().IsRegular() { + size += info.Size() + } + return nil + }) + return size, err +} + +func canonical(name string) bool { + if len(name) != 64 { + return false + } + for _, c := range name { + if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') { + return false + } + } + return true +} diff --git a/core/internal/runnablestage/stage_test.go b/core/internal/runnablestage/stage_test.go new file mode 100644 index 00000000..42e006b9 --- /dev/null +++ b/core/internal/runnablestage/stage_test.go @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package runnablestage + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestRootAndBytes(t *testing.T) { + root := t.TempDir() + t.Setenv("BREWLET_RUNNABLE_STAGE", root) + if Root() != root { + t.Fatal("environment override not used") + } + t.Setenv("BREWLET_RUNNABLE_STAGE", "") + t.Setenv("TMPDIR", t.TempDir()) + want := filepath.Join(os.TempDir(), "brewlet-runnable") + if runtime.GOOS == "linux" { + want = "/tmp/brewlet-runnable" + } + if Root() != want { + t.Fatalf("default root = %q, want %q", Root(), want) + } + for name, data := range map[string]string{ + "legacy/app.jar": "123", + "immutable-v2/.pending/app.jar": "12345", + "immutable-v2/" + strings.Repeat("a", 64) + "/app.jar": "1234567", + } { + writeFile(t, filepath.Join(root, name), data) + } + outside := t.TempDir() + writeFile(t, filepath.Join(outside, "large.jar"), strings.Repeat("x", 1024)) + if err := os.Symlink(outside, filepath.Join(root, "linked")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(outside, "large.jar"), filepath.Join(root, "linked.jar")); err != nil { + t.Fatal(err) + } + got, err := Bytes(root) + if err != nil || got != 15 { + t.Fatalf("Bytes = %d, %v; want 15", got, err) + } + if got, err := Bytes(filepath.Join(root, "missing")); got != 0 || err != nil { + t.Fatalf("missing Bytes = %d, %v", got, err) + } + if _, err := Bytes(filepath.Join(root, "linked")); err == nil { + t.Fatal("accepted symlink root") + } + if _, err := Bytes(filepath.Join(outside, "large.jar")); err == nil { + t.Fatal("accepted regular-file root") + } +} + +func writeFile(t *testing.T, path, data string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(data), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestCanonical(t *testing.T) { + for _, name := range []string{"", ".pending", strings.Repeat("A", 64), strings.Repeat("g", 64), strings.Repeat("a", 63), strings.Repeat("a", 65)} { + if canonical(name) { + t.Errorf("accepted %q", name) + } + } + if !canonical(strings.Repeat("0123456789abcdef", 4)) { + t.Fatal("rejected lowercase digest") + } +} diff --git a/core/internal/runnablestage/unsupported.go b/core/internal/runnablestage/unsupported.go new file mode 100644 index 00000000..16fd9e2e --- /dev/null +++ b/core/internal/runnablestage/unsupported.go @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build !linux + +package runnablestage + +import ( + "context" + "fmt" + "io" +) + +type noopLock struct{} + +func (noopLock) Close() error { return nil } + +// Acquire is a no-op outside Linux, where Reap is unsupported. +func Acquire(root string) (io.Closer, error) { return noopLock{}, nil } + +func Reap(context.Context, string, Options) (Result, error) { + return Result{}, fmt.Errorf("runnable stage reaping is supported only on Linux") +} diff --git a/core/internal/runtime/bundle.go b/core/internal/runtime/bundle.go index b765c74b..2187dd59 100644 --- a/core/internal/runtime/bundle.go +++ b/core/internal/runtime/bundle.go @@ -396,6 +396,12 @@ func GenerateBundleWithIdentityAndRegen(cfg artifact.JVMConfig, jdkRoot, launche return os.WriteFile(filepath.Join(outDir, "config.json"), b, 0o644) } +// Validate checks resource limits before a caller creates bundle-owned files. +func (res Resources) Validate() error { + _, err := buildResources(res) + return err +} + func buildResources(res Resources) (ociResources, error) { out := ociResources{} if res.MemoryLimit != "" { diff --git a/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go b/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go index 5c34e93f..3c60e3e8 100644 --- a/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go +++ b/core/shim/cmd/containerd-shim-brewlet-v2/service_linux.go @@ -36,6 +36,7 @@ import ( "google.golang.org/protobuf/types/known/anypb" "github.com/microsoft/brewlet/internal/artifact" + "github.com/microsoft/brewlet/internal/runnablestage" kcruntime "github.com/microsoft/brewlet/internal/runtime" "github.com/microsoft/brewlet/internal/telemetry" ) @@ -175,6 +176,12 @@ func (s *brewletTaskService) Create(ctx context.Context, r *taskAPI.CreateTaskRe return nil, err } if !sandbox { + // Hold the stage guard until runc has installed the workload's mounts. + guard, err := runnablestage.Acquire(runnablestage.Root()) + if err != nil { + return nil, err + } + defer guard.Close() info, err := assembleBrewletBundle(ctx, r, s.imageIdentity) if err != nil { emitLaunch(info, err) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 9b13c984..784cf4ee 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -23,6 +23,7 @@ brewlet keygen [flags] generate an ECDSA P-256 key pair brewlet inspect [flags] show the artifact manifest + config brewlet run [flags] pull + launch java -jar on this node brewlet bundle [flags] emit an OCI runc bundle (the shim path) +brewlet stage-gc [flags] reclaim orphaned runnable-image stages (Linux) brewlet jdks [flags] list JDKs available across the cluster brewlet doctor [flags] diagnose cluster and developer readiness brewlet k8s [flags] manage and inspect Brewlet on Kubernetes @@ -269,6 +270,56 @@ cat ./bundle/config.json --- +## `brewlet stage-gc` + +Perform one reference-aware cleanup sweep on a Linux node. Run as root in the +host PID, mount, and initial user namespaces, with complete host `/proc` +visibility and access to containerd. + +```text +brewlet stage-gc [--stage-root DIR] [--address SOCKET] [--min-age DURATION] [--dry-run] +``` + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--stage-root` | `BREWLET_RUNNABLE_STAGE`, otherwise `/tmp/brewlet-runnable` on Linux (ignores `TMPDIR`) or `os.TempDir()/brewlet-runnable` elsewhere | Administrator-owned staging root; must match the shim. | +| `--address` | `/run/containerd/containerd.sock` | containerd endpoint; references are checked across all namespaces. | +| `--min-age` | `24h` | Positive Go duration. Only published stages with directory modification times older than this floor are eligible; this is not time since last use. | +| `--dry-run` | `false` | Apply the same eligibility checks and report candidates without deleting them. | + +On a node installed by the provisioner, the CLI is packaged under a helper name: + +```bash +sudo /usr/local/bin/brewlet-stage-gc stage-gc --dry-run +``` + +With the standalone `brewlet` CLI installed on the host: + +```bash +sudo brewlet stage-gc --dry-run +# Only after completing the migration prerequisites: +sudo brewlet stage-gc --min-age 24h +``` + +Only canonical `immutable-v2/` directories are considered. Image +and content references, live mounts, or an active staging guard prevent +reclamation. Incomplete reference inspection fails closed; legacy and abandoned +pending directories are not removed. The command reports stage counts, logical +bytes reclaimed (or eligible in dry-run), and remaining logical stage bytes. +In dry-run, remaining bytes describe the current tree, not projected usage. +Errors, including lock contention, exit nonzero. A five-minute context deadline +and TERM/INT cancellation bound cooperative work; filesystem operations already +in progress may take longer to return. + +**Manual invocation does not check `stageGC.enabled`, migration acknowledgment, +node ownership, or the provisioner's compatibility record.** Complete the +[migration prerequisites](runnable-image.md#upgrading-existing-nodes) before +deleting anything. Helm-managed nodes already have a periodic loop; do not add +a second timer. For standalone installations only, arrange a host timer after +those checks. See [Runnable stage cleanup](runnable-image.md#reclaiming-unused-stages). + +--- + ## `brewlet jdks` List the JDKs available across the cluster — **vendor, major version, minor diff --git a/docs/configuration.md b/docs/configuration.md index 60a06b54..8946aa4a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -9,6 +9,8 @@ and autoscaling, see [Capability labels and autoscaling](capability-labels-and-autoscaling.md). For the operational workflow behind the metrics values, see [Runtime metrics and Grafana dashboards](runtime-metrics.md). +For runnable-image cache reclamation and migration prerequisites, see +[Runnable stage cleanup](runnable-image.md#reclaiming-unused-stages). --- @@ -75,6 +77,10 @@ with `--set key=value` or a values file. | `operator.replicas` | `1` | Operator replica count. | | `operator.leaderElect` | `true` | Enable leader election for HA. | | `operator.resources` | requests `50m/64Mi`, limits `200m/128Mi` | Operator pod resources. | +| `stageGC.enabled` | `true` | Run periodic orphaned runnable-image stage cleanup in every managed provisioner, independently of metrics. Fresh nodes activate automatically; existing installations require a matching compatibility record or migration acknowledgment. | +| `stageGC.interval` | `5m` | Delay after each attempt, plus up to 10% jitter. Sweeps do not overlap. | +| `stageGC.minAge` | `24h` | Minimum stage directory age, not time since last use or since becoming unreferenced. | +| `stageGC.upgradeAcknowledged` | `false` | Confirm older unguarded shims and stage-dependent exported bundles have been retired across all managed nodes. Reset after the rollout; compatible nodes retain their approval. | | `uninstall.timeoutSeconds` | `240` | Pre-delete cleanup coordinator timeout, 1-86400 whole seconds. Configure before uninstalling; Helm's `--timeout` must exceed this plus 30 seconds. Failure retains the control plane. See [Uninstall](installation.md#uninstall). | | `uninstall.imagePullSecrets` | `[]` | Namespaced registry Secret references for the cleanup Job, which uses the operator image and its own service account. | | `metrics.enabled` | `false` | Opt in to Brewlet runtime and control-plane Prometheus endpoints. Enables the operator and admission listeners plus the node exporter sidecar and Services. | @@ -125,6 +131,27 @@ component-image overrides match it. For an existing installation, follow > tagless SHA-256 digest references. For air-gapped clusters, mirror the exact > digest and approve the destination host explicitly. +### Runnable-image stage cleanup + +`stageGC.*` is operator-wide policy, not a `NodeProfile.spec` field. It applies +to chart-rendered and externally managed profiles alike. Configure it through +Helm values or operator flags, not by patching a reconciled DaemonSet. +`interval` and `minAge` accept Go durations that resolve to positive whole +seconds, at most `2147483647s`; the operator rejects invalid values. + +Kubelet and containerd still own image retention and content reclamation. +Brewlet removes only old, unreferenced, unmounted published stages; it does not +copy kubelet's disk thresholds or change kubelet configuration. Containerd +references in any namespace can delay reclamation. Legacy and abandoned pending +trees are not automatically removed. + +For existing nodes, follow the [GC activation procedure](installation.md#activating-runnable-stage-gc) +before setting `stageGC.upgradeAcknowledged=true`. Readiness does not imply +that GC is active. Setting `stageGC.enabled=false` stops automatic sweeps, not +separately scheduled or manually invoked cleanup commands. + +### AppCDS regeneration + AppCDS regeneration is default-deny. Enable it for the default profile with `--set provisioner.appCDS.regenerationEnabled=true`, or on a named profile with: @@ -220,6 +247,10 @@ When you install via Helm, the chart populates them for you. | `--health-probe-bind-address` | `:8081` | Health/readiness endpoint. | | `--node-metrics-enabled` | `false` | Run the node-local exporter sidecar in managed provisioner pods. | | `--node-metrics-port` | `9090` | Exporter port when node metrics are enabled. | +| `--stage-gc-enabled` | `true` | Enable periodic stage cleanup after successful provisioning and compatibility checks. | +| `--stage-gc-interval` | `5m` | Delay between GC attempts, plus jitter; positive whole seconds in Go duration syntax. | +| `--stage-gc-min-age` | `24h` | Minimum stage directory age; positive whole seconds in Go duration syntax. | +| `--stage-gc-upgrade-acknowledged` | `false` | Acknowledge retirement of unguarded stage consumers across the managed fleet. Reset after rollout. | For a local operator built from the same release as the installed CLI, use that CLI's version to select the matching provisioner image. For custom source builds, @@ -261,6 +292,11 @@ only touch them directly if you hand-wire the DaemonSet. | `LAUNCHER_SOURCE__PATH` | required per entry | Clean absolute launcher path inside the image. | | `JDKS` / `LAUNCHERS` | derived | Internal comma-separated inventories derived from the indexed entries. They are not accepted as independent inputs. | | `BREWLET_APP_CDS_REGENERATION_ENABLED` | `false` | Internal operator-to-provisioner policy transport. When true, atomically creates the root-owned AppCDS authorization sentinel and publishes `brewlet.sh/appcds-regeneration=true`; when false or during cleanup/failure, removes both. Configure `spec.appCDS.regenerationEnabled`, not this variable directly. | +| `BREWLET_STAGE_GC_ENABLED` | `true` | Enable the periodic GC loop after provisioning, subject to compatibility checks. Rendered from operator-wide `stageGC.enabled`. | +| `BREWLET_STAGE_GC_INTERVAL_SECONDS` | `300` | Delay between GC attempts, plus up to 10% jitter; integer seconds in `1..2147483647`. | +| `BREWLET_STAGE_GC_MIN_AGE_SECONDS` | `86400` | Minimum stage directory age; integer seconds in `1..2147483647`. | +| `BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED` | `false` | Confirm older unguarded shims and stage-dependent exported bundles have been retired; rendered from operator-wide policy. | +| `BREWLET_RUNNABLE_STAGE` | `/tmp/brewlet-runnable` (provisioner) | Host stage root used by automatic GC. A custom location must also be configured separately for the shim and exporter's read-only mount; this variable does not reconfigure those processes. | | `NODE_NAME` | (downward API) | The node to label; injected from `spec.nodeName`. | | `BREWLET_PROFILE_NAME` | `default` | Profile name paired with its UID in managed worker authority checks. | | `BREWLET_PROFILE_UID` | *(empty)* | Operator-managed profile UID paired with node ownership and persisted writer authority. | @@ -270,8 +306,8 @@ only touch them directly if you hand-wire the DaemonSet. | `CONTAINERD_CONFIG` | `/etc/containerd/config.toml` | Primary containerd configuration. Validated mode uses an imported drop-in when supported and otherwise patches this file with a backup. | | `CONTAINERD_DROPIN_DIR` | `/etc/containerd/config.toml.d` | Drop-in directory used when the primary config imports `*.toml` from it. | | `CONTAINERD_DROPIN_FILE` | `/etc/containerd/config.toml.d/99-brewlet.toml` | Brewlet-managed runtime drop-in. | -| `CONTAINERD_ADDRESS` | `/run/containerd/containerd.sock` | Host containerd socket (used for copy-from-image). | -| `CONTAINERD_NAMESPACE` | `k8s.io` | containerd namespace for image pulls. | +| `CONTAINERD_ADDRESS` | `/run/containerd/containerd.sock` | Host containerd socket for copy-from-image and stage GC reference inspection. | +| `CONTAINERD_NAMESPACE` | `k8s.io` | containerd namespace for image pulls; does not restrict GC, which checks all namespaces. | | `BREWLET_MODE` | `provision` | `provision` installs the runtime; `cleanup` reverses recorded host state for a retiring target or deleted profile. Managed cleanup uses the frozen per-node containerd policy and runs in `brewlet-cleanup-` (§5.6). | | `BREWLET_CONTAINERD_RESTART` | `validated` | `validated` smoke-tests the runtime inventory, validates the effective config with `containerd config dump`, restarts the host service only when needed, checks containerd and the live `brewlet` handler, and restores known-good config on activation failure. `sighup` preserves the legacy in-place render/reload path without the config-dump gate. `none` neither mutates nor signals containerd. Rendered from `spec.rollout.containerdRestart`. | | `BREWLET_VALIDATE` | `true` | Run `java -version` for every JDK and verify every staged launcher is executable before publishing runtime or capability labels. Arbitrary launchers are not executed. `false` skips both sets of checks. Rendered from `spec.rollout.validate`. | diff --git a/docs/installation.md b/docs/installation.md index 95e54ac3..e84c1b16 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -359,6 +359,11 @@ For source builds, use the local chart instead and retain your digest-pinned component overrides. For incompatible legacy profiles, use the maintenance sequence below instead of an ordinary upgrade. +An upgrade that introduces automatic runnable-stage GC can leave existing nodes +Ready while cleanup remains blocked pending migration acknowledgment. +`helm upgrade --wait` does not prove GC is active. After the runtime rollout, +follow [Activating runnable-stage GC](#activating-runnable-stage-gc). + An older CRD prunes unsupported fields when a resource is saved. Updating the CRD cannot restore those values; reapply the original JavaApplication manifests afterward. @@ -419,6 +424,68 @@ helm upgrade brewlet oci://ghcr.io/microsoft/charts/brewlet \ --wait ``` +#### Activating runnable-stage GC + +Fresh nodes enable periodic cleanup automatically. On an existing installation, +the default `stageGC.upgradeAcknowledged=false` keeps cleanup blocked unless +the node has a matching compatibility record. A blocked node continues serving +workloads and logs `stage GC blocked`. + +First roll out the matching operator and provisioner, then retire older +unguarded shims and finish or quiesce their launches. Merely replacing the shim +binary does not retire already running shim processes; use your workload +restart/drain procedure where needed. Retire or regenerate exported OCI bundles +that point into the stage cache, and stop or upgrade other unguarded stage +consumers. Complete these prerequisites across **all operator-managed nodes**, +including externally managed NodeProfiles, before acknowledging migration: + +```bash +helm upgrade brewlet oci://ghcr.io/microsoft/charts/brewlet \ + --version "$RELEASE_VERSION" \ + --namespace brewlet \ + --values values.yaml \ + --values my-jdks.yaml \ + --set stageGC.upgradeAcknowledged=true \ + --wait +``` + +Keep `stageGC.enabled=true` for activation. For source builds, use the local chart +and retain the matching component overrides as above. Check the provisioner +logs for each managed node, not just pod readiness: + +```bash +kubectl get pods -n brewlet -l app=brewlet-node-provisioner -o wide +kubectl logs -n brewlet -c provisioner --tail=100 +``` + +Look for `stage GC: successful_sweeps=...` with a recent `last_success` timestamp; +zero removed stages is a valid successful sweep. `last_success=never` with +failed attempts means cleanup has not succeeded. Investigate inspection, +namespace, socket-access, and lock errors; do not bypass them by deleting stage +directories. Neither GC failure counters nor compatibility blocking are part of +the pod readiness gate. + +After the rollout, reset the fleet-wide acknowledgment: + +```bash +helm upgrade brewlet oci://ghcr.io/microsoft/charts/brewlet \ + --version "$RELEASE_VERSION" \ + --namespace brewlet \ + --values values.yaml \ + --values my-jdks.yaml \ + --set stageGC.upgradeAcknowledged=false \ + --wait +``` + +Compatible nodes retain their approval, tied to the installed shims and stage +path. Disable automatic GC with `stageGC.enabled=false` before manually rolling +back shims or introducing unguarded consumers. Do not add a separate timer or +invoke destructive manual cleanup to bypass blocked activation: manual commands +do not enforce the provisioner's migration gate. See +[Runnable stage cleanup](runnable-image.md#reclaiming-unused-stages) for the +reclamation rules and [Configuration](configuration.md#runnable-image-stage-cleanup) +for all settings. + ### What the chart deploys vs. what the operator creates | Deployed by the chart | Created/reconciled by the operator at runtime | diff --git a/docs/runnable-image.md b/docs/runnable-image.md index 8f874081..ab5760de 100644 --- a/docs/runnable-image.md +++ b/docs/runnable-image.md @@ -118,7 +118,7 @@ New shims publish stages under an `immutable-v2` subdirectory of `BREWLET_RUNNABLE_STAGE` (or the default temporary staging root). They leave legacy stages untouched because running workloads may still mount those files. Allow extra disk capacity during rollout: legacy stages and staging directories left -by abruptly terminated processes are not automatically garbage-collected. +by abruptly terminated processes are not removed by the stage reaper. Do not remove staging trees while workloads still reference them. The stage retains descriptor-verified packed layers as well as extracted @@ -131,6 +131,134 @@ layers are already missing, or an upgrade with only a legacy stage, requires a verified image re-pull; this does not repair missing content from an unpacked snapshot. Allow disk capacity for the retained compressed bytes per image. +### Reclaiming unused stages + +Run the Linux node command as root, in the host PID, mount, and initial user +namespaces, with complete visibility of the host's `/proc` and access to the +containerd socket. On nodes installed by the provisioner, use its host helper: + +```sh +sudo /usr/local/bin/brewlet-stage-gc stage-gc --dry-run +``` + +If you installed the standalone `brewlet` CLI on the host, use: + +```sh +sudo brewlet stage-gc --dry-run +# Only after completing the migration prerequisites below: +sudo brewlet stage-gc --min-age 24h +# Non-default locations (use the same stage root as the shim): +sudo brewlet stage-gc --stage-root /var/lib/brewlet/runnable \ + --address /run/containerd/containerd.sock --min-age 48h +``` + +Manual commands do not enforce `stageGC.enabled`, migration acknowledgment, +node ownership, or the provisioner's compatibility record. The reaper's +reference, mount, and locking checks still apply, but they cannot make +unguarded legacy consumers safe. Do not use a manual deletion or an additional +timer to bypass blocked automatic activation. + +`--stage-root` defaults to `BREWLET_RUNNABLE_STAGE`, then +`/tmp/brewlet-runnable` on Linux (independent of `TMPDIR`, so it matches the +shim even when containerd sets `TMPDIR`) or `os.TempDir()/brewlet-runnable` +elsewhere. `--min-age` must be positive and defaults to +24 hours. Keep the stage root administrator-owned and do not rename or replace +it while launchers or cleanup are running. Only published +`immutable-v2/` directories older than +that floor are eligible. The command preserves stages whose manifest remains +in containerd image or content metadata in **any namespace**, including platform +manifests referenced by image indexes. Removing an image alone may not free its +stage until containerd also garbage-collects the manifest content. + +The reaper checks live references across process mount namespaces, including +read-only bind mounts of individual staged files. It fails closed if it cannot +establish that reference information is complete. A shared staging guard covers +resolution and workload mount creation; cleanup takes the exclusive guard and +fails if a launch is in progress. Retry on the next scheduled run rather than +removing trees manually. Running workloads do not need to stop. + +The Helm chart enables periodic cleanup by default on fresh nodes through the +existing provisioner, independently of `metrics.enabled`. The global settings +apply to every operator-managed NodeProfile, including externally managed CRs: + +```yaml +stageGC: + enabled: true + interval: 5m + minAge: 24h + upgradeAcknowledged: false +``` + +The provisioner enters the host PID and mount namespaces for each sweep and +rechecks node/profile ownership before it runs. If the Kubernetes API cannot be +read, that sweep is skipped and retried; the worker exits only when a successful +read shows it no longer owns the node. Sweeps start after successful +provisioning, do not overlap, and wait the configured interval plus up to 10% +jitter between attempts. Each attempt has a five-minute context deadline; +filesystem operations already in progress may take longer to return. Lock contention +or inspection failures are logged and retried on the next interval; shutdown +signals cancel the active reaper. Logs include removed stage counts, logical +bytes, process-local success/failure counters, and the last successful sweep. +GC never runs in the provisioner's teardown mode. Disable it with +`--set stageGC.enabled=false`. + +This follows kubelet's image-retention decisions rather than competing with +them: kubelet removes unused image records via CRI, containerd eventually +releases unreferenced content, and Brewlet reclaims the orphaned stages. Brewlet +does not delete containerd images, read or modify kubelet configuration, or apply +another set of disk-pressure thresholds. The `minAge` floor measures stage +directory age, **not** time since last use or since becoming unreferenced, and +is not kubelet's `imageMinimumGCAge` or `imageMaximumGCAge`. This is eventual +orphan reclamation, not guaranteed immediate disk-pressure relief. + +### Upgrading existing nodes + +Existing installations without a matching compatibility record stay provisioned +but log `stage GC blocked` instead of deleting stages. Before acknowledging the +migration, retire older shims that do not participate in the guard, finish or +quiesce their launches, and retire or regenerate previously exported OCI bundles +that name stage paths. Merely installing a new shim does not retire already +running older shims. New `brewlet bundle` outputs retain their own payloads. +Then upgrade with `stageGC.upgradeAcknowledged=true` in addition to your normal +release values. This acknowledges those prerequisites for **all managed nodes**; +do not set it before the whole selected fleet is ready. + +After the rollout, reset `stageGC.upgradeAcknowledged=false`. Each compatible +node retains a root-owned `/opt/brewlet/.stage-gc-compatible` record tied to both +installed shim copies and the staging path. A changed shim identity or path +without a matching record blocks activation again at provisioner startup. An interrupted installation +can also require acknowledgment again. Disable GC before manually rolling back +shims or introducing consumers that do not participate in the guard. +Earlier Linux shims derived their default stage root from `TMPDIR`. If +containerd set a non-default `TMPDIR`, stages created before the upgrade remain +under that old location; new launches use `/tmp/brewlet-runnable`. Automatic GC +does not sweep the old location. Remove it manually only after the migration +checks above confirm nothing still references it. +For complete Helm commands and per-node activation checks, see +[Activating runnable-stage GC](installation.md#activating-runnable-stage-gc). + +Standalone shim installation still does not schedule GC. Outside the provisioner, +use a host timer or root cron job, such as an hourly invocation of +`/usr/local/bin/brewlet stage-gc --min-age 24h`, after the same migration checks. +Legacy trees and abandoned pending trees remain outside automated eviction. +External consumers that read stages without mounts must participate in the +staging guard or be stopped during cleanup; open file descriptors alone are +not tracked. + +### Observing stage usage and cleanup + +The metrics exporter exposes `brewlet_runnable_stage_bytes`, the logical size +of regular files remaining under its stage root (including legacy and pending +trees, without following symlinks). This is not filesystem-allocated space or +free disk space. It is refreshed on each scrape; inspection errors fail the +scrape rather than reporting a misleading zero. The operator's exporter mounts +the default host stage root read-only. For a custom staging location, configure +the exporter's `--stage-root` and its read-only host mount to point to that same +location. Alert on sustained growth and monitor node filesystem free space too. +Sweep success/failure counters and last-success timestamps are currently +provisioner logs, not Prometheus metrics. See the +[metric catalog](runtime-metrics.md#runtime-and-node-metrics). + ## 5. Operator & webhook - The `JavaApplication` controller sets the Deployment's container `image:` to diff --git a/docs/runtime-metrics.md b/docs/runtime-metrics.md index 2d9919d7..2b48edf4 100644 --- a/docs/runtime-metrics.md +++ b/docs/runtime-metrics.md @@ -105,7 +105,8 @@ observability must never block or fail workload launch. The `metrics-exporter` sidecar runs in every profile-managed provisioner DaemonSet. It receives shim events, exposes them as Prometheus counters and histograms, and derives inventory gauges from the node's active JDK and launcher -state under `/opt/brewlet`. +state under `/opt/brewlet`. It also measures runnable-image stage bytes through +a read-only host mount of `/tmp/brewlet-runnable`. The sidecar is deliberately narrow: it runs with a read-only root filesystem, all capabilities dropped, and no privilege escalation, and it mounts @@ -113,6 +114,9 @@ all capabilities dropped, and no privilege escalation, and it mounts `DirectoryOrCreate` mount of `/opt/brewlet/metrics`, which exists solely so the exporter can bind the telemetry socket. That volume is present only when the sidecar itself is — a metrics-disabled or cleanup DaemonSet carries neither. +The read-only stage volume is also omitted without the exporter. Automatic +stage GC runs separately in the provisioner through the host namespaces and +does not depend on exporter mounts or `metrics.enabled`. The operator and admission webhook use controller-runtime's Prometheus registry. Their endpoints include standard controller-runtime/process metrics alongside @@ -205,6 +209,19 @@ If you changed `metrics.nodePort`, replace `9090` with that value. | `brewlet_jdk_info` | Gauge | `distribution`, `feature`, `version`, `vendor`, `arch`, `source` | Value `1` identifies each active JDK build on a node. | | `brewlet_jdk_installed_timestamp_seconds` | Gauge | `distribution`, `feature`, `version` | Unix timestamp when that JDK root was installed on the node. It is **not** the upstream JDK patch release date. | | `brewlet_launcher_info` | Gauge | `launcher` | Value `1` identifies an available launcher. Vanilla `java` is always emitted; additional installed launchers such as `jaz` are also reported. | +| `brewlet_runnable_stage_bytes` | Gauge | None | Logical regular-file bytes under the host staging root, including legacy and pending stages. Refreshed on each scrape without following symlinks. This is not allocated disk space, free space, or reclaimable bytes. | + +Stage inspection errors fail the scrape instead of reporting a misleading zero. +Monitor node filesystem free space alongside stage usage; protected, legacy, +and pending trees can keep the gauge high even when GC is working correctly. +The exporter's `--stage-root` and read-only host mount must match the shim for a +custom stage location. + +Sweep success/failure counters and last-success timestamps are currently +reported in **provisioner logs**, not Prometheus metrics. The exporter only +observes usage; it never deletes stages. See +[Runnable stage cleanup](runnable-image.md#observing-stage-usage-and-cleanup) +and the [activation checks](installation.md#activating-runnable-stage-gc). `brewlet_sandbox_launches_total` uses these bounded failure reasons: `NoCompatibleJDK`, `NoCompatibleLauncher`, `NoCompatibleArch`, diff --git a/docs/third-party-notices.md b/docs/third-party-notices.md index a065125a..5f30c943 100644 --- a/docs/third-party-notices.md +++ b/docs/third-party-notices.md @@ -27,7 +27,7 @@ only inside a built artifact and cannot be browsed in the source tree. | `brewlet` CLI archive | `NOTICE.txt` beside the executable | `./scripts/generate-go-notice.sh core /tmp/cli-notice.txt ./cmd/brewlet` | | Operator image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh kubernetes /tmp/operator-notice.txt ./cmd/manager` | | Admission image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh kubernetes /tmp/admission-notice.txt ./cmd/admission` | -| Node provisioner image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh core /tmp/provisioner-notice.txt ./shim/cmd/containerd-shim-brewlet-v2 ./cmd/brewlet-metrics-exporter ./cmd/brewlet-source-policy` | +| Node provisioner image | `/NOTICE.txt` | `./scripts/generate-go-notice.sh core /tmp/provisioner-notice.txt ./shim/cmd/containerd-shim-brewlet-v2 ./cmd/brewlet-metrics-exporter ./cmd/brewlet-source-policy ./cmd/brewlet` | | Helm chart | No third-party code is embedded in the chart package | — | The provisioner image build appends the `kubectl`, `ctr`, and `crictl` licenses diff --git a/integration-tests/AGENTS.md b/integration-tests/AGENTS.md index cedad591..eb6ccb1d 100644 --- a/integration-tests/AGENTS.md +++ b/integration-tests/AGENTS.md @@ -40,15 +40,15 @@ The harness does not switch branches or modify component sources. It uses | Tool | Tiers | |---|---| | Go | all | -| Python 3 | 2, 4, 12, 13, 16 | -| JDK 21+ | 2, 3, 7, 8, 9, 12, 14, 15, 16 | -| Docker | 3, 6, 7, 8-12, 14, 15, 16 | -| kubectl and a reachable cluster | 4-16 | -| Helm | 4 (optional), 10, 15 | +| Python 3 | 2, 4, 12, 13, 16, 17 | +| JDK 21+ | 2, 3, 7, 8, 9, 12, 14, 15, 16, 17 | +| Docker | 3, 6, 7, 8-12, 14-17 | +| kubectl and a reachable cluster | 4-17 | +| Helm | 4 (optional), 10, 15, 17 | | OpenSSL | 4, 5, 6, 10, 11 | Host-only tiers 1-3 need no cluster. Tiers 4-7 and 13 exercise API-server -behavior. Tiers 6, 8-12, 14, 15, and 16 require local containerd nodes that +behavior. Tiers 6, 8-12, and 14-17 require local containerd nodes that Docker can enter, such as kind. Managed clusters skip those node-side paths. Tier 13 also proves the control-plane guard: a NodeProfile that does not set `nodePool.includeControlPlane` never counts or schedules onto a control-plane @@ -69,6 +69,15 @@ a silently non-running pod. It moves the node's shim aside for the last case and it both inline and from its cleanup trap. §14's remaining row, the cgroup-v1 refusal, cannot be produced on a cgroup-v2 CI node and is covered deterministically by `provisioner/entrypoint_test.sh` over `require_cgroup_v2`. +Tier 17 installs the chart with default `stageGC` values, verifies they reach +the provisioner DaemonSet without the metrics exporter, and proves the upgrade +gate blocks GC on a node whose stage root already holds data. It then +acknowledges the upgrade with `interval=5s` and `minAge=1s`, resets the +acknowledgment to prove the per-node compatibility record persists, and shows +that a runnable-image stage survives while its pod runs and while containerd +still holds the image, then is reclaimed after the image is removed. Once GC is +active, it may also reclaim unreferenced stages left on the node by earlier +tiers. **Architecture coverage.** Tiers pick up the node's architecture automatically, but hosted runners are amd64, so `.github/workflows/e2e.yml` runs the host-only @@ -106,7 +115,7 @@ Common environment-specific skips: - Tier 5 skips when a cluster cannot reach a host-bound webhook; tier 6 covers the same assertions in-cluster. -- Tiers 8, 9, 12, 14, 15, and 16 skip if no schedulable local containerd node +- Tiers 8, 9, 12, and 14-17 skip if no schedulable local containerd node can be provisioned. - Tier 12 skips when the node's `ctr` supports neither `images unpack` nor import-time unpack; tier 16 applies the same rule. diff --git a/integration-tests/README.md b/integration-tests/README.md index 565bf70f..aad34627 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -7,7 +7,7 @@ manifests, and specifications remain in their owning monorepo directories. ## Layout ```text -e2e/ runner, reset helper, shared library, and tiers 1-15 +e2e/ runner, reset helper, shared library, and tiers 1-17 fixtures/ harness-owned Java demo applications and PetClinic build fixture ``` @@ -26,7 +26,7 @@ testing external checkouts. ## Running ```bash -integration-tests/e2e/run.sh # all 15 tiers +integration-tests/e2e/run.sh # all 17 tiers integration-tests/e2e/run.sh --list # tier catalog integration-tests/e2e/run.sh --tier 4 # one tier integration-tests/e2e/run.sh --reset # remove Brewlet test state @@ -51,6 +51,8 @@ an exercised capability fails. The suite covers: | 13 | NodeProfile lifecycle | Kubernetes | | 14 | custom JDK + jaz NodeProfile, live workload, and broken-launcher readiness failure | both + fixtures | | 15 | live opt-in Prometheus metrics through Helm, provisioner, shim, and exporter | both + fixtures | +| 16 | SPECIFICATION §14 failure modes on a live node | core + fixtures | +| 17 | default-enabled runnable-stage GC: chart defaults, upgrade gate, reference protection, and reclamation | both + fixtures | See [AGENTS.md](AGENTS.md) for cluster requirements, cleanup, and troubleshooting. diff --git a/integration-tests/e2e/reset.sh b/integration-tests/e2e/reset.sh index 3e360018..ad09fe55 100755 --- a/integration-tests/e2e/reset.sh +++ b/integration-tests/e2e/reset.sh @@ -31,7 +31,7 @@ fi # Fixed namespaces the tiers use for their fixtures. Throwaway per-run # namespaces (tiers 5/6/10/11) are random and self-cleaned by their RETURN # traps; we only need to sweep the deterministic ones here. -E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e) +E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e brewlet-stagegc-e2e) # scrub_node_labels: remove every brewlet.sh/* label and annotation from every # node. These are advertised by the provisioner (or simulated by tiers) and are diff --git a/integration-tests/e2e/run.sh b/integration-tests/e2e/run.sh index 75941800..9f981fd3 100755 --- a/integration-tests/e2e/run.sh +++ b/integration-tests/e2e/run.sh @@ -32,7 +32,7 @@ mkdir -p "$WORK" for arg in "$@"; do case "$arg" in --list) - printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster)\n' + printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster)\n' exit 0 ;; esac @@ -97,6 +97,7 @@ source "$E2E_DIR/tier13-nodeprofile.sh" source "$E2E_DIR/tier14-custom-jdk.sh" source "$E2E_DIR/tier15-metrics-incluster.sh" source "$E2E_DIR/tier16-failure-modes.sh" +source "$E2E_DIR/tier17-stage-gc.sh" declare -a TIERS=() DO_RESET=0 @@ -106,7 +107,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --tier) TIERS+=("$2"); shift 2 ;; --reset) DO_RESET=1; shift ;; - --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster)\n'; exit 0 ;; + --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster)\n'; exit 0 ;; -h|--help) usage; exit 0 ;; *) warn "unknown arg: $1"; usage; exit 2 ;; esac @@ -119,7 +120,7 @@ if [[ "$DO_RESET" -eq 1 ]]; then [[ ${#TIERS[@]} -eq 0 ]] && exit 0 fi -[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16) +[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17) section "Brewlet E2E — environment" info "harness : $REPO_DIR" @@ -177,6 +178,7 @@ for t in "${TIERS[@]}"; do 14) tier14_custom_jdk ;; 15) tier15_metrics_incluster ;; 16) tier16_failure_modes ;; + 17) tier17_stage_gc ;; *) warn "no such tier: $t" ;; esac done diff --git a/integration-tests/e2e/tier17-stage-gc.sh b/integration-tests/e2e/tier17-stage-gc.sh new file mode 100644 index 00000000..526d8bdd --- /dev/null +++ b/integration-tests/e2e/tier17-stage-gc.sh @@ -0,0 +1,580 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Tier 17 — default-enabled runnable-stage GC through the real Helm chart, +# operator, provisioner DaemonSet, shim, kubelet, and containerd. +# +# What this tier proves on a local containerd node (kind/CI): +# (1) the chart's defaults enable GC and propagate interval/min-age/ack to the +# provisioner DaemonSet without the metrics exporter; +# (2) the upgrade gate blocks GC on a node whose stage root already holds data +# and no compatibility record exists, and never deletes anything; +# (3) stageGC.upgradeAcknowledged=true activates sweeps and writes the per-node +# compatibility record, which keeps GC active after the acknowledgment is +# reset; +# (4) a stage stays while its image is referenced by containerd and mounted by +# a running pod, and still stays after the pod is gone but the image +# remains; +# (5) once the image is removed and containerd collects its content, a sweep +# deletes the orphaned stage and leaves non-canonical trees alone. +# +# The tier uses minAge=1s and interval=5s to finish quickly. With GC activated, +# the reaper may also remove other unreferenced canonical stages left on the +# test node by earlier tiers; that is exactly its production behavior. + +T17_RELEASE="brewlet-stagegc-e2e" +T17_RELEASE_NS="default" +T17_NS="brewlet" +T17_APP_NS="brewlet-stagegc-e2e" +T17_PROFILE="stagegc" +T17_POOL_KEY="brewlet.sh/e2e-pool" +T17_POOL="stagegc" +T17_JDK="temurin-21" +T17_REF="demo/hello:stagegc-e2e-$$" +T17_APP="stagegc-orders" +T17_OP_IMG="brewlet.local/brewlet-operator:stagegc-e2e" +T17_ADM_IMG="brewlet.local/brewlet-admission:stagegc-e2e" +T17_PROV_IMG="brewlet.local/brewlet-node-provisioner:stagegc-e2e" +T17_STAGE_ROOT="/tmp/brewlet-runnable" +T17_RECORD="/opt/brewlet/.stage-gc-compatible" +T17_SENTINEL="$T17_STAGE_ROOT/t17-legacy-sentinel" +T17_NODE="" +T17_ARCH="" +T17_HELM_INSTALLED="" +T17_PROFILE_CREATED="" +T17_APP_NS_CREATED="" +T17_NODE_TOUCHED="" +T17_JDK_PREEXISTING="" +T17_JDK_ACTIVE_PREEXISTING="" +T17_RECORD_PREEXISTING="" +T17_SENTINEL_CREATED="" +T17_IMAGE_DIGEST="" +declare -a T17_LOADED_NODES=() +declare -a T17_BUILT_IMAGES=() + +# Snapshot/restore host files the provisioner may mutate. An empty snapshot +# plus a missing ".present" marker means the file did not exist. +_t17_snapshot_file() { + local path="$1" out="$2" + rm -f "$out" "$out.present" + if docker exec "$T17_NODE" test -f "$path" >/dev/null 2>&1; then + docker exec "$T17_NODE" cat "$path" >"$out" || return 1 + : >"$out.present" + else + : >"$out" + fi +} + +_t17_restore_file() { + local path="$1" snapshot="$2" + [[ -f "$snapshot" ]] || return 0 + if [[ -f "$snapshot.present" ]]; then + docker exec "$T17_NODE" mkdir -p "$(dirname "$path")" >/dev/null 2>&1 && + docker exec -i "$T17_NODE" sh -c 'cat > "$1"' sh "$path" <"$snapshot" >/dev/null 2>&1 + else + docker exec "$T17_NODE" rm -f "$path" >/dev/null 2>&1 + fi +} + +_t17_restore_containerd() { + local before="$WORK/t17-containerd-state-before" after="$WORK/t17-containerd-state-after" tries=30 + _t17_restore_file /etc/containerd/config.toml "$WORK/t17-containerd.toml" || return 1 + _t17_restore_file /etc/containerd/config.toml.brewlet.bak "$WORK/t17-containerd-bak.toml" || return 1 + _t17_restore_file /etc/containerd/config.toml.d/99-brewlet.toml "$WORK/t17-containerd-dropin.toml" || return 1 + docker exec "$T17_NODE" sh -c \ + 'cat /etc/containerd/config.toml /etc/containerd/config.toml.d/99-brewlet.toml 2>/dev/null' \ + >"$after" 2>/dev/null || true + cmp -s "$before" "$after" && return 0 + docker exec "$T17_NODE" systemctl restart containerd >/dev/null 2>&1 || return 1 + while (( tries-- > 0 )); do + docker exec "$T17_NODE" ctr version >/dev/null 2>&1 && return 0 + sleep 1 + done + return 1 +} + +_t17_image_refs() { + docker exec "$T17_NODE" ctr -n k8s.io images ls 2>/dev/null | + awk -v d="$1" 'NR > 1 && $3 == d { print $1 }' +} + +_t17_remove_image() { + local refs + refs="$(_t17_image_refs "$1")" + [[ -z "$refs" ]] && return 0 + # shellcheck disable=SC2086 + docker exec "$T17_NODE" ctr -n k8s.io images rm --sync $refs >>"$WORK/t17-app.log" 2>&1 +} + +_t17_cleanup() { + info "tier17: cleaning up" + if [[ -n "$T17_APP_NS_CREATED" ]]; then + kubectl delete ns "$T17_APP_NS" --ignore-not-found --wait=true --timeout=120s \ + >/dev/null 2>&1 || true + fi + if [[ -n "$T17_PROFILE_CREATED" ]] && + kubectl get nodeprofile "$T17_PROFILE" >/dev/null 2>&1; then + kubectl delete nodeprofile "$T17_PROFILE" --wait=false >/dev/null 2>&1 || true + if ! wait_for_seconds 120 bash -c "! kubectl get nodeprofile '$T17_PROFILE'"; then + warn "tier17: profile cleanup did not finish; removing the test finalizer" + docker exec "$T17_NODE" rm -f /opt/brewlet/bin/containerd-shim-brewlet-v2 \ + /usr/local/bin/containerd-shim-brewlet-v2 /usr/local/bin/brewlet-ctr \ + /usr/local/bin/brewlet-stage-gc "$T17_RECORD" >/dev/null 2>&1 || true + kubectl patch nodeprofile "$T17_PROFILE" --type=merge \ + -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || true + fi + fi + if [[ -n "$T17_HELM_INSTALLED" ]]; then + helm uninstall "$T17_RELEASE" -n "$T17_RELEASE_NS" >/dev/null 2>&1 || true + kubectl delete runtimeclass brewlet --ignore-not-found >/dev/null 2>&1 || true + kubectl delete crd javaapplications.apps.brewlet.sh nodeprofiles.node.brewlet.sh \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + kubectl delete mutatingwebhookconfiguration brewlet-admission \ + --ignore-not-found >/dev/null 2>&1 || true + kubectl delete validatingwebhookconfiguration brewlet-nodeprofiles \ + --ignore-not-found >/dev/null 2>&1 || true + kubectl delete ns "$T17_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true + wait_for_seconds 120 bash -c "! kubectl get namespace '$T17_NS'" || true + fi + + if [[ -n "$T17_NODE_TOUCHED" ]]; then + _t17_restore_containerd || + fail "tier17: restore the node's original containerd configuration" + if [[ -n "$T17_RECORD_PREEXISTING" ]]; then + _t17_restore_file "$T17_RECORD" "$WORK/t17-record" || true + fi + [[ -n "$T17_SENTINEL_CREATED" ]] && + docker exec "$T17_NODE" rm -rf "$T17_SENTINEL" >/dev/null 2>&1 || true + label_node "$T17_NODE" "$T17_POOL_KEY-" brewlet.sh/provision- \ + brewlet.sh/runtime- "brewlet.sh/jdk.$T17_JDK-" \ + "brewlet.sh/jdk-feature.${T17_JDK##*-}-" brewlet.sh/launcher.java- \ + >/dev/null 2>&1 || true + annotate_node "$T17_NODE" brewlet.sh/jdks- brewlet.sh/jdks-info- \ + brewlet.sh/launchers- brewlet.sh/profile- brewlet.sh/profile-generation- \ + brewlet.sh/provision-state- brewlet.sh/provision-error- >/dev/null 2>&1 || true + if [[ -z "$T17_JDK_PREEXISTING" ]]; then + docker exec "$T17_NODE" sh -c \ + 'chmod -R u+w "$1" 2>/dev/null || true; rm -rf "$1"' \ + sh "/opt/brewlet/jdks/$T17_JDK" >/dev/null 2>&1 || true + fi + _t17_restore_file /opt/brewlet/jdks/.brewlet-active "$WORK/t17-jdk-active" || true + [[ -n "$T17_IMAGE_DIGEST" ]] && _t17_remove_image "$T17_IMAGE_DIGEST" || true + fi + + local n + for n in ${T17_LOADED_NODES[@]+"${T17_LOADED_NODES[@]}"}; do + docker exec "$n" ctr -n k8s.io images rm \ + "$T17_OP_IMG" "$T17_ADM_IMG" "$T17_PROV_IMG" >/dev/null 2>&1 || true + done + if [[ -n "${T17_BUILT_IMAGES[*]-}" ]]; then + docker rmi ${T17_BUILT_IMAGES[@]+"${T17_BUILT_IMAGES[@]}"} >/dev/null 2>&1 || true + fi +} + +_t17_build_load() { + local image="$1" dockerfile="$2" nodes="$3" name="$4"; shift 4 + local tarball="$WORK/t17-$name.tar" n + if ! docker build --provenance=false "$@" -t "$image" -f "$dockerfile" "$MONOREPO_DIR" \ + >>"$WORK/t17-build.log" 2>&1; then + docker build "$@" -t "$image" -f "$dockerfile" "$MONOREPO_DIR" \ + >>"$WORK/t17-build.log" 2>&1 || return 1 + fi + T17_BUILT_IMAGES+=("$image") + docker save "$image" -o "$tarball" >>"$WORK/t17-load.log" 2>&1 || return 1 + for n in $nodes; do + docker exec -i "$n" ctr -n k8s.io images import - <"$tarball" \ + >>"$WORK/t17-load.log" 2>&1 || return 1 + done +} + +# Newest non-terminating provisioner pod on the test node. +_t17_provisioner_pod() { + kubectl get pods -n "$T17_NS" -l "brewlet.sh/nodeprofile=$T17_PROFILE" \ + --field-selector "spec.nodeName=$T17_NODE" \ + --sort-by=.metadata.creationTimestamp \ + -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.deletionTimestamp}{"\n"}{end}' \ + 2>/dev/null | awk -F'|' '$1 != "" && $2 == "" { name = $1 } END { print name }' +} + +# _t17_platform_stage STORE REF ARCH: the stage path for the platform manifest +# the shim selects (stages are keyed by manifest digest, not index digest). +_t17_platform_stage() { + local hex + hex="$(python3 - "$1" "$2" "$3" <<'PY' +import json, sys +root, ref, arch = sys.argv[1:4] +tag = ref.rsplit(":", 1)[-1] +def blob(digest): + with open(f"{root}/blobs/{digest.replace(':', '/')}", encoding="utf-8") as stream: + return json.load(stream) +with open(f"{root}/index.json", encoding="utf-8") as stream: + index = json.load(stream) +top = next((m for m in index["manifests"] + if m.get("annotations", {}).get("org.opencontainers.image.ref.name") in (ref, tag)), + index["manifests"][0]) +if top["mediaType"].endswith("image.index.v1+json"): + top = next(m for m in blob(top["digest"])["manifests"] + if m.get("platform", {}).get("os") == "linux" + and m.get("platform", {}).get("architecture") == arch) +print(top["digest"].split(":", 1)[1]) +PY +)" || return 1 + [[ -n "$hex" ]] && printf '%s/immutable-v2/%s' "$T17_STAGE_ROOT" "$hex" +} + +_t17_logs() { + kubectl logs "$1" -n "$T17_NS" -c provisioner 2>/dev/null +} + +_t17_last_sweeps() { + _t17_logs "$1" | sed -n 's/.*stage GC: successful_sweeps=\([0-9]*\).*/\1/p' | tail -1 +} + +# _t17_wait_sweeps POD N: wait until POD has logged at least N successful sweeps. +_t17_wait_sweeps() { + local pod="$1" want="$2" deadline=$(( $(date +%s) + 120 )) got + while (( $(date +%s) < deadline )); do + got="$(_t17_last_sweeps "$pod")" + [[ -n "$got" ]] && (( got >= want )) && return 0 + sleep 2 + done + return 1 +} + +_t17_ds_env() { + kubectl get ds "brewlet-node-provisioner-$T17_PROFILE" -n "$T17_NS" \ + -o jsonpath="{.spec.template.spec.containers[0].env[?(@.name==\"$1\")].value}" 2>/dev/null +} + +# _t17_wait_rollout ACK: after a Helm change, wait for the operator to render ACK +# into the DaemonSet, the rollout to finish, and the new pod to provision. +_t17_wait_rollout() { + local ack="$1" ds="brewlet-node-provisioner-$T17_PROFILE" pod + wait_for_seconds 180 bash -c \ + "[[ \"\$(kubectl get ds '$ds' -n '$T17_NS' -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name==\"BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED\")].value}')\" == '$ack' ]]" || + return 1 + kubectl rollout status ds/"$ds" -n "$T17_NS" --timeout=300s >>"$WORK/t17-profile.log" 2>&1 || + return 1 + local deadline=$(( $(date +%s) + 180 )) + while (( $(date +%s) < deadline )); do + pod="$(_t17_provisioner_pod)" + if [[ -n "$pod" ]] && _t17_logs "$pod" | grep -Fq "node ${T17_NODE} provisioned successfully"; then + printf '%s' "$pod" + return 0 + fi + sleep 2 + done + return 1 +} + +tier17_stage_gc() { + section "Tier 17 — default-enabled runnable-stage GC in-cluster" + if ! have kubectl || ! k8s_reachable; then skip "tier17: stage GC" "no reachable cluster"; return 0; fi + if ! have docker || ! docker info >/dev/null 2>&1; then + skip "tier17: stage GC" "docker daemon not available"; return 0 + fi + if ! have helm; then skip "tier17: stage GC" "helm not installed"; return 0; fi + if ! have go; then skip "tier17: stage GC" "go not installed"; return 0; fi + if ! have java; then skip "tier17: stage GC" "JDK not installed"; return 0; fi + if ! have python3; then skip "tier17: stage GC" "python3 not installed"; return 0; fi + + local nodes n + nodes="$(kubectl get nodes -o name 2>/dev/null | sed 's#node/##')" + for n in $nodes; do + if ! node_provisionable "$n"; then + skip "tier17: stage GC" "node '$n' is not a local containerd docker container"; return 0 + fi + done + T17_NODE="$(pick_provisionable_node)" + if [[ -z "$T17_NODE" ]] || ! node_schedulable "$T17_NODE"; then + skip "tier17: stage GC" "no schedulable local containerd node"; return 0 + fi + T17_ARCH="$(_t15_node_arch "$T17_NODE")" + [[ -n "$T17_ARCH" ]] || { skip "tier17: stage GC" "unknown node architecture"; return 0; } + if [[ -n "$(detect_leftovers)" ]] || + helm status "$T17_RELEASE" -n "$T17_RELEASE_NS" >/dev/null 2>&1 || + kubectl get ns "$T17_NS" >/dev/null 2>&1 || + kubectl get ns "$T17_APP_NS" >/dev/null 2>&1; then + fail "tier17: clean Brewlet cluster state" "run ./run.sh --reset before tier 17" + return 0 + fi + + # --- snapshot and prepare host state -------------------------------------- + docker exec "$T17_NODE" test -e "/opt/brewlet/jdks/$T17_JDK" >/dev/null 2>&1 && + T17_JDK_PREEXISTING=1 + if ! _t17_snapshot_file /etc/containerd/config.toml "$WORK/t17-containerd.toml" || + ! _t17_snapshot_file /etc/containerd/config.toml.brewlet.bak "$WORK/t17-containerd-bak.toml" || + ! _t17_snapshot_file /etc/containerd/config.toml.d/99-brewlet.toml "$WORK/t17-containerd-dropin.toml" || + ! _t17_snapshot_file /opt/brewlet/jdks/.brewlet-active "$WORK/t17-jdk-active" || + ! _t17_snapshot_file "$T17_RECORD" "$WORK/t17-record"; then + fail "tier17: snapshot node state"; return 0 + fi + docker exec "$T17_NODE" sh -c \ + 'cat /etc/containerd/config.toml /etc/containerd/config.toml.d/99-brewlet.toml 2>/dev/null' \ + >"$WORK/t17-containerd-state-before" 2>/dev/null || true + [[ -f "$WORK/t17-record.present" ]] && T17_RECORD_PREEXISTING=1 + T17_NODE_TOUCHED=1 + trap _t17_cleanup RETURN + + # An existing, record-less stage root makes this an "upgraded" node. The + # sentinel is a non-canonical tree, which the reaper must never delete. + docker exec "$T17_NODE" rm -f "$T17_RECORD" >/dev/null 2>&1 || true + if ! docker exec "$T17_NODE" test -e "$T17_SENTINEL" >/dev/null 2>&1; then + docker exec "$T17_NODE" sh -c 'mkdir -p "$1" && echo keep >"$1/app.jar"' sh "$T17_SENTINEL" \ + >/dev/null 2>&1 || { fail "tier17: plant pre-existing stage data"; return 0; } + T17_SENTINEL_CREATED=1 + fi + + # --- build and load images ------------------------------------------------ + for n in $nodes; do T17_LOADED_NODES+=("$n"); done + : >"$WORK/t17-build.log"; : >"$WORK/t17-load.log" + info "tier17: building and side-loading operator, admission, and provisioner images" + if ! _t17_build_load "$T17_OP_IMG" "$BREWLET_KUBERNETES_DIR/Dockerfile" "$nodes" manager --build-arg CMD=manager || + ! _t17_build_load "$T17_ADM_IMG" "$BREWLET_KUBERNETES_DIR/Dockerfile" "$nodes" admission --build-arg CMD=admission || + ! _t17_build_load "$T17_PROV_IMG" "$MONOREPO_DIR/provisioner/Dockerfile" "$nodes" provisioner --platform "linux/$T17_ARCH"; then + fail "tier17: build and load images" "see t17-build.log and t17-load.log"; return 0 + fi + if docker run --rm --platform "linux/$T17_ARCH" --entrypoint /bin/test "$T17_PROV_IMG" \ + -x /opt/brewlet-dist/brewlet; then + pass "tier17: provisioner image ships the stage-GC CLI" + else + fail "tier17: provisioner image ships the stage-GC CLI"; return 0 + fi + + local jar="$FIXTURES_DIR/demo-app/target/app.jar" store="$WORK/t17-oci" jh image_ref + if [[ ! -f "$jar" ]]; then + jh="$(resolve_java_home)" + env JAVA_HOME="$jh" PATH="$jh/bin:$PATH" "$FIXTURES_DIR/demo-app/build.sh" \ + >>"$WORK/t17-app.log" 2>&1 || { fail "tier17: build demo JAR" "see $WORK/t17-app.log"; return 0; } + fi + rm -rf "$store" + if ! (cd "$BREWLET_CORE_DIR" && go build -o "$WORK/t17-brewlet" ./cmd/brewlet) >>"$WORK/t17-app.log" 2>&1 || + ! "$WORK/t17-brewlet" push "$jar" "$T17_REF" --store "$store" --format=image >>"$WORK/t17-app.log" 2>&1; then + fail "tier17: build runnable image" "see $WORK/t17-app.log"; return 0 + fi + local digest + digest="$(oci_layout_digest "$store" "$T17_REF")" + if [[ -z "$digest" ]] || ! import_oci_layout "$T17_NODE" "$store" "$WORK/t17-app.log" || + ! image_ref="$(pin_image_for_cri "$T17_NODE" "$T17_REF" "$digest" "$WORK/t17-app.log")"; then + fail "tier17: import runnable image into node" "see $WORK/t17-app.log"; return 0 + fi + T17_IMAGE_DIGEST="$digest" + + # --- (1)+(2): chart defaults, blocked on an upgraded node ----------------- + info "tier17: installing the chart with default stageGC values" + T17_HELM_INSTALLED=1 + if ! helm install "$T17_RELEASE" "$BREWLET_KUBERNETES_DIR/charts/brewlet" \ + --namespace "$T17_RELEASE_NS" \ + --set images.operator="$T17_OP_IMG" \ + --set images.admission="$T17_ADM_IMG" \ + --set images.provisioner="$T17_PROV_IMG" \ + --set images.pullPolicy=IfNotPresent \ + --set defaultProfile.enabled=false \ + --set operator.leaderElect=false \ + --wait --timeout 180s >"$WORK/t17-install.log" 2>&1; then + save_pod_diag t17-install "$T17_NS" >>"$WORK/t17-install.log" 2>&1 || true + fail "tier17: install chart" "see $WORK/t17-install.log"; return 0 + fi + pass "tier17: chart installed with default stageGC values" + + label_node "$T17_NODE" --overwrite "$T17_POOL_KEY=$T17_POOL" brewlet.sh/provision=true \ + brewlet.sh/runtime- >>"$WORK/t17-profile.log" 2>&1 + annotate_node "$T17_NODE" brewlet.sh/provision-state- brewlet.sh/provision-error- \ + >>"$WORK/t17-profile.log" 2>&1 || true + T17_PROFILE_CREATED=1 + if ! kubectl apply -f - >>"$WORK/t17-profile.log" 2>&1 <>"$WORK/t17-install.log" 2>&1 || + ! pod="$(_t17_wait_rollout true)"; then + fail "tier17: roll out upgradeAcknowledged=true" \ + "diag: $(save_pod_diag t17-ack "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + if _t17_wait_sweeps "$pod" 1; then + pass "tier17: acknowledgment activates periodic sweeps" + else + fail "tier17: acknowledgment activates periodic sweeps" \ + "diag: $(save_pod_diag t17-ack-sweep "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + check "tier17: compatible node persists its compatibility record" \ + docker exec "$T17_NODE" test -f "$T17_RECORD" + + if ! helm upgrade "$T17_RELEASE" "$BREWLET_KUBERNETES_DIR/charts/brewlet" \ + --namespace "$T17_RELEASE_NS" --reuse-values \ + --set stageGC.upgradeAcknowledged=false \ + --wait --timeout 180s >>"$WORK/t17-install.log" 2>&1 || + ! pod="$(_t17_wait_rollout false)"; then + fail "tier17: roll out upgradeAcknowledged=false" \ + "diag: $(save_pod_diag t17-reset "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + if _t17_wait_sweeps "$pod" 1; then + pass "tier17: compatibility record keeps GC active after resetting the acknowledgment" + else + fail "tier17: compatibility record keeps GC active after resetting the acknowledgment" \ + "diag: $(save_pod_diag t17-reset-sweep "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + + # --- (4): referenced and mounted stages survive --------------------------- + local stage sweeps + if ! stage="$(_t17_platform_stage "$store" "$T17_REF" "$T17_ARCH")"; then + fail "tier17: resolve the platform manifest stage path"; return 0 + fi + T17_APP_NS_CREATED=1 + kubectl create namespace "$T17_APP_NS" >/dev/null 2>&1 || true + if ! kubectl apply -n "$T17_APP_NS" -f - >>"$WORK/t17-app.log" 2>&1 <>"$WORK/t17-app.log" 2>&1; then + fail "tier17: runnable-image workload launched" \ + "diag: $(save_pod_diag t17-workload "$T17_APP_NS" "app=$T17_APP")" + return 0 + fi + if ! docker exec "$T17_NODE" test -d "$stage"; then + fail "tier17: workload published its runnable stage" "missing $stage" + return 0 + fi + info "tier17: workload stage $stage" + + sweeps="$(_t17_last_sweeps "$pod")" + if _t17_wait_sweeps "$pod" $(( ${sweeps:-0} + 2 )) && + docker exec "$T17_NODE" test -d "$stage"; then + pass "tier17: stage of a running, referenced image survives sweeps" + else + fail "tier17: stage of a running, referenced image survives sweeps"; return 0 + fi + + kubectl delete deploy "$T17_APP" -n "$T17_APP_NS" --wait=true --timeout=60s \ + >>"$WORK/t17-app.log" 2>&1 || true + wait_for_seconds 60 bash -c "[[ -z \"\$(kubectl get pods -n '$T17_APP_NS' -o name)\" ]]" || true + local containers + containers="$(docker exec "$T17_NODE" crictl ps -a -q \ + --label "io.kubernetes.pod.namespace=$T17_APP_NS" 2>/dev/null || true)" + if [[ -n "$containers" ]]; then + # shellcheck disable=SC2086 + docker exec "$T17_NODE" crictl rm $containers >>"$WORK/t17-app.log" 2>&1 || true + fi + sweeps="$(_t17_last_sweeps "$pod")" + if _t17_wait_sweeps "$pod" $(( ${sweeps:-0} + 2 )) && + docker exec "$T17_NODE" test -d "$stage"; then + pass "tier17: stage survives while containerd still holds its image" + else + fail "tier17: stage survives while containerd still holds its image"; return 0 + fi + + # --- (5): image removal makes the stage an orphan ------------------------- + if ! _t17_remove_image "$T17_IMAGE_DIGEST" || [[ -n "$(_t17_image_refs "$T17_IMAGE_DIGEST")" ]]; then + fail "tier17: remove the workload image from containerd" "see $WORK/t17-app.log"; return 0 + fi + if wait_for_seconds 120 docker exec "$T17_NODE" test ! -e "$stage"; then + pass "tier17: sweep reclaims the stage after its image is removed" + else + fail "tier17: sweep reclaims the stage after its image is removed" \ + "diag: $(save_pod_diag t17-reclaim "$T17_NS" "brewlet.sh/nodeprofile=$T17_PROFILE")" + return 0 + fi + if wait_for_seconds 30 bash -c \ + "kubectl logs '$pod' -n '$T17_NS' -c provisioner | grep -Eq 'removed [1-9][0-9]* stages'"; then + pass "tier17: provisioner logs the reclaimed stage" + else + fail "tier17: provisioner logs the reclaimed stage" + fi + check "tier17: non-canonical stage data is never reclaimed" \ + docker exec "$T17_NODE" test -f "$T17_SENTINEL/app.jar" +} diff --git a/kubernetes/charts/brewlet/README.md b/kubernetes/charts/brewlet/README.md index fe2b6a10..363b5d74 100644 --- a/kubernetes/charts/brewlet/README.md +++ b/kubernetes/charts/brewlet/README.md @@ -104,6 +104,10 @@ for upgrades; pass them again rather than substituting example defaults. | `provisioner.launchers` | `[]` | Optional entries with `name`, digest-pinned `source.image`, and absolute `source.path` (§5.4). Empty = vanilla `java` only. | | `provisioner.registry.mirrors` | `{}` | Upstream host → approved mirror host/path map. Rewrites preserve the source digest. | | `provisioner.appCDS.regenerationEnabled` | `false` | Authorize node-side AppCDS regeneration for the default profile. | +| `stageGC.enabled` | `true` | Automatically reclaim orphaned runnable-image stages on managed nodes, independently of metrics. | +| `stageGC.interval` | `5m` | Time between sweeps, plus up to 10% jitter. | +| `stageGC.minAge` | `24h` | Minimum stage directory age, not time since last use or loss of references. | +| `stageGC.upgradeAcknowledged` | `false` | Confirm older unguarded shims and stage-dependent bundles have been retired across the managed fleet. Reset after rollout. | | `operator.leaderElect` | `true` | Enable operator leader election. | | `profiles` | `[]` | Additional profiles, each with a unique name, nonempty named `pools`, and explicit JDK sources. | | `uninstall.timeoutSeconds` | `240` | Cleanup coordinator timeout, 1-86400 whole seconds. Configure before uninstall; Helm's `--timeout` must exceed this plus 30 seconds. | @@ -128,6 +132,29 @@ for upgrades; pass them again rather than substituting example defaults. | `admission.certManager.renewBefore` | `720h` | How early cert-manager renews the serving certificate. | | `admission.certManager.caRenewBefore` | `2160h` | How early a chart-managed CA renews; must exceed the serving certificate renewal interval. | +### Automatic stage cleanup + +Stage GC settings are operator-wide defaults for **all** managed NodeProfiles, +including profiles created outside Helm. Fresh nodes activate after successful +provisioning; no extra DaemonSet, CronJob, kubelet access, or RBAC is needed. +The provisioner cleans only eligible unreferenced, unmounted stages; kubelet and +containerd remain responsible for image retention and content reclamation. +Durations must be positive whole seconds (Go duration syntax, at most +`2147483647s`); invalid settings are rejected by the operator. + +Existing nodes without a matching compatibility record log `stage GC blocked` +and continue serving workloads. First retire older unguarded shims and exported +bundles that reference the stage cache across the selected fleet. Then include +`--set stageGC.upgradeAcknowledged=true` in your normal Helm upgrade. Reset it +to `false` after the rollout; compatible nodes remember activation. Installing +the new shim alone is not proof that older consumers have stopped. +To opt out, set `stageGC.enabled=false`. + +See [runnable stage cleanup](../../../docs/runnable-image.md#reclaiming-unused-stages) +for safety constraints, migration details, and observability. These settings do +not copy or alter kubelet image GC thresholds, and reclamation may lag +containerd content GC. + All JDK distributions use the structured inventory form: ```yaml diff --git a/kubernetes/charts/brewlet/templates/operator.yaml b/kubernetes/charts/brewlet/templates/operator.yaml index ba0d18c6..5ff59b67 100644 --- a/kubernetes/charts/brewlet/templates/operator.yaml +++ b/kubernetes/charts/brewlet/templates/operator.yaml @@ -142,6 +142,10 @@ spec: - --provisioner-image={{ include "brewlet.image" (dict "root" . "component" "provisioner" "name" "brewlet-node-provisioner") }} - --node-metrics-port={{ .Values.metrics.nodePort }} - --node-metrics-enabled={{ .Values.metrics.enabled }} + - {{ printf "--stage-gc-enabled=%v" .Values.stageGC.enabled | quote }} + - {{ printf "--stage-gc-interval=%v" .Values.stageGC.interval | quote }} + - {{ printf "--stage-gc-min-age=%v" .Values.stageGC.minAge | quote }} + - {{ printf "--stage-gc-upgrade-acknowledged=%v" .Values.stageGC.upgradeAcknowledged | quote }} - {{ printf "--allowed-source-mirror-hosts=%s" (join "," .Values.security.allowedSourceMirrorHosts) | quote }} {{- if .Values.metrics.enabled }} - --metrics-bind-address=:8080 diff --git a/kubernetes/charts/brewlet/values.yaml b/kubernetes/charts/brewlet/values.yaml index fa883472..43c9832a 100644 --- a/kubernetes/charts/brewlet/values.yaml +++ b/kubernetes/charts/brewlet/values.yaml @@ -131,6 +131,17 @@ defaultProfile: # registry: { mirrors: { "mcr.microsoft.com": "mirror.internal/mcr" } } profiles: [] +# Applies to every operator-managed NodeProfile, independently of metrics. +stageGC: + enabled: true + interval: 5m + # Stage directory age, not time since last use or since becoming unreferenced. + minAge: 24h + # On existing installations, first retire older unguarded shims and exported + # bundles that point into the stage cache. Fresh nodes activate automatically. + # Reset to false after the rollout; compatible nodes remember activation. + upgradeAcknowledged: false + operator: replicas: 1 leaderElect: true diff --git a/kubernetes/cmd/manager/main.go b/kubernetes/cmd/manager/main.go index 9a3df467..87382d97 100644 --- a/kubernetes/cmd/manager/main.go +++ b/kubernetes/cmd/manager/main.go @@ -12,6 +12,7 @@ import ( "flag" "fmt" "os" + "time" nodev1alpha1 "brewlet-operator/api/nodeprofile/v1alpha1" appsv1alpha1 "brewlet-operator/api/v1alpha1" @@ -57,6 +58,10 @@ func main() { flag.StringVar(&cfg.ProvisionerImage, "provisioner-image", "ghcr.io/microsoft/brewlet-node-provisioner:0.1.0", "brewlet-node-provisioner image to run") flag.IntVar(&cfg.MetricsPort, "node-metrics-port", 9090, "node provisioner metrics exporter port") flag.BoolVar(&cfg.MetricsEnabled, "node-metrics-enabled", false, "run the node-local metrics exporter in provisioner pods") + flag.BoolVar(&cfg.StageGCEnabled, "stage-gc-enabled", true, "periodically reclaim unreferenced runnable-image stages") + flag.DurationVar(&cfg.StageGCInterval, "stage-gc-interval", 5*time.Minute, "stage GC interval (positive whole seconds)") + flag.DurationVar(&cfg.StageGCMinAge, "stage-gc-min-age", 24*time.Hour, "minimum stage age (positive whole seconds)") + flag.BoolVar(&cfg.StageGCUpgradeAcknowledged, "stage-gc-upgrade-acknowledged", false, "confirm older unguarded shims and stage-dependent exported bundles have been retired") flag.StringVar(&allowedMirrors, "allowed-source-mirror-hosts", "", "comma-separated exact registry hosts approved as runtime source mirror destinations") flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "address the metric endpoint binds to; 0 disables metrics") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "address the health probe endpoint binds to") @@ -84,6 +89,10 @@ func main() { fmt.Fprintf(os.Stderr, "invalid --allowed-source-mirror-hosts: %v\n", err) os.Exit(1) } + if err := cfg.ValidateStageGC(); err != nil { + fmt.Fprintf(os.Stderr, "invalid stage GC configuration: %v\n", err) + os.Exit(1) + } ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog := ctrl.Log.WithName("setup") diff --git a/kubernetes/internal/chart/stage_gc_test.go b/kubernetes/internal/chart/stage_gc_test.go new file mode 100644 index 00000000..5c43322b --- /dev/null +++ b/kubernetes/internal/chart/stage_gc_test.go @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package chart_test + +import ( + "slices" + "testing" + + appsv1 "k8s.io/api/apps/v1" +) + +func TestStageGCOperatorArguments(t *testing.T) { + for _, tc := range []struct { + name string + args []string + want []string + }{ + { + name: "defaults without metrics or chart-managed profiles", + want: []string{"--stage-gc-enabled=true", "--stage-gc-interval=5m", "--stage-gc-min-age=24h", "--stage-gc-upgrade-acknowledged=false", "--node-metrics-enabled=false"}, + }, + { + name: "overrides", + args: []string{"--set", "stageGC.enabled=false", "--set", "stageGC.interval=10m", "--set", "stageGC.minAge=48h", "--set", "stageGC.upgradeAcknowledged=true"}, + want: []string{"--stage-gc-enabled=false", "--stage-gc-interval=10m", "--stage-gc-min-age=48h", "--stage-gc-upgrade-acknowledged=true"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + found := false + for _, object := range render(t, tc.args...) { + if object.GetKind() != "Deployment" || object.GetName() != "brewlet-operator" { + continue + } + found = true + operator := convert[appsv1.Deployment](t, object) + args := operator.Spec.Template.Spec.Containers[0].Args + for _, want := range tc.want { + if !slices.Contains(args, want) { + t.Errorf("missing %q in %v", want, args) + } + } + } + if !found { + t.Fatal("operator Deployment missing") + } + }) + } +} diff --git a/kubernetes/internal/controller/nodeprofile_resources.go b/kubernetes/internal/controller/nodeprofile_resources.go index 6b85e55f..9e8f5e9b 100644 --- a/kubernetes/internal/controller/nodeprofile_resources.go +++ b/kubernetes/internal/controller/nodeprofile_resources.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "time" nodev1alpha1 "brewlet-operator/api/nodeprofile/v1alpha1" "brewlet-operator/internal/brewlet" @@ -285,6 +286,10 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv {Name: "BREWLET_PROFILE_UID", Value: string(profile.UID)}, {Name: "BREWLET_PROFILE_GENERATION", Value: strconv.FormatInt(profile.Generation, 10)}, {Name: "SOURCE_ALLOWED_MIRROR_HOSTS", Value: strings.Join(cfg.AllowedSourceMirrorHosts, ",")}, + {Name: "BREWLET_STAGE_GC_ENABLED", Value: strconv.FormatBool(cfg.StageGCEnabled)}, + {Name: "BREWLET_STAGE_GC_INTERVAL_SECONDS", Value: strconv.FormatInt(int64(cfg.StageGCInterval/time.Second), 10)}, + {Name: "BREWLET_STAGE_GC_MIN_AGE_SECONDS", Value: strconv.FormatInt(int64(cfg.StageGCMinAge/time.Second), 10)}, + {Name: "BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED", Value: strconv.FormatBool(cfg.StageGCUpgradeAcknowledged)}, } if profile.UID != "" { env = append(env, corev1.EnvVar{Name: "BREWLET_REQUIRE_NODE_CLAIM", Value: "true"}) @@ -346,6 +351,7 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv Command: []string{"/opt/brewlet-dist/brewlet-metrics-exporter"}, Args: []string{ "--root=/opt/brewlet", + "--stage-root=/tmp/brewlet-runnable", fmt.Sprintf("--listen-address=:%d", metricsPort), }, Ports: []corev1.ContainerPort{{ @@ -368,6 +374,7 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv // nested bind mount lands on an existing path. {Name: "host-opt", MountPath: "/opt/brewlet", ReadOnly: true}, {Name: metricsSocketVolume, MountPath: "/opt/brewlet/metrics"}, + {Name: "runnable-stage", MountPath: "/tmp/brewlet-runnable", ReadOnly: true}, }, }, }, @@ -381,6 +388,7 @@ func buildProfileDaemonSet(cfg Config, profile *nodev1alpha1.NodeProfile, resolv // writable, and only when the sidecar is scheduled; // dropMetricsExporter removes it otherwise. hostPathVolume(metricsSocketVolume, "/opt/brewlet/metrics", &hostPathDirOrCreate), + hostPathVolume("runnable-stage", "/tmp/brewlet-runnable", &hostPathDirOrCreate), hostPathVolume("containerd-conf", "/etc/containerd", &hostPathDir), hostPathVolume("host-bin", "/usr/local/bin", &hostPathDir), hostPathVolume("containerd-sock", "/run/containerd/containerd.sock", &hostPathSocket), @@ -454,7 +462,7 @@ func dropMetricsExporter(ds *appsv1.DaemonSet) { spec.Containers = spec.Containers[:1] kept := spec.Volumes[:0] for _, vol := range spec.Volumes { - if vol.Name == metricsSocketVolume { + if vol.Name == metricsSocketVolume || vol.Name == "runnable-stage" { continue } kept = append(kept, vol) diff --git a/kubernetes/internal/controller/nodeprofile_scheduling_test.go b/kubernetes/internal/controller/nodeprofile_scheduling_test.go index 7009a383..0fa5a0b0 100644 --- a/kubernetes/internal/controller/nodeprofile_scheduling_test.go +++ b/kubernetes/internal/controller/nodeprofile_scheduling_test.go @@ -166,8 +166,8 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { // The metrics exporter only reads what the provisioner wrote, except for // the one directory holding the telemetry socket it must bind. exporter := spec.Containers[1] - if len(exporter.VolumeMounts) != 2 { - t.Fatalf("metrics-exporter mounts = %+v, want the read-only tree plus the socket directory", exporter.VolumeMounts) + if len(exporter.VolumeMounts) != 3 { + t.Fatalf("metrics-exporter mounts = %+v, want read-only inventory and stages plus the socket directory", exporter.VolumeMounts) } // The read-only parent must be listed first so the nested socket mount // lands on a path that already exists inside it. @@ -177,6 +177,9 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { if exporter.VolumeMounts[1].MountPath != "/opt/brewlet/metrics" || exporter.VolumeMounts[1].ReadOnly { t.Errorf("second exporter mount = %+v, want a writable /opt/brewlet/metrics", exporter.VolumeMounts[1]) } + if exporter.VolumeMounts[2].MountPath != "/tmp/brewlet-runnable" || !exporter.VolumeMounts[2].ReadOnly { + t.Errorf("third exporter mount = %+v, want read-only runnable staging", exporter.VolumeMounts[2]) + } if exporter.SecurityContext == nil || exporter.SecurityContext.ReadOnlyRootFilesystem == nil || !*exporter.SecurityContext.ReadOnlyRootFilesystem { @@ -193,6 +196,7 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { wantTypes := map[string]corev1.HostPathType{ "/opt/brewlet": corev1.HostPathDirectoryOrCreate, "/opt/brewlet/metrics": corev1.HostPathDirectoryOrCreate, + "/tmp/brewlet-runnable": corev1.HostPathDirectoryOrCreate, "/etc/containerd": corev1.HostPathDirectory, "/usr/local/bin": corev1.HostPathDirectory, "/run/containerd/containerd.sock": corev1.HostPathSocket, @@ -217,6 +221,13 @@ func TestProvisionerHostMountsMinimizeWrite(t *testing.T) { t.Error("provisioner must not carry the exporter's socket mount") } } + cfg.MetricsEnabled = false + withoutMetrics := buildProfileDaemonSet(cfg, &p, "agentpool", nil).Spec.Template.Spec + for _, volume := range withoutMetrics.Volumes { + if volume.Name == "runnable-stage" { + t.Error("disabled metrics must not mount the staging tree") + } + } } func TestMetricsSocketVolumeOnlyWithExporter(t *testing.T) { diff --git a/kubernetes/internal/controller/resources.go b/kubernetes/internal/controller/resources.go index d48417e4..c2b73b86 100644 --- a/kubernetes/internal/controller/resources.go +++ b/kubernetes/internal/controller/resources.go @@ -4,6 +4,9 @@ package controller import ( + "fmt" + "time" + "brewlet-operator/internal/brewlet" corev1 "k8s.io/api/core/v1" @@ -24,12 +27,31 @@ type Config struct { MetricsPort int // MetricsEnabled controls whether managed provisioner pods run the node-local // metrics exporter sidecar. - MetricsEnabled bool + MetricsEnabled bool + StageGCEnabled bool + StageGCInterval time.Duration + StageGCMinAge time.Duration + // StageGCUpgradeAcknowledged authorizes activation on previously provisioned + // nodes after the administrator has retired unguarded stage consumers. + StageGCUpgradeAcknowledged bool // AllowedSourceMirrorHosts is the exact operator-level allowlist for registry // mirror destination hosts, including any explicit port. AllowedSourceMirrorHosts []string } +// ValidateStageGC keeps the duration-to-shell-seconds conversion exact and bounded. +func (c Config) ValidateStageGC() error { + for name, value := range map[string]time.Duration{ + "stage-gc-interval": c.StageGCInterval, + "stage-gc-min-age": c.StageGCMinAge, + } { + if value < time.Second || value%time.Second != 0 || value/time.Second > 2147483647 { + return fmt.Errorf("%s must be a positive whole number of seconds, at most 2147483647s", name) + } + } + return nil +} + // buildRuntimeClass returns the desired brewlet RuntimeClass: it schedules only // onto provisioned nodes (LabelRuntimeReady=ready) and reserves JVM overhead so // the scheduler accounts for the runtime baseline (§7). diff --git a/kubernetes/internal/controller/stage_gc_test.go b/kubernetes/internal/controller/stage_gc_test.go new file mode 100644 index 00000000..3f9bac56 --- /dev/null +++ b/kubernetes/internal/controller/stage_gc_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package controller + +import ( + "testing" + "time" +) + +func TestStageGCValidation(t *testing.T) { + for _, duration := range []time.Duration{0, -time.Second, time.Millisecond, time.Second + time.Nanosecond, 2147483648 * time.Second} { + for _, field := range []string{"interval", "minAge"} { + cfg := Config{StageGCInterval: 5 * time.Minute, StageGCMinAge: 24 * time.Hour} + if field == "interval" { + cfg.StageGCInterval = duration + } else { + cfg.StageGCMinAge = duration + } + if err := cfg.ValidateStageGC(); err == nil { + t.Errorf("%s accepted %s", field, duration) + } + } + } + for _, duration := range []time.Duration{time.Second, 5 * time.Minute, 24 * time.Hour, 2147483647 * time.Second} { + cfg := Config{StageGCInterval: duration, StageGCMinAge: duration} + if err := cfg.ValidateStageGC(); err != nil { + t.Errorf("rejected %s: %v", duration, err) + } + } +} + +func TestStageGCProvisionerConfiguration(t *testing.T) { + for _, metrics := range []bool{false, true} { + for _, enabled := range []bool{false, true} { + cfg := testConfig() + cfg.MetricsEnabled = metrics + cfg.StageGCEnabled = enabled + cfg.StageGCInterval = 7 * time.Minute + cfg.StageGCMinAge = 48 * time.Hour + cfg.StageGCUpgradeAcknowledged = true + p := profileNamed("external", []string{"java"}, jdk("temurin", 21)) + ds := buildProfileDaemonSet(cfg, &p, "agentpool", nil) + env := map[string]string{} + for _, item := range ds.Spec.Template.Spec.Containers[0].Env { + env[item.Name] = item.Value + } + wantEnabled := "false" + if enabled { + wantEnabled = "true" + } + for key, want := range map[string]string{ + "BREWLET_STAGE_GC_ENABLED": wantEnabled, + "BREWLET_STAGE_GC_INTERVAL_SECONDS": "420", + "BREWLET_STAGE_GC_MIN_AGE_SECONDS": "172800", + "BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED": "true", + } { + if env[key] != want { + t.Errorf("metrics=%t GC=%t: %s=%q, want %q", metrics, enabled, key, env[key], want) + } + } + if !ds.Spec.Template.Spec.HostPID { + t.Fatal("GC requires host PID visibility") + } + if !metrics && len(ds.Spec.Template.Spec.Containers) != 1 { + t.Fatal("GC must not depend on an exporter sidecar") + } + } + } +} diff --git a/provisioner/Dockerfile b/provisioner/Dockerfile index 4a4a5fd2..67da903d 100644 --- a/provisioner/Dockerfile +++ b/provisioner/Dockerfile @@ -46,11 +46,15 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ && CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ go build -trimpath -ldflags='-s -w' \ -o /out/brewlet-source-policy ./cmd/brewlet-source-policy \ + && CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags='-s -w' \ + -o /out/brewlet ./cmd/brewlet \ && GOOS=linux GOARCH=${TARGETARCH} \ /usr/local/bin/generate-go-notice /src /out/NOTICE.txt \ ./shim/cmd/containerd-shim-brewlet-v2 \ ./cmd/brewlet-metrics-exporter \ - ./cmd/brewlet-source-policy + ./cmd/brewlet-source-policy \ + ./cmd/brewlet # ---- stage 2: download and verify remote assets ---------------------------- FROM --platform=$BUILDPLATFORM debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 AS downloaded-assets @@ -163,6 +167,7 @@ COPY --from=verified-assets /verified/NOTICE.txt /NOTICE.txt COPY --from=shim /out/containerd-shim-brewlet-v2 /opt/brewlet-dist/containerd-shim-brewlet-v2 COPY --from=shim /out/brewlet-metrics-exporter /opt/brewlet-dist/brewlet-metrics-exporter COPY --from=shim /out/brewlet-source-policy /opt/brewlet-dist/brewlet-source-policy +COPY --from=shim /out/brewlet /opt/brewlet-dist/brewlet COPY LICENSE.txt /LICENSE.txt COPY legal/CONTAINER-LEGAL-NOTICE.txt /CONTAINER-LEGAL-NOTICE.txt COPY provisioner/entrypoint.sh /usr/local/bin/brewlet-provision diff --git a/provisioner/README.md b/provisioner/README.md index 9697ad48..f39ff7c6 100644 --- a/provisioner/README.md +++ b/provisioner/README.md @@ -32,7 +32,7 @@ make provisioner-image PROVISIONER_IMAGE=ghcr.io/acme/brewlet-provisioner:1.2.3 ``` The multi-stage Docker build compiles `containerd-shim-brewlet-v2`, -`brewlet-source-policy`, and `brewlet-metrics-exporter` for the target Linux +`brewlet`, `brewlet-source-policy`, and `brewlet-metrics-exporter` for the target Linux architecture and packages them with the entrypoint. ## Runtime source model @@ -94,6 +94,11 @@ the checksum gate to reject each build. | `LAUNCHER_SOURCE__PATH` | required per entry | Clean absolute binary path inside that image | | `JDKS` / `LAUNCHERS` | derived | Internal inventories; never accepted as independent input | | `BREWLET_APP_CDS_REGENERATION_ENABLED` | `false` | Authorize node-side AppCDS regeneration for the active profile | +| `BREWLET_STAGE_GC_ENABLED` | `true` | Periodically reclaim unreferenced runnable-image stages after successful provisioning | +| `BREWLET_STAGE_GC_INTERVAL_SECONDS` | `300` | Positive whole seconds between sweeps, plus up to 10% jitter | +| `BREWLET_STAGE_GC_MIN_AGE_SECONDS` | `86400` | Minimum stage directory age, not unused age; positive whole seconds | +| `BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED` | `false` | Confirm all older unguarded shims and stage-dependent exported bundles have been retired | +| `BREWLET_RUNNABLE_STAGE` | `/tmp/brewlet-runnable` | Host staging path for GC; must match the shim's staging root | | `NODE_NAME` | downward API | Kubernetes node to label | | `BREWLET_PROFILE_NAME` | `default` | Profile name paired with its UID for managed writer authority | | `BREWLET_PROFILE_UID` | empty | Operator-managed profile UID bound to durable writer authority | @@ -130,6 +135,30 @@ root-owned, mode `0444` provisioning failure removes both. The node label is a scheduling hint; the shim checks the sentinel authoritatively. +After readiness, the provisioner runs `brewlet stage-gc` through the installed +`/usr/local/bin/brewlet-stage-gc` helper in the host PID/mount namespaces. It +rechecks managed node/profile ownership before every sweep and stops when a +successful read shows loss of authority; if the API is unreachable, it skips +that sweep and retries. Cleanup mode never starts this loop. Failures, including lock +contention, are logged and retried; successful runs report logical bytes +reclaimed, and every attempt reports process-local counters and last success. +TERM/INT stops the active sweep's process group, or the timer, before the +provisioner exits. +This is independent of the optional metrics exporter. + +Fresh nodes activate automatically. Existing shim installations or nonempty +staging roots require migration acknowledgment unless a saved +`$PREFIX/.stage-gc-compatible` record matches both installed shims and the stage +path. Compatibility is recorded even when GC is disabled so later enabling it +on a fresh, guarded installation needs no migration acknowledgment. The record +is invalidated before replacing binaries and removed on teardown. Never +acknowledge an upgrade until older unguarded shims and stage-dependent exported +bundles have been retired; reset the acknowledgment after rollout. +See [runnable stage cleanup](../docs/runnable-image.md#reclaiming-unused-stages). +The shim's environment and metrics exporter's stage mount must also be aligned +when using a nondefault staging location; setting the provisioner variable alone +does not configure those processes. + Mirror destinations are approved outside `NodeProfile` by the manager/admission `--allowed-source-mirror-hosts` setting and passed into each provisioner pod. Destination hosts, including explicit ports, must match exactly. Schemes, diff --git a/provisioner/entrypoint.sh b/provisioner/entrypoint.sh index 138f8921..98f2e6a5 100755 --- a/provisioner/entrypoint.sh +++ b/provisioner/entrypoint.sh @@ -18,8 +18,8 @@ # JDKs/launchers via annotations. # # The script is idempotent: it is safe to re-run, and only does work that is -# still missing. It publishes a container-local completion marker, then sleeps -# forever so the DaemonSet readiness probe can observe successful completion. +# still missing. It publishes a container-local completion marker, then runs +# reference-aware stage GC while the DaemonSet stays Ready. # # It also runs in a reversal mode (BREWLET_MODE=cleanup): a short-lived # brewlet-cleanup DaemonSet the operator launches for a deleted NodeProfile. @@ -57,6 +57,20 @@ JDK_ACTIVE_INVENTORY=".brewlet-active" LAUNCHER_SOURCE_METADATA=".brewlet-source" LAUNCHER_ACTIVE_INVENTORY=".brewlet-active" SOURCE_POLICY_BIN="${SOURCE_POLICY_BIN:-/opt/brewlet-dist/brewlet-source-policy}" +STAGE_GC_SRC="${STAGE_GC_SRC:-/opt/brewlet-dist/brewlet}" +HOST_STAGE_GC_PATH="${HOST_STAGE_GC_PATH:-/usr/local/bin/brewlet-stage-gc}" +STAGE_GC_ENABLED="${BREWLET_STAGE_GC_ENABLED:-true}" +STAGE_GC_INTERVAL_SECONDS="${BREWLET_STAGE_GC_INTERVAL_SECONDS:-300}" +STAGE_GC_MIN_AGE_SECONDS="${BREWLET_STAGE_GC_MIN_AGE_SECONDS:-86400}" +STAGE_GC_UPGRADE_ACKNOWLEDGED="${BREWLET_STAGE_GC_UPGRADE_ACKNOWLEDGED:-false}" +STAGE_GC_ROOT="${BREWLET_RUNNABLE_STAGE:-/tmp/brewlet-runnable}" +STAGE_GC_COMPATIBLE=false +STAGE_GC_CHILD="" +STAGE_GC_GROUP="" +# Never add --no-fork: setns(CLONE_NEWPID) only affects children. The pod is +# hostPID today, but the reaper must not depend on that to join the host PID +# namespace it verifies. +STAGE_GC_NSENTER_FLAGS=(--mount --pid) # How long a rotated-out JDK/launcher root must age before it may be reclaimed. # Belt-and-braces alongside the live-mount check in reclaim_retired_roots. RETIRED_GRACE_SECONDS="${BREWLET_RETIRED_GRACE_SECONDS:-3600}" @@ -539,6 +553,151 @@ install_shim() { log "installed shim and host containerd/CRI helpers" } +stage_gc_identity() { + printf '%s\n' "$STAGE_GC_ROOT" + sha256sum "$HOST_BIN/$SHIM_NAME" "$PREFIX/bin/$SHIM_NAME" +} + +prepare_stage_gc() { + local value contents identity + for value in "$STAGE_GC_ENABLED" "$STAGE_GC_UPGRADE_ACKNOWLEDGED"; do + case "$value" in + true|false) ;; + *) die invalid-stage-gc-config "stage GC booleans must be true or false" ;; + esac + done + for value in "$STAGE_GC_INTERVAL_SECONDS" "$STAGE_GC_MIN_AGE_SECONDS"; do + [[ "$value" =~ ^[1-9][0-9]{0,9}$ ]] && (( value <= 2147483647 )) \ + || die invalid-stage-gc-config "stage GC durations must be 1..2147483647 whole seconds" + done + [[ "$STAGE_GC_ROOT" == /* && "$STAGE_GC_ROOT" != *$'\n'* ]] \ + || die invalid-stage-gc-config "stage root must be an absolute single-line host path" + STAGE_GC_COMPATIBLE=false + if [[ "$STAGE_GC_UPGRADE_ACKNOWLEDGED" == true ]]; then + STAGE_GC_COMPATIBLE=true + elif [[ -f "$PREFIX/.stage-gc-compatible" && ! -L "$PREFIX/.stage-gc-compatible" && + -f "$HOST_BIN/$SHIM_NAME" && -f "$PREFIX/bin/$SHIM_NAME" ]]; then + identity="$(stage_gc_identity)" \ + || die stage-gc-state-failed "could not verify installed shim identities" + contents="$(cat "$PREFIX/.stage-gc-compatible")" \ + || die stage-gc-state-failed "could not read stage GC compatibility record" + [[ "$contents" != "$identity" ]] || STAGE_GC_COMPATIBLE=true + elif [[ ! -e "$HOST_BIN/$SHIM_NAME" && ! -L "$HOST_BIN/$SHIM_NAME" && + ! -e "$PREFIX/bin/$SHIM_NAME" && ! -L "$PREFIX/bin/$SHIM_NAME" && + ! -e "$PREFIX/.stage-gc-compatible" && ! -L "$PREFIX/.stage-gc-compatible" ]]; then + contents="$(host_exec sh -c ' + if [ -L "$1" ]; then exit 1 + elif [ -d "$1" ]; then find "$1" -mindepth 1 -maxdepth 1 -print -quit + elif [ -e "$1" ]; then exit 1 + fi + ' sh "$STAGE_GC_ROOT")" \ + || die stage-gc-state-failed "could not inspect the host stage root before installation" + [[ -n "$contents" ]] || STAGE_GC_COMPATIBLE=true + fi + # Invalidate before replacing binaries: an interrupted upgrade must not leave + # an approval associated with a different shim installation. + rm -f "$PREFIX/.stage-gc-compatible" \ + || die stage-gc-state-failed "could not invalidate stage GC compatibility record" +} + +install_stage_gc() { + local tmp + [[ -x "$STAGE_GC_SRC" ]] \ + || die shim-image-incomplete "stage GC CLI not found in image at $STAGE_GC_SRC" + tmp="$(mktemp "$HOST_BIN/.brewlet-stage-gc.XXXXXX")" \ + || die stage-gc-state-failed "could not stage GC executable" + if ! install -m 0755 "$STAGE_GC_SRC" "$tmp" || + ! mv -f "$tmp" "$HOST_BIN/brewlet-stage-gc"; then + rm -f "$tmp" + die stage-gc-state-failed "could not install GC executable" + fi + if [[ "$STAGE_GC_COMPATIBLE" == true ]]; then + tmp="$(mktemp "$PREFIX/.stage-gc-compatible.XXXXXX")" \ + || die stage-gc-state-failed "could not stage compatibility record" + if ! stage_gc_identity >"$tmp" || ! mv -f "$tmp" "$PREFIX/.stage-gc-compatible"; then + rm -f "$tmp" + die stage-gc-state-failed "could not record stage GC compatibility" + fi + fi +} + +# Each sweep runs as the leader of its own session so shutdown can signal the +# whole tree, including the reaper that nsenter forks. +stop_stage_gc_child() { + local pid="$STAGE_GC_CHILD" group="$STAGE_GC_GROUP" attempt + STAGE_GC_CHILD="" + STAGE_GC_GROUP="" + [[ -n "$pid" ]] || return 0 + if [[ -n "$group" ]]; then + kill -TERM -- "-$group" 2>/dev/null || true + else + kill -TERM "$pid" 2>/dev/null || true + fi + wait "$pid" 2>/dev/null || true + [[ -n "$group" ]] || return 0 + for attempt in {1..50}; do + kill -0 -- "-$group" 2>/dev/null || return 0 + command sleep 0.1 + done + kill -KILL -- "-$group" 2>/dev/null || true +} + +stop_stage_gc() { + stop_stage_gc_child + exit "$1" +} + +run_stage_gc_sweep() { + setsid nsenter --target 1 "${STAGE_GC_NSENTER_FLAGS[@]}" -- "$HOST_STAGE_GC_PATH" stage-gc \ + --stage-root "$STAGE_GC_ROOT" --address "$CONTAINERD_ADDRESS" \ + --min-age "${STAGE_GC_MIN_AGE_SECONDS}s" & + STAGE_GC_CHILD=$! + # Without job control, a background child is never a group leader, so + # setsid execs in place and the child PID is also the new group ID. + STAGE_GC_GROUP="$STAGE_GC_CHILD" + local status=0 + wait "$STAGE_GC_CHILD" || status=$? + # Reap anything the sweep left behind before the next interval. + stop_stage_gc_child + return "$status" +} + +run_stage_gc_loop() { + if [[ "$STAGE_GC_ENABLED" != true ]]; then + log "stage GC disabled" + exec sleep infinity + fi + local successes=0 failures=0 last_success=never delay ownership + trap 'stop_stage_gc 143' TERM + trap 'stop_stage_gc 130' INT + while true; do + # Recheck authority after every wait and before any sweep. The operator + # drains this worker before allowing a replacement owner or cleanup worker. + # A failed API read skips this sweep; only a proven loss of authority exits. + ownership=0 + verify_node_ownership --allow-inconclusive || ownership=$? + if (( ownership != 0 )); then + log "WARN: stage GC skipped: could not verify node ownership; retrying next interval" + elif [[ "$STAGE_GC_COMPATIBLE" != true ]]; then + log "WARN: stage GC blocked: retire older unguarded shims and stage-dependent bundles, then set stageGC.upgradeAcknowledged=true" + else + if run_stage_gc_sweep; then + successes=$((successes + 1)) + last_success="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + else + failures=$((failures + 1)) + log "WARN: stage GC attempt failed (including lock contention); retrying next interval" + fi + log "stage GC: successful_sweeps=$successes failed_attempts=$failures last_success=$last_success" + fi + delay=$((STAGE_GC_INTERVAL_SECONDS + RANDOM % (STAGE_GC_INTERVAL_SECONDS / 10 + 1))) + sleep "$delay" & + STAGE_GC_CHILD=$! + wait "$STAGE_GC_CHILD" + STAGE_GC_CHILD="" + done +} + # --------------------------------------------------------------------------- # Step 2 — install JDK runtime roots. Each inventory directory is a complete # image rootfs plus metadata pointing at the JDK or jlink runtime within it. @@ -1479,23 +1638,38 @@ verify_profile_identity() { || die profile-changed "profile ${BREWLET_PROFILE_NAME} identity changed before readiness publication" } +# With --allow-inconclusive, a failed API read restores the previous authority +# and returns 75 instead of dying. Only a successful read can revoke authority, +# so a transient control-plane outage cannot crash a provisioned worker. verify_node_ownership() { [[ "$BREWLET_REQUIRE_NODE_CLAIM" == "true" ]] || return 0 + local allow_inconclusive=false previous_authorization="$NODE_WRITE_AUTHORIZED" + [[ "${1:-}" != "--allow-inconclusive" ]] || allow_inconclusive=true NODE_WRITE_AUTHORIZED=false [[ -n "$BREWLET_PROFILE_UID" && -n "$NODE_NAME" ]] \ || die ownership-fence-failed "managed writers require profile and node identities" local node_identity node_uid owner_uid owner_node_uid owner_name node_identity="$(kubectl get node "$NODE_NAME" -o \ - 'jsonpath={.metadata.uid}|{.metadata.labels.brewlet\.sh/owner-uid}|{.metadata.labels.brewlet\.sh/owner-node-uid}|{.metadata.annotations.brewlet\.sh/owner-name}')" \ - || die ownership-fence-failed "could not read the node ownership claim" + 'jsonpath={.metadata.uid}|{.metadata.labels.brewlet\.sh/owner-uid}|{.metadata.labels.brewlet\.sh/owner-node-uid}|{.metadata.annotations.brewlet\.sh/owner-name}' --request-timeout=10s)" || { + if [[ "$allow_inconclusive" == true ]]; then + NODE_WRITE_AUTHORIZED="$previous_authorization" + return 75 + fi + die ownership-fence-failed "could not read the node ownership claim" + } IFS='|' read -r node_uid owner_uid owner_node_uid owner_name <<<"$node_identity" [[ -n "$node_uid" && "$owner_uid" == "$BREWLET_PROFILE_UID" && "$owner_node_uid" == "$node_uid" && "$owner_name" == "$BREWLET_PROFILE_NAME" ]] \ || die ownership-fence-failed "node identity or ownership no longer authorizes this container" local identity uid generation deleting target_uid claimed retirement_generation retirement_phase retiring_uid retiring_claimed target_restart retiring_restart local query='{.metadata.uid}|{.metadata.generation}|{.metadata.deletionTimestamp}|{.status.targets[?(@.name=="'"$NODE_NAME"'")].uid}|{.status.targets[?(@.name=="'"$NODE_NAME"'")].claimed}|{.status.retirement.generation}|{.status.retirement.phase}|{.status.retirement.targets[?(@.name=="'"$NODE_NAME"'")].uid}|{.status.retirement.targets[?(@.name=="'"$NODE_NAME"'")].claimed}|{.status.targets[?(@.name=="'"$NODE_NAME"'")].containerdRestart}|{.status.retirement.targets[?(@.name=="'"$NODE_NAME"'")].containerdRestart}' - identity="$(kubectl get nodeprofile "$BREWLET_PROFILE_NAME" -o "jsonpath=$query")" \ - || die ownership-fence-failed "could not read the durable target ledger" + identity="$(kubectl get nodeprofile "$BREWLET_PROFILE_NAME" -o "jsonpath=$query" --request-timeout=10s)" || { + if [[ "$allow_inconclusive" == true ]]; then + NODE_WRITE_AUTHORIZED="$previous_authorization" + return 75 + fi + die ownership-fence-failed "could not read the durable target ledger" + } IFS='|' read -r uid generation deleting target_uid claimed retirement_generation retirement_phase retiring_uid retiring_claimed target_restart retiring_restart <<<"$identity" [[ "$uid" == "$BREWLET_PROFILE_UID" && "$target_uid" == "$node_uid" && "$claimed" == "true" ]] \ || die ownership-fence-failed "node target was not durably authorized by this profile incarnation" @@ -1595,7 +1769,7 @@ unpatch_containerd() { remove_shim() { rm -f "$PREFIX/bin/$SHIM_NAME" "$HOST_BIN/$SHIM_NAME" \ - "$HOST_CTR" "$HOST_CRICTL" + "$HOST_CTR" "$HOST_CRICTL" "$HOST_BIN/brewlet-stage-gc" "$PREFIX/.stage-gc-compatible" log "removed shim binaries and host containerd/CRI helpers" } @@ -1673,7 +1847,9 @@ main() { parse_runtime_sources require_cgroup_v2 preflight_sources + prepare_stage_gc install_shim + install_stage_gc install_source_mount_traps cleanup_stale_source_mounts require_containerd_image_identity @@ -1696,8 +1872,7 @@ main() { # Keep provisioning independent from observability. The exporter runs as a # sidecar in the same pod, so an exporter failure cannot reprovision the node. - log "entering idle loop; the pod stays Ready to keep the node provisioned" - exec sleep infinity + run_stage_gc_loop } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then diff --git a/provisioner/entrypoint_test.sh b/provisioner/entrypoint_test.sh index 1f11febe..eaf7ff00 100644 --- a/provisioner/entrypoint_test.sh +++ b/provisioner/entrypoint_test.sh @@ -465,6 +465,8 @@ if ( parse_runtime_sources() { return 0; } require_cgroup_v2() { return 0; } preflight_sources() { return 0; } + prepare_stage_gc() { return 0; } + install_stage_gc() { return 0; } install_shim() { printf 'install-shim\n' >>"$calls"; } install_source_mount_traps() { printf 'install-mount-traps\n' >>"$calls"; } cleanup_stale_source_mounts() { printf 'cleanup-stale-mounts\n' >>"$calls"; } @@ -1499,6 +1501,8 @@ completion_case() ( parse_runtime_sources() { completion_step parse-sources; } require_cgroup_v2() { completion_step require-cgroups; } preflight_sources() { completion_step preflight-sources; } + prepare_stage_gc() { completion_step prepare-stage-gc; } + install_stage_gc() { completion_step install-stage-gc; } install_shim() { completion_step install-shim; } install_source_mount_traps() { completion_step install-traps; } cleanup_stale_source_mounts() { completion_step cleanup-mounts; } @@ -1513,6 +1517,11 @@ completion_case() ( configure_appcds_regeneration_policy() { completion_step configure-policy; } label_node() { completion_step label-node; } cleanup_host() { completion_step cleanup-host; } + run_stage_gc_loop() { + [[ -f "$COMPLETION_FILE" ]] || exit 1 + printf 'gc-ready\n' >>"$calls" + exec sleep infinity + } main ) @@ -1524,9 +1533,15 @@ for mode in provision cleanup; do } [[ -f "$COMPLETION_FILE" ]] grep -Fxq "idle-ready" "$calls" + if [[ "$mode" == provision ]]; then + grep -Fxq "gc-ready" "$calls" + elif grep -Fq "stage-gc" "$calls" || grep -Fxq "gc-ready" "$calls"; then + echo "cleanup mode must not install or start stage GC" >&2 + exit 1 + fi done -for scenario in provision:parse-sources provision:install-sources \ +for scenario in provision:parse-sources provision:prepare-stage-gc provision:install-stage-gc provision:install-sources \ provision:activate-containerd provision:label-node \ cleanup:cleanup-mounts cleanup:cleanup-host; do if completion_case "${scenario%%:*}" "${scenario#*:}" >/dev/null 2>&1; then @@ -1636,3 +1651,5 @@ if output="$( exit 1 fi grep -Fq "ERROR: completion-state-failed" <<<"$output" + +bash "$repo_root/provisioner/stage_gc_test.sh" diff --git a/provisioner/stage_gc_test.sh b/provisioner/stage_gc_test.sh new file mode 100644 index 00000000..91225971 --- /dev/null +++ b/provisioner/stage_gc_test.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +set -euo pipefail +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +source "$repo_root/provisioner/entrypoint.sh" +work="$(mktemp -d)" +loop_pid="" +trap 'if [[ -n "$loop_pid" ]]; then kill -TERM "$loop_pid" 2>/dev/null || true; wait "$loop_pid" 2>/dev/null || true; fi; rm -rf "$work"' EXIT + +fixture() { + local dir + dir="$(mktemp -d "$work/case.XXXXXX")" + PREFIX="$dir/prefix" + HOST_BIN="$dir/bin" + STAGE_GC_ROOT="$dir/stages" + STAGE_GC_SRC="$dir/brewlet" + SHIM_SRC="$dir/shim" + STAGE_GC_ENABLED=true + STAGE_GC_INTERVAL_SECONDS=1 + STAGE_GC_MIN_AGE_SECONDS=86400 + STAGE_GC_UPGRADE_ACKNOWLEDGED=false + STAGE_GC_COMPATIBLE=false + STAGE_GC_CHILD="" + STAGE_GC_GROUP="" + NODE_NAME="" + setsid() { "$@"; } + mkdir -p "$PREFIX/bin" "$HOST_BIN" + printf '#!/bin/sh\nexit 0\n' >"$STAGE_GC_SRC" + chmod +x "$STAGE_GC_SRC" + printf 'guarded-shim\n' >"$SHIM_SRC" + calls="$dir/calls" + : >"$calls" + host_exec() { "$@"; } + die() { printf '%s: %s\n' "$1" "$*" >&2; exit 1; } +} + +install_fixture() { + cp "$SHIM_SRC" "$HOST_BIN/$SHIM_NAME" + cp "$SHIM_SRC" "$PREFIX/bin/$SHIM_NAME" + install_stage_gc +} + +# Fresh nodes activate even when the optional exporter created an empty root. +for root in absent empty; do + ( + fixture + [[ "$root" != empty ]] || mkdir "$STAGE_GC_ROOT" + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + install_fixture + [[ -x "$HOST_BIN/brewlet-stage-gc" ]] + [[ -s "$PREFIX/.stage-gc-compatible" ]] + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + printf 'next-guarded-shim\n' >"$SHIM_SRC" + install_fixture + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + ) +done + +# Existing shims or any retained staging tree require explicit acknowledgment. +for existing in shim stage; do + ( + fixture + if [[ "$existing" == shim ]]; then + printf 'old-shim\n' >"$HOST_BIN/$SHIM_NAME" + else + mkdir -p "$STAGE_GC_ROOT/immutable-v2/retained" + fi + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == false ]] + install_fixture + [[ ! -e "$PREFIX/.stage-gc-compatible" ]] + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == false ]] + STAGE_GC_UPGRADE_ACKNOWLEDGED=true + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + install_fixture + STAGE_GC_UPGRADE_ACKNOWLEDGED=false + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + ) +done + +# A rollback or changed staging location invalidates previously saved approval. +for changed in host-shim prefix-shim root; do + ( + fixture + prepare_stage_gc + install_fixture + case "$changed" in + host-shim) printf 'old-shim\n' >"$HOST_BIN/$SHIM_NAME" ;; + prefix-shim) printf 'old-shim\n' >"$PREFIX/bin/$SHIM_NAME" ;; + root) STAGE_GC_ROOT="$STAGE_GC_ROOT/other" ;; + esac + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == false ]] + [[ ! -e "$PREFIX/.stage-gc-compatible" ]] + ) +done + +for invalid in zero negative fractional overflow boolean age inspection symlink; do + if ( + fixture + case "$invalid" in + zero) STAGE_GC_INTERVAL_SECONDS=0 ;; + negative) STAGE_GC_INTERVAL_SECONDS=-1 ;; + fractional) STAGE_GC_INTERVAL_SECONDS=1.5 ;; + overflow) STAGE_GC_INTERVAL_SECONDS=2147483648 ;; + boolean) STAGE_GC_ENABLED=yes ;; + age) STAGE_GC_MIN_AGE_SECONDS=0 ;; + inspection) host_exec() { return 1; } ;; + symlink) ln -s "$PREFIX" "$STAGE_GC_ROOT" ;; + esac + prepare_stage_gc + ) >"$work/invalid.log" 2>&1; then + echo "stage GC accepted invalid configuration or inspection: $invalid" >&2 + exit 1 + fi +done + +# Retry failures, recheck authority on every tick, and preserve all host flags. +( + fixture + STAGE_GC_COMPATIBLE=true + checks=0 + verify_node_ownership() { + checks=$((checks + 1)) + if (( checks == 3 )); then + [[ "$(grep -c '^sweep ' "$calls")" == 2 ]] + grep -Fq -- '--target 1 --mount --pid -- /usr/local/bin/brewlet-stage-gc stage-gc' "$calls" + ! grep -Fq -- '--no-fork' "$calls" + grep -Fq -- "--stage-root $STAGE_GC_ROOT --address $CONTAINERD_ADDRESS --min-age 86400s" "$calls" + exit 0 + fi + } + nsenter() { + printf 'sweep %s\n' "$*" >>"$calls" + [[ "$(grep -c '^sweep ' "$calls")" != 1 ]] + } + sleep() { [[ "$1" == 1 ]]; } + run_stage_gc_loop +) >"$work/loop.log" +grep -Eq 'successful_sweeps=1 failed_attempts=1 last_success=[0-9]{4}-' "$work/loop.log" + +( + fixture + STAGE_GC_COMPATIBLE=true + STAGE_GC_INTERVAL_SECONDS=300 + checks=0 + verify_node_ownership() { + checks=$((checks + 1)) + [[ "$checks" != 2 ]] || exit 0 + } + nsenter() { return 0; } + sleep() { (( $1 >= 300 && $1 <= 330 )); } + run_stage_gc_loop +) >"$work/jitter.log" + +for state in blocked lost-ownership; do + ( + fixture + [[ "$state" != lost-ownership ]] || STAGE_GC_COMPATIBLE=true + checks=0 + verify_node_ownership() { + checks=$((checks + 1)) + [[ "$state" != lost-ownership && "$checks" != 2 ]] || exit 42 + } + nsenter() { echo 'unexpected-sweep' >"$calls"; } + sleep() { return 0; } + run_stage_gc_loop + ) >"$work/$state.log" 2>&1 && { + echo "expected worker to stop when ownership verification exits" >&2 + exit 1 + } +done +# An inconclusive API read skips the sweep but keeps the worker running. +( + fixture + STAGE_GC_COMPATIBLE=true + checks=0 + verify_node_ownership() { + [[ "$1" == --allow-inconclusive ]] + checks=$((checks + 1)) + (( checks != 1 )) || return 75 + [[ "$(grep -c '^sweep' "$calls")" == 0 ]] + exit 0 + } + nsenter() { echo 'sweep' >>"$calls"; } + sleep() { return 0; } + run_stage_gc_loop +) >"$work/inconclusive.log" 2>&1 +grep -Fq 'stage GC skipped: could not verify node ownership' "$work/inconclusive.log" + +# The real fence restores prior authority on failed reads in background mode +# and still dies at startup. +( + fixture + BREWLET_REQUIRE_NODE_CLAIM=true + BREWLET_PROFILE_UID=profile-uid + BREWLET_PROFILE_NAME=profile + NODE_NAME=node + NODE_WRITE_AUTHORIZED=true + kubectl() { return 1; } + status=0 + verify_node_ownership --allow-inconclusive || status=$? + [[ "$status" == 75 && "$NODE_WRITE_AUTHORIZED" == true ]] + if (verify_node_ownership) 2>/dev/null; then exit 1; fi +) + +if grep -Rq 'unexpected-sweep' "$work"; then + echo "blocked or unauthorized worker invoked GC" >&2 + exit 1 +fi +grep -Fq 'stage GC blocked' "$work/blocked.log" + +( + fixture + STAGE_GC_ENABLED=false + prepare_stage_gc + install_fixture + STAGE_GC_ENABLED=true + prepare_stage_gc + [[ "$STAGE_GC_COMPATIBLE" == true ]] + STAGE_GC_ENABLED=false + verify_node_ownership() { exit 1; } + nsenter() { exit 1; } + exec() { [[ "$*" == "sleep infinity" ]]; exit 0; } + run_stage_gc_loop +) + +# Signal the supervisor during a sweep and while sleeping; no descendant may +# outlive it. The fake nsenter forks like the real one and does not forward +# TERM, so only the process-group shutdown reaches the grandchild. +test_child_script="$work/child.sh" +fake_bin="$work/fake-bin" +mkdir -p "$fake_bin" +export test_child_script +cat >"$test_child_script" <<'EOF' +printf '%s\n' "$$" >"$child_file" +exec /bin/sleep 60 +EOF +cat >"$fake_bin/nsenter" <<'EOF' +#!/usr/bin/env bash +trap 'exit 143' TERM +/bin/sh "$test_child_script" & +wait +EOF +chmod +x "$fake_bin/nsenter" +for phase in sweep sleep; do + child_file="$work/$phase.pid" + export phase child_file + PATH="$fake_bin:$PATH" bash -c ' + source "$1" + STAGE_GC_COMPATIBLE=true + STAGE_GC_ENABLED=true + verify_node_ownership() { :; } + if [[ "$phase" == sleep ]]; then + nsenter() { return 0; } + setsid() { "$@"; } + fi + sleep() { + exec /bin/sh "$test_child_script" + } + run_stage_gc_loop + ' bash "$repo_root/provisioner/entrypoint.sh" >"$work/signal.log" 2>&1 & + loop_pid=$! + for attempt in {1..100}; do + [[ ! -s "$child_file" ]] || break + /bin/sleep 0.05 + done + [[ -s "$child_file" ]] || { cat "$work/signal.log" >&2; exit 1; } + child_pid="$(cat "$child_file")" + kill -TERM "$loop_pid" + status=0 + wait "$loop_pid" || status=$? + loop_pid="" + [[ "$status" == 143 ]] || { echo "unexpected shutdown status $status" >&2; exit 1; } + if kill -0 "$child_pid" 2>/dev/null; then + echo "GC supervisor left child $child_pid running" >&2 + exit 1 + fi +done + +# Prove the configured nsenter flags place the helper itself, not just its +# children, in the target PID namespace. setns(2) can only enter a descendant +# PID namespace, so target a nested one. Requires root and util-linux. +if [[ "$(id -u)" == 0 ]] && command -v unshare >/dev/null && command -v nsenter >/dev/null \ + && unshare --pid --fork true 2>/dev/null; then + unshare --pid --fork /bin/sleep 30 & + unshare_pid=$! + target="" + for attempt in {1..100}; do + target="$(pgrep -P "$unshare_pid" || true)" + [[ -z "$target" ]] || break + /bin/sleep 0.05 + done + [[ -n "$target" ]] || { echo "nested PID namespace did not start" >&2; exit 1; } + target_ns="$(readlink "/proc/$target/ns/pid")" + entered="$(nsenter --target "$target" "${STAGE_GC_NSENTER_FLAGS[@]}" -- readlink /proc/self/ns/pid)" + pkill -P "$unshare_pid" || true + wait "$unshare_pid" 2>/dev/null || true + [[ "$target_ns" != "$(readlink /proc/self/ns/pid)" && "$entered" == "$target_ns" ]] || { + echo "nsenter flags did not place the helper in the target PID namespace" >&2 + exit 1 + } +else + echo "skipping PID namespace test (requires root, unshare, and nsenter)" +fi + +echo "Stage GC provisioner tests passed." diff --git a/specs/SPECIFICATION.md b/specs/SPECIFICATION.md index 3b169dd7..bbdbfd10 100644 --- a/specs/SPECIFICATION.md +++ b/specs/SPECIFICATION.md @@ -787,16 +787,73 @@ or readiness advertisement. Brewlet has no built-in runtime catalog. Their exact keys, token grammar, presence semantics, compatibility guarantees, and autoscaler integration are defined by the public [capability-label contract](CAPABILITY_LABELS.md). -9. Publishes the container-local `/tmp/brewlet-complete` marker and idles. +9. Publishes the container-local `/tmp/brewlet-complete` marker, then runs + periodic runnable-stage cleanup when enabled and compatible (§5.2.1). + Disabled provisioners and completed cleanup-mode workers idle instead. Provisioner and cleanup containers have an exec readiness probe for that marker. Each entrypoint invocation removes stale completion state before doing any work; failures do not publish completion. This gate is independent of the optional runtime smoke checks and has no startup deadline that could kill a legitimately slow installation. + Readiness certifies provisioning, not GC activation or successful sweeps. The `brewlet.sh/runtime=ready` label is used by the `RuntimeClass` `nodeSelector` so workloads only schedule onto provisioned nodes. +#### 5.2.1 Runnable-stage cleanup lifecycle + +The existing provisioner owns the automatic stage-GC loop; no additional +DaemonSet, CronJob, or kubelet API access is required. Policy is operator-wide, +including externally managed NodeProfiles, and is independent of +`metrics.enabled`: + +| Helm setting | Default | Contract | +|---|---|---| +| `stageGC.enabled` | `true` | Enable automatic sweeps after successful provisioning and compatibility checks | +| `stageGC.interval` | `5m` | Delay after each attempt, plus up to 10% jitter; sweeps do not overlap | +| `stageGC.minAge` | `24h` | Minimum published stage directory age, not unused age or time since references disappeared | +| `stageGC.upgradeAcknowledged` | `false` | Confirm retirement of older unguarded shims, stage-dependent exported bundles, and other unguarded consumers across the managed fleet | + +Durations must resolve to positive whole seconds, at most `2147483647s`. +The operator validates and passes them to the provisioner; this is not a +per-profile `spec` field. The complete flags and environment mapping is in +[Configuration](../docs/configuration.md#runnable-image-stage-cleanup). + +Fresh installations with no installed shim copies and an absent or empty stage +root activate automatically. Existing installations need a matching saved +compatibility record or explicit migration acknowledgment. The root-owned +record (§14.5) binds approval to both installed shim copies and the stage path, +is invalidated before replacement, and is renewed only for a compatible +installation. It is maintained even when sweeps are disabled. A detected +identity/path change or interrupted installation can require acknowledgment +again at startup. Reset the fleet-wide acknowledgment after rollout; compatible +nodes retain approval. Installing new binaries alone does not prove that older +running shim processes or exported bundles have been retired. + +Each managed sweep rechecks node/profile ownership before invoking the reaper +in the host PID and mount namespaces; `nsenter` MUST fork so the reaper itself +joins the host PID namespace, and each sweep runs in its own process group. A +successful read showing loss of authority stops the worker; an unreadable node +or NodeProfile (API timeout or outage) skips that sweep without exiting or +changing node advertisements; +cleanup-mode workers never start this loop. Reference-inspection errors and +lock contention are logged and retried at the next interval, without treating a +failed sweep as a provisioning failure. Blocked compatibility also leaves the +node provisioned but logs `stage GC blocked`. TERM/INT signals the active +sweep's whole process group, or the timer, before supervisor exit. Each reaper uses a five-minute context deadline; +filesystem operations already in progress may take longer to return. + +Kubelet and containerd remain responsible for image eviction and content +reclamation. Brewlet neither deletes their images nor reads, copies, or changes +kubelet GC configuration. It follows released references through eventual +orphan reclamation (§6.3), not a second disk-pressure eviction policy. +There is no guarantee of immediate disk relief or a bounded total stage size. + +Manual `stage-gc` invocations apply the same reaper safety checks but do not +enforce the provisioner's enablement, ownership, or migration gate. Do not +schedule a second timer on managed nodes or use manual deletion to bypass +blocked activation. See the [upgrade procedure](../docs/installation.md#activating-runnable-stage-gc). + ### 5.3 Installing JDK runtime roots on nodes Which JDKs a node offers is **declarative**: the platform team lists them in the @@ -948,11 +1005,15 @@ directory (`Dockerfile` + `entrypoint.sh`) and deployed by [`deploy/node-provisioner.yaml`](../kubernetes/deploy/node-provisioner.yaml): - **Image** — a multi-stage build that compiles - `containerd-shim-brewlet-v2` and `brewlet-source-policy` from the + `containerd-shim-brewlet-v2`, `brewlet`, `brewlet-source-policy`, and + `brewlet-metrics-exporter` from the [core runtime](https://github.com/microsoft/brewlet) for the target architecture (so installed binaries always match the node arch), then assembles a small - Debian-based runtime carrying the entrypoint, source-policy validator, and - `bash`/`curl`/`kubectl`. Build with `make provisioner-image` + Debian-based runtime carrying the entrypoint, compiled binaries, and host + tools including `bash`, `kubectl`, `ctr`, `crictl`, and `nsenter`. + The CLI is installed on the host as `/usr/local/bin/brewlet-stage-gc` for + one-shot sweeps; it is invoked with the `stage-gc` subcommand. + Build with `make provisioner-image` (single arch) or `make provisioner-image-push` (multi-arch via buildx). - **Entrypoint** — an idempotent script that performs all §5.2 steps: validates all indexed JDK/launcher sources before host mutation; installs @@ -969,7 +1030,8 @@ directory (`Dockerfile` + `entrypoint.sh`) and deployed by annotates the installed JDKs/launchers, and emits the per-capability scheduling labels the admission webhook uses (`brewlet.sh/jdk.*`, `brewlet.sh/jdk-feature.*`, `brewlet.sh/launcher.*`). The manifest ships the `ServiceAccount` + `ClusterRole`/binding - (`get`/`patch` on nodes) the labelling step needs. + (`get`/`patch` on nodes) the labelling step needs. After publishing completion, + the provisioner continues with the cleanup lifecycle in §5.2.1. Node provisioning is driven by the **operator** (§8.1) and admission is handled by the **pod webhook** (§8.3). The whole set is packaged by @@ -1320,6 +1382,25 @@ and builds/runs on Linux: source bytes. This warm-reuse protection does not guarantee cold startup after source-layer GC. Re-pull affected digest-pinned images with packed-layer retention enabled, and allow extra disk capacity for retained layer bytes. +- Runnable-stage reclamation is reference-aware and Linux-only. Only canonical + published `immutable-v2/` directories with modification times + older than the positive age floor are eligible. Containerd image targets, + index-referenced platform manifests, content records and content-reference + labels in **any namespace** protect their stages. Removing an image record + alone may not release a stage until containerd also collects its content. + Live mounts across process mount namespaces, including individual file bind + mounts, protect the corresponding stage. + Launchers hold a shared staging guard through resolution and mount creation + (or direct execution); cleanup requires the exclusive guard. Reclamation + requires complete reference inspection in the initial host PID/user and host + mount namespaces with a complete `/proc`; ambiguous or unreadable evidence + prevents deletion. The root is administrator-owned and must not be renamed + or replaced while consumers or cleanup run. Legacy and abandoned pending + trees remain outside automated eviction. Open file descriptors alone are + not tracked: direct readers must hold the guard or be stopped. New exported + runnable-image bundles retain their own payload copies rather than depend on + the evictable stage; legacy bundles require migration before GC activation. + These rules govern both automatic and manual sweeps. The workload image reference and manifest digest hints are managed **cluster-side, not in the shim**: the `brewlet-admission` webhook (§8.3) overwrites the @@ -1820,6 +1901,14 @@ other, so each shape behaves as plain Kubernetes does: NodeProfile readiness/provisioning metrics and admission outcomes (including `NoCompatibleJDK`, `NoCompatibleLauncher`, and `NoCompatibleArch`) to their controller-runtime endpoints. +- **Stage cleanup observability:** the optional exporter exposes + `brewlet_runnable_stage_bytes` from a read-only host stage mount. It measures + logical regular-file bytes, including legacy and pending trees, not allocated + space, free space, or reclaimable bytes. It does not follow symlinks; inspection + errors fail the scrape rather than report zero. Automatic cleanup runs + independently of metrics. Removed stage counts/bytes, process-local + success/failure counters, and last-success timestamps are provisioner logs, + not additional Prometheus GC metrics. Monitor filesystem free space as well. - **Metric discovery:** with `metrics.enabled=true`, the Helm chart creates Services for node, operator, and admission metrics. `metrics.serviceMonitor.enabled` and @@ -1961,6 +2050,8 @@ repurposed. | `invalid-jdk-source` | A JDK source entry is missing fields, has a malformed `-` token, or is duplicated | | `invalid-launcher-source` | A launcher source entry is missing fields, has a malformed or reserved name, or is duplicated | | `invalid-restart-mode` | `rollout.containerdRestart` is not `validated` / `sighup` / `none` | +| `invalid-stage-gc-config` | Provisioner GC booleans, integer-second durations, or the host staging path are invalid | +| `stage-gc-state-failed` | GC compatibility inspection/recording or helper installation failed; distinct from a periodic sweep failure, which is logged and retried | | `unsupported-architecture` | The node's architecture is not supported | | `host-tooling-missing` | A required host tool (`nsenter`, the host `ctr` helper) is unavailable | | `completion-state-failed` | The container-local readiness marker could not be reset or published; the operation exits unsuccessfully | @@ -1969,7 +2060,7 @@ repurposed. | `containerd-version-invalid` | The reported containerd server version could not be parsed | | `unsupported-containerd-version` | containerd is older than 2.0 (protected CRI requested-image metadata is required) | | `cgroup-v2-required` | The node is cgroup-v1 only | -| `shim-image-incomplete` | The provisioner image is missing the shim / `ctr` / `crictl` binary | +| `shim-image-incomplete` | The provisioner image is missing the shim / `ctr` / `crictl` binary or stage-GC CLI | | `shim-install-failed` | The shim was not present after installation | | `jdk-source-missing` | No configured source exists for a requested JDK token (there is no built-in catalog — §5.2) | | `jdk-install-failed` | Mounting, copying, activating, or retaining a JDK root failed | @@ -2024,6 +2115,7 @@ All are opt-in (`metrics.enabled=true`) and bounded-cardinality. | `brewlet_artifact_resolution_duration_seconds` | Shim → node exporter | | `brewlet_cds_regeneration_decisions_total`, `brewlet_cds_archive_mapped` | Shim → node exporter | | `brewlet_jdk_info`, `brewlet_jdk_installed_timestamp_seconds`, `brewlet_launcher_info` | Node exporter | +| `brewlet_runnable_stage_bytes` | Node exporter; logical regular-file bytes under the host stage root, refreshed per scrape, with no metric-specific labels | | `brewlet_telemetry_events_invalid_total` | Node exporter | | `brewlet_node_provision_transitions_total` | Operator | | `brewlet_nodeprofile_condition`, `brewlet_nodeprofile_nodes` | Operator | @@ -2047,6 +2139,23 @@ read-only mounts §6.1 describes. | `metrics/telemetry.sock` | Unix datagram socket the shim writes best-effort telemetry to | | `.retired../` | A rotated-out JDK/launcher root, retained while live mounts may reference it (§12) | | `.image-mount-*` | Transient source-image mount points; reclaimed on the next pass | +| `.stage-gc-compatible` | Root-owned record binding GC compatibility to the installed shim copies and stage path; checked at provisioner startup, invalidated before replacing binaries, and removed during teardown | + +Related host paths outside this prefix: + +| Path | Contents | +|---|---| +| `/usr/local/bin/brewlet-stage-gc` | Provisioner-installed CLI helper, invoked as `brewlet-stage-gc stage-gc`; removed during teardown | +| `/tmp/brewlet-runnable/immutable-v2//` | Published immutable runnable-image stages eligible for reference-aware reclamation (§6.3) | + +The provisioner's default host stage root is `/tmp/brewlet-runnable`, overridden +by `BREWLET_RUNNABLE_STAGE`. When that variable is unset, the shim and the +standalone CLI on Linux use the same fixed `/tmp/brewlet-runnable`, ignoring +`TMPDIR` so that a containerd service `TMPDIR` cannot diverge the shim's root +from the reaper's; on other platforms the CLI uses +`os.TempDir()/brewlet-runnable`. A custom root must +be configured consistently for the shim, GC, and the exporter's read-only host +mount; configuring the provisioner alone does not reconfigure other processes. ---