diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9fd8e83..131c5f8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,6 +173,11 @@ jobs: - name: Test release specification version guard run: bash scripts/check-release-version_test.sh + - name: Test isolated live fixture safeguards (offline) + env: + PYTHONDONTWRITEBYTECODE: "1" + run: python3 -m unittest discover -s integration-tests/e2e/live -p '*test*.py' -v + container-security: name: Container build integrity runs-on: ubuntu-latest diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 61936bbd..1e87c536 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -7,6 +7,22 @@ on: schedule: - cron: "30 6 * * *" workflow_dispatch: + inputs: + suite: + description: Scenarios to run + type: choice + options: [all, live, legacy] + default: all + candidate: + description: Live runtime (release reproduces the 0.5.0 packed-layer GC failure) + type: choice + options: [shim, release] + default: shim + scenario: + description: Independently runnable live scenario (ignored by legacy suite) + type: choice + options: [both, admission, hpa] + default: both concurrency: group: e2e-${{ github.workflow }}-${{ github.ref }} @@ -17,6 +33,7 @@ permissions: jobs: tiers: + if: github.event_name != 'workflow_dispatch' || inputs.suite != 'live' name: ${{ matrix.name }} runs-on: ubuntu-latest strategy: @@ -146,6 +163,7 @@ jobs: # shim-to-runc sandbox assembly (3), which is where an arch-specific # regression would surface. arm64: + if: github.event_name != 'workflow_dispatch' || inputs.suite != 'live' name: "Tier 01-03 on arm64" runs-on: ubuntu-24.04-arm steps: @@ -181,3 +199,55 @@ jobs: - name: Run host-only tiers run: integration-tests/e2e/run.sh --tier 1 --tier 2 --tier 3 + + live: + name: "Live ${{ matrix.scenario }} (two fresh clusters)" + if: github.event_name != 'workflow_dispatch' || inputs.suite != 'legacy' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + scenario: ${{ fromJSON(inputs.scenario == 'hpa' && '["hpa"]' || inputs.scenario == 'admission' && '["admission"]' || '["admission","hpa"]') }} + env: + PYTHONDONTWRITEBYTECODE: "1" + BREWLET_LIVE_CANDIDATE: ${{ inputs.candidate || 'shim' }} + steps: + - name: Select isolated evidence directory + run: echo "BREWLET_LIVE_OUTPUT=$RUNNER_TEMP/brewlet-live-evidence" >> "$GITHUB_ENV" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: admission/ratify-verifier/go.mod + cache-dependency-path: admission/ratify-verifier/go.sum + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + distribution: temurin + java-version: "21" + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + - name: Install pinned kind + run: | + go install sigs.k8s.io/kind@v0.30.0 + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + if ! command -v htpasswd; then + sudo apt-get update -qq + sudo apt-get install -y -qq apache2-utils + fi + - name: Test strict fixture safeguards + run: python3 -m unittest discover -s integration-tests/e2e/live -p '*test*.py' -v + - name: Run two consecutive fresh-cluster scenarios + env: + SCENARIO: ${{ matrix.scenario }} + run: | + python3 "integration-tests/e2e/live/${SCENARIO}.py" + python3 "integration-tests/e2e/live/${SCENARIO}.py" + - name: Retain redacted evidence even on failure + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-${{ matrix.scenario }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/brewlet-live-evidence + if-no-files-found: error + retention-days: 14 diff --git a/README.md b/README.md index 77a7c852..e88e9079 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,13 @@ Use a disposable evaluation environment, not a production or shared cluster. The source and GitHub release downloads are public; no repository credentials are needed to clone the project or install the CLI. -In 0.5.0, managed-dependency admission and CPU HPA support are implemented, but -live end-to-end validation remains pending for -[Ratify/Gatekeeper enforcement (#95)](https://github.com/microsoft/brewlet/issues/95) -and [metrics-driven scaling (#94)](https://github.com/microsoft/brewlet/issues/94). +Live [Ratify/Gatekeeper enforcement (#95)](https://github.com/microsoft/brewlet/issues/95) +validation remains pending. Real CPU HPA scale-up/down has passed twice on fresh +disposable clusters with a **fixed-shim candidate** over the 0.5.0 components. +The unmodified release exposed a packed-layer GC failure during scale-out; +[metrics-driven scaling (#94)](https://github.com/microsoft/brewlet/issues/94) +and the [live runbook](docs/live-validation.md) distinguish that baseline from +the candidate and its remaining cold-start limits. See [preview status and validation](https://brewlet.sh/docs/#preview-status-and-validation) for the distinction between release smoke, component, and live cluster coverage. diff --git a/core/internal/artifact/blobs.go b/core/internal/artifact/blobs.go index 372a28c1..6946d4e8 100644 --- a/core/internal/artifact/blobs.go +++ b/core/internal/artifact/blobs.go @@ -218,13 +218,28 @@ func ResolveRunnableBlobs(src BlobSource, man Manifest, manifestDigest string) ( return ResolvedBlobs{}, err } defer os.RemoveAll(pending) - if err := extractGzTar(src, appLayer, filepath.Join(pending, "app")); err != nil { + // CRI may discard packed layers after unpacking. Retain verified bytes in + // the atomically published stage so later replicas can verify them again. + retained := Store{Root: filepath.Join(pending, "content")} + for _, layer := range runnableLayers(man, appLayer) { + raw, err := ReadVerifiedBlob(src, layer) + if err != nil { + if os.IsNotExist(err) { + return ResolvedBlobs{}, fmt.Errorf("runnable image layer %s is unavailable before verified staging; re-pull the digest-pinned image with packed-layer retention enabled: %w", layer.Digest, err) + } + return ResolvedBlobs{}, fmt.Errorf("retain runnable image layer: %w", err) + } + if _, err := retained.writeBlob(raw); err != nil { + return ResolvedBlobs{}, fmt.Errorf("retain runnable image layer: %w", err) + } + } + if err := extractGzTar(retained, appLayer, filepath.Join(pending, "app")); err != nil { return ResolvedBlobs{}, fmt.Errorf("stage app layer: %w", err) } - if _, err := stageLayerTars(src, man.RunnableClasspathLayers(), pending, "cp"); err != nil { + if _, err := stageLayerTars(retained, man.RunnableClasspathLayers(), pending, "cp"); err != nil { return ResolvedBlobs{}, err } - if _, err := stageLayerTars(src, man.RunnableModulepathLayers(), pending, "mp"); err != nil { + if _, err := stageLayerTars(retained, man.RunnableModulepathLayers(), pending, "mp"); err != nil { return ResolvedBlobs{}, err } if _, err := runnableStagedBlobs(cfg, man, manifestDigest, pending); err != nil { @@ -250,16 +265,23 @@ func reuseRunnableStage(src BlobSource, cfg JVMConfig, man Manifest, manifestDig if err != nil { return ResolvedBlobs{}, err } - layers := append([]Descriptor{appLayer}, man.RunnableClasspathLayers()...) - layers = append(layers, man.RunnableModulepathLayers()...) - for _, layer := range layers { - if _, err := ReadVerifiedBlob(src, layer); err != nil { + retained := Store{Root: filepath.Join(stageDir, "content")} + for _, layer := range runnableLayers(man, appLayer) { + if _, err := ReadVerifiedBlob(retained, layer); err != nil { + return ResolvedBlobs{}, fmt.Errorf("verify retained runnable image layer: %w", err) + } + if _, err := ReadVerifiedBlob(src, layer); err != nil && !os.IsNotExist(err) { return ResolvedBlobs{}, fmt.Errorf("verify cached runnable image layer: %w", err) } } return runnableStagedBlobs(cfg, man, manifestDigest, stageDir) } +func runnableLayers(man Manifest, appLayer Descriptor) []Descriptor { + layers := append([]Descriptor{appLayer}, man.RunnableClasspathLayers()...) + return append(layers, man.RunnableModulepathLayers()...) +} + func runnableStagedBlobs(cfg JVMConfig, man Manifest, manifestDigest, stageDir string) (ResolvedBlobs, error) { appDir := filepath.Join(stageDir, "app") for _, dir := range []string{stageDir, appDir} { @@ -360,9 +382,9 @@ func runnableStageDir(manifestDigest string) (string, error) { if base == "" { base = filepath.Join(os.TempDir(), "brewlet-runnable") } - // Older shims rewrite / in place. Never reuse or migrate those - // files: they may still be mounted by a running workload during an upgrade. - return filepath.Join(base, "immutable-v1", hex), nil + // Previous stages may still be mounted and lack retained verification blobs. + // Never rewrite or migrate a live stage during an upgrade. + return filepath.Join(base, "immutable-v2", hex), nil } // stageLayerTars gunzips each layer blob to an uncompressed -.tar diff --git a/core/internal/artifact/staging_test.go b/core/internal/artifact/staging_test.go index 412567b0..f802031c 100644 --- a/core/internal/artifact/staging_test.go +++ b/core/internal/artifact/staging_test.go @@ -6,6 +6,7 @@ package artifact import ( "bytes" "encoding/json" + "errors" "io" "os" "os/exec" @@ -353,6 +354,151 @@ func TestRunnableStageCacheHitStillVerifiesDescriptors(t *testing.T) { } } +func TestRunnableStageSurvivesPackedLayerGC(t *testing.T) { + t.Setenv("BREWLET_RUNNABLE_STAGE", t.TempDir()) + store, man, digest, expected := runnableLayersFixture(t) + first, err := ResolveRunnableBlobs(store, man, digest) + if err != nil { + t.Fatal(err) + } + appLayer, err := man.RunnableAppLayer() + if err != nil { + t.Fatal(err) + } + for _, layer := range runnableLayers(man, appLayer) { + path, err := store.BlobPath(layer.Digest) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + } + for range 3 { + got, err := ResolveRunnableBlobs(store, man, digest) + if err != nil { + t.Fatalf("launch after packed-layer GC: %v", err) + } + if !reflect.DeepEqual(got, first) { + t.Fatalf("GC changed immutable stage paths: %+v != %+v", got, first) + } + checkStagedContents(t, digest, expected) + } +} + +func TestRunnableStageRejectsLostOrCorruptRetainedLayers(t *testing.T) { + for _, failure := range []string{"missing", "corrupt"} { + t.Run(failure, func(t *testing.T) { + t.Setenv("BREWLET_RUNNABLE_STAGE", t.TempDir()) + store, man, digest, _ := runnableLayersFixture(t) + if _, err := ResolveRunnableBlobs(store, man, digest); err != nil { + t.Fatal(err) + } + root, err := runnableStageDir(digest) + if err != nil { + t.Fatal(err) + } + layer := man.RunnableClasspathLayers()[0] + path, err := (Store{Root: filepath.Join(root, "content")}).BlobPath(layer.Digest) + if err != nil { + t.Fatal(err) + } + if failure == "missing" { + err = os.Remove(path) + } else { + err = os.WriteFile(path, []byte("corrupt retained bytes"), 0o644) + } + if err != nil { + t.Fatal(err) + } + if _, err := ResolveRunnableBlobs(store, man, digest); err == nil { + t.Fatal("unverifiable retained layer was accepted") + } + }) + } +} + +func TestRunnableStageColdMissingLayerFailsClosed(t *testing.T) { + t.Setenv("BREWLET_RUNNABLE_STAGE", t.TempDir()) + store, man, digest, _ := runnableLayersFixture(t) + layer, err := man.RunnableAppLayer() + if err != nil { + t.Fatal(err) + } + path, err := store.BlobPath(layer.Digest) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if _, err := ResolveRunnableBlobs(store, man, digest); err == nil || !strings.Contains(err.Error(), "re-pull the digest-pinned image") { + t.Fatalf("cold missing source must fail with repair guidance: %v", err) + } +} + +type failingLayerSource struct { + BlobSource + digest string + err error +} + +func (s failingLayerSource) ReadBlob(digest string) ([]byte, error) { + if digest == s.digest { + return nil, s.err + } + return s.BlobSource.ReadBlob(digest) +} + +func TestRunnableStageOnlyToleratesSourceNotExist(t *testing.T) { + t.Setenv("BREWLET_RUNNABLE_STAGE", t.TempDir()) + store, man, digest, _ := runnableLayersFixture(t) + if _, err := ResolveRunnableBlobs(store, man, digest); err != nil { + t.Fatal(err) + } + layer := man.RunnableClasspathLayers()[0] + for _, sourceErr := range []error{os.ErrPermission, errors.New("I/O failure")} { + source := failingLayerSource{store, layer.Digest, sourceErr} + if _, err := ResolveRunnableBlobs(source, man, digest); !errors.Is(err, sourceErr) { + t.Fatalf("source error was hidden by retained bytes: %v", err) + } + } +} + +func TestRunnableStageUpgradeLeavesV1StageUntouched(t *testing.T) { + stage := t.TempDir() + t.Setenv("BREWLET_RUNNABLE_STAGE", stage) + store, man, digest, expected := runnableLayersFixture(t) + old := filepath.Join(stage, "immutable-v1", strings.TrimPrefix(digest, "sha256:"), "app", "orders.jar") + if err := os.MkdirAll(filepath.Dir(old), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(old, []byte("already mounted by 0.5.0"), 0o644); err != nil { + t.Fatal(err) + } + file, err := os.Open(old) + if err != nil { + t.Fatal(err) + } + defer file.Close() + before, err := file.Stat() + if err != nil { + t.Fatal(err) + } + if _, err := ResolveRunnableBlobs(store, man, digest); err != nil { + t.Fatal(err) + } + after, err := os.Stat(old) + if err != nil { + t.Fatal(err) + } + raw, err := io.ReadAll(file) + if err != nil || !os.SameFile(before, after) || string(raw) != "already mounted by 0.5.0" { + t.Fatalf("upgrade changed an in-use v1 stage: %q, %v", raw, err) + } + checkStagedContents(t, digest, expected) +} + func TestRunnableStageRejectsIncompletePublishedTree(t *testing.T) { t.Setenv("BREWLET_RUNNABLE_STAGE", t.TempDir()) store, man, digest, _ := runnableLayersFixture(t) diff --git a/docs/README.md b/docs/README.md index e91b55db..f6657358 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,7 +37,7 @@ Implemented functionality and live end-to-end validation are different: | Public release access and local use | The Pages release smoke exercises the released CLI, local Java example, Maven plugin, anonymous chart download, component manifest access, and release provenance. | It renders but does not install the chart or provision nodes. | | Kubernetes runtime | The source-built E2E tiers exercise provisioning, serving, manual scaling, and runtime telemetry. | These scenarios do not establish the admission or autoscaling loops below, or production readiness. | | Managed-dependency admission | Component tests exercise real DSSE verification and Ratify policy logic with substituted registry access and plugin transport. | Live registry discovery, external plugin execution, Ratify/Gatekeeper wiring, and Kubernetes admission outcomes remain pending in [#95](https://github.com/microsoft/brewlet/issues/95). | -| CPU autoscaling | Tests cover HPA resource creation, simulated HPA ownership of replicas, and manual Deployment scaling. | Real metrics-server-driven CPU scale-up and scale-down remain pending in [#94](https://github.com/microsoft/brewlet/issues/94). | +| CPU autoscaling | Alongside HPA creation and simulated HPA ownership tests, two fresh local arm64 clusters passed real metrics-server-driven 1-to-3-to-1 scaling, Ready Pods, serving endpoints and three ownership reconciliations with a fixed-shim candidate. | Unmodified 0.5.0 exposed a packed-layer GC scale-out failure. The candidate fixes warm reuse, not cold startup after missing-source GC; see [#94](https://github.com/microsoft/brewlet/issues/94) and the [runbook](live-validation.md). | [The validation tracker (#93)](https://github.com/microsoft/brewlet/issues/93) requires both live scenarios to succeed on two consecutive fresh disposable @@ -46,9 +46,13 @@ The existing E2E harness permits skips; a successful run alone is not evidence that every assertion executed. Broader strict-mode work is tracked in [#13](https://github.com/microsoft/brewlet/issues/13). -These are coverage gaps, not confirmed failures of the implemented features. -Until they are closed, do not treat component tests or release smoke results as -proof of live admission enforcement or CPU-driven autoscaling. +Admission remains a coverage gap. CPU validation demonstrated both a release +defect and the scoped candidate correction; it is not an unmodified 0.5.0 pass. +Do not treat component tests or release smoke results as proof of either live loop, +or these disposable-cluster runs as production certification. + +The [disposable live-validation runbook](live-validation.md) describes the +independent required scenarios, pinned release baseline, evidence and limits. --- diff --git a/docs/concepts.md b/docs/concepts.md index ef968061..40405f20 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -152,7 +152,7 @@ directories. Each implementation maps to a section of the and hands it to **runc**. 6. The JVM runs in a pod with a real IP via CNI, `kubectl logs`/`exec`, probes, and Services. The controller can also create a CPU HPA; - [live metrics-driven scaling validation is pending](deploying-workloads.md#autoscaling). + [live validation is scoped to a fixed-shim candidate](deploying-workloads.md#autoscaling). ```mermaid sequenceDiagram diff --git a/docs/deploying-workloads.md b/docs/deploying-workloads.md index b988f90b..4eac584c 100644 --- a/docs/deploying-workloads.md +++ b/docs/deploying-workloads.md @@ -71,7 +71,7 @@ Because the shim is runc-backed, this pod is a **first-class Kubernetes citizen* - real pod IP via CNI → Services/Ingress/NetworkPolicy work; - `kubectl logs` / `kubectl exec` work; - readiness/liveness/startup probes (`httpGet`, `tcpSocket`, `exec`) work; -- CPU HPA configuration is available; [live metrics-driven validation is pending](#autoscaling). +- CPU HPA configuration is available; [live validation is scoped to the fixed candidate](#autoscaling). Ordinary-image ephemeral debug containers are not supported by the current Brewlet handler. Use `kubectl exec` with tools present in the selected runtime, @@ -316,14 +316,16 @@ and resources belonging to an older same-name application are not deleted. ### Autoscaling -!!! warning "Preview: live CPU autoscaling validation pending" +!!! warning "Preview: fixed-candidate CPU validation, not an unmodified release pass" - Tests cover HPA resource creation, preserving a simulated HPA-selected replica - count, and manual Deployment scaling. They do not yet prove real CPU-driven - scale-up and scale-down through metrics-server and the Kubernetes HPA - controller. [Issue #94](https://github.com/microsoft/brewlet/issues/94) - tracks that loop, including Ready Pods, Service endpoints, and controller - ownership across reconciliation cycles. Evaluate only in a disposable cluster. + A fixed-shim candidate over 0.5.0 components passed real CPU-driven + 1-to-3-to-1 scaling twice on fresh local arm64 clusters, including Ready Pods, + Service endpoints and ownership across three reconciliation cycles. + Unmodified 0.5.0 failed scale-out after containerd discarded packed layers. + The candidate repairs verified warm reuse, not cold startup with missing + source bytes. See [issue #94](https://github.com/microsoft/brewlet/issues/94) + and the [runbook and evidence](live-validation.md). Evaluate only in a + disposable cluster; this is not production certification. Set `spec.autoscaling.enabled: true` and the controller manages a `HorizontalPodAutoscaler` (`autoscaling/v1`) targeting the generated Deployment: @@ -350,6 +352,9 @@ spec: Brewlet's runtime exporter is a separate telemetry path; enabling it does not provide the Kubernetes resource metrics required by CPU HPA. +Disabling autoscaling resumes Brewlet's replica ownership; it does not safely +hand a JavaApplication to an external HPA. For advanced scaling, use a separately +managed ordinary Deployment with `runtimeClassName: brewlet`. --- diff --git a/docs/live-validation.md b/docs/live-validation.md new file mode 100644 index 00000000..f5950050 --- /dev/null +++ b/docs/live-validation.md @@ -0,0 +1,204 @@ +# Disposable live validation + +The independent admission and CPU HPA scenarios in +[`integration-tests/e2e/live/`](https://github.com/microsoft/brewlet/tree/main/integration-tests/e2e/live) +target the existing native managed-dependency and basic CPU-autoscaling +contracts. They are not production certification, a performance benchmark, or +the broader zero-skip rewrite tracked in +[#13](https://github.com/microsoft/brewlet/issues/13). + +**Coverage status:** CPU HPA passed twice on fresh local arm64 clusters with the +fixed-shim candidate described below. Admission remains pending. The consolidated +acceptance work remains tracked in +[#93](https://github.com/microsoft/brewlet/issues/93), +[#94](https://github.com/microsoft/brewlet/issues/94), and +[#95](https://github.com/microsoft/brewlet/issues/95). A harness implementation or +successful component test is not evidence that the live matrix passed. See +[preview status](README.md#preview-status-and-validation) for the public +coverage boundary. + +## Prerequisites and invocation + +Use an otherwise idle Docker engine with at least **4 CPUs and 7 GiB RAM**. +The fixture bounds its single kind node to 3 CPUs/6 GiB and the registry to +0.5 CPU/256 MiB. Run the two scenarios sequentially on that host. Linux amd64 +is the hosted-workflow target; local native Linux/arm64 and macOS Docker Desktop +are supported fixture targets, not additional architecture acceptance claims. +Cross-architecture emulation is deliberately not selected. + +Required host tools: Python 3, Docker, **kind 0.30.0**, kubectl, Helm, Go +(the toolchain required by the pinned verifier), JDK 21 or newer, Maven, Git, +curl, tar, OpenSSL, and `htpasswd` (Apache utilities, for admission's private +registry fixture). The demo is compiled with `--release 21`. Internet +access to GitHub release assets, GHCR, Docker Hub, registry.k8s.io, and Maven +Central is required. Missing prerequisites fail; mandatory cases never skip. + +From a full checkout containing release commit +`f0b9334f7b29177d2ba4b49b044163ef69e16af7`: + +```bash +go install sigs.k8s.io/kind@v0.30.0 +export PATH="$(go env GOPATH)/bin:$PATH" +export PYTHONDONTWRITEBYTECODE=1 +export BREWLET_LIVE_OUTPUT="$(mktemp -d "${TMPDIR:-/tmp}/brewlet-live-evidence.XXXXXX")" +export BREWLET_LIVE_CANDIDATE=shim +python3 -m unittest discover -s integration-tests/e2e/live -p '*test*.py' -v +python3 integration-tests/e2e/live/hpa.py +python3 integration-tests/e2e/live/admission.py +``` + +The output directory must not exist as an invocation directory: each invocation +creates its own random child. Keep evidence out of commits. No `--reset`, current +kube context, existing cluster, existing registry, or pre-existing runtime is +used. Do not point the legacy tier reset helper at these fixtures. + +The **E2E** workflow remains scheduled/manual-only. Its `live` selection runs +two separate jobs, each executing its scenario twice consecutively with fresh +clusters. The first failure stops that job; the other scenario is independent. +The manual `scenario` selector can run only `hpa` or only `admission`; the +scheduled default is both. +Ordinary PR CI executes only offline fixture safeguards, not the live jobs. + +## Release baseline and reproducibility + +Both scenarios begin with the **published 0.5.0** CLI, Maven plugin, chart and +component images. CLI/plugin/chart bytes are checked against fixed SHA-256 +checksums, and component, kind-node, registry and JDK images are digest-pinned. +The external Brewlet verifier is built from the matching release source, not +silently from the current checkout. `versions.json` records the baseline, +component digests, host Java, and fixture revision. The public fixture source +archive and its per-file hash manifest identify the exact executed fixture +even for diagnostic runs from an uncommitted checkout. + +The runnable demo is a test fixture, built from the invocation's checkout. +It is not a new Spring Boot application. The opt-in CPU endpoint is disabled +unless the fixture supplies `-Dbrewlet.test.cpuLoad=true`; load expires after +20 seconds unless renewed, and only accepts POST with `seconds=0` or +`seconds=20`. + +Record final acceptance on a clean committed fixture revision. If a scenario +exposes a product defect, preserve the release failure and record any patched +candidate's exact source revision and image digest separately. A candidate run +must not be called unmodified 0.5.0. + +### Demonstrated 0.5.0 scale-out defect and candidate + +On the stock pinned kind configuration (`discard_unpacked_layers=true`), +unmodified 0.5.0 served the first Pod and real CPU caused an HPA recommendation +from 1 to 3. Additional Pods failed to create their tasks: cached runnable-image +verification tried to read a packed layer already garbage-collected by +containerd. No replica writes or synthetic metrics produced that transition. + +`BREWLET_LIVE_CANDIDATE=release` preserves that baseline; it is expected to fail +this configuration's scale-out regression. The documented invocation and hosted +live jobs explicitly select `shim`: they compile the current shim and overlay +**only that binary** on the digest-pinned released provisioner image, keeping +the released operator, admission webhook, Maven plugin, CLI and chart. +`versions.json` identifies the candidate image manifest, binary SHA-256, Go +version and complete source snapshot. These runs are **0.5.0 plus the fixed +shim**, not unmodified release validation. + +The candidate retains descriptor-verified compressed layers in an atomically +published `immutable-v2` stage and verifies them on reuse. Only missing source +bytes can use the retained evidence; present-but-corrupt source bytes and other +I/O errors still fail. The HPA scenario explicitly waits for natural +containerd source-layer GC and checks retained hashes before applying load. +It never deletes source blobs to manufacture this condition. + +This is a **warm-reuse fix**, not a complete packed-layer retention policy. +Cold startup when the source has already disappeared and no verified v2 stage +exists still fails closed with re-pull guidance. Upgrades leave in-use older +stages untouched and build a separate v2 stage when source bytes are available. +For reliable cold starts, operators must retain packed layers in their effective +containerd configuration and re-pull already affected images. No node-wide +containerd policy is silently changed by this candidate. + +## Isolation, trust, and cleanup + +Every invocation creates a uniquely named Docker network, registry and kind +cluster. All Kubernetes commands carry the private kubeconfig and explicit +context. Cleanup checks immutable container IDs and ownership labels, removes +only those containers and their anonymous volumes, and removes only the +identified network. It does not clear production finalizers or call the old +reset helper. Destroying the disposable node is **not** proof of the production +NodeProfile cleanup lifecycle. + +Kubeconfig, Maven settings/repository, signing keys and source/build scratch +files live outside the uploaded evidence tree and are removed at teardown. +Do not enable shell tracing around credentials. Fixture keys are generated per +invocation and never committed. + +The registry uses plain HTTP **only inside the owned fixture network and on a +loopback-bound host port**, with exact containerd host mappings. This is not a +production registry configuration. The Brewlet compatibility webhook boots +with the chart's default `Ignore`, then switches to `Fail` after readiness and +before any workload or admission assertion. Signature enforcement is a separate +Ratify/Gatekeeper policy and must not be weakened to make a test pass. + +## CPU HPA contract and timing + +CPU metrics come from pinned **metrics-server 0.8.0** and the real kubelet +resource-metrics API, not the Brewlet runtime exporter. Only this disposable +kind fixture passes `--kubelet-insecure-tls` for its self-signed kubelet +certificate. The metrics-server deployment is digest-pinned on its first apply. + +The JavaApplication requests 100m CPU/128 MiB, limits each Pod to 500m/256 MiB, +targets 50% CPU, and bounds replicas to 1-3. Kind's controller-manager is +explicitly tuned for this test: 10-second HPA synchronization, 60-second +downscale stabilization and 30-second CPU initialization. Production defaults +and Brewlet's autoscaling API are unchanged. + +The scenario waits up to 180 seconds for usable per-Pod CPU samples, 420 seconds +for idle minimum convergence, 600 seconds for load-driven scale-up, 120 seconds +per ownership reconciliation and 420 seconds for scale-down. Assertions use +observed conditions, not a fixed sleep. Requests to the real Service are +recorded throughout both transitions, including failures; no zero-downtime +guarantee is inferred from eventual readiness. + +While loaded, the test changes `JavaApplication.spec.replicas` to stale values +7, 8 and 9 and waits for each observed generation. The managed Deployment must +remain within the HPA's 1-3 bounds. No manual Deployment scaling or synthetic +metric can substitute for the Kubernetes HPA decision. + +The existing CPU HPA convenience remains optional: Brewlet creates an +`autoscaling/v1` HPA and stops writing Deployment replicas while enabled. +Disabling it resumes ownership of `spec.replicas` (default 1); it is **not** +an external-HPA ownership mode. Advanced scaling uses separately managed +ordinary Deployments with `runtimeClassName: brewlet`. Custom metrics, KEDA, +scale-to-zero, JVM scaling algorithms and node/cluster autoscaling are outside +this validation. + +## Evidence and failure diagnosis + +The printed per-invocation directory contains `versions.json`, owned resource +identities, `assertions.json`, `result.json`, and failure diagnostics. +`result.json` reports success only after the scenario and cleanup succeed. +Hosted jobs upload this directory on both success and failure. + +For CPU scaling, inspect `scaling-samples.json` and `requests.json`: they retain +raw metrics, HPA conditions, selected/ready replica counts, JavaApplication +status and individual serving/load outcomes. `events.json`, per-Pod logs and +`node.log` retain controller, kubelet, containerd and workload context. + +Required evidence for admission includes actual API responses and real +candidate-verification reports, not only successful subprocess exit codes. +Negative candidates must remain registry-discoverable. Key rotation and outage +checks must account for both Ratify and Gatekeeper provider caches; a cached +admission decision is not evidence that a new candidate was processed. + +| Acceptance group | Required evidence | +|---|---| +| #94 metrics and capacity | Real per-Pod CPU samples plus node allocatable capacity | +| #94 scaling up/down | Low baseline, CPU above target, extra Ready Brewlet Pods, low CPU and minimum convergence | +| #94 ownership/status/serving | Three observed reconciliations, exact ready endpoints, transition request outcomes | +| #94 bounded cleanup | Expiring CPU leases, stopped clients, identity-checked fixture teardown | +| #95 trusted deployment | Digest-pinned JavaApplication-generated Pods Ready and serving | +| #95 negative evidence | Discoverable unsigned/wrong-key/builder/subject/malformed/tampered/incomplete cases and API denials | +| #95 rotation and verifier identity | Current-only/current+obsolete admission, obsolete-only denial, real candidate reports, other-verifier isolation | +| #95 fail-closed dependencies | Fresh subjects denied on missing evidence, registry auth/fetch failure and unavailable provider | +| #95 API scope | Valid regular/init/ephemeral image requests, CREATE/UPDATE, non-Brewlet and namespace exclusions | +| #93 repeatability | Two consecutive successful fresh-cluster results per scenario on an identified revision | + +Admission coverage does not imply support for executing ordinary-image ephemeral +debug containers through Brewlet. Neither scenario covers broader signature +formats, keyless identity, production hardening or performance claims. diff --git a/docs/observability.md b/docs/observability.md index 6967b3f4..58a5d7c6 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -2,7 +2,7 @@ The runc-backed shim uses Kubernetes mechanisms for networking, logs, probes, and resource accounting. This page covers those interfaces, the remaining -autoscaling validation gap, and Brewlet-specific day‑2 tasks. +autoscaling validation limits, and Brewlet-specific day‑2 tasks. See also [SPECIFICATION §12](https://github.com/microsoft/brewlet/blob/main/specs/SPECIFICATION.md). @@ -40,7 +40,7 @@ kubectl logs --previous # after a restart - **JFR (Java Flight Recorder)** can be enabled via `jvm.args` (e.g. `-XX:StartFlightRecording=...`). - **CPU HPA** requires Kubernetes resource metrics from metrics-server. - [Live metrics-driven scaling validation is pending](#autoscaling). + [Live validation is scoped to the fixed candidate](#autoscaling). These are application and Kubernetes resource signals. Brewlet's own control-plane, launch-path, and node inventory telemetry is a separate, @@ -99,12 +99,15 @@ Deployment replica count. It requires metrics-server and CPU requests on the workload. Brewlet's runtime exporter is not a replacement for that resource metrics path. -Existing coverage exercises HPA creation, simulated replica ownership, and -manual scaling, not the real CPU-driven scale-up and scale-down loop. -[Issue #94](https://github.com/microsoft/brewlet/issues/94) tracks live validation. +Two fresh local arm64 clusters passed real CPU scale-up/down with a fixed-shim +candidate over 0.5.0 components. Unmodified 0.5.0 exposed a packed-layer GC +scale-out failure; the candidate fixes verified warm reuse, not cold startup +with missing source bytes. See [issue #94](https://github.com/microsoft/brewlet/issues/94) +and the [live-validation runbook](live-validation.md). See [Autoscaling configuration](deploying-workloads.md#autoscaling) and use a disposable evaluation cluster. Memory or custom-metric HPAs require separately -managed HPA resources and the appropriate metrics providers; they are not +managed ordinary Deployments with `runtimeClassName: brewlet`, HPA resources +and the appropriate metrics providers; they are not configured by `JavaApplication` or covered by that validation milestone. --- diff --git a/docs/runnable-image.md b/docs/runnable-image.md index 39a5485b..8f874081 100644 --- a/docs/runnable-image.md +++ b/docs/runnable-image.md @@ -19,8 +19,9 @@ shim runs with the node-resident JDK. This page documents the delivery contract payload has to reach the node **out of band**, such as with `ctr images import`. - **Runnable-image mode fixes this without changing the native format.** `brewlet push --format=image` publishes the *same* JAR as a **standard, kubelet-pullable OCI - image**. containerd/kubelet pull + unpack it with no special configuration; the shim - recognizes it and runs it on the node-resident JDK. + image**. containerd/kubelet pull + unpack it through the standard OCI path; the shim + recognizes it and runs it on the node-resident JDK. Keep packed layers available + until verified staging completes, as described below. - **The developer experience becomes the WASI/SpinKube one:** `image: ` + `runtimeClassName: brewlet` and nothing else. kubelet pulls, containerd unpacks, the shim launches `java -jar` under the pod's cgroup. @@ -113,13 +114,23 @@ CLI / prepare-bundle workflows only. For a runnable image the shim: Nothing about JVM launch, cgroup-awareness, JDK/launcher selection, or Brewlet's overlay rootfs (shared read-only JDK lower + per-container upper) changes. -New shims publish stages under an `immutable-v1` subdirectory of +New shims publish stages under an `immutable-v2` subdirectory of `BREWLET_RUNNABLE_STAGE` (or the default temporary staging root). They leave legacy stages untouched because running workloads may still mount those files. Allow extra disk capacity during rollout: legacy stages and staging directories left by abruptly terminated processes are not automatically garbage-collected. Do not remove staging trees while workloads still reference them. +The stage retains descriptor-verified packed layers as well as extracted +payloads. Every cache reuse verifies those retained bytes against the manifest +digests. This permits later replicas to start after containerd garbage-collects +its packed layers (`discard_unpacked_layers=true`) without bypassing blob +verification. A corrupt source blob that remains present still causes failure. +Missing or corrupt retained evidence fails closed. A cold launch whose packed +layers are already missing, or an upgrade with only a legacy stage, requires a +verified image re-pull; this does not repair missing content from an unpacked +snapshot. Allow disk capacity for the retained compressed bytes per image. + ## 5. Operator & webhook - The `JavaApplication` controller sets the Deployment's container `image:` to diff --git a/integration-tests/AGENTS.md b/integration-tests/AGENTS.md index 44812a8d..cedad591 100644 --- a/integration-tests/AGENTS.md +++ b/integration-tests/AGENTS.md @@ -1,5 +1,27 @@ # Brewlet E2E runbook +## Isolated admission and CPU HPA scenarios + +The strict scenarios in `e2e/live/` are independent of the legacy tiers below. +Run `python3 integration-tests/e2e/live/hpa.py` or +`python3 integration-tests/e2e/live/admission.py` from the repository root. +They create their own uniquely named clusters and registries; never pass a +shared kube context or run the tier reset helper for them. Mandatory assertions +fail instead of skipping. `.github/workflows/e2e.yml` has separate scheduled/manual +live jobs, each running twice on fresh clusters; it has no push/PR triggers. + +See [the live-validation runbook](../docs/live-validation.md) for release pins, +capacity, load leases, stabilization, fixture-only TLS/HTTP exceptions, evidence, +cleanup, and the acceptance mapping. Keep component-test and live-run claims +distinct; preserve the preview limitations until the corresponding runs pass. + +Offline safeguard checks: + +```bash +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover \ + -s integration-tests/e2e/live -p '*test*.py' -v +``` + ## Reliable invocation Run against the monorepo checkout: diff --git a/integration-tests/e2e/live/common.py b/integration-tests/e2e/live/common.py new file mode 100644 index 00000000..d8406ecb --- /dev/null +++ b/integration-tests/e2e/live/common.py @@ -0,0 +1,607 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Strict, invocation-owned fixtures; deliberately independent of the tier reset.""" + +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import signal +import subprocess +import tarfile +import tempfile +import time +import uuid +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[3] +RELEASE = "0.5.0" +RELEASE_COMMIT = "f0b9334f7b29177d2ba4b49b044163ef69e16af7" +KIND_IMAGE = "kindest/node@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a" +REGISTRY_IMAGE = "registry@sha256:6c5666b861f3505b116bb9aa9b25175e71210414bd010d92035ff64018f9457e" +JDK_IMAGE = "docker.io/library/eclipse-temurin@sha256:85f00967bcc624fc19fa9c2cf124ea426a5363898e267141726f31f358c2e14b" +CHART_SHA = "c270ac8ec08067fe6044bba2bd856e40d010d7692447e1e7d95b39200afd26a5" +COMPONENTS = { + "operator": "ghcr.io/microsoft/brewlet-operator@sha256:40719bb42728a50a20655223365597e6e632bab5f5fd97425a8a7985bd487f69", + "admission": "ghcr.io/microsoft/brewlet-admission@sha256:351cf755b60d9de01048a9c3127824a4a803dc1bb2e50b46f510f2c72fbc2267", + "provisioner": "ghcr.io/microsoft/brewlet-node-provisioner@sha256:783c9f88c73c750eb2a195cbcbceccbc8e45a69b8b3e4a4397946780c3d2396f", +} +ASSETS = { + "darwin_arm64": "67ff59526fa3c448d9e976615ea63c7f70ec7b22795524389912afe782cfcff1", + "darwin_amd64": "dbccf7bc9b3417f1c773373386120ef0eaf35df2dc3cd811616d72453d61763f", + "linux_arm64": "0e4a4e55da038d05ce5fc11226fba6eae9957d38ea93dc803f5360f40a436687", + "linux_amd64": "888fc032fc5f3a773585f014ddf23695ee63e97cb9dbf54b56884e7000385ed1", +} +OWNER_LABEL = "sh.brewlet.live-owner" + + +def redact(text): + text = re.sub(r"-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----", + "[REDACTED PRIVATE KEY]", str(text), flags=re.S) + return re.sub(r'(?i)(authorization[": =]+(?:bearer|basic)\s+)\S+', + r"\1[REDACTED]", text) + + +def run(argv, *, input=None, check=True, timeout=300, cwd=None, env=None): + argv = [str(value) for value in argv] + process = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, cwd=cwd, env=env, + start_new_session=True) + try: + stdout, stderr = process.communicate(input=input, timeout=timeout) + except BaseException: + # Own process group only; never terminate by executable name. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + raise + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + if check and result.returncode: + raise RuntimeError(redact(f"Command failed ({result.returncode}): " + f"{' '.join(argv)}\n{result.stdout}\n{result.stderr}")) + return result + + +def wait(description, predicate, timeout=300, interval=5): + deadline = time.monotonic() + timeout + while True: + value = predicate() + if value: + return value + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"Timed out after {timeout}s: {description}") + time.sleep(min(interval, remaining)) + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def download(url, path, digest): + run(["curl", "--fail", "--silent", "--show-error", "--location", + "--retry", "3", "--max-time", "180", url, "-o", path]) + if sha256(path) != digest: + raise RuntimeError(f"Checksum mismatch: {path.name}") + + +def owned_container(info, identifier, label, owner): + return (info["Id"] == identifier and + info["Config"].get("Labels", {}).get(label) == owner) + + +class Fixture: + def __init__(self, scenario): + if scenario not in ("hpa", "admission"): + raise ValueError("scenario must be hpa or admission") + self.candidate = os.environ.get("BREWLET_LIVE_CANDIDATE", "release") + if self.candidate not in ("release", "shim"): + raise ValueError("BREWLET_LIVE_CANDIDATE must be release or shim") + self.scenario = scenario + self.name = f"brewlet-live-{scenario}-{uuid.uuid4().hex[:12]}" + base = Path(os.environ.get("BREWLET_LIVE_OUTPUT", tempfile.gettempdir())) + base.mkdir(parents=True, exist_ok=True) + self.work = base / self.name + self.work.mkdir(mode=0o700) + # Sibling, not a child of the uploaded diagnostics directory. + self.private = Path(tempfile.mkdtemp(prefix=f"{self.name}-private-")) + self.kubeconfig = self.private / "kubeconfig" + self.context = f"kind-{self.name}" + self.namespace = "live-e2e" + self.node = f"{self.name}-control-plane" + self.node_id = None + self.registry_id = None + self.network_id = None + self.candidate_image_id = None + self.children = [] + self.evidence = [] + self.env = dict(os.environ, KUBECONFIG=str(self.kubeconfig), + KIND_EXPERIMENTAL_DOCKER_NETWORK=self.name) + docker_config = self.private / "docker" + docker_config.mkdir() + (docker_config / "config.json").write_text("{}") + self.env["DOCKER_CONFIG"] = str(docker_config) + self.env["HELM_REGISTRY_CONFIG"] = str(self.private / "helm-registry.json") + Path(self.env["HELM_REGISTRY_CONFIG"]).write_text("{}") + self.source = self.private / "source" + self.cli = self.private / "bin" / "brewlet" + self.maven_args = ["mvn", "-B", "--no-transfer-progress", "-q", + f"-Dmaven.repo.local={self.private / 'm2'}", + "--settings", str(self.private / "settings.xml")] + (self.private / "settings.xml").write_text("") + self.old_signals = {} + print(f"Evidence: {self.work}", flush=True) + + def run(self, argv, **kwargs): + kwargs.setdefault("env", self.env) + return run(argv, **kwargs) + + def save(self, name, data): + if Path(name).name != name: + raise ValueError("diagnostic names must be basenames") + text = data if isinstance(data, str) else json.dumps(data, indent=2) + (self.work / name).write_text(redact(text) + "\n") + + def record(self, name, details): + self.evidence.append({"assertion": name, "time": time.time(), "details": details}) + self.save("assertions.json", self.evidence) + print(f"PASS: {name}", flush=True) + + def kube(self, *args, **kwargs): + if any(str(arg).startswith(("--context", "--kubeconfig", "--server", "--cluster")) + for arg in args): + raise ValueError("fixture Kubernetes target cannot be overridden") + return self.run(["kubectl", "--kubeconfig", self.kubeconfig, + "--context", self.context, "--request-timeout=45s", *args], + **kwargs) + + def get(self, *args): + return json.loads(self.kube("get", *args, "-o", "json").stdout) + + def apply(self, obj): + return self.kube("apply", "-f", "-", input=json.dumps(obj)) + + def own_container(self, name): + info = json.loads(self.run(["docker", "inspect", name]).stdout)[0] + if name == self.node: + valid = owned_container(info, self.node_id, "io.x-k8s.kind.cluster", self.name) + else: + valid = owned_container(info, self.registry_id, OWNER_LABEL, self.name) + if not valid: + raise RuntimeError(f"Refusing to touch foreign/replaced container: {name}") + return info + + def __enter__(self): + def interrupted(signum, _frame): + raise InterruptedError(f"Fixture interrupted by signal {signum}") + for sig in (signal.SIGINT, signal.SIGTERM): + self.old_signals[sig] = signal.signal(sig, interrupted) + try: + self.start() + except BaseException as error: + try: + self.save("failure.txt", str(error)) + finally: + self.finish(False) + raise + return self + + def __exit__(self, exc_type, exc, _traceback): + try: + if exc: + self.save("failure.txt", str(exc)) + finally: + self.finish(exc_type is None) + return False + + def start(self): + for tool in ("kind", "docker", "kubectl", "helm", "java", "javac", + "jar", "mvn", "curl", "openssl", "git", "tar"): + if not shutil.which(tool): + raise RuntimeError(f"Required prerequisite missing: {tool}; no assertions skipped") + if "v0.30.0" not in self.run(["kind", "version"]).stdout: + raise RuntimeError("This fixture requires kind v0.30.0") + engine = json.loads(self.run(["docker", "info", "--format", "{{json .}}"]).stdout) + if engine["NCPU"] < 4 or engine["MemTotal"] < 7 * 1024 ** 3: + raise RuntimeError("Live fixture requires at least 4 Docker CPUs and 7 GiB RAM") + self.arch = {"aarch64": "arm64", "arm64": "arm64", "x86_64": "amd64", + "amd64": "amd64"}[engine["Architecture"]] + self.env["DOCKER_DEFAULT_PLATFORM"] = f"linux/{self.arch}" + self.release() + try: + self.run(["docker", "network", "create", "--label", + f"{OWNER_LABEL}={self.name}", self.name]) + finally: + result = self.run(["docker", "network", "inspect", self.name], check=False) + if result.returncode == 0: + network = json.loads(result.stdout)[0] + if network.get("Labels", {}).get(OWNER_LABEL) != self.name: + raise RuntimeError("Refusing to adopt a foreign fixture network") + self.network_id = network["Id"] + self.registry_name = self.name + "-registry" + try: + self.run([ + "docker", "create", "--platform", f"linux/{self.arch}", "--name", + self.registry_name, "--label", f"{OWNER_LABEL}={self.name}", + "--network", self.name, "--cpus", "0.5", "--memory", "256m", + "-p", "127.0.0.1::5000", REGISTRY_IMAGE]) + finally: + result = self.run(["docker", "inspect", self.registry_name], check=False) + if result.returncode == 0: + info = json.loads(result.stdout)[0] + if info["Config"].get("Labels", {}).get(OWNER_LABEL) != self.name: + raise RuntimeError("Refusing to adopt a foreign fixture registry") + self.registry_id = info["Id"] + self.save("registry-identity.json", {"id": self.registry_id, + "name": self.registry_name, "mounts": info["Mounts"]}) + self.run(["docker", "start", self.registry_id]) + info = self.own_container(self.registry_name) + self.save("registry-identity.json", {"id": self.registry_id, + "name": self.registry_name, "mounts": info["Mounts"]}) + port = info["NetworkSettings"]["Ports"]["5000/tcp"][0]["HostPort"] + self.registry = f"localhost:{port}" + self.registry_internal = f"{self.registry_name}:5000" + self.registry_ip = info["NetworkSettings"]["Networks"][self.name]["IPAddress"] + wait("registry /v2/", lambda: self.run( + ["curl", "-fsS", "--max-time", "5", f"http://{self.registry}/v2/"], + check=False).returncode == 0, timeout=60) + config = self.private / "kind.yaml" + config.write_text(f"""kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +nodes: +- role: control-plane + labels: + sh.brewlet/live-owner: {self.name} + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + controllerManager: + extraArgs: + horizontal-pod-autoscaler-downscale-stabilization: "60s" + horizontal-pod-autoscaler-sync-period: "10s" + horizontal-pod-autoscaler-cpu-initialization-period: "30s" +""") + try: + result = self.run(["kind", "create", "cluster", "--name", self.name, + "--kubeconfig", self.kubeconfig, "--image", KIND_IMAGE, + "--config", config, "--wait", "180s", "--retain"], timeout=360) + self.save("kind-create.log", result.stdout + result.stderr) + finally: + result = self.run(["docker", "inspect", self.node], check=False) + if result.returncode == 0: + info = json.loads(result.stdout)[0] + if info["Config"].get("Labels", {}).get("io.x-k8s.kind.cluster") != self.name: + raise RuntimeError("Unexpected kind node ownership") + self.node_id = info["Id"] + self.save("node-identity.json", {"id": self.node_id, + "name": self.node, "mounts": info["Mounts"]}) + self.own_container(self.node) + self.run(["docker", "update", "--cpus", "3", "--memory", "6g", + "--memory-swap", "6g", self.node_id]) + self.kube("wait", "--for=condition=Ready", "nodes", "--all", "--timeout=180s") + nodes = self.get("nodes")["items"] + if len(nodes) != 1 or nodes[0]["metadata"]["labels"].get("sh.brewlet/live-owner") != self.name: + raise RuntimeError("Kubeconfig does not identify the owned disposable node") + self.cluster_uid = self.get("namespace", "kube-system")["metadata"]["uid"] + for host in (self.registry, self.registry_internal): + target = f"/etc/containerd/certs.d/{host}" + self.run(["docker", "exec", self.node_id, "mkdir", "-p", target]) + self.run(["docker", "exec", "-i", self.node_id, "tee", f"{target}/hosts.toml"], + input=f'[host."http://{self.registry_internal}"]\n' + ' capabilities = ["pull", "resolve"]\n') + self.save("identity.json", {"name": self.name, "clusterUID": self.cluster_uid, + "nodeID": self.node_id, "registryID": self.registry_id, + "networkID": self.network_id, "arch": self.arch}) + self.apply({"apiVersion": "v1", "kind": "Namespace", + "metadata": {"name": self.namespace}}) + self.provision() + + def release(self): + host_arch = {"aarch64": "arm64", "arm64": "arm64", + "x86_64": "amd64", "amd64": "amd64"}[platform.machine()] + host = platform.system().lower() + "_" + host_arch + base = "https://github.com/microsoft/brewlet/releases/download/v0.5.0/" + archive = self.private / "cli.tar.gz" + download(base + f"brewlet_0.5.0_{host}.tar.gz", archive, ASSETS[host]) + self.cli.parent.mkdir() + self.run(["tar", "-xzf", archive, "-C", self.cli.parent]) + if self.run([self.cli, "version"]).stdout.strip() != RELEASE: + raise RuntimeError("Downloaded CLI is not 0.5.0") + for suffix, digest in ( + ("jar", "e2bd00643f6d5a89ef448ae6f9724d472a9d09ef58159994a05c928df64764e1"), + ("pom", "760abd6fc426d6b7e4a278527d4751071e6e4e4c20737f9ad7ce75d75a5b615e"), + ): + download(base + f"brewlet-maven-plugin-0.5.0.{suffix}", + self.private / f"plugin.{suffix}", digest) + result = self.run([*self.maven_args, + "org.apache.maven.plugins:maven-install-plugin:3.1.4:install-file", + f"-Dfile={self.private / 'plugin.jar'}", + f"-DpomFile={self.private / 'plugin.pom'}"], timeout=360) + self.save("maven-install.log", result.stdout + result.stderr) + self.source.mkdir() + self.run(["git", "archive", RELEASE_COMMIT, "-o", self.private / "source.tar"], + cwd=ROOT) + self.run(["tar", "-xf", self.private / "source.tar", "-C", self.source]) + result = self.run(["helm", "pull", "oci://ghcr.io/microsoft/charts/brewlet", + "--version", RELEASE, "--destination", self.private]) + self.save("chart-pull.log", result.stdout + result.stderr) + self.chart = self.private / "brewlet-0.5.0.tgz" + if sha256(self.chart) != CHART_SHA: + raise RuntimeError("Released chart checksum changed") + sources = sorted( + [p for p in (ROOT / "integration-tests/e2e/live").iterdir() + if p.suffix in (".py", ".go") and p.is_file()] + + [p for p in (ROOT / "integration-tests/fixtures/demo-app").rglob("*") + if p.is_file() and "target" not in p.parts]) + if self.candidate == "shim": + sources += [ROOT / p for p in self.run( + ["git", "ls-files", "core"], cwd=ROOT).stdout.splitlines()] + source_hashes = {str(p.relative_to(ROOT)): sha256(p) for p in sources} + self.save("fixture-source-hashes.json", source_hashes) + with tarfile.open(self.work / "fixture-source.tar.gz", "w:gz") as archive: + for path in sources: + if path.is_symlink(): + raise RuntimeError(f"Fixture source must not be a symlink: {path}") + archive.add(path, arcname=str(path.relative_to(ROOT))) + self.save("versions.json", { + "release": RELEASE, "releaseSource": RELEASE_COMMIT, + "fixtureSource": self.run(["git", "rev-parse", "HEAD"], cwd=ROOT).stdout.strip(), + "fixtureDirty": bool(self.run(["git", "status", "--porcelain"], cwd=ROOT).stdout), + "fixtureSourceArchiveSHA256": sha256(self.work / "fixture-source.tar.gz"), + "fixtureSourceManifestSHA256": sha256(self.work / "fixture-source-hashes.json"), + "cliSHA256": ASSETS[host], "chartSHA256": CHART_SHA, "components": COMPONENTS, + "kind": "0.30.0", "kindImage": KIND_IMAGE, "registryImage": REGISTRY_IMAGE, + "jdkImage": JDK_IMAGE, "hostJava": self.run(["java", "-version"]).stderr, + }) + + def provision(self): + images = dict(COMPONENTS) + if self.candidate == "shim": + images["provisioner"] = self.build_candidate_shim() + values = {"defaultProfile": {"enabled": False}, + "images": images, + "operator": {"leaderElect": False}, + "admission": {"failurePolicy": "Ignore", "nodeProfileFailurePolicy": "Fail"}} + path = self.private / "values.json" + path.write_text(json.dumps(values)) + result = self.run(["helm", "--kubeconfig", self.kubeconfig, + "--kube-context", self.context, "install", "brewlet", + self.chart, "--namespace", "default", "-f", path, + "--wait", "--timeout", "240s"], timeout=300) + self.save("helm-install.log", result.stdout + result.stderr) + # Bootstrap with the shipped default; fail closed before creating any workload. + self.kube("patch", "mutatingwebhookconfiguration", "brewlet-admission", + "--type=json", "-p", json.dumps([ + {"op": "replace", "path": "/webhooks/0/failurePolicy", "value": "Fail"}])) + self.kube("label", "node", self.node, "brewlet.sh/e2e-pool=live") + self.apply({ + "apiVersion": "node.brewlet.sh/v1alpha1", "kind": "NodeProfile", + "metadata": {"name": "live"}, + "spec": {"nodePool": {"key": "brewlet.sh/e2e-pool", "names": ["live"], + "includeControlPlane": True}, + "jdks": [{"distribution": "temurin", "feature": 21, + "source": {"image": JDK_IMAGE, "javaHome": "/opt/java/openjdk"}}], + "rollout": {"validate": True, "containerdRestart": "validated"}}, + }) + wait("real provisioner advertises temurin-21", + lambda: self.get("node", self.node)["metadata"]["labels"].get( + "brewlet.sh/jdk.temurin-21") == "true", timeout=480) + self.kube("rollout", "status", "daemonset/brewlet-node-provisioner-live", + "-n", "brewlet", "--timeout=180s") + self.record("runtime-provisioned", {"mode": self.candidate, + "status": self.get("nodeprofile", "live")["status"]}) + + def build_candidate_shim(self): + build = self.private / "candidate" + build.mkdir() + env = dict(self.env, CGO_ENABLED="0", GOOS="linux", GOARCH=self.arch) + result = self.run(["go", "build", "-trimpath", "-o", + build / "containerd-shim-brewlet-v2", + "./shim/cmd/containerd-shim-brewlet-v2"], + cwd=ROOT / "core", env=env, timeout=600) + self.save("candidate-build.log", result.stdout + result.stderr) + (build / "Dockerfile").write_text( + f"FROM {COMPONENTS['provisioner']}\n" + "COPY --chmod=0755 containerd-shim-brewlet-v2 " + "/opt/brewlet-dist/containerd-shim-brewlet-v2\n") + tag = f"brewlet.local/{self.name}-provisioner:candidate" + result = self.run(["docker", "build", "--platform", f"linux/{self.arch}", + "--label", f"{OWNER_LABEL}={self.name}", "-t", tag, build], + timeout=600) + self.save("candidate-image.log", result.stdout + result.stderr) + image = json.loads(self.run(["docker", "image", "inspect", tag]).stdout)[0] + self.candidate_image_id = image["Id"] + self.load_image(tag) + rows = self.run(["docker", "exec", self.node_id, "ctr", "-n", "k8s.io", + "images", "ls"]).stdout.splitlines() + digest = next((row.split()[2] for row in rows if row.split()[0] == tag), "") + if not re.fullmatch(r"sha256:[a-f0-9]{64}", digest): + raise RuntimeError("Could not pin the imported candidate provisioner manifest") + pinned = tag.split(":")[0] + "@" + digest + self.run(["docker", "exec", self.node_id, "ctr", "-n", "k8s.io", + "images", "tag", tag, pinned]) + versions = json.loads((self.work / "versions.json").read_text()) + versions["candidate"] = { + "component": "shim", "baseProvisioner": COMPONENTS["provisioner"], + "image": pinned, "dockerImageID": self.candidate_image_id, + "binarySHA256": sha256(build / "containerd-shim-brewlet-v2"), + "goVersion": self.run(["go", "version"], env=env).stdout.strip(), + "sourceManifestSHA256": versions["fixtureSourceManifestSHA256"], + } + self.save("versions.json", versions) + return pinned + + def load_image(self, ref): + self.own_container(self.node) + return self.run(["kind", "load", "docker-image", "--name", self.name, ref], + timeout=300) + + def publish_demo(self): + app = self.private / "demo-app" + shutil.copytree(ROOT / "integration-tests/fixtures/demo-app", app, + ignore=shutil.ignore_patterns("target")) + result = self.run([*self.maven_args, "-f", app / "pom.xml", "package", + "sh.brewlet:brewlet-maven-plugin:0.5.0:push", + f"-Dbrewlet.image={self.registry}/apps/demo:live", + "-Dbrewlet.jdk=21"], timeout=360) + self.save("publish-demo.log", result.stdout + result.stderr) + with urlopen(f"http://{self.registry}/v2/apps/demo/manifests/live", timeout=10) as response: + digest = response.headers["Docker-Content-Digest"] + if not re.fullmatch(r"sha256:[a-f0-9]{64}", digest or ""): + raise RuntimeError("Registry omitted immutable demo digest") + return f"{self.registry_internal}/apps/demo@{digest}" + + def java_application(self, name, image, autoscaling=False): + obj = {"apiVersion": "apps.brewlet.sh/v1alpha1", "kind": "JavaApplication", + "metadata": {"name": name, "namespace": self.namespace}, + "spec": {"artifact": {"image": image, "pullPolicy": "IfNotPresent"}, + "replicas": 1, "jvm": {"version": 21, "distribution": "temurin"}, + "ports": [{"name": "http", "containerPort": 8080}], + "resources": {"requests": {"cpu": "100m", "memory": "128Mi"}, + "limits": {"cpu": "500m", "memory": "256Mi"}}, + "probes": {"readiness": {"httpGet": {"path": "/healthz", "port": 8080}, + "periodSeconds": 2, "timeoutSeconds": 2}}}} + if autoscaling: + obj["spec"]["autoscaling"] = {"enabled": True, "minReplicas": 1, "maxReplicas": 3, + "targetCPUUtilizationPercentage": 50} + obj["spec"]["jvm"]["args"] = ["-Dbrewlet.test.cpuLoad=true"] + self.apply(obj) + return obj + + def ready(self, name, count=None): + dep = self.get("deployment", name, "-n", self.namespace) + app = self.get("javaapplication", name, "-n", self.namespace) + n = dep["spec"]["replicas"] + if count is not None and n != count: + return False + if dep.get("status", {}).get("readyReplicas", 0) != n: + return False + if app.get("status", {}).get("readyReplicas", 0) != n: + return False + if app["status"].get("observedGeneration") != app["metadata"]["generation"]: + return False + if not any(c["type"] == "Ready" and c["status"] == "True" and + c.get("observedGeneration") == app["metadata"]["generation"] + for c in app["status"].get("conditions", [])): + return False + pods = self.get("pods", "-n", self.namespace, + "-l", f"app.kubernetes.io/name={name}")["items"] + active = [p for p in pods if not p["metadata"].get("deletionTimestamp")] + if len(active) != n or any( + p["spec"].get("runtimeClassName") != "brewlet" or + not any(c["type"] == "Ready" and c["status"] == "True" + for c in p.get("status", {}).get("conditions", [])) for p in active): + return False + endpoints = self.get("endpointslice", "-n", self.namespace, + "-l", f"kubernetes.io/service-name={name}")["items"] + ready_ips = {address for s in endpoints for e in s.get("endpoints", []) + if e.get("conditions", {}).get("ready") + for address in e["addresses"]} + return ready_ips == {p["status"]["podIP"] for p in active} and n > 0 + + def wait_ready(self, name): + self.kube("rollout", "status", f"deployment/{name}", "-n", self.namespace, + "--timeout=240s") + wait(f"{name} status, runtime pods and endpoints agree", lambda: self.ready(name)) + + def service_get(self, name, path="/hello"): + return self.kube("get", "--raw", + f"/api/v1/namespaces/{self.namespace}/services/{name}:8080/proxy" + + path, timeout=50).stdout + + def diagnostics(self): + if self.node_id: + self.own_container(self.node) + for filename, args in ( + ("resources.json", ["get", "pods,deploy,hpa,javaapplication,nodeprofile,endpointslice", "-A", "-o", "json"]), + ("events.json", ["get", "events", "-A", "-o", "json"]), + ("metrics.json", ["get", "--raw", "/apis/metrics.k8s.io/v1beta1/pods"]), + ("nodes.json", ["get", "nodes", "-o", "json"]), + ): + result = self.kube(*args, check=False, timeout=60) + self.save(filename, result.stdout + result.stderr) + pods = self.kube("get", "pods", "-A", "-o", "json", check=False) + if pods.returncode == 0: + for pod in json.loads(pods.stdout)["items"]: + meta = pod["metadata"] + result = self.kube("logs", "-n", meta["namespace"], meta["name"], + "--all-containers", "--tail=250", check=False, timeout=60) + self.save(f"pod-{meta['namespace']}-{meta['name']}.log", + result.stdout + result.stderr) + result = self.run(["docker", "exec", self.node_id, "journalctl", "-u", "kubelet", + "-u", "containerd", "--no-pager", "-n", "500"], check=False) + self.save("node.log", result.stdout + result.stderr) + if self.registry_id: + self.own_container(self.registry_name) + result = self.run(["docker", "logs", "--tail", "500", self.registry_id], check=False) + self.save("registry.log", result.stdout + result.stderr) + + def finish(self, passed): + errors = [] + for child in self.children: + try: + if child.poll() is None: + child.terminate() + try: + child.wait(timeout=10) + except subprocess.TimeoutExpired: + child.kill() + child.wait(timeout=10) + except ProcessLookupError: + # The child exited between poll and terminate. + pass + except (OSError, subprocess.TimeoutExpired) as error: + errors.append(f"child cleanup: {error}") + try: + self.diagnostics() + except (RuntimeError, subprocess.TimeoutExpired, OSError) as error: + errors.append(f"diagnostics: {error}") + for name, identifier in ((self.node, self.node_id), + (getattr(self, "registry_name", ""), self.registry_id)): + if identifier: + try: + self.own_container(name) + self.run(["docker", "rm", "-f", "--volumes", identifier]) + except (RuntimeError, subprocess.TimeoutExpired, OSError) as error: + errors.append(f"cleanup: {error}") + if self.network_id: + try: + network = json.loads(self.run(["docker", "network", "inspect", self.name]).stdout)[0] + if network["Id"] != self.network_id or network["Labels"].get(OWNER_LABEL) != self.name: + raise RuntimeError("Refusing to remove foreign/replaced network") + self.run(["docker", "network", "rm", self.network_id]) + except (RuntimeError, subprocess.TimeoutExpired, OSError) as error: + errors.append(f"cleanup: {error}") + if self.candidate_image_id: + try: + image = json.loads(self.run( + ["docker", "image", "inspect", self.candidate_image_id]).stdout)[0] + if image["Config"].get("Labels", {}).get(OWNER_LABEL) != self.name: + raise RuntimeError("Refusing to remove foreign candidate image") + self.run(["docker", "image", "rm", self.candidate_image_id]) + except (RuntimeError, subprocess.TimeoutExpired, OSError) as error: + errors.append(f"candidate cleanup: {error}") + for sig, handler in self.old_signals.items(): + signal.signal(sig, handler) + try: + shutil.rmtree(self.private) + except OSError as error: + errors.append(f"private material cleanup: {error}") + try: + self.save("result.json", {"passed": passed and not errors, "errors": errors, + "scenario": self.scenario, "assertions": len(self.evidence)}) + except OSError as error: + errors.append(f"result persistence: {error}") + if errors: + raise RuntimeError("; ".join(errors)) diff --git a/integration-tests/e2e/live/hpa.py b/integration-tests/e2e/live/hpa.py new file mode 100644 index 00000000..c5e30e23 --- /dev/null +++ b/integration-tests/e2e/live/hpa.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Real resource-metrics CPU scaling. No replica writes or synthetic metrics.""" + +import json +import hashlib +import re +import time +from urllib.request import Request, urlopen + +from common import Fixture, download, wait + +METRICS_IMAGE = "registry.k8s.io/metrics-server/metrics-server@sha256:89258156d0e9af60403eafd44da9676fd66f600c7934d468ccc17e42b199aee2" +METRICS_MANIFEST_SHA = "ff64d1a13b9ac3b0635f0dd985815fb44c23eed4706c04e5db1daadf6bc0a83b" +APP = "cpu-demo" + + +def cpu_millicores(quantity): + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(n|u|m|)?", quantity) + if not match: + raise ValueError(f"Unrecognized CPU quantity: {quantity}") + return float(match[1]) * {"n": 0.000001, "u": 0.001, "m": 1, "": 1000}[match[2] or ""] + + +def hpa_cpu_utilization(status): + if "currentCPUUtilizationPercentage" in status: + return status["currentCPUUtilizationPercentage"] + for metric in status.get("currentMetrics", []): + if metric.get("resource", {}).get("name") == "cpu": + return metric["resource"].get("current", {}).get("averageUtilization", 0) + return 0 + + +def install_metrics(fixture): + manifest = fixture.private / "metrics-server.yaml" + download("https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.8.0/components.yaml", + manifest, METRICS_MANIFEST_SHA) + # The exception is confined to this disposable kind fixture's self-signed kubelet. + text = manifest.read_text() + image = "registry.k8s.io/metrics-server/metrics-server:v0.8.0" + if text.count(image) != 1 or text.count(" - --cert-dir=/tmp") != 1: + raise RuntimeError("Pinned metrics manifest no longer matches fixture substitutions") + text = text.replace(image, METRICS_IMAGE).replace( + " - --cert-dir=/tmp", + " - --cert-dir=/tmp\n - --kubelet-insecure-tls") + fixture.kube("apply", "-f", "-", input=text) + fixture.kube("rollout", "status", "deployment/metrics-server", "-n", "kube-system", + "--timeout=180s") + fixture.kube("wait", "--for=condition=Available", "apiservice/v1beta1.metrics.k8s.io", + "--timeout=180s") + fixture.record("real-resource-metrics-server", {"image": METRICS_IMAGE, + "manifestSHA256": METRICS_MANIFEST_SHA, + "fixtureOnlyKubeletInsecureTLS": True, + "downscaleStabilizationSeconds": 60, "hpaSyncSeconds": 10}) + + +class Scaling: + def __init__(self, fixture): + self.f = fixture + self.samples = [] + self.requests = [] + self.loading = False + self.loaded = set() + self.load_failures = 0 + + def pods(self): + return [p for p in self.f.get("pods", "-n", self.f.namespace, + "-l", f"app.kubernetes.io/name={APP}")["items"] + if not p["metadata"].get("deletionTimestamp")] + + def load(self, seconds): + successes = 0 + for pod in self.pods(): + if pod.get("status", {}).get("phase") != "Running": + continue + name = pod["metadata"]["name"] + result = self.f.kube( + "create", "--raw", + f"/api/v1/namespaces/{self.f.namespace}/pods/{name}:8080/proxy/load?seconds={seconds}", + "-f", "-", input="", check=False, timeout=50) + self.requests.append({"time": time.time(), "kind": "load", + "pod": name, "seconds": seconds, + "success": result.returncode == 0, + "response": result.stdout + result.stderr}) + if result.returncode == 0 and seconds: + self.loaded.add(name) + successes += 1 + if seconds: + self.load_failures = 0 if successes else self.load_failures + 1 + if self.load_failures >= 3: + self.f.save("requests.json", self.requests) + raise AssertionError("CPU load failed for every running Pod three times") + + def metrics(self): + result = self.f.kube("get", "--raw", + f"/apis/metrics.k8s.io/v1beta1/namespaces/{self.f.namespace}/pods", + check=False) + if result.returncode: + return None + metrics = json.loads(result.stdout) + names = {pod["metadata"]["name"] for pod in self.pods()} + values = {pod["metadata"]["name"]: + sum(cpu_millicores(c["usage"]["cpu"]) for c in pod["containers"]) + for pod in metrics["items"] if pod["metadata"]["name"] in names} + if values.keys() != names or not names: + return None + return {"millicores": values, "raw": metrics} + + def sample(self, phase): + if self.loading: + self.load(20) + hpa = self.f.get("hpa", APP, "-n", self.f.namespace) + dep = self.f.get("deployment", APP, "-n", self.f.namespace) + app = self.f.get("javaapplication", APP, "-n", self.f.namespace) + result = self.f.kube("get", "--raw", + f"/api/v1/namespaces/{self.f.namespace}/services/{APP}:8080/proxy/hello", + check=False) + serving = result.returncode == 0 and "Hello from a JAR" in result.stdout + self.requests.append({"time": time.time(), "phase": phase, "kind": "serving", + "success": serving, "response": result.stdout + result.stderr}) + sample = {"time": time.time(), "phase": phase, "hpa": hpa["status"], + "desired": dep["spec"]["replicas"], "deployment": dep.get("status", {}), + "javaApplication": app.get("status", {}), "metrics": self.metrics(), + "serving": serving} + if not 1 <= sample["desired"] <= 3: + raise AssertionError(f"HPA replica count outside [1,3]: {sample['desired']}") + self.samples.append(sample) + self.f.save("scaling-samples.json", self.samples) + self.f.save("requests.json", self.requests) + return sample + + def idle(self, phase): + sample = self.sample(phase) + metrics = sample["metrics"] + return (sample["desired"] == 1 and metrics is not None and + max(metrics["millicores"].values()) < 50 and sample["serving"] and + self.f.ready(APP, 1)) + + def observe_source_gc(self, image): + repository, digest = image.split("/", 1)[1].split("@") + + def manifest(digest): + request = Request(f"http://{self.f.registry}/v2/{repository}/manifests/{digest}", + headers={"Accept": "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json"}) + with urlopen(request, timeout=10) as response: + raw = response.read() + if "sha256:" + hashlib.sha256(raw).hexdigest() != digest: + raise AssertionError("Registry returned a mismatched manifest") + return json.loads(raw) + + index = manifest(digest) + matches = [entry for entry in index["manifests"] + if entry.get("platform", {}).get("architecture") == self.f.arch] + if len(matches) != 1: + raise AssertionError("Expected one platform manifest for the fixture node") + selected = matches[0]["digest"] + layers = [entry["digest"] for entry in manifest(selected)["layers"]] + if not layers or any(not re.fullmatch(r"sha256:[a-f0-9]{64}", d) for d in layers): + raise AssertionError("Invalid runnable layer descriptors") + self.f.own_container(self.f.node) + + def collected(): + return all(self.f.run([ + "docker", "exec", self.f.node_id, "test", "!", "-e", + "/var/lib/containerd/io.containerd.content.v1.content/blobs/sha256/" + d[7:]], + check=False).returncode == 0 for d in layers) + wait("containerd naturally discards unpacked source layers", collected, timeout=180) + if self.f.candidate == "shim": + for layer in layers: + path = (f"/tmp/brewlet-runnable/immutable-v2/{selected[7:]}/content/" + f"blobs/sha256/{layer[7:]}") + result = self.f.run(["docker", "exec", self.f.node_id, "sha256sum", path]) + if result.stdout.split()[0] != layer[7:]: + raise AssertionError("Retained layer no longer matches its descriptor") + self.f.record("post-unpack-source-gc-observed", + {"platformManifest": selected, "missingSourceLayers": layers, + "retainedBytesVerified": self.f.candidate == "shim"}) + + def up(self): + sample = self.sample("scale-up") + metrics = sample["metrics"] + hpa = sample["hpa"] + return (sample["desired"] > 1 and metrics is not None and + min(metrics["millicores"].values()) > 50 and + hpa_cpu_utilization(hpa) > 50 and + hpa.get("desiredReplicas") == sample["desired"] and sample["serving"] and + self.f.ready(APP, sample["desired"])) + + def reconcile_ownership(self): + for cycle in range(3): + # Change a real spec field ignored while HPA is enabled. ObservedGeneration + # then proves the operator reconciled, rather than merely waiting. + stale = 7 + cycle + self.f.kube("patch", "javaapplication", APP, "-n", self.f.namespace, + "--type=merge", "-p", json.dumps({"spec": {"replicas": stale}})) + generation = self.f.get("javaapplication", APP, "-n", self.f.namespace)["metadata"]["generation"] + + def reconciled(): + sample = self.sample("ownership") + return (sample["javaApplication"].get("observedGeneration") == generation and + sample["desired"] > 1 and self.f.ready(APP, sample["desired"])) + wait(f"HPA ownership after reconciliation {cycle + 1}", reconciled, + timeout=120, interval=5) + self.f.record("operator-preserves-live-hpa-replicas", {"cycles": 3, + "javaApplicationReplicas": [7, 8, 9], "hpaBounds": [1, 3]}) + + def execute(self, image): + try: + wait("usable pod CPU metrics", self.metrics, timeout=180) + wait("idle minimum, low real CPU, serving and endpoints", lambda: self.idle("baseline"), + timeout=420, interval=5) + self.f.record("metrics-and-idle-preflight", self.samples[-1]) + self.observe_source_gc(image) + self.loading = True + wait("real CPU causes HPA scale-up and Ready Brewlet pods", self.up, + timeout=600, interval=5) + if len(self.loaded) < 2: + raise AssertionError("Additional Brewlet pods did not accept load") + self.f.record("cpu-driven-scale-up", self.samples[-1]) + self.reconcile_ownership() + self.loading = False + self.load(0) + wait("real CPU falls and HPA returns to minimum", lambda: self.idle("scale-down"), + timeout=420, interval=5) + self.f.record("cpu-driven-scale-down", self.samples[-1]) + for phase in ("scale-up", "scale-down"): + requests = [r for r in self.requests if r.get("phase") == phase] + self.f.record(f"{phase}-serving-outcomes", + {"attempts": len(requests), + "successes": sum(r["success"] for r in requests), + "failures": sum(not r["success"] for r in requests)}) + finally: + self.loading = False + self.load(0) + self.f.save("requests.json", self.requests) + + +def main(): + with Fixture("hpa") as fixture: + install_metrics(fixture) + capacity = fixture.get("node", fixture.node)["status"]["allocatable"] + if cpu_millicores(capacity["cpu"]) < 3000: + raise RuntimeError("Insufficient node CPU for the 3x500m bounded workload") + fixture.record("capacity-preflight", capacity) + image = fixture.publish_demo() + fixture.java_application(APP, image, autoscaling=True) + wait("operator creates Deployment", + lambda: fixture.kube("get", "deployment", APP, "-n", fixture.namespace, + check=False).returncode == 0) + fixture.wait_ready(APP) + if "Hello from a JAR" not in fixture.service_get(APP): + raise AssertionError("Ready workload did not serve through its Service") + fixture.record("initial-serving-brewlet-workload", {"image": image}) + Scaling(fixture).execute(image) + + +if __name__ == "__main__": + main() diff --git a/integration-tests/e2e/live/test_common.py b/integration-tests/e2e/live/test_common.py new file mode 100644 index 00000000..187e9ffb --- /dev/null +++ b/integration-tests/e2e/live/test_common.py @@ -0,0 +1,103 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import Mock, patch + +from common import Fixture, owned_container, redact, wait +from hpa import cpu_millicores, hpa_cpu_utilization + + +class FixtureSafetyTests(unittest.TestCase): + def test_container_requires_both_identity_and_owner(self): + info = {"Id": "ours", "Config": {"Labels": {"owner": "invocation"}}} + self.assertTrue(owned_container(info, "ours", "owner", "invocation")) + self.assertFalse(owned_container(info, "replaced", "owner", "invocation")) + self.assertFalse(owned_container(info, "ours", "owner", "foreign")) + + def test_target_is_explicit_and_cannot_be_overridden(self): + fixture = Fixture.__new__(Fixture) + fixture.kubeconfig = Path("/owned/private/kubeconfig") + fixture.context = "kind-owned" + with patch.object(fixture, "run") as execute: + fixture.kube("get", "nodes") + command = execute.call_args.args[0] + self.assertEqual(command[1:5], ["--kubeconfig", fixture.kubeconfig, + "--context", "kind-owned"]) + for arg in ("--context=shared", "--kubeconfig", "--server=x", "--cluster=x"): + with self.assertRaises(ValueError): + fixture.kube("get", "nodes", arg) + + def test_wait_is_bounded_and_does_not_swallow_failures(self): + with self.assertRaises(TimeoutError): + wait("mandatory", lambda: False, timeout=0) + with self.assertRaisesRegex(RuntimeError, "API unavailable"): + wait("mandatory", lambda: (_ for _ in ()).throw(RuntimeError("API unavailable"))) + + def test_private_material_is_redacted(self): + secret = "-----BEGIN EC PRIVATE KEY-----\nSECRET\n-----END EC PRIVATE KEY-----" + self.assertNotIn("SECRET", redact(secret)) + self.assertNotIn("credential", redact("Authorization: Bearer credential")) + + def test_cpu_quantities(self): + for value, expected in [("50000000n", 50), ("50000u", 50), ("50m", 50), ("0.5", 500)]: + self.assertEqual(cpu_millicores(value), expected) + with self.assertRaises(ValueError): + cpu_millicores("synthetic") + + def test_preferred_hpa_api_cpu_utilization(self): + self.assertEqual(hpa_cpu_utilization({"currentCPUUtilizationPercentage": 70}), 70) + self.assertEqual(hpa_cpu_utilization({"currentMetrics": [ + {"resource": {"name": "cpu", "current": {"averageUtilization": 70}}}]}), 70) + self.assertEqual(hpa_cpu_utilization({"currentMetrics": [{"type": ""}]}), 0) + + def test_cleanup_refuses_replaced_container(self): + fixture = Fixture.__new__(Fixture) + fixture.node = "owned-node" + fixture.node_id = "original" + result = subprocess.CompletedProcess([], 0, json.dumps([ + {"Id": "replacement", "Config": {"Labels": {"io.x-k8s.kind.cluster": "owned"}}} + ]), "") + fixture.name = "owned" + with patch.object(fixture, "run", return_value=result): + with self.assertRaisesRegex(RuntimeError, "foreign/replaced"): + fixture.own_container("owned-node") + + def test_failure_log_error_still_cleans_up(self): + fixture = Fixture.__new__(Fixture) + with patch.object(fixture, "save", side_effect=OSError("disk full")), \ + patch.object(fixture, "finish") as finish: + with self.assertRaises(OSError): + fixture.__exit__(RuntimeError, RuntimeError("failed"), None) + finish.assert_called_once_with(False) + + def test_finish_removes_owned_volumes_after_child_race(self): + fixture = Fixture.__new__(Fixture) + fixture.node, fixture.node_id = "owned-node", "node-id" + fixture.registry_name, fixture.registry_id = "owned-registry", "registry-id" + fixture.network_id = None + fixture.candidate_image_id = None + fixture.old_signals = {} + fixture.scenario = "hpa" + fixture.evidence = [] + fixture.private = Path(tempfile.mkdtemp()) + child = Mock() + child.poll.return_value = None + child.terminate.side_effect = ProcessLookupError("already exited") + fixture.children = [child] + with patch.object(fixture, "diagnostics"), patch.object(fixture, "own_container"), \ + patch.object(fixture, "save"), patch.object(fixture, "run") as execute: + fixture.finish(False) + self.assertEqual([c.args[0] for c in execute.call_args_list], [ + ["docker", "rm", "-f", "--volumes", "node-id"], + ["docker", "rm", "-f", "--volumes", "registry-id"], + ]) + self.assertFalse(fixture.private.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/integration-tests/fixtures/demo-app/src/com/example/Hello.java b/integration-tests/fixtures/demo-app/src/com/example/Hello.java index 4d993bc7..c8801194 100644 --- a/integration-tests/fixtures/demo-app/src/com/example/Hello.java +++ b/integration-tests/fixtures/demo-app/src/com/example/Hello.java @@ -11,6 +11,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; /** * Minimal dependency-free HTTP server used to prove the Brewlet PoC: @@ -18,6 +20,9 @@ */ public final class Hello { + private static final AtomicLong loadUntil = new AtomicLong(); + private static volatile long loadResult; + public static void main(String[] args) throws IOException { int port = Integer.getInteger("server.port", 8080); HttpServer server = HttpServer.create(new InetSocketAddress(port), 0); @@ -51,6 +56,48 @@ public static void main(String[] args) throws IOException { server.createContext("/healthz", exchange -> respond(exchange, 200, "ok\n")); + // Opt-in, leased CPU work: a failed client cannot leave permanent load. + if (Boolean.getBoolean("brewlet.test.cpuLoad")) { + server.createContext("/load", exchange -> { + if (!"POST".equals(exchange.getRequestMethod())) { + respond(exchange, 405, "POST required\n"); + return; + } + String rawQuery = exchange.getRequestURI().getRawQuery(); + String[] parameters = rawQuery == null ? new String[0] : rawQuery.split("&"); + String[] leases = Arrays.stream(parameters) + .filter(parameter -> parameter.startsWith("seconds=")) + .toArray(String[]::new); + String query = leases.length == 1 ? leases[0] : ""; + if ("seconds=0".equals(query)) { + loadUntil.set(0); + } else if ("seconds=20".equals(query)) { + loadUntil.set(System.nanoTime() + 20_000_000_000L); + } else { + respond(exchange, 400, "seconds must be 0 or 20\n"); + return; + } + respond(exchange, 200, "ok\n"); + }); + Thread.ofPlatform().daemon().name("fixture-cpu-load").start(() -> { + long value = 1; + while (!Thread.currentThread().isInterrupted()) { + if (System.nanoTime() < loadUntil.get()) { + for (int i = 0; i < 10_000; i++) { + value = value * 2862933555777941757L + 3037000493L; + } + loadResult = value; + } else { + try { + Thread.sleep(50); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + } + }); + } + server.setExecutor(null); System.out.printf("[demo-app] listening on :%d (pid=%d, java=%s)%n", port, ProcessHandle.current().pid(), System.getProperty("java.version")); diff --git a/site/index-value-prop.html b/site/index-value-prop.html index c0e3324a..e042397d 100644 --- a/site/index-value-prop.html +++ b/site/index-value-prop.html @@ -443,7 +443,7 @@

Evaluate the changes that matter.

What needs attention in an evaluation?

Brewlet is a pre-1.0 preview. Use a disposable evaluation environment. - Live admission enforcement and CPU-driven autoscaling validation remain pending; + Live admission enforcement remains pending; CPU scaling passed with a fixed-shim candidate, not unmodified 0.5.0; see validation coverage and remaining gaps. Privileged node provisioning adds platform responsibilities. Current capability admission does not support waking a completely zero-sized pool; diff --git a/site/index.html b/site/index.html index dc13e999..acb4400e 100644 --- a/site/index.html +++ b/site/index.html @@ -245,7 +245,7 @@

Run

Pods use CNI, kubectl logs/exec, probes, and Services. CPU HPA configuration is available; - live metrics-driven validation is pending. + live validation uses a fixed-shim candidate.

@@ -331,8 +331,9 @@

See Brewlet run locally and on Kubernetes

The JVM sees 1 CPU and a heap bounded under 256Mi because the - containerd shim wires the pod's cgroup limits into the process. Existing E2E - coverage exercises manual scaling, not the live CPU autoscaling loop. + containerd shim wires the pod's cgroup limits into the process. + Live CPU scaling passed with a fixed-shim candidate; + unmodified 0.5.0 exposed a packed-layer GC scale-out failure. With Brewlet's default runnable-image format, image: <ref> is all the pod needs; kubelet pulls it like any container image. @@ -361,7 +362,7 @@

Brewlet vs. containers vs. SpinKube

Runtime locationinside the imageshared node JDKnode-installed Wasm runtime Patch the runtimeupdate images and roll workloadsupdate node JDKs, then roll workloadsupdate node runtime Isolationnamespaces + cgroups (runc)namespaces + cgroups (runc)Wasm sandbox - K8s integrationfullServices, probes, logs; CPU HPA validation pendingfull + K8s integrationfullServices, probes, logs; CPU HPA candidate validatedfull Best fitgeneral-purpose workloadsJVM compatibility without full imagessmall, fast-starting Wasm workloads diff --git a/site/mkdocs.yml b/site/mkdocs.yml index ae536a66..1eb00771 100644 --- a/site/mkdocs.yml +++ b/site/mkdocs.yml @@ -98,6 +98,7 @@ nav: - Admission enforcement: admission-enforcement.md - Observability: observability.md - Runtime metrics and Grafana dashboards: runtime-metrics.md + - Disposable live validation: live-validation.md - Multi-architecture fleets: multi-arch.md - Troubleshooting: troubleshooting.md - Reference: diff --git a/site/scripts/test_site_contracts.py b/site/scripts/test_site_contracts.py index d10dcad7..6ca3ca45 100644 --- a/site/scripts/test_site_contracts.py +++ b/site/scripts/test_site_contracts.py @@ -213,13 +213,15 @@ def test_validation_status_distinguishes_smoke_component_and_live_coverage(self) for boundary in ("does not install the chart or provision nodes", "substituted registry access and plugin transport", "simulated HPA ownership", + "fixed-shim candidate", + "packed-layer GC scale-out failure", "two consecutive fresh disposable clusters", "existing E2E harness permits skips"): self.assertIn(boundary, text) for issue in (13, 93, 94, 95): self.assertIn(f"https://github.com/microsoft/brewlet/issues/{issue}", document) - def test_operational_guides_link_pending_live_validation(self): + def test_operational_guides_distinguish_pending_and_candidate_validation(self): guides = { "docs/admission-enforcement.md": 95, "admission/README.md": 95, @@ -231,7 +233,12 @@ def test_operational_guides_link_pending_live_validation(self): with self.subTest(document=filename): text = " ".join((ROOT / filename).read_text(encoding="utf-8").split()) self.assertIn(f"https://github.com/microsoft/brewlet/issues/{issue}", text) - self.assertIn("pending", text) + if issue == 95: + self.assertIn("pending", text) + else: + self.assertIn("fixed-shim candidate", text) + self.assertIn("0.5.0", text) + self.assertIn("cold startup", text) self.assertIn("disposable", text) self.assertNotIn("production admission integration", text) self.assertNotIn("Production admission policy that", text) diff --git a/specs/SPECIFICATION.md b/specs/SPECIFICATION.md index 61dc9e8b..abd939ba 100644 --- a/specs/SPECIFICATION.md +++ b/specs/SPECIFICATION.md @@ -66,7 +66,9 @@ capability model. node, is shared across workloads, and is upgraded independently of app artifacts. - **G6 — First-class Kubernetes citizen.** Use standard Kubernetes Services, Ingress, probes, autoscaling, logs, and metrics interfaces. Live CPU HPA - validation remains pending in [#94](https://github.com/microsoft/brewlet/issues/94). + validation passed twice on fresh local arm64 clusters with a fixed-shim + candidate over 0.5.0, not the unmodified release; see + [#94](https://github.com/microsoft/brewlet/issues/94). ### 2.2 Non-Goals (for v1) - Replacing OCI *images* for apps that legitimately need OS packages/native deps. @@ -330,7 +332,11 @@ publishes the *same* JAR as a **standard, kubelet-pullable OCI image**: portable bytecode JAR, so any provisioned node matches; narrowed to `--arch` for a JAR carrying native libraries). -containerd/kubelet pull and unpack this image with **no special configuration**. +containerd/kubelet pull and unpack this image using the standard OCI image path. +Packed layers must remain available until the shim publishes a verified stage; +for reliable cold starts, retain them in the effective containerd configuration +(`discard_unpacked_layers=false`). An unpacked snapshot alone is not verified +input to the Brewlet blob resolver. Kubernetes execution requires a digest-pinned request. The shim takes that exact manifest/index target from protected CRI requested-image metadata, requires the containerd-owned `io.kubernetes.cri.image-name` OCI annotation to name the same @@ -1300,6 +1306,14 @@ and builds/runs on Linux: Repeated and concurrent resolutions, including separate shim processes, reuse the published stage without truncating or replacing files held by existing workloads. Failed extraction never publishes an incomplete stage. + The `immutable-v2` stage also retains the exact descriptor-verified packed + layers. Cache reuse verifies those retained bytes and tolerates source-layer + removal by containerd GC, but not other source I/O errors or present corrupt + source bytes. Missing/corrupt retained evidence fails closed. Older stage + directories remain untouched during upgrades; a new stage requires available + source bytes. This warm-reuse protection does not guarantee cold startup + after source-layer GC. Re-pull affected digest-pinned images with packed-layer + retention enabled, and allow extra disk capacity for retained layer bytes. The workload image reference and manifest digest hints are managed **cluster-side, not in the shim**: the `brewlet-admission` webhook (§8.3) overwrites the @@ -1496,7 +1510,10 @@ as per deployment descriptor.”* The descriptor is the `JavaApplication`. > reference and garbage-collected with the `JavaApplication`. > > Existing tests cover HPA resource creation, simulated HPA ownership, and manual -> scaling. Real metrics-server-driven CPU scale-up and scale-down remain pending +> scaling. Real metrics-server-driven 1-to-3-to-1 scaling passed twice on fresh +> local arm64 clusters with the fixed-shim candidate over 0.5.0. Unmodified +> 0.5.0 failed warm scale-out after packed-layer GC; cold missing-source startup +> remains outside the correction. Evidence and remaining scope are tracked > validation in [#94](https://github.com/microsoft/brewlet/issues/94). ### 8.3 Pod admission/scheduling webhook