diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5ea85554..58d8b9e0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -107,6 +107,11 @@ jobs: java: true cluster: true helm: true + - name: "Tier 18: Patched JDK rollout" + tiers: --tier 18 + java: true + cluster: true + helm: false steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/integration-tests/AGENTS.md b/integration-tests/AGENTS.md index eb6ccb1d..d3da3017 100644 --- a/integration-tests/AGENTS.md +++ b/integration-tests/AGENTS.md @@ -41,14 +41,14 @@ The harness does not switch branches or modify component sources. It uses |---|---| | Go | all | | Python 3 | 2, 4, 12, 13, 16, 17 | -| JDK 21+ | 2, 3, 7, 8, 9, 12, 14, 15, 16, 17 | -| Docker | 3, 6, 7, 8-12, 14-17 | -| kubectl and a reachable cluster | 4-17 | +| JDK 21+ | 2, 3, 7, 8, 9, 12, 14-18 | +| Docker | 3, 6, 7, 8-12, 14-18 | +| kubectl and a reachable cluster | 4-18 | | Helm | 4 (optional), 10, 15, 17 | | OpenSSL | 4, 5, 6, 10, 11 | Host-only tiers 1-3 need no cluster. Tiers 4-7 and 13 exercise API-server -behavior. Tiers 6, 8-12, and 14-17 require local containerd nodes that +behavior. Tiers 6, 8-12, and 14-18 require local containerd nodes that Docker can enter, such as kind. Managed clusters skip those node-side paths. Tier 13 also proves the control-plane guard: a NodeProfile that does not set `nodePool.includeControlPlane` never counts or schedules onto a control-plane @@ -78,6 +78,15 @@ that a runnable-image stage survives while its pod runs and while containerd still holds the image, then is reclaimed after the image is removed. Once GC is active, it may also reclaim unreferenced stages left on the node by earlier tiers. +Tier 18 covers a patched-JDK rollout. It provisions a Temurin 21 digest, +runs a Brewlet workload on it, then replaces the NodeProfile digest with a +patched release. It asserts that the node re-advertises the new version for +the new profile generation, that the DaemonSet and `.brewlet-source` carry the +new digest, and that the previous root is retired rather than deleted. It also +checks that the running pod keeps its original JDK and container, and that a +retired-root sweep keeps the root that pod still uses. Finally, it asserts +that `kubectl rollout restart` moves the same application image onto the +patched JDK, after which an idle sweep reclaims the retired root. **Architecture coverage.** Tiers pick up the node's architecture automatically, but hosted runners are amd64, so `.github/workflows/e2e.yml` runs the host-only @@ -115,7 +124,7 @@ Common environment-specific skips: - Tier 5 skips when a cluster cannot reach a host-bound webhook; tier 6 covers the same assertions in-cluster. -- Tiers 8, 9, 12, and 14-17 skip if no schedulable local containerd node +- Tiers 8, 9, 12, and 14-18 skip if no schedulable local containerd node can be provisioned. - Tier 12 skips when the node's `ctr` supports neither `images unpack` nor import-time unpack; tier 16 applies the same rule. diff --git a/integration-tests/e2e/reset.sh b/integration-tests/e2e/reset.sh index ad09fe55..feebe70b 100755 --- a/integration-tests/e2e/reset.sh +++ b/integration-tests/e2e/reset.sh @@ -31,7 +31,7 @@ fi # Fixed namespaces the tiers use for their fixtures. Throwaway per-run # namespaces (tiers 5/6/10/11) are random and self-cleaned by their RETURN # traps; we only need to sweep the deterministic ones here. -E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e brewlet-stagegc-e2e) +E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e brewlet-stagegc-e2e brewlet-jdk-patch) # scrub_node_labels: remove every brewlet.sh/* label and annotation from every # node. These are advertised by the provisioner (or simulated by tiers) and are diff --git a/integration-tests/e2e/run.sh b/integration-tests/e2e/run.sh index 9f981fd3..78aec1c0 100755 --- a/integration-tests/e2e/run.sh +++ b/integration-tests/e2e/run.sh @@ -32,7 +32,7 @@ mkdir -p "$WORK" for arg in "$@"; do case "$arg" in --list) - printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster)\n' + printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster) 18 jdk-patch(in-cluster)\n' exit 0 ;; esac @@ -98,6 +98,7 @@ source "$E2E_DIR/tier14-custom-jdk.sh" source "$E2E_DIR/tier15-metrics-incluster.sh" source "$E2E_DIR/tier16-failure-modes.sh" source "$E2E_DIR/tier17-stage-gc.sh" +source "$E2E_DIR/tier18-jdk-patch.sh" declare -a TIERS=() DO_RESET=0 @@ -107,7 +108,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --tier) TIERS+=("$2"); shift 2 ;; --reset) DO_RESET=1; shift ;; - --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster)\n'; exit 0 ;; + --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster) 18 jdk-patch(in-cluster)\n'; exit 0 ;; -h|--help) usage; exit 0 ;; *) warn "unknown arg: $1"; usage; exit 2 ;; esac @@ -120,7 +121,7 @@ if [[ "$DO_RESET" -eq 1 ]]; then [[ ${#TIERS[@]} -eq 0 ]] && exit 0 fi -[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17) +[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18) section "Brewlet E2E — environment" info "harness : $REPO_DIR" @@ -179,6 +180,7 @@ for t in "${TIERS[@]}"; do 15) tier15_metrics_incluster ;; 16) tier16_failure_modes ;; 17) tier17_stage_gc ;; + 18) tier18_jdk_patch ;; *) warn "no such tier: $t" ;; esac done diff --git a/integration-tests/e2e/tier18-jdk-patch.sh b/integration-tests/e2e/tier18-jdk-patch.sh new file mode 100644 index 00000000..e4f1c704 --- /dev/null +++ b/integration-tests/e2e/tier18-jdk-patch.sh @@ -0,0 +1,559 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Tier 18 — patched JDK rollout (docs/jdk-management.md#patching-upgrading-jdks) +# through the real NodeProfile -> provisioner -> node -> shim -> workload path: +# +# 1. provision temurin-21 from an older patch digest and run a workload on it; +# 2. replace the NodeProfile source digest with a newer patch release; +# 3. the node re-advertises the new build for the new profile generation, +# while the running pod keeps its original JVM and retired root; +# 4. a retired-root sweep must not reclaim the root that pod still uses; +# 5. restarting the unchanged application image moves it to the patched JDK; +# 6. once nothing uses it, the retired root is reclaimed. +# +# Prereqs: kubectl + reachable cluster, Docker with a local containerd node +# (kind / Docker Desktop worker), Go, a host JDK 21+, and network access. + +T18_NS_OP="brewlet" +T18_NS_APP="brewlet-jdk-patch" +T18_PROFILE="jdk-patch" +T18_POOL_KEY="brewlet.sh/e2e-pool" +T18_POOL="jdk-patch" +T18_JDK="temurin-21" +T18_JAVA_HOME="/opt/java/openjdk" +# Multi-platform indexes for two Temurin 21 patch releases. +T18_OLD_IMAGE="docker.io/library/eclipse-temurin@sha256:8a79c84cdf6967ae437eba13c8859d74d41aeccb1e65a42191ca57b1992ad0b8" +T18_OLD_VERSION="21.0.11" +T18_NEW_IMAGE="docker.io/library/eclipse-temurin@sha256:85f00967bcc624fc19fa9c2cf124ea426a5363898e267141726f31f358c2e14b" +T18_NEW_VERSION="21.0.12" +T18_REF="demo/hello:jdk-patch-e2e" +T18_APP="patched-orders" +T18_PORT=8080 +T18_PROVISIONER_IMAGE="localhost/brewlet-node-provisioner:jdk-patch-e2e-$$" +T18_SWEEP_POD="brewlet-retired-sweep" +T18_MGR_PID="" +T18_NODE="" +T18_CONTAINERD_SNAPSHOT="" +T18_JDK_SNAPSHOT="" +T18_JDK_ACTIVE_SNAPSHOT="" +T18_JDK_ACTIVE_STATE="" +T18_PRE_RETIRED="" + +_t18_cleanup() { + info "tier18: cleaning up" + kubectl delete ns "$T18_NS_APP" --ignore-not-found --timeout=60s >/dev/null 2>&1 || true + kubectl delete pod -n "$T18_NS_OP" "$T18_SWEEP_POD" \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + if kubectl get nodeprofile "$T18_PROFILE" >/dev/null 2>&1; then + kubectl delete nodeprofile "$T18_PROFILE" --wait=false >/dev/null 2>&1 || true + wait_for bash -c "! kubectl get nodeprofile '$T18_PROFILE' >/dev/null 2>&1" || \ + kubectl patch nodeprofile "$T18_PROFILE" --type=merge -p '{"metadata":{"finalizers":[]}}' >/dev/null 2>&1 || true + fi + [[ -n "$T18_MGR_PID" ]] && kill "$T18_MGR_PID" 2>/dev/null || true + force_delete_nodeprofiles + kubectl delete daemonset -n "$T18_NS_OP" \ + "brewlet-node-provisioner-$T18_PROFILE" "brewlet-cleanup-$T18_PROFILE" \ + --ignore-not-found >/dev/null 2>&1 || true + kubectl delete clusterrolebinding brewlet-node-provisioner --ignore-not-found >/dev/null 2>&1 || true + kubectl delete clusterrole brewlet-node-provisioner --ignore-not-found >/dev/null 2>&1 || true + kubectl delete runtimeclass brewlet --ignore-not-found >/dev/null 2>&1 || true + kubectl delete ns "$T18_NS_OP" --ignore-not-found --wait=false >/dev/null 2>&1 || true + kubectl delete crd nodeprofiles.node.brewlet.sh javaapplications.apps.brewlet.sh \ + --ignore-not-found --wait=false >/dev/null 2>&1 || true + if [[ -n "$T18_NODE" ]]; then + if [[ -n "$T18_CONTAINERD_SNAPSHOT" && -f "$T18_CONTAINERD_SNAPSHOT" ]] && + docker cp "$T18_NODE:/etc/containerd/config.toml" "$WORK/t18-containerd-after.toml" >/dev/null 2>&1 && + ! cmp -s "$T18_CONTAINERD_SNAPSHOT" "$WORK/t18-containerd-after.toml"; then + docker cp "$T18_CONTAINERD_SNAPSHOT" \ + "$T18_NODE:/etc/containerd/config.toml" >/dev/null 2>&1 || true + docker exec "$T18_NODE" systemctl restart containerd >/dev/null 2>&1 || true + fi + docker exec "$T18_NODE" rm -f /etc/containerd/config.toml.brewlet.bak >/dev/null 2>&1 || true + # Remove the root and any retired copies this tier created; keep pre-existing ones. + local dir + while IFS= read -r dir; do + [[ -n "$dir" ]] || continue + grep -Fxq -- "$dir" <<<"$T18_PRE_RETIRED" && continue + docker exec "$T18_NODE" sh -c 'chmod -R u+w "$1" 2>/dev/null || true; rm -rf "$1"' \ + sh "$dir" >/dev/null 2>&1 || true + done < <(_t18_retired_roots) + docker exec "$T18_NODE" sh -c 'chmod -R u+w "$1" 2>/dev/null || true; rm -rf "$1"' \ + sh "/opt/brewlet/jdks/$T18_JDK" >/dev/null 2>&1 || true + if [[ -n "$T18_JDK_SNAPSHOT" && -f "$T18_JDK_SNAPSHOT" ]]; then + docker exec "$T18_NODE" mkdir -p /opt/brewlet/jdks >/dev/null 2>&1 || true + docker exec -i "$T18_NODE" tar -C /opt/brewlet/jdks -xf - \ + <"$T18_JDK_SNAPSHOT" >/dev/null 2>&1 || true + fi + docker exec "$T18_NODE" mkdir -p /opt/brewlet/jdks >/dev/null 2>&1 || true + case "$T18_JDK_ACTIVE_STATE" in + present) + docker cp "$T18_JDK_ACTIVE_SNAPSHOT" \ + "$T18_NODE:/opt/brewlet/jdks/.brewlet-active.t18" >/dev/null 2>&1 || true + docker exec "$T18_NODE" mv \ + /opt/brewlet/jdks/.brewlet-active.t18 /opt/brewlet/jdks/.brewlet-active >/dev/null 2>&1 || true ;; + absent) + docker exec "$T18_NODE" rm -f /opt/brewlet/jdks/.brewlet-active >/dev/null 2>&1 || true ;; + esac + label_node "$T18_NODE" "$T18_POOL_KEY-" brewlet.sh/runtime- \ + "brewlet.sh/jdk.$T18_JDK-" "brewlet.sh/jdk-feature.${T18_JDK##*-}-" \ + brewlet.sh/launcher.java- >/dev/null 2>&1 || true + annotate_node "$T18_NODE" brewlet.sh/jdks- brewlet.sh/jdks-info- \ + brewlet.sh/launchers- brewlet.sh/profile- brewlet.sh/profile-generation- \ + brewlet.sh/provision-error- >/dev/null 2>&1 || true + docker exec "$T18_NODE" ctr -n k8s.io images rm "$T18_PROVISIONER_IMAGE" \ + >/dev/null 2>&1 || true + fi + docker rmi "$T18_PROVISIONER_IMAGE" >/dev/null 2>&1 || true +} + +_t18_retired_roots() { + docker exec "$T18_NODE" sh -c \ + 'for d in /opt/brewlet/jdks/"$1".retired.*; do [ -d "$d" ] && echo "$d"; done; true' \ + sh "$T18_JDK" 2>/dev/null +} + +# _t18_node_at VERSION GENERATION: the node is ready, advertises the given +# profile generation, and its inventory reports the given temurin-21 build. +_t18_node_at() { + local version="$1" generation="$2" + [[ "$(kubectl get node "$T18_NODE" -o jsonpath='{.metadata.labels.brewlet\.sh/runtime}' 2>/dev/null)" == "ready" ]] && + [[ "$(kubectl get node "$T18_NODE" -o jsonpath='{.metadata.labels.brewlet\.sh/jdk\.temurin-21}' 2>/dev/null)" == "true" ]] && + [[ "$(kubectl get node "$T18_NODE" -o jsonpath='{.metadata.annotations.brewlet\.sh/profile-generation}' 2>/dev/null)" == "$generation" ]] && + kubectl get node "$T18_NODE" -o jsonpath='{.metadata.annotations.brewlet\.sh/jdks-info}' 2>/dev/null | + grep -q "\"version\":\"${version}" +} + +_t18_curl() { + kubectl exec -n "$T18_NS_APP" t18-client -- \ + wget -q -O- -T 5 "http://$T18_APP.$T18_NS_APP.svc.cluster.local:$T18_PORT$1" 2>/dev/null +} + +# _t18_pod_info POD_IP: query one specific pod rather than the Service, so the +# answer cannot come from a different replica during a rollout. +_t18_pod_info() { + local ip="$1" tries="${2:-40}" body + while (( tries-- > 0 )); do + if body="$(kubectl exec -n "$T18_NS_APP" t18-client -- \ + wget -q -O- -T 5 "http://$ip:$T18_PORT/info" 2>/dev/null)" && [[ -n "$body" ]]; then + printf '%s' "$body" + return 0 + fi + sleep 1 + done + return 1 +} + +_t18_app_pod() { + kubectl get pod -n "$T18_NS_APP" -l app="$T18_APP" \ + --field-selector=status.phase=Running \ + -o jsonpath='{range .items[*]}{.metadata.name}{"|"}{.metadata.uid}{"|"}{.status.podIP}{"|"}{.spec.containers[0].image}{"|"}{.status.containerStatuses[0].restartCount}{"\n"}{end}' \ + 2>/dev/null +} + +# _t18_sweep LOG: run the real provisioner's retired-root sweep on the node +# with no grace period, so only the live-mount safeguard decides what is kept. +_t18_sweep() { + local log="$1" + kubectl delete pod -n "$T18_NS_OP" "$T18_SWEEP_POD" \ + --ignore-not-found --wait=true >/dev/null 2>&1 || true + kubectl apply -f - >>"$log" 2>&1 </dev/null)\" == Succeeded ]]"; then + kubectl logs -n "$T18_NS_OP" "$T18_SWEEP_POD" >>"$log" 2>&1 || true + return 1 + fi + kubectl logs -n "$T18_NS_OP" "$T18_SWEEP_POD" >>"$log" 2>&1 || true + kubectl delete pod -n "$T18_NS_OP" "$T18_SWEEP_POD" \ + --ignore-not-found --wait=true >/dev/null 2>&1 || true +} + +tier18_jdk_patch() { + section "Tier 18 — patched JDK rollout (NodeProfile digest replacement)" + if ! have kubectl || ! k8s_reachable; then skip "tier18: JDK patch rollout" "no reachable cluster"; return 0; fi + if ! have docker || ! docker info >/dev/null 2>&1; then skip "tier18: JDK patch rollout" "docker daemon not available"; return 0; fi + if ! have go; then skip "tier18: JDK patch rollout" "go not installed"; return 0; fi + if ! have java; then skip "tier18: JDK patch rollout" "host JDK not installed"; return 0; fi + + T18_NODE="$(pick_provisionable_node)" + if [[ -z "$T18_NODE" ]]; then + skip "tier18: JDK patch rollout" "no local containerd node (need kind/CI or Docker Desktop)"; return 0 + fi + if ! node_schedulable "$T18_NODE"; then + skip "tier18: JDK patch rollout" "only provisionable node is unschedulable"; return 0 + fi + local arch; arch="$(_t9_node_arch "$T18_NODE")" + if [[ -z "$arch" ]]; then skip "tier18: JDK patch rollout" "unknown node architecture"; return 0; fi + info "tier18: node=$T18_NODE arch=$arch" + + # Snapshot node state this tier replaces so cleanup can restore it. + T18_CONTAINERD_SNAPSHOT="$WORK/t18-containerd-before.toml" + if ! docker cp "$T18_NODE:/etc/containerd/config.toml" "$T18_CONTAINERD_SNAPSHOT" >/dev/null 2>&1; then + fail "tier18: snapshot containerd config"; return 0 + fi + if docker exec "$T18_NODE" test -f /opt/brewlet/jdks/.brewlet-active; then + T18_JDK_ACTIVE_SNAPSHOT="$WORK/t18-jdk-active-before" + if docker cp "$T18_NODE:/opt/brewlet/jdks/.brewlet-active" \ + "$T18_JDK_ACTIVE_SNAPSHOT" >/dev/null 2>&1; then + T18_JDK_ACTIVE_STATE=present + else + fail "tier18: snapshot existing JDK active inventory"; return 0 + fi + else + T18_JDK_ACTIVE_STATE=absent + fi + if docker exec "$T18_NODE" test -e "/opt/brewlet/jdks/$T18_JDK"; then + T18_JDK_SNAPSHOT="$WORK/t18-jdk-before.tar" + if ! docker exec "$T18_NODE" tar -C /opt/brewlet/jdks -cf - "$T18_JDK" \ + >"$T18_JDK_SNAPSHOT" 2>/dev/null; then + fail "tier18: snapshot existing JDK directory"; return 0 + fi + fi + T18_PRE_RETIRED="$(_t18_retired_roots)" + trap _t18_cleanup RETURN + docker exec "$T18_NODE" sh -c 'chmod -R u+w "$1" 2>/dev/null || true; rm -rf "$1"' \ + sh "/opt/brewlet/jdks/$T18_JDK" >/dev/null 2>&1 || true + + # Build the real provisioner and load it into the node's k8s.io namespace. + local -a build_args=(--platform "linux/$arch" -t "$T18_PROVISIONER_IMAGE") + if [[ -n "${CAROOT:-}" && -f "$CAROOT/rootCA.pem" ]]; then + build_args+=(--secret "id=additional-ca,src=$CAROOT/rootCA.pem") + fi + if docker build "${build_args[@]}" \ + -f "$MONOREPO_DIR/provisioner/Dockerfile" "$MONOREPO_DIR" \ + >"$WORK/t18-provisioner-build.log" 2>&1 && + docker save "$T18_PROVISIONER_IMAGE" | + docker exec -i "$T18_NODE" ctr -n k8s.io images import - \ + >"$WORK/t18-provisioner-import.log" 2>&1; then + pass "tier18: built and loaded the real node provisioner" + else + fail "tier18: build/load node provisioner" "see t18-provisioner-build.log and t18-provisioner-import.log" + return 0 + fi + + # Install APIs and the provisioner's node-patching identity. + if ! T14_NS_OP="$T18_NS_OP" _t14_ensure_operator_namespace; then + fail "tier18: prepare operator namespace" "namespace $T18_NS_OP remained terminating"; return 0 + fi + _t14_wait_crd_not_terminating nodeprofiles.node.brewlet.sh || { + fail "tier18: wait for previous NodeProfile CRD deletion"; return 0 + } + _t14_wait_crd_not_terminating javaapplications.apps.brewlet.sh || { + fail "tier18: wait for previous JavaApplication CRD deletion"; return 0 + } + if kubectl apply -f "$BREWLET_KUBERNETES_DIR/deploy/nodeprofile-crd.yaml" >"$WORK/t18-control.log" 2>&1 && + kubectl apply -f "$BREWLET_KUBERNETES_DIR/deploy/javaapplication-crd.yaml" >>"$WORK/t18-control.log" 2>&1 && + kubectl wait --for=condition=Established --timeout=30s crd/nodeprofiles.node.brewlet.sh >>"$WORK/t18-control.log" 2>&1; then + pass "tier18: installed the NodeProfile API" + else + fail "tier18: install NodeProfile API" "see $WORK/t18-control.log"; return 0 + fi + kubectl apply -f - >>"$WORK/t18-control.log" 2>&1 <>"$WORK/t18-control.log" 2>&1; then + fail "tier18: build operator" "see $WORK/t18-control.log"; return 0 + fi + local probe; probe="$(free_port)" + "$WORK/t18-manager" \ + --namespace "$T18_NS_OP" \ + --provisioner-image "$T18_PROVISIONER_IMAGE" \ + --leader-elect=false --metrics-bind-address=0 --health-probe-bind-address=":$probe" \ + >"$WORK/t18-manager.log" 2>&1 & + T18_MGR_PID=$! + if ! retry_curl "http://localhost:$probe/readyz" 40 0.5 >/dev/null; then + fail "tier18: operator readyz" "see $WORK/t18-manager.log"; return 0 + fi + label_node "$T18_NODE" --overwrite "$T18_POOL_KEY=$T18_POOL" >/dev/null 2>&1 + + # --- 1. Provision the older patch release ------------------------------- + kubectl apply -f - >"$WORK/t18-profile.log" 2>&1 </dev/null)" + if wait_for_seconds 240 _t18_node_at "$T18_OLD_VERSION" "$gen_old"; then + pass "tier18: node advertised temurin-21 $T18_OLD_VERSION for generation $gen_old" + else + kubectl logs -n "$T18_NS_OP" -l "brewlet.sh/nodeprofile=$T18_PROFILE" -c provisioner --tail=200 \ + >"$WORK/t18-provisioner.log" 2>&1 || true + fail "tier18: node advertised temurin-21 $T18_OLD_VERSION" \ + "provision-error=$(kubectl get node "$T18_NODE" -o jsonpath='{.metadata.annotations.brewlet\.sh/provision-error}' 2>/dev/null); see $WORK/t18-provisioner.log" + return 0 + fi + assert_eq "tier18: installed root records the original source digest" \ + "$(docker exec "$T18_NODE" head -n 1 "/opt/brewlet/jdks/$T18_JDK/.brewlet-source" 2>/dev/null)" \ + "$T18_OLD_IMAGE" + + # --- 2. Run a workload on the older build ------------------------------- + local jar="$FIXTURES_DIR/demo-app/target/app.jar" + local jh; jh="$(resolve_java_home)" + if [[ ! -f "$jar" ]] && + ! env JAVA_HOME="$jh" PATH="$jh/bin:$PATH" "$FIXTURES_DIR/demo-app/build.sh" >>"$WORK/t18-app.log" 2>&1; then + fail "tier18: build demo JAR" "see $WORK/t18-app.log"; return 0 + fi + if ! (cd "$BREWLET_CORE_DIR" && go build -o "$WORK/t18-brewlet" ./cmd/brewlet) >>"$WORK/t18-app.log" 2>&1; then + fail "tier18: build CLI" "see $WORK/t18-app.log"; return 0 + fi + local store="$WORK/t18-oci"; rm -rf "$store" + if ! "$WORK/t18-brewlet" push "$jar" "$T18_REF" --store "$store" \ + --format=image >>"$WORK/t18-app.log" 2>&1; then + fail "tier18: build demo runnable image" "see $WORK/t18-app.log"; return 0 + fi + local digest image_ref + digest="$(oci_layout_digest "$store" "$T18_REF")" + if [[ -z "$digest" ]] || ! import_oci_layout "$T18_NODE" "$store" "$WORK/t18-app.log"; then + fail "tier18: import demo runnable image" "see $WORK/t18-app.log"; return 0 + fi + if ! image_ref="$(pin_image_for_cri "$T18_NODE" "$T18_REF" "$digest" "$WORK/t18-app.log")"; then + fail "tier18: create digest-pinned CRI image reference" "see $WORK/t18-app.log"; return 0 + fi + + kubectl create namespace "$T18_NS_APP" >/dev/null 2>&1 || true + kubectl apply -n "$T18_NS_APP" -f - >>"$WORK/t18-app.log" 2>&1 <>"$WORK/t18-app.log" 2>&1 || true + if ! kubectl rollout status -n "$T18_NS_APP" deploy/"$T18_APP" --timeout=180s >>"$WORK/t18-app.log" 2>&1; then + fail "tier18: workload became Ready on the older JDK" "diag: $(save_pod_diag "$T18_APP" "$T18_NS_APP" "app=$T18_APP")" + return 0 + fi + if ! kubectl wait -n "$T18_NS_APP" --for=condition=Ready pod/t18-client --timeout=60s >>"$WORK/t18-app.log" 2>&1; then + fail "tier18: in-cluster client became Ready" "see $WORK/t18-app.log"; return 0 + fi + local old_pod old_name old_uid old_ip old_image old_restarts body + old_pod="$(_t18_app_pod | head -n 1)" + IFS='|' read -r old_name old_uid old_ip old_image old_restarts <<<"$old_pod" + if [[ -z "$old_ip" ]]; then + fail "tier18: locate the running workload pod" "see $WORK/t18-app.log"; return 0 + fi + if body="$(_t18_pod_info "$old_ip")"; then + assert_contains "tier18: workload runs on temurin-21 $T18_OLD_VERSION" "$body" \ + "java.version = $T18_OLD_VERSION" + else + fail "tier18: query workload JVM version" "no response from $old_name /info"; return 0 + fi + + # --- 3. Replace the source digest with the patched release -------------- + if ! kubectl patch nodeprofile "$T18_PROFILE" --type=json -p \ + "[{\"op\":\"replace\",\"path\":\"/spec/jdks/0/source/image\",\"value\":\"$T18_NEW_IMAGE\"}]" \ + >>"$WORK/t18-profile.log" 2>&1; then + fail "tier18: replace the NodeProfile JDK digest" "see $WORK/t18-profile.log"; return 0 + fi + local gen_new + gen_new="$(kubectl get nodeprofile "$T18_PROFILE" -o jsonpath='{.metadata.generation}' 2>/dev/null)" + if [[ "$gen_new" == "$gen_old" ]]; then + fail "tier18: digest replacement advanced the profile generation" "generation stayed $gen_old"; return 0 + fi + if wait_for_seconds 240 _t18_node_at "$T18_NEW_VERSION" "$gen_new"; then + pass "tier18: node re-advertised temurin-21 $T18_NEW_VERSION for generation $gen_new" + else + kubectl logs -n "$T18_NS_OP" -l "brewlet.sh/nodeprofile=$T18_PROFILE" -c provisioner --tail=200 \ + >"$WORK/t18-provisioner.log" 2>&1 || true + fail "tier18: node re-advertised temurin-21 $T18_NEW_VERSION" \ + "provision-error=$(kubectl get node "$T18_NODE" -o jsonpath='{.metadata.annotations.brewlet\.sh/provision-error}' 2>/dev/null); see $WORK/t18-provisioner.log" + return 0 + fi + assert_eq "tier18: operator passed the patched digest to the provisioner" \ + "$(kubectl get ds "brewlet-node-provisioner-$T18_PROFILE" -n "$T18_NS_OP" \ + -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="JDK_SOURCE_0_IMAGE")].value}')" \ + "$T18_NEW_IMAGE" + assert_eq "tier18: active root records the patched source digest" \ + "$(docker exec "$T18_NODE" head -n 1 "/opt/brewlet/jdks/$T18_JDK/.brewlet-source" 2>/dev/null)" \ + "$T18_NEW_IMAGE" + + local retired="" dir + while IFS= read -r dir; do + [[ -n "$dir" ]] || continue + grep -Fxq -- "$dir" <<<"$T18_PRE_RETIRED" && continue + if [[ "$(docker exec "$T18_NODE" head -n 1 "$dir/.brewlet-source" 2>/dev/null)" == "$T18_OLD_IMAGE" ]]; then + retired="$dir" + fi + done < <(_t18_retired_roots) + if [[ -n "$retired" ]]; then + pass "tier18: previous root was retired rather than deleted ($retired)" + else + fail "tier18: previous root was retired rather than deleted" \ + "no $T18_JDK.retired.* root records $T18_OLD_IMAGE" + fi + + # --- 4. The running pod keeps its JVM and its retired root -------------- + local cur cur_name cur_uid cur_ip cur_image cur_restarts + cur="$(_t18_app_pod | head -n 1)" + IFS='|' read -r cur_name cur_uid cur_ip cur_image cur_restarts <<<"$cur" + assert_eq "tier18: JDK rotation did not replace the running pod" "$cur_uid" "$old_uid" + assert_eq "tier18: JDK rotation did not restart the running container" "$cur_restarts" "$old_restarts" + if body="$(_t18_pod_info "$old_ip")"; then + assert_contains "tier18: running pod still reports temurin-21 $T18_OLD_VERSION" "$body" \ + "java.version = $T18_OLD_VERSION" + else + fail "tier18: running pod stayed serving after the JDK rotation" "no response from $old_name /info" + fi + + if [[ -n "$retired" ]]; then + if _t18_sweep "$WORK/t18-sweep-live.log"; then + if docker exec "$T18_NODE" test -d "$retired"; then + pass "tier18: retired-root sweep kept the root a running pod still uses" + else + fail "tier18: retired-root sweep kept the root a running pod still uses" \ + "$retired was reclaimed while $old_name ran; see $WORK/t18-sweep-live.log" + fi + else + fail "tier18: run the retired-root sweep" "see $WORK/t18-sweep-live.log" + fi + if body="$(_t18_pod_info "$old_ip")"; then + assert_contains "tier18: running pod still serves after the sweep" "$body" "java.version" + else + fail "tier18: running pod still serves after the sweep" "no response from $old_name /info" + fi + fi + + # --- 5. Restarting the unchanged image moves it to the patched JDK ------ + if ! kubectl rollout restart -n "$T18_NS_APP" deploy/"$T18_APP" >>"$WORK/t18-app.log" 2>&1 || + ! kubectl rollout status -n "$T18_NS_APP" deploy/"$T18_APP" --timeout=180s >>"$WORK/t18-app.log" 2>&1; then + fail "tier18: restarted workload became Ready" "diag: $(save_pod_diag "$T18_APP" "$T18_NS_APP" "app=$T18_APP")" + return 0 + fi + wait_for_seconds 60 bash -c \ + "! kubectl get pod -n '$T18_NS_APP' '$old_name' >/dev/null 2>&1" || true + local new_pod new_name new_uid new_ip new_image new_restarts + new_pod="$(_t18_app_pod | head -n 1)" + IFS='|' read -r new_name new_uid new_ip new_image new_restarts <<<"$new_pod" + if [[ -n "$new_uid" && "$new_uid" != "$old_uid" ]]; then + pass "tier18: restart replaced the workload pod" + else + fail "tier18: restart replaced the workload pod" "pod=$new_pod" + fi + assert_eq "tier18: restarted pod uses the same application image" "$new_image" "$old_image" + if body="$(_t18_pod_info "$new_ip")"; then + assert_contains "tier18: restarted pod runs the patched temurin-21 $T18_NEW_VERSION" "$body" \ + "java.version = $T18_NEW_VERSION" + else + fail "tier18: query restarted workload JVM version" "no response from $new_name /info" + fi + + # --- 6. Unused retired roots are reclaimed ------------------------------ + if [[ -n "$retired" ]] && docker exec "$T18_NODE" test -d "$retired"; then + if _t18_sweep "$WORK/t18-sweep-idle.log"; then + if docker exec "$T18_NODE" test -d "$retired"; then + fail "tier18: sweep reclaimed the retired root after its pod stopped" \ + "$retired remains; see $WORK/t18-sweep-idle.log" + else + pass "tier18: sweep reclaimed the retired root after its pod stopped" + fi + else + fail "tier18: run the retired-root sweep after restart" "see $WORK/t18-sweep-idle.log" + fi + check "tier18: patched root is still usable after the sweep" \ + docker exec "$T18_NODE" test -x "/opt/brewlet/jdks/$T18_JDK$T18_JAVA_HOME/bin/java" + fi +} diff --git a/provisioner/entrypoint.sh b/provisioner/entrypoint.sh index 98f2e6a5..95f91646 100755 --- a/provisioner/entrypoint.sh +++ b/provisioner/entrypoint.sh @@ -38,6 +38,8 @@ set -euo pipefail NODE_NAME="${NODE_NAME:-$(hostname)}" # Keep this container-local path aligned with the DaemonSet readiness probe. COMPLETION_FILE="/tmp/brewlet-complete" +SERVICE_ACCOUNT_DIR="${SERVICE_ACCOUNT_DIR:-/var/run/secrets/kubernetes.io/serviceaccount}" +IN_CLUSTER_KUBECONFIG="${IN_CLUSTER_KUBECONFIG:-/tmp/brewlet-kubeconfig}" PREFIX="${BREWLET_PREFIX:-/opt/brewlet}" # host-mounted at $PREFIX HOST_BIN="${HOST_BIN:-/host/usr/local/bin}" # host /usr/local/bin (on containerd PATH) CONTAINERD_CONFIG="${CONTAINERD_CONFIG:-/etc/containerd/config.toml}" @@ -931,27 +933,112 @@ preflight_sources() { # references them. # # Two independent gates, both must pass: -# 1. no current mount entry mentions the path (fail safe: if the mount table -# cannot be read, the root is treated as referenced and kept); +# 1. no mount in any mount namespace is backed by the root (fail safe: if +# the mount tables cannot be read, the root is treated as referenced); # 2. the root has aged past a grace period, so a sandbox being created right # now cannot race the sweep. # +# Mount *option strings* cannot answer gate 1: overlay lowerdir= and +# /proc/mounts keep the path the root had when the sandbox started, which after +# rotation is the NEW root's name. Every Brewlet sandbox also bind-mounts its +# JDK home (and launcher root) from inside the runtime root, and the kernel +# renders a bind mount's mountinfo "root" field from the live dentry, so it +# follows the rename. Matching that field plus the device number identifies a +# retired root that a running container still uses. +# # This runs BEFORE new roots are copied so a node that is already tight on disk # reclaims space ahead of a multi-hundred-megabyte copy-from-image. +PROC_ROOT="${BREWLET_PROC_ROOT:-/proc}" + +# Print ": " for a host +# path, resolved through the host mount namespace's (PID 1) mount table. +path_device_number() { + stat -c %d -- "$1" 2>/dev/null || stat -f %d -- "$1" 2>/dev/null +} + +mount_identity() { + local path="$1" dev major minor + dev="$(path_device_number "$path")" || return 1 + [[ "$dev" =~ ^[0-9]+$ ]] || return 1 + major=$(( ((dev >> 8) & 0xfff) | ((dev >> 32) & ~0xfff) )) + minor=$(( (dev & 0xff) | ((dev >> 12) & ~0xff) )) + awk -v dev="$major:$minor" -v path="$path" ' + function unescape(s) { + gsub(/\\040/, " ", s); gsub(/\\011/, "\t", s) + gsub(/\\012/, "\n", s); gsub(/\\134/, "\\", s) + return s + } + $3 == dev { + mountpoint = unescape($5) + if (mountpoint == "/") rest = path + else if (path == mountpoint) rest = "" + else if (index(path, mountpoint "/") == 1) rest = substr(path, length(mountpoint) + 1) + else next + if (found && length(mountpoint) <= best) next + found = 1; best = length(mountpoint) + root = unescape($4) + relative = (root == "/" ? "" : root) rest + if (relative == "") relative = "/" + } + END { if (!found) exit 1; print dev " " relative } + ' "$PROC_ROOT/1/mountinfo" 2>/dev/null +} + +# Write " " for every mount in every mount namespace +# to $1, reading each namespace once per sweep. A namespace only counts as +# seen once one of its PIDs yielded a mount table, so a PID exiting mid-scan +# cannot hide a namespace that other PIDs still hold. Fails if no mount table +# was readable, so callers can fail safe. +snapshot_mount_roots() { + local out="$1" file ns seen=" " + : >"$out" || return 1 + for file in "$PROC_ROOT"/[0-9]*/mountinfo; do + ns="$(readlink "${file%/mountinfo}/ns/mnt" 2>/dev/null || printf '%s' "$file")" + [[ "$seen" != *" $ns "* ]] || continue + if awk '{ print $3 " " $4; n++ } END { exit n ? 0 : 1 }' "$file" >>"$out" 2>/dev/null; then + seen+="$ns " + fi + done + [[ "$seen" != " " ]] +} + +# Succeeds when a mount in the snapshot is backed by the root (or anything +# beneath it); an unresolvable root or missing snapshot counts as referenced. root_referenced_by_mount() { - local root="$1" mounts - mounts="$(host_exec cat /proc/mounts 2>/dev/null || true)" - [[ -n "$mounts" ]] || return 0 - grep -Fq -- "$root" <<<"$mounts" + local root="$1" snapshot="$2" identity dev relative + [[ -n "$snapshot" && -s "$snapshot" ]] || return 0 + identity="$(mount_identity "$root")" || return 0 + dev="${identity%% *}" + relative="${identity#* }" + awk -v dev="$dev" -v relative="$relative" ' + function unescape(s) { + gsub(/\\040/, " ", s); gsub(/\\011/, "\t", s) + gsub(/\\012/, "\n", s); gsub(/\\134/, "\\", s) + return s + } + $1 == dev { + root = unescape($2) + if (root == relative || index(root, relative "/") == 1) { hit = 1; exit } + } + END { exit hit ? 0 : 1 } + ' "$snapshot" } reclaim_retired_roots() { - local now dir age mtime + local now dir age mtime snapshot="" snapshot_taken=false now="$(date +%s)" while IFS= read -r dir; do [[ -n "$dir" && -d "$dir" ]] || continue - if root_referenced_by_mount "$dir"; then + if [[ "$snapshot_taken" == false ]]; then + snapshot_taken=true + snapshot="$(mktemp "${TMPDIR:-/tmp}/brewlet-mounts.XXXXXX")" || snapshot="" + if [[ -n "$snapshot" ]] && ! snapshot_mount_roots "$snapshot"; then + rm -f -- "$snapshot" + snapshot="" + fi + fi + if root_referenced_by_mount "$dir" "$snapshot"; then log "retaining retired root $dir (still referenced by a live mount)" continue fi @@ -965,6 +1052,7 @@ reclaim_retired_roots() { chmod -R u+w "$dir" 2>/dev/null || true rm -rf "$dir" || log "WARN: could not reclaim retired root $dir" done < <(find "$PREFIX/jdks" "$PREFIX/launchers" -maxdepth 1 -type d -name '*.retired.*' 2>/dev/null) + [[ -z "$snapshot" ]] || rm -f -- "$snapshot" # Staging directories are only ever visible mid-install, so one left behind # belongs to an interrupted run and is never referenced by a sandbox. @@ -1828,10 +1916,46 @@ cleanup_node() { exec sleep infinity } +# kubectl only falls back to in-cluster config when the client config is +# otherwise default; any explicit flag such as --request-timeout makes it use +# the empty default (localhost:8080) instead. An explicit kubeconfig that +# reads the projected service-account token keeps such flags working and +# still picks up token rotation. +ensure_in_cluster_kubeconfig() { + [[ -z "${KUBECONFIG:-}" && -n "${KUBERNETES_SERVICE_HOST:-}" ]] || return 0 + [[ -r "$SERVICE_ACCOUNT_DIR/token" && -r "$SERVICE_ACCOUNT_DIR/ca.crt" ]] || return 0 + local host="$KUBERNETES_SERVICE_HOST" + [[ "$host" != *:* ]] || host="[$host]" + if ! cat >"$IN_CLUSTER_KUBECONFIG" <"$fake_proc/1/mountinfo" + { + printf '600 500 0:99 / / rw,relatime - overlay overlay rw,lowerdir=%s/jdks/temurin-21,upperdir=/u,workdir=/w\n' "$reclaim_root" + local root + for root in "$@"; do + printf '601 600 %s %s /opt/jdk ro,relatime - ext4 /dev/root rw\n' "$reclaim_majmin" "$root" + done + } >"$fake_proc/42/mountinfo" +} + # Unreferenced and past the grace period: reclaimed. gone="$(new_retired "$reclaim_root/jdks/temurin-21.retired.100.1")" +set_container_mounts "$reclaim_root/jdks/temurin-21/home" ( PREFIX="$reclaim_root" + PROC_ROOT="$fake_proc" RETIRED_GRACE_SECONDS=0 - host_exec() { printf 'overlay / overlay rw,lowerdir=/opt/brewlet/jdks/temurin-25\n'; } reclaim_retired_roots ) [[ ! -e "$gone" ]] || { echo "unreferenced retired root should have been reclaimed" >&2; exit 1; } -# Referenced by a live overlay mount: retained even though it is past grace. +# Used by a running container whose bind mount followed the rename: retained +# even though it is past grace and no mount option string names it. kept="$(new_retired "$reclaim_root/jdks/temurin-21.retired.200.2")" +# A sibling whose name merely extends the used root's name is not protected. +sibling="$(new_retired "$reclaim_root/jdks/temurin-21.retired.200.20")" +set_container_mounts "$kept/home" ( PREFIX="$reclaim_root" + PROC_ROOT="$fake_proc" RETIRED_GRACE_SECONDS=0 - host_exec() { printf 'overlay / overlay rw,lowerdir=%s\n' "$kept"; } reclaim_retired_roots ) -[[ -e "$kept" ]] || { echo "retired root still referenced by a mount must be retained" >&2; exit 1; } +[[ -e "$kept" ]] || { echo "retired root still used by a container mount must be retained" >&2; exit 1; } +[[ ! -e "$sibling" ]] || { echo "path-prefix sibling of a used root should have been reclaimed" >&2; exit 1; } rm -rf "$kept" +# A PID that exits mid-scan must not hide its mount namespace: PID 41 shares +# the container's namespace but its mount table is gone, so the sweep must +# still read the namespace through PID 42. +raced="$(new_retired "$reclaim_root/jdks/temurin-21.retired.250.25")" +set_container_mounts "$raced/home" +mkdir -p "$fake_proc/41/ns" "$fake_proc/42/ns" +: >"$fake_proc/41/mountinfo" +ln -s 'mnt:[4026532000]' "$fake_proc/41/ns/mnt" +ln -s 'mnt:[4026532000]' "$fake_proc/42/ns/mnt" +( + PREFIX="$reclaim_root" + PROC_ROOT="$fake_proc" + RETIRED_GRACE_SECONDS=0 + reclaim_retired_roots +) +[[ -e "$raced" ]] || { echo "an exited PID must not hide a live mount namespace" >&2; exit 1; } +rm -rf "$raced" + # Within the grace period: retained, so a sandbox being created right now # cannot race the sweep. fresh="$(new_retired "$reclaim_root/jdks/temurin-21.retired.300.3")" +set_container_mounts ( PREFIX="$reclaim_root" + PROC_ROOT="$fake_proc" RETIRED_GRACE_SECONDS=99999 - host_exec() { printf 'overlay / overlay rw,lowerdir=/opt/brewlet/jdks/temurin-25\n'; } reclaim_retired_roots ) [[ -e "$fresh" ]] || { echo "retired root within grace must be retained" >&2; exit 1; } rm -rf "$fresh" -# Mount table unreadable: fail safe and keep the root. +# Mount tables unreadable: fail safe and keep the root. unknown="$(new_retired "$reclaim_root/jdks/temurin-21.retired.400.4")" ( PREFIX="$reclaim_root" + PROC_ROOT="$TEST_TMP_ROOT/missing-proc" RETIRED_GRACE_SECONDS=0 - host_exec() { return 1; } reclaim_retired_roots ) [[ -e "$unknown" ]] || { echo "unreadable mount table must fail safe and retain the root" >&2; exit 1; } @@ -1279,10 +1325,11 @@ rm -rf "$unknown" launcher_gone="$(new_retired "$reclaim_root/launchers/jaz.retired.100.5")" # A staging directory only exists mid-install, so one left behind is orphaned. staging="$(new_retired "$reclaim_root/jdks/temurin-21.staging.999")" +set_container_mounts ( PREFIX="$reclaim_root" + PROC_ROOT="$fake_proc" RETIRED_GRACE_SECONDS=0 - host_exec() { printf 'overlay / overlay rw,lowerdir=/opt/brewlet/jdks/temurin-25\n'; } reclaim_retired_roots ) [[ ! -e "$launcher_gone" ]] || { echo "retired launcher root should have been reclaimed" >&2; exit 1; } @@ -1652,4 +1699,38 @@ if output="$( fi grep -Fq "ERROR: completion-state-failed" <<<"$output" +# In-cluster kubectl calls pass explicit flags (--request-timeout), which +# disable kubectl's implicit in-cluster fallback, so the provisioner must +# point KUBECONFIG at the projected service-account credentials. +sa_dir="$TEST_TMP_ROOT/serviceaccount" +mkdir -p "$sa_dir" +: >"$sa_dir/token" +: >"$sa_dir/ca.crt" +( + unset KUBECONFIG + SERVICE_ACCOUNT_DIR="$sa_dir" + IN_CLUSTER_KUBECONFIG="$TEST_TMP_ROOT/in-cluster-kubeconfig" + KUBERNETES_SERVICE_HOST=fd00::1 + KUBERNETES_SERVICE_PORT=6443 + ensure_in_cluster_kubeconfig + [[ "$KUBECONFIG" == "$IN_CLUSTER_KUBECONFIG" ]] + grep -Fxq " server: https://[fd00::1]:6443" "$KUBECONFIG" + grep -Fxq " certificate-authority: $sa_dir/ca.crt" "$KUBECONFIG" + grep -Fxq " tokenFile: $sa_dir/token" "$KUBECONFIG" +) +( + KUBECONFIG=/explicit/kubeconfig + SERVICE_ACCOUNT_DIR="$sa_dir" + IN_CLUSTER_KUBECONFIG="$TEST_TMP_ROOT/unused-kubeconfig" + KUBERNETES_SERVICE_HOST=10.96.0.1 + ensure_in_cluster_kubeconfig + [[ "$KUBECONFIG" == /explicit/kubeconfig && ! -e "$IN_CLUSTER_KUBECONFIG" ]] +) +( + unset KUBECONFIG KUBERNETES_SERVICE_HOST + IN_CLUSTER_KUBECONFIG="$TEST_TMP_ROOT/out-of-cluster-kubeconfig" + ensure_in_cluster_kubeconfig + [[ -z "${KUBECONFIG:-}" && ! -e "$IN_CLUSTER_KUBECONFIG" ]] +) + bash "$repo_root/provisioner/stage_gc_test.sh" diff --git a/specs/SPECIFICATION.md b/specs/SPECIFICATION.md index bbdbfd10..9c285f51 100644 --- a/specs/SPECIFICATION.md +++ b/specs/SPECIFICATION.md @@ -1927,8 +1927,11 @@ other, so each shape behaves as plain Kubernetes does: deleting it, because overlayfs resolves `lowerdir` at mount time — so running pods keep the root they started with. A later provisioning pass reclaims a rotated-out root once no live mount references it *and* it has aged past a - grace period (`BREWLET_RETIRED_GRACE_SECONDS`, default 1h); if the mount table - cannot be read the root is retained. Deleting a `NodeProfile` removes the + grace period (`BREWLET_RETIRED_GRACE_SECONDS`, default 1h). Mount option + strings keep the pre-rename path, so references are found by device number + and the mountinfo root field of each sandbox's JDK/launcher bind mount, which + the kernel renders from the renamed directory, across every mount namespace. + If the mount tables cannot be read, the root is retained. Deleting a `NodeProfile` removes the profile's installed roots during cleanup unless `BREWLET_CLEANUP_RUNTIME_ROOTS=false`. Roots that merely drop out of a profile's inventory while the profile still exists are not yet reclaimed —