Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions docs/generated-vpa-lifecycle.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ the prevention is deployed. Dispatch it from main with mode assess first and
confirmation cleanup-orphan-generated-vpas. Review its aggregate result, then
dispatch mode apply with the same confirmation. A rerun requires a fresh dispatch.

Apply compares all three deployed generators with the reviewed dispatch revision
before considering deletion. The complete census is limited to 250 VPAs. Only
Apply renders the reviewed dispatch revision's production infrastructure and
compares all three complete deployed generators with that render before
considering deletion. This retains exact expression comparison after Kustomize's
multiline-scalar serialization; it does not strip or normalize CEL text. A failed
render, missing or duplicate generator, or empty expression list stops cleanup.
The complete census is limited to 250 VPAs. Only
autoscalers with consistent Kyverno policy and trigger provenance are eligible;
objects with owner references, finalizers, deletion in progress or an existing
target are preserved. An existing target is preserved even when its UID differs
Expand Down
16 changes: 13 additions & 3 deletions scripts/cleanup-orphan-generated-vpas.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,21 @@ k() { kubectl --context admin@prod --request-timeout=10s "$@" 2>/dev/null; }
readonly resource=verticalpodautoscalers.autoscaling.k8s.io

# Do not clean up before the reviewed prevention is deployed. Compare the
# complete generator, including the required real UID expression, to Git.
# complete generator, including the required real UID expression, to the
# reviewed production render. Kustomize can reserialize multiline CEL scalars;
# comparing raw source strings would reject the deployed prevention.
if [[ "$mode" == apply ]]; then
kubectl kustomize "$root/k8s/providers/hetzner/infrastructure" >"$work/reviewed.yaml" 2>/dev/null ||
unknown 'cannot render the reviewed production policies'
for kind in deployment statefulset daemonset; do
yq -o=json '.spec.generate' "$root/k8s/bases/infrastructure/cluster-policies/best-practices/auto-vpa-$kind.yaml" >"$work/expected.json" 2>/dev/null ||
unknown 'cannot read the reviewed generator'
if ! yq -o=json "select(.kind == \"GeneratingPolicy\" and .metadata.name == \"auto-vpa-$kind\") | .spec.generate" \
"$work/reviewed.yaml" 2>/dev/null |
jq -ces 'if length==1 and (.[0] | type=="array" and length>0 and
all(.[]; type=="object" and (.expression | type=="string" and length>0)))
then .[0] else error("incomplete reviewed generator") end' \
>"$work/expected.json" 2>/dev/null; then
unknown 'cannot read exactly one complete reviewed generator'
fi
k get generatingpolicies.policies.kyverno.io "auto-vpa-$kind" -o json >"$work/policy.json" ||
unknown 'cannot read the deployed generator'
jq -e --slurpfile expected "$work/expected.json" '.spec.generate == $expected[0]' "$work/policy.json" >/dev/null 2>&1 ||
Expand Down
20 changes: 18 additions & 2 deletions scripts/tests/test-cleanup-orphan-generated-vpas.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
work="$(mktemp -d)"
trap 'rm -rf "$work"' EXIT
mkdir -p "$work/bin"
# Use the real offline production render as the API representation. Raw source
# scalars can change serialization when Kustomize emits them.
kubectl kustomize "$root/k8s/providers/hetzner/infrastructure" >"$work/rendered.yaml"
export VPA_TEST_RENDER="$work/rendered.yaml"
cat >"$work/base.json" <<'JSON'
{"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"name":"old-deployment","namespace":"apps-ns","uid":"00000000-0000-0000-0000-000000000001","resourceVersion":"42","labels":{"app.kubernetes.io/managed-by":"kyverno","generate.kyverno.io/policy-name":"auto-vpa-deployment","generate.kyverno.io/trigger-group":"apps","generate.kyverno.io/trigger-version":"v1","generate.kyverno.io/trigger-kind":"Deployment","generate.kyverno.io/trigger-name":"old","generate.kyverno.io/trigger-namespace":"apps-ns","generate.kyverno.io/trigger-uid":"00000000-0000-0000-0000-000000000002"}},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"old"}}}
JSON
Expand All @@ -17,6 +21,18 @@ while [[ "$1" == --* ]]; do
if [[ "$1" == --context ]]; then shift 2; else shift; fi
done
verb="$1"; shift
if [[ "$verb" == kustomize ]]; then
[[ "$SCENARIO" != render-failed ]] || exit 1
case "$SCENARIO" in
render-missing) yq 'select(.kind != "GeneratingPolicy" or .metadata.name != "auto-vpa-deployment")' "$VPA_TEST_RENDER" ;;
render-duplicate) cat "$VPA_TEST_RENDER" "$VPA_TEST_RENDER" ;;
render-malformed) printf ': invalid: [\n' ;;
render-empty) yq '.spec.generate=[]' "$VPA_TEST_RENDER" ;;
render-empty-entry) yq '.spec.generate=[{}]' "$VPA_TEST_RENDER" ;;
*) cat "$VPA_TEST_RENDER" ;;
esac
exit 0
fi
if [[ "$verb" == delete ]]; then
[[ "$*" == --raw\ /apis/autoscaling.k8s.io/v1/namespaces/apps-ns/verticalpodautoscalers/old-deployment\ --filename\ * ]] || exit 80
jq -e '.apiVersion=="v1" and .kind=="DeleteOptions" and .preconditions.uid=="00000000-0000-0000-0000-000000000001" and .preconditions.resourceVersion=="42"' "${@: -1}" >/dev/null
Expand All @@ -30,7 +46,7 @@ if [[ "$resource" == generatingpolicies.policies.kyverno.io ]]; then
if [[ "$SCENARIO" == undeployed ]]; then
printf '{"spec":{"generate":[]}}'
else
yq -o=json '.' "$ROOT/k8s/bases/infrastructure/cluster-policies/best-practices/$1.yaml"
yq -o=json "select(.kind == \"GeneratingPolicy\" and .metadata.name == \"$1\")" "$VPA_TEST_RENDER"
fi
elif [[ "$resource" == verticalpodautoscalers.autoscaling.k8s.io && "$1" == --all-namespaces ]]; then
[[ "$SCENARIO" != bad-list ]] || { printf '{"items":null}'; exit 0; }
Expand Down Expand Up @@ -91,7 +107,7 @@ run_case orphan apply 0 yes
run_case live apply 0 no
run_case replacement apply 0 no
run_case owned apply 0 no
for scenario in forbidden bad-list partial-read malformed-target; do run_case "$scenario" apply 2 no; done
for scenario in forbidden bad-list partial-read malformed-target render-failed render-missing render-duplicate render-malformed render-empty render-empty-entry; do run_case "$scenario" apply 2 no; done
for scenario in wrong-label wrong-target invalid-uid changed-uid changed-version target-returned undeployed; do run_case "$scenario" apply 1 no; done
run_case delete-failed apply 2 no
run_case leftover apply 1 yes
Expand Down
Loading