diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4b4ab77aa6..e96d66c962 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -185,6 +185,7 @@ jobs: - 'scripts/tests/test-business-disclosure-reader.sh' - 'scripts/tests/test-business-staging-hold.sh' - 'scripts/check-business-disclosure-reader/**' + - 'scripts/tests/vault-backup-recovery/**' - 'scripts/openbao-backup-recovery/**' - 'scripts/run-openbao-backup-sandbox.sh' - 'scripts/tests/test-openbao-standby-canary.sh' @@ -3321,6 +3322,12 @@ jobs: key: kubeconform-schema-cache-v1-render-regressions-${{ github.run_id }} restore-keys: | kubeconform-schema-cache-v1- + - name: 🔐 Prove snapshot readback with pinned static tooling + if: needs.changes.outputs.k8s == 'true' + timeout-minutes: 5 + env: + VAULT_BACKUP_TOOLING_NATIVE_TEST: '1' + run: go test -count=1 -v ./scripts/tests/vault-backup-recovery - name: 🐤 Validate staged OpenBao standby repair if: needs.changes.outputs.k8s == 'true' run: | diff --git a/docs/dr/openbao.md b/docs/dr/openbao.md index 70a6410af7..6bf8e136cc 100644 --- a/docs/dr/openbao.md +++ b/docs/dr/openbao.md @@ -6,7 +6,7 @@ OpenBao runs as a raft (Integrated Storage) cluster. Three artifacts make it recoverable: 1. **Raft snapshots** — the `vault-snapshot` CronJob saves a - `bao operator raft snapshot` to the `vault-snapshots` PVC daily (newest 14 + `bao operator raft snapshot` daily (newest 14 kept) and mirrors them to the S3 backup target under `openbao-snapshots/` (Cloudflare R2 in prod, MinIO locally). The mirror exists because Velero's file-system backup only captures volumes mounted by *running* pods — @@ -31,6 +31,36 @@ downloads only that object. Missing projected credentials or an empty or unreachable mirror listing leave the scratch volume empty. When recovery needs a snapshot and none is available, the restore guard refuses re-initialization. +## Snapshot storage and verification + +Production's initial and scheduled snapshot jobs use the retained +`vault-snapshots-recovery` claim. The original `vault-snapshots` claim remains +protected and unmodified. Local deployments keep their original claim. Active +claims must bind; the temporary preparation marker no longer bypasses readiness. + +The snapshot container completes before the mirror starts. The mirror copies +without remote removal, then downloads each nonempty snapshot and compares its +SHA-256 with the local bytes. A failed or partial GET fails the run, including a +GET that delivers a complete body and then returns an error. Production requires +the projected backup credentials; an empty source also fails. Fresh local +bootstrap can report `VAULT_SNAPSHOT_REMOTE=UNSEEDED` until credentials arrive. + +The fixed `VAULT_SNAPSHOT_REMOTE=VERIFIED count=N` verdict requires matching +downloaded bytes for every snapshot and successful age-based retention. Retention +starts after upload and readback; the mirror never uses `--remove`. Credentials, +client configuration and copied static tools use bounded memory-backed volumes. +The existing concurrency exclusion, execution deadlines and snapshot resources +remain in place. + +Only the snapshot init container receives the explicitly projected Kubernetes +token used for its existing OpenBao login. Automatic token mounting is disabled; +the static-tool installer and remote mirror receive no Kubernetes identity. + +Complete recovery acceptance with an isolated restore of the new off-cluster +snapshot and verification of its credential records using the matching recovery +keys. Keep #5376 open until that native restore proof is observed, before moving +Email credentials or retiring legacy custody. + ## Recovery Scenarios ### Scenario 1: Single pod restart diff --git a/k8s/bases/infrastructure/vault-backup/cron-job.yaml b/k8s/bases/infrastructure/vault-backup/cron-job.yaml index e8ac2855fb..e2754ee2b7 100644 --- a/k8s/bases/infrastructure/vault-backup/cron-job.yaml +++ b/k8s/bases/infrastructure/vault-backup/cron-job.yaml @@ -75,11 +75,9 @@ spec: app: vault-snapshot spec: serviceAccountName: vault-snapshot - # The token IS this pod's credential: the snapshot container reads - # /var/run/secrets/kubernetes.io/serviceaccount/token and presents it - # as the `jwt` for `auth/kubernetes/login`. Left at the mount-by-default - # value rather than set explicitly — an explicit `true` reads identically - # to Kubernetes and trips Kubescape C-0034. + # Only the authenticating snapshot container receives a short-lived + # projected token. The tool installer and remote mirror need no identity. + automountServiceAccountToken: false restartPolicy: OnFailure # The full restricted-PSS context is set explicitly (not left to the # add-security-context Kyverno mutation) so the pod stays hardened @@ -103,8 +101,42 @@ spec: secretName: vault-snapshot-r2 optional: true - name: mc-config - emptyDir: {} + emptyDir: + medium: Memory + sizeLimit: 8Mi + - name: mirror-tools + emptyDir: + medium: Memory + sizeLimit: 8Mi + - name: kubernetes-identity + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 initContainers: + - name: install-mirror-tools + image: docker.io/library/busybox:1.38.0-musl@sha256:ea2b9914a16a4ac1981994af97b318f7c7d4db76b580c56177f08bf76f4a0be8 + command: ["/bin/sh", "-ec", "cp /bin/busybox /tools/busybox; /tools/busybox --install -s /tools"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 10m + memory: 8Mi + limits: + cpu: 100m + memory: 32Mi + volumeMounts: + - name: mirror-tools + mountPath: /tools - name: snapshot image: quay.io/openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806 securityContext: @@ -140,6 +172,9 @@ spec: volumeMounts: - name: snapshots mountPath: /snapshots + - name: kubernetes-identity + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true command: - /bin/sh - -ec @@ -210,6 +245,8 @@ spec: env: - name: MC_CONFIG_DIR value: /tmp/.mc + - name: REQUIRE_REMOTE_SNAPSHOT + value: "false" resources: requests: cpu: 10m @@ -227,25 +264,52 @@ spec: readOnly: true - name: mc-config mountPath: /tmp/.mc + - name: mirror-tools + mountPath: /tools + readOnly: true command: - - /bin/sh - - -ec + - /tools/sh + - -euc - | - if [ ! -f /r2/access_key_id ]; then - echo "vault-snapshot-r2 Secret not synced yet — skipping the" - echo "off-cluster mirror. The local PVC snapshot above still ran." + set -o pipefail + umask 077 + fail() { + printf 'VAULT_SNAPSHOT_REMOTE=FAIL stage=%s\n' "$1" >&2 + exit 1 + } + case "$REQUIRE_REMOTE_SNAPSHOT" in true|false) ;; *) fail configuration ;; esac + [ -d "$MC_CONFIG_DIR" ] || fail configuration + if [ ! -s /r2/access_key_id ] || [ ! -s /r2/secret_access_key ]; then + if [ "$REQUIRE_REMOTE_SNAPSHOT" = true ]; then + fail credentials + fi + printf '%s\n' 'VAULT_SNAPSHOT_REMOTE=UNSEEDED' exit 0 fi mc alias set backup "${r2_endpoint}" \ - "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)" - # Copy-only mirror: NO --remove, so an empty/recreated PVC can - # never wipe the off-cluster history. Remote retention is - # pruned by age instead, and only after a successful upload in - # the same run (this container only starts when the snapshot - # initContainer succeeded), so the newest snapshot always - # survives the prune. - mc mirror --overwrite /snapshots/ "backup/${r2_bucket}/openbao-snapshots/" - mc rm --recursive --force --older-than 14d \ - "backup/${r2_bucket}/openbao-snapshots/" || true - echo "Mirrored snapshots:" - mc ls "backup/${r2_bucket}/openbao-snapshots/" + "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)" \ + > /dev/null 2>"$MC_CONFIG_DIR/error" || fail authentication + remote="backup/${r2_bucket}/openbao-snapshots" + # Copy-only: never remove remote history because a local volume is empty. + mc mirror --overwrite /snapshots/ "$remote/" \ + > /dev/null 2>"$MC_CONFIG_DIR/error" || fail upload + verified=0 + for snapshot in /snapshots/openbao-*.snap; do + [ -e "$snapshot" ] || continue + [ -f "$snapshot" ] && [ -s "$snapshot" ] && [ ! -L "$snapshot" ] || fail source + name=$(/tools/basename "$snapshot") || fail source + /tools/sha256sum < "$snapshot" > "$MC_CONFIG_DIR/expected.sha256" || fail local_hash + # Hash an actual GET, not object metadata or an upload acknowledgement. + # pipefail also rejects a GET that emits a full body before returning error. + if ! mc cat "$remote/$name" 2>"$MC_CONFIG_DIR/error" | + /tools/sha256sum > "$MC_CONFIG_DIR/actual.sha256"; then + fail readback + fi + /tools/cmp -s "$MC_CONFIG_DIR/expected.sha256" "$MC_CONFIG_DIR/actual.sha256" || fail integrity + verified=$((verified + 1)) + done + [ "$verified" -gt 0 ] || fail source + # Retention runs only after upload AND byte-for-byte readback succeeded. + mc rm --recursive --force --older-than 14d "$remote/" \ + > /dev/null 2>"$MC_CONFIG_DIR/error" || fail retention + printf 'VAULT_SNAPSHOT_REMOTE=VERIFIED count=%s\n' "$verified" diff --git a/k8s/bases/infrastructure/vault-backup/job.yaml b/k8s/bases/infrastructure/vault-backup/job.yaml index e22af6c019..9fc273ac5e 100644 --- a/k8s/bases/infrastructure/vault-backup/job.yaml +++ b/k8s/bases/infrastructure/vault-backup/job.yaml @@ -69,11 +69,9 @@ spec: app: vault-snapshot spec: serviceAccountName: vault-snapshot - # The token IS this pod's credential: the snapshot container reads - # /var/run/secrets/kubernetes.io/serviceaccount/token and presents it as - # the `jwt` for `auth/kubernetes/login`. Left at the mount-by-default - # value rather than set explicitly — an explicit `true` reads identically - # to Kubernetes and trips Kubescape C-0034. + # Only the authenticating snapshot container receives a short-lived + # projected token. The tool installer and remote mirror need no identity. + automountServiceAccountToken: false restartPolicy: OnFailure # The full restricted-PSS context is set explicitly (not left to the # add-security-context Kyverno mutation) so the pod stays hardened @@ -97,8 +95,42 @@ spec: secretName: vault-snapshot-r2 optional: true - name: mc-config - emptyDir: {} + emptyDir: + medium: Memory + sizeLimit: 8Mi + - name: mirror-tools + emptyDir: + medium: Memory + sizeLimit: 8Mi + - name: kubernetes-identity + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + path: token + expirationSeconds: 3600 initContainers: + - name: install-mirror-tools + image: docker.io/library/busybox:1.38.0-musl@sha256:ea2b9914a16a4ac1981994af97b318f7c7d4db76b580c56177f08bf76f4a0be8 + command: ["/bin/sh", "-ec", "cp /bin/busybox /tools/busybox; /tools/busybox --install -s /tools"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 10m + memory: 8Mi + limits: + cpu: 100m + memory: 32Mi + volumeMounts: + - name: mirror-tools + mountPath: /tools - name: snapshot image: quay.io/openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806 securityContext: @@ -132,6 +164,9 @@ spec: volumeMounts: - name: snapshots mountPath: /snapshots + - name: kubernetes-identity + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true command: - /bin/sh - -ec @@ -202,6 +237,8 @@ spec: env: - name: MC_CONFIG_DIR value: /tmp/.mc + - name: REQUIRE_REMOTE_SNAPSHOT + value: "false" resources: requests: cpu: 10m @@ -219,23 +256,52 @@ spec: readOnly: true - name: mc-config mountPath: /tmp/.mc + - name: mirror-tools + mountPath: /tools + readOnly: true command: - - /bin/sh - - -ec + - /tools/sh + - -euc - | - if [ ! -f /r2/access_key_id ]; then - echo "vault-snapshot-r2 Secret not synced yet — skipping the" - echo "off-cluster mirror. The local PVC snapshot above still ran." + set -o pipefail + umask 077 + fail() { + printf 'VAULT_SNAPSHOT_REMOTE=FAIL stage=%s\n' "$1" >&2 + exit 1 + } + case "$REQUIRE_REMOTE_SNAPSHOT" in true|false) ;; *) fail configuration ;; esac + [ -d "$MC_CONFIG_DIR" ] || fail configuration + if [ ! -s /r2/access_key_id ] || [ ! -s /r2/secret_access_key ]; then + if [ "$REQUIRE_REMOTE_SNAPSHOT" = true ]; then + fail credentials + fi + printf '%s\n' 'VAULT_SNAPSHOT_REMOTE=UNSEEDED' exit 0 fi mc alias set backup "${r2_endpoint}" \ - "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)" - # Copy-only mirror (no --remove) + age-based prune, same - # rationale as cron-job.yaml: an empty/recreated PVC can never - # wipe the off-cluster history, and the prune only runs after a - # successful upload in the same pod. - mc mirror --overwrite /snapshots/ "backup/${r2_bucket}/openbao-snapshots/" - mc rm --recursive --force --older-than 14d \ - "backup/${r2_bucket}/openbao-snapshots/" || true - echo "Mirrored snapshots:" - mc ls "backup/${r2_bucket}/openbao-snapshots/" + "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)" \ + > /dev/null 2>"$MC_CONFIG_DIR/error" || fail authentication + remote="backup/${r2_bucket}/openbao-snapshots" + # Copy-only: never remove remote history because a local volume is empty. + mc mirror --overwrite /snapshots/ "$remote/" \ + > /dev/null 2>"$MC_CONFIG_DIR/error" || fail upload + verified=0 + for snapshot in /snapshots/openbao-*.snap; do + [ -e "$snapshot" ] || continue + [ -f "$snapshot" ] && [ -s "$snapshot" ] && [ ! -L "$snapshot" ] || fail source + name=$(/tools/basename "$snapshot") || fail source + /tools/sha256sum < "$snapshot" > "$MC_CONFIG_DIR/expected.sha256" || fail local_hash + # Hash an actual GET, not object metadata or an upload acknowledgement. + # pipefail also rejects a GET that emits a full body before returning error. + if ! mc cat "$remote/$name" 2>"$MC_CONFIG_DIR/error" | + /tools/sha256sum > "$MC_CONFIG_DIR/actual.sha256"; then + fail readback + fi + /tools/cmp -s "$MC_CONFIG_DIR/expected.sha256" "$MC_CONFIG_DIR/actual.sha256" || fail integrity + verified=$((verified + 1)) + done + [ "$verified" -gt 0 ] || fail source + # Retention runs only after upload AND byte-for-byte readback succeeded. + mc rm --recursive --force --older-than 14d "$remote/" \ + > /dev/null 2>"$MC_CONFIG_DIR/error" || fail retention + printf 'VAULT_SNAPSHOT_REMOTE=VERIFIED count=%s\n' "$verified" diff --git a/k8s/clusters/base/flux-kustomization-infrastructure.yaml b/k8s/clusters/base/flux-kustomization-infrastructure.yaml index e04539b1f8..c4b84626f8 100644 --- a/k8s/clusters/base/flux-kustomization-infrastructure.yaml +++ b/k8s/clusters/base/flux-kustomization-infrastructure.yaml @@ -38,21 +38,13 @@ spec: # KRO reports readiness via `status.state: Active|Inactive`, so teach Flux to # health-check RGDs by that field via a CEL expression. healthCheckExprs: - # Recovery preparation has no consumer, so hcloud WFFC remains Pending. - # Activation removes this marker and must wait for actual volume binding. + # Active snapshot storage must bind. The preparation-only Pending allowance + # is retired with the first consumers of its main-owned retained claim. - apiVersion: v1 kind: PersistentVolumeClaim failed: has(status.phase) && status.phase == 'Lost' current: >- - !has(metadata.deletionTimestamp) && has(status.phase) && - (status.phase == 'Bound' || - (status.phase == 'Pending' && - metadata.name == 'vault-snapshots-recovery' && metadata.namespace == 'openbao' && - has(metadata.labels) && 'platform.devantler.tech/storage-stage' in metadata.labels && - metadata.labels['platform.devantler.tech/storage-stage'] == 'prepared' && - has(spec.storageClassName) && spec.storageClassName == 'hcloud' && - !has(spec.volumeName) && - (!has(metadata.annotations) || !('volume.kubernetes.io/selected-node' in metadata.annotations)))) + !has(metadata.deletionTimestamp) && has(status.phase) && status.phase == 'Bound' - apiVersion: kro.run/v1alpha1 kind: ResourceGraphDefinition current: status.state == 'Active' diff --git a/k8s/providers/hetzner/infrastructure/kustomization.yaml b/k8s/providers/hetzner/infrastructure/kustomization.yaml index 7161a053b3..e2e5041581 100644 --- a/k8s/providers/hetzner/infrastructure/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/kustomization.yaml @@ -211,6 +211,20 @@ components: - ../../../bases/components/helmrelease-drift-detection - ../../../bases/components/helmrelease-flux-defaults patches: + - path: vault-backup-recovery/patches/use-recovery-storage-for-initial-snapshot.yaml + target: + group: batch + version: v1 + kind: Job + name: vault-snapshot-init + namespace: openbao + - path: vault-backup-recovery/patches/use-recovery-storage-for-daily-snapshot.yaml + target: + group: batch + version: v1 + kind: CronJob + name: vault-snapshot + namespace: openbao # UniFi exists only on Hetzner and applies Crossplane Managed Resources whose # default deletion policy cascades to live network objects. Keep its deliberate # non-pruning exception in this provider overlay so the shared admission policy diff --git a/k8s/providers/hetzner/infrastructure/vault-backup-recovery/patches/use-recovery-storage-for-daily-snapshot.yaml b/k8s/providers/hetzner/infrastructure/vault-backup-recovery/patches/use-recovery-storage-for-daily-snapshot.yaml new file mode 100644 index 0000000000..75be916ece --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/vault-backup-recovery/patches/use-recovery-storage-for-daily-snapshot.yaml @@ -0,0 +1,23 @@ +# Guarded patch operations, not a second incomplete workload. +# Test names and existing values before replacing an indexed field. +- op: test + path: /spec/jobTemplate/spec/template/spec/volumes/0/name + value: snapshots +- op: test + path: /spec/jobTemplate/spec/template/spec/volumes/0/persistentVolumeClaim/claimName + value: vault-snapshots +- op: replace + path: /spec/jobTemplate/spec/template/spec/volumes/0/persistentVolumeClaim/claimName + value: vault-snapshots-recovery +- op: test + path: /spec/jobTemplate/spec/template/spec/containers/0/name + value: mirror +- op: test + path: /spec/jobTemplate/spec/template/spec/containers/0/env/1/name + value: REQUIRE_REMOTE_SNAPSHOT +- op: test + path: /spec/jobTemplate/spec/template/spec/containers/0/env/1/value + value: "false" +- op: replace + path: /spec/jobTemplate/spec/template/spec/containers/0/env/1/value + value: "true" diff --git a/k8s/providers/hetzner/infrastructure/vault-backup-recovery/patches/use-recovery-storage-for-initial-snapshot.yaml b/k8s/providers/hetzner/infrastructure/vault-backup-recovery/patches/use-recovery-storage-for-initial-snapshot.yaml new file mode 100644 index 0000000000..9f38ab585a --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/vault-backup-recovery/patches/use-recovery-storage-for-initial-snapshot.yaml @@ -0,0 +1,23 @@ +# Guarded patch operations, not a second incomplete workload. +# Test names and existing values before replacing an indexed field. +- op: test + path: /spec/template/spec/volumes/0/name + value: snapshots +- op: test + path: /spec/template/spec/volumes/0/persistentVolumeClaim/claimName + value: vault-snapshots +- op: replace + path: /spec/template/spec/volumes/0/persistentVolumeClaim/claimName + value: vault-snapshots-recovery +- op: test + path: /spec/template/spec/containers/0/name + value: mirror +- op: test + path: /spec/template/spec/containers/0/env/1/name + value: REQUIRE_REMOTE_SNAPSHOT +- op: test + path: /spec/template/spec/containers/0/env/1/value + value: "false" +- op: replace + path: /spec/template/spec/containers/0/env/1/value + value: "true" diff --git a/k8s/providers/hetzner/infrastructure/vault-backup-recovery/persistent-volume-claim.yaml b/k8s/providers/hetzner/infrastructure/vault-backup-recovery/persistent-volume-claim.yaml index 1fd5de2631..b2c1109493 100644 --- a/k8s/providers/hetzner/infrastructure/vault-backup-recovery/persistent-volume-claim.yaml +++ b/k8s/providers/hetzner/infrastructure/vault-backup-recovery/persistent-volume-claim.yaml @@ -5,8 +5,7 @@ metadata: name: vault-snapshots-recovery namespace: openbao labels: - # No consumer is switched until this retained claim is owned by main. - platform.devantler.tech/storage-stage: prepared + backup.platform.devantler.tech/volume-data: independently-mirrored annotations: kustomize.toolkit.fluxcd.io/prune: disabled spec: diff --git a/scripts/tests/test-velero-data-mover-resources.sh b/scripts/tests/test-velero-data-mover-resources.sh index b24645fa89..c8b9d8edb2 100755 --- a/scripts/tests/test-velero-data-mover-resources.sh +++ b/scripts/tests/test-velero-data-mover-resources.sh @@ -122,9 +122,8 @@ hcloud_pvcs="$( )" || fail 'the production hcloud PVC inventory could not be read' readonly hcloud_pvcs -# The replacement is deliberately unused until main owns its retained claim. -# It has no independently-mirrored marker yet, so the existing labelled skip -# rule below still exempts only the original snapshot cache. +# The retained original and active replacement snapshot caches use the same +# independently mirrored route. Other hcloud data must retain filesystem backup. expected_hcloud_pvcs=$'openbao/vault-snapshots\nopenbao/vault-snapshots-recovery' readonly expected_hcloud_pvcs @@ -142,8 +141,8 @@ independently_mirrored_pvcs="$( )" || fail 'the independently mirrored PVC inventory could not be read' readonly independently_mirrored_pvcs -[[ "${independently_mirrored_pvcs}" == 'openbao/vault-snapshots' ]] || - fail "only openbao/vault-snapshots may opt out of Velero volume-data backup; found: ${independently_mirrored_pvcs:-none}" +[[ "${independently_mirrored_pvcs}" == "${expected_hcloud_pvcs}" ]] || + fail "only the retained original and active replacement snapshot caches must use independent mirrors; found: ${independently_mirrored_pvcs:-none}" controller_hcloud_claims="$({ yq -r ' diff --git a/scripts/tests/vault-backup-recovery/readback_test.go b/scripts/tests/vault-backup-recovery/readback_test.go new file mode 100644 index 0000000000..639e7cbd92 --- /dev/null +++ b/scripts/tests/vault-backup-recovery/readback_test.go @@ -0,0 +1,262 @@ +package recovery_test + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" +) + +// readbackSourceRoot locates the checkout or an explicitly selected test fixture. +func readbackSourceRoot() string { + if root := os.Getenv("BACKUP_SOURCE_ROOT"); root != "" { + return root + } + _, file, _, _ := runtime.Caller(0) + return filepath.Clean(filepath.Join(filepath.Dir(file), "../../..")) +} + +// mirrorProgram reads the exact program embedded in each snapshot workload. +func mirrorProgram(t *testing.T, filename string) string { + t.Helper() + query := `.spec.template.spec.containers[] | select(.name == "mirror") | .command[-1]` + if filename == "cron-job.yaml" { + query = `.spec.jobTemplate.spec.template.spec.containers[] | select(.name == "mirror") | .command[-1]` + } + out, err := exec.Command("yq", "-o=json", query, filepath.Join(readbackSourceRoot(), "k8s/bases/infrastructure/vault-backup", filename)).Output() + if err != nil { + t.Fatal("cannot read the actual declared mirror program", err) + } + var script string + if err := json.Unmarshal(out, &script); err != nil || script == "" { + t.Fatal("missing declared mirror program") + } + return script +} + +// exerciseMirror exercises the declared program with synthetic storage responses. +func exerciseMirror(t *testing.T, filename, reply string, missingCredentials, empty, uploadFailure bool) (string, []string, error) { + t.Helper() + return exerciseMirrorMode(t, filename, reply, missingCredentials, empty, uploadFailure, false) +} + +// exerciseMirrorMode also supports the real installer and mirror image without networking. +func exerciseMirrorMode(t *testing.T, filename, reply string, missingCredentials, empty, uploadFailure, native bool) (string, []string, error) { + t.Helper() + dir := t.TempDir() + for _, name := range []string{"bin", "r2", "snapshots", "tools", "config"} { + if err := os.Mkdir(filepath.Join(dir, name), 0o700); err != nil { + t.Fatal(err) + } + } + snapshot := filepath.Join(dir, "snapshots/openbao-20260101-000000.snap") + if !empty { + if err := os.WriteFile(snapshot, []byte("synthetic-snapshot-bytes\x00preserve-verbatim\n"), 0o600); err != nil { + t.Fatal(err) + } + } + if reply == "matching-two" || reply == "missing-second" { + if err := os.WriteFile(filepath.Join(dir, "snapshots/openbao-20260101-000001.snap"), []byte("different second snapshot\n"), 0o600); err != nil { + t.Fatal(err) + } + } + if !missingCredentials { + for _, name := range []string{"access_key_id", "secret_access_key"} { + if err := os.WriteFile(filepath.Join(dir, "r2", name), []byte("synthetic-credential"), 0o600); err != nil { + t.Fatal(err) + } + } + } + shim := `#!/bin/sh +set -eu +printf '%s\n' "$1" >> "$FIXTURE_TRACE" +case "$1" in + alias|ls) exit 0;; + rm) [ "$FIXTURE_REPLY" != retention-failed ];; + mirror) [ "$FIXTURE_UPLOAD_FAILURE" != true ];; + cat) + name=$(basename "$2") + [ "$2" = "backup/fixture-bucket/openbao-snapshots/$name" ] || exit 95 + case "$FIXTURE_REPLY" in + matching|matching-two|retention-failed) cat "$FIXTURE_SOURCE_DIR/$name";; + missing-second) [ "$name" != openbao-20260101-000001.snap ] || exit 1; cat "$FIXTURE_SOURCE_DIR/$name";; + corrupt) printf 'wrong bytes';; + failed) exit 1;; + full-then-error) cat "$FIXTURE_SNAPSHOT"; exit 1;; + *) exit 93;; + esac;; + *) exit 94;; +esac +` + if err := os.WriteFile(filepath.Join(dir, "bin/mc"), []byte(shim), 0o700); err != nil { + t.Fatal(err) + } + // Use a real native SHA-256 implementation, including on macOS. The + // production static tooling is separately exercised by Linux CI. + hasher, err := exec.LookPath("sha256sum") + if !native && err == nil { + if err := os.Symlink(hasher, filepath.Join(dir, "tools/sha256sum")); err != nil { + t.Fatal(err) + } + } else if !native { + if err := os.WriteFile(filepath.Join(dir, "tools/sha256sum"), []byte("#!/bin/bash\nexec shasum -a 256 \"$@\"\n"), 0o700); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{"basename", "cmp"} { + if native { + continue + } + path, err := exec.LookPath(name) + if err != nil { + t.Fatal(err) + } + if err := os.Symlink(path, filepath.Join(dir, "tools", name)); err != nil { + t.Fatal(err) + } + } + script := mirrorProgram(t, filename) + for original, replacement := range map[string]string{ + "/snapshots/": filepath.Join(dir, "snapshots") + "/", + "/r2/": filepath.Join(dir, "r2") + "/", + "/tools/": filepath.Join(dir, "tools") + "/", + "${r2_endpoint}": "https://fixture.invalid", + "${r2_bucket}": "fixture-bucket", + } { + script = strings.ReplaceAll(script, original, replacement) + } + trace := filepath.Join(dir, "trace") + environment := []string{ + "PATH=" + filepath.Join(dir, "bin") + ":" + os.Getenv("PATH"), "HOME=" + dir, + "MC_CONFIG_DIR=" + filepath.Join(dir, "config"), "REQUIRE_REMOTE_SNAPSHOT=true", + "FIXTURE_TRACE=" + trace, "FIXTURE_REPLY=" + reply, "FIXTURE_SNAPSHOT=" + snapshot, + "FIXTURE_SOURCE_DIR=" + filepath.Join(dir, "snapshots"), + "FIXTURE_UPLOAD_FAILURE=" + map[bool]string{true: "true", false: "false"}[uploadFailure], + } + command := exec.Command("bash", "-eu", "-c", script) + if native { + if runtime.GOOS != "linux" || os.Getuid() == 0 { + t.Fatal("native tooling proof requires the nonroot Linux CI host") + } + query := `.spec.template.spec` + if filename == "cron-job.yaml" { + query = `.spec.jobTemplate.spec.template.spec` + } + query += ` | {"installer": (.initContainers[] | select(.name == "install-mirror-tools")), "mirror": (.containers[] | select(.name == "mirror"))}` + data, err := exec.Command("yq", "-o=json", query, filepath.Join(readbackSourceRoot(), "k8s/bases/infrastructure/vault-backup", filename)).Output() + var spec struct { + Installer, Mirror struct { + Image string + Command []string + } + } + if err != nil || json.Unmarshal(data, &spec) != nil || !strings.HasPrefix(spec.Installer.Image, "docker.io/library/busybox:") || !strings.Contains(spec.Installer.Image, "@sha256:") || !strings.HasPrefix(spec.Mirror.Image, "quay.io/minio/aistor/mc:") || !strings.Contains(spec.Mirror.Image, "@sha256:") || len(spec.Installer.Command) != 3 || len(spec.Mirror.Command) != 3 || spec.Mirror.Command[0] != "/tools/sh" { + t.Fatal("native proof must use the declared pinned installer and actual mirror image") + } + // Execute the real installation, then use the copied bytes in the real + // mirror image. Running only the BusyBox source image would not detect a + // dynamic-linker incompatibility in the destination image. + installArgs := []string{"run", "--rm", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--memory=64m", "--pids-limit=64", "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), "--mount", "type=bind,source=" + filepath.Join(dir, "tools") + ",target=/tools", "--entrypoint", spec.Installer.Command[0], spec.Installer.Image} + installArgs = append(installArgs, spec.Installer.Command[1:]...) + install := exec.Command("docker", installArgs...) + install.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir} + if out, err := install.CombinedOutput(); err != nil { + t.Fatalf("pinned static-tool installation failed: %v %s", err, out) + } + args := []string{"run", "--rm", "--network=none", "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--memory=64m", "--cpus=0.5", "--pids-limit=64", "--user", fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()), "--mount", "type=bind,source=" + dir + ",target=" + dir, "--mount", "type=bind,source=" + filepath.Join(dir, "tools") + ",target=/tools,readonly", "--entrypoint", spec.Mirror.Command[0]} + for _, value := range environment { + if strings.HasPrefix(value, "PATH=") { + value = "PATH=" + filepath.Join(dir, "bin") + ":/tools:/bin:/usr/bin" + } + args = append(args, "--env", value) + } + args = append(args, spec.Mirror.Image, spec.Mirror.Command[1], script) + command = exec.Command("docker", args...) + } + command.Env = environment + out, runErr := command.CombinedOutput() + if strings.Contains(string(out), "synthetic-credential") || strings.Contains(string(out), "synthetic-snapshot-bytes") { + t.Fatal("mirror exposed private inputs") + } + verbs, _ := os.ReadFile(trace) + return string(out), strings.Fields(string(verbs)), runErr +} + +// TestPinnedSnapshotReadbackTooling proves cross-image tool compatibility and pipefail. +func TestPinnedSnapshotReadbackTooling(t *testing.T) { + if os.Getenv("VAULT_BACKUP_TOOLING_NATIVE_TEST") != "1" { + t.Skip("pinned static BusyBox execution runs in Linux CI") + } + for _, filename := range []string{"job.yaml", "cron-job.yaml"} { + for _, reply := range []string{"matching", "corrupt", "full-then-error"} { + t.Run(filename+"/"+reply, func(t *testing.T) { + out, _, err := exerciseMirrorMode(t, filename, reply, false, false, false, true) + if reply == "matching" { + if err != nil || !strings.Contains(out, "VAULT_SNAPSHOT_REMOTE=VERIFIED count=1") { + t.Fatalf("pinned tooling did not verify actual matching bytes: %v %s", err, out) + } + } else { + stage := "integrity" + if reply == "full-then-error" { + stage = "readback" + } + if err == nil || !strings.Contains(out, "VAULT_SNAPSHOT_REMOTE=FAIL stage="+stage) || strings.Contains(out, "VAULT_SNAPSHOT_REMOTE=VERIFIED") { + t.Fatalf("pinned shell did not reject the intended failure: %v %s", err, out) + } + } + }) + } + } +} + +// TestRemoteReadbackRequiresDownloadedMatchingBytes rejects metadata-only success. +func TestRemoteReadbackRequiresDownloadedMatchingBytes(t *testing.T) { + for _, filename := range []string{"job.yaml", "cron-job.yaml"} { + t.Run(filename, func(t *testing.T) { + out, verbs, err := exerciseMirror(t, filename, "matching", false, false, false) + if err != nil || !strings.Contains(out, "VAULT_SNAPSHOT_REMOTE=VERIFIED count=1") || !strings.Contains(strings.Join(verbs, " "), "mirror cat") { + t.Fatalf("backup needs actual matching GET readback: %v; verbs=%v; %s", err, verbs, out) + } + }) + t.Run(filename+"/all snapshots", func(t *testing.T) { + out, verbs, err := exerciseMirror(t, filename, "matching-two", false, false, false) + if err != nil || !strings.Contains(out, "VAULT_SNAPSHOT_REMOTE=VERIFIED count=2") || strings.Count(strings.Join(verbs, " "), "cat") != 2 { + t.Fatalf("every different snapshot needs its own GET verification: %v %v %s", err, verbs, out) + } + }) + } +} + +// TestRemoteReadbackRejectsUnprovenProductionBackups binds each refusal to its actual stage. +func TestRemoteReadbackRejectsUnprovenProductionBackups(t *testing.T) { + for _, filename := range []string{"job.yaml", "cron-job.yaml"} { + for _, test := range []struct { + name, reply, stage string + missing, empty, failedUpload bool + }{ + {"different bytes", "corrupt", "integrity", false, false, false}, + {"failed download", "failed", "readback", false, false, false}, + {"full body then failure", "full-then-error", "readback", false, false, false}, + {"missing credentials", "matching", "credentials", true, false, false}, + {"empty source", "matching", "source", false, true, false}, + {"failed upload", "matching", "upload", false, false, true}, + {"missing second copy", "missing-second", "readback", false, false, false}, + {"failed retention", "retention-failed", "retention", false, false, false}, + } { + t.Run(filename+"/"+test.name, func(t *testing.T) { + out, verbs, err := exerciseMirror(t, filename, test.reply, test.missing, test.empty, test.failedUpload) + if err == nil || !strings.Contains(out, "VAULT_SNAPSHOT_REMOTE=FAIL stage="+test.stage) || strings.Contains(out, "VAULT_SNAPSHOT_REMOTE=VERIFIED") { + t.Fatalf("unproven backup reported success: %v; verbs=%v; %s", err, verbs, out) + } + if test.stage != "retention" && slices.Contains(verbs, "rm") { + t.Fatal("retention must wait for successful upload and complete readback") + } + }) + } + } +} diff --git a/scripts/tests/vault-backup-recovery/storage_test.go b/scripts/tests/vault-backup-recovery/storage_test.go index 51972e9c05..ced4c6965d 100644 --- a/scripts/tests/vault-backup-recovery/storage_test.go +++ b/scripts/tests/vault-backup-recovery/storage_test.go @@ -47,7 +47,8 @@ func object(t *testing.T, objects []map[string]any, kind, name string) map[strin return nil } -func TestPreparationPreservesExistingStorageAndConsumers(t *testing.T) { +// TestCutoverPreservesOldStorageAndUsesRetainedReplacement fences the storage handoff. +func TestCutoverPreservesOldStorageAndUsesRetainedReplacement(t *testing.T) { production := render(t, "k8s/providers/hetzner/infrastructure") for _, name := range []string{"vault-snapshots", "vault-snapshots-recovery"} { claim := object(t, production, "PersistentVolumeClaim", name) @@ -66,6 +67,15 @@ func TestPreparationPreservesExistingStorageAndConsumers(t *testing.T) { if _, assigned := spec["volumeName"]; assigned { t.Fatal("recovery must not rebind the missing provider volume") } + if name == "vault-snapshots-recovery" { + labels, _ := metadata["labels"].(map[string]any) + if labels["backup.platform.devantler.tech/volume-data"] != "independently-mirrored" { + t.Fatal("active snapshot storage must use the existing independently mirrored backup route") + } + if _, marked := labels["platform.devantler.tech/storage-stage"]; marked { + t.Fatal("active recovery storage must leave the prepared-only health exemption") + } + } } for _, workload := range []struct{ kind, name string }{{"Job", "vault-snapshot-init"}, {"CronJob", "vault-snapshot"}} { spec := object(t, production, workload.kind, workload.name)["spec"].(map[string]any) @@ -79,17 +89,31 @@ func TestPreparationPreservesExistingStorageAndConsumers(t *testing.T) { if !ok { continue } - if claim["claimName"] == "vault-snapshots-recovery" { - t.Fatalf("%s must not activate speculative recovery storage", workload.name) + if claim["claimName"] == "vault-snapshots" { + t.Fatalf("%s must stop mounting the missing provider volume", workload.name) } - found = found || claim["claimName"] == "vault-snapshots" + found = found || claim["claimName"] == "vault-snapshots-recovery" } if !found { - t.Fatalf("%s lost its existing snapshot mount", workload.name) + t.Fatalf("%s must mount the main-owned retained replacement", workload.name) + } + strictMirror := false + for _, container := range pod["containers"].([]any) { + c := container.(map[string]any) + if c["name"] != "mirror" { + continue + } + for _, entry := range c["env"].([]any) { + e := entry.(map[string]any) + strictMirror = strictMirror || e["name"] == "REQUIRE_REMOTE_SNAPSHOT" && e["value"] == "true" + } + } + if !strictMirror { + t.Fatal("production snapshots must not succeed without their off-cluster mirror") } } // Bootstrap reads the offsite mirror into scratch space. Keep it independent - // of both the failed RWO claim and its deliberately inactive replacement. + // of both the failed RWO claim and its retained replacement. bootstrap := object(t, production, "Job", "vault-config")["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any) foundScratch := false for _, volume := range bootstrap["volumes"].([]any) { @@ -111,10 +135,11 @@ func TestPreparationPreservesExistingStorageAndConsumers(t *testing.T) { } } -func TestOnlyUnconsumedPreparedRecoveryStorageCanBePending(t *testing.T) { +// TestActiveSnapshotStorageRequiresBinding evaluates the deployed CEL predicates. +func TestActiveSnapshotStorageRequiresBinding(t *testing.T) { infrastructure := object(t, render(t, "k8s/clusters/prod"), "Kustomization", "infrastructure")["spec"].(map[string]any) if infrastructure["wait"] != true || infrastructure["force"] != false { - t.Fatal("recovery preparation must retain health waiting and immutable-conflict protection") + t.Fatal("recovery must retain health waiting and immutable-conflict protection") } var current, failed string for _, item := range infrastructure["healthCheckExprs"].([]any) { @@ -125,7 +150,7 @@ func TestOnlyUnconsumedPreparedRecoveryStorageCanBePending(t *testing.T) { } } if current == "" || failed == "" { - t.Fatal("prepared WFFC recovery storage needs an explicit health predicate") + t.Fatal("active snapshot storage needs its binding health predicate") } env, err := cel.NewEnv(cel.Variable("metadata", cel.DynType), cel.Variable("spec", cel.DynType), cel.Variable("status", cel.DynType)) if err != nil { @@ -149,7 +174,9 @@ func TestOnlyUnconsumedPreparedRecoveryStorageCanBePending(t *testing.T) { want bool wantFailed bool }{ - {name: "unconsumed prepared claim", phase: "Pending", want: true}, + {name: "old prepared marker cannot bypass binding", phase: "Pending", mutate: func(m, _ map[string]any) { + m["labels"] = map[string]any{"platform.devantler.tech/storage-stage": "prepared"} + }}, {name: "bound claim", phase: "Bound", want: true}, {name: "lost claim", phase: "Lost", wantFailed: true}, {name: "unobserved status"}, diff --git a/scripts/tests/vault-backup-recovery/token_test.go b/scripts/tests/vault-backup-recovery/token_test.go new file mode 100644 index 0000000000..a082e4379f --- /dev/null +++ b/scripts/tests/vault-backup-recovery/token_test.go @@ -0,0 +1,62 @@ +package recovery_test + +import "testing" + +// TestOnlySnapshotContainerReceivesKubernetesIdentity prevents token access by tools and mirror. +func TestOnlySnapshotContainerReceivesKubernetesIdentity(t *testing.T) { + objects := render(t, "k8s/providers/hetzner/infrastructure") + for _, workload := range []struct{ kind, name string }{{"Job", "vault-snapshot-init"}, {"CronJob", "vault-snapshot"}} { + t.Run(workload.name, func(t *testing.T) { + spec := object(t, objects, workload.kind, workload.name)["spec"].(map[string]any) + if workload.kind == "CronJob" { + spec = spec["jobTemplate"].(map[string]any)["spec"].(map[string]any) + } + pod := spec["template"].(map[string]any)["spec"].(map[string]any) + if value, ok := pod["automountServiceAccountToken"].(bool); !ok || value { + t.Fatal("automatic identity must not reach every container") + } + found := false + for _, entry := range pod["volumes"].([]any) { + volume := entry.(map[string]any) + if volume["name"] != "kubernetes-identity" { + continue + } + projected, ok := volume["projected"].(map[string]any) + if !ok || projected["defaultMode"] != 288 { + t.Fatal("identity projection must remain group-readable and private") + } + sources, ok := projected["sources"].([]any) + if !ok || len(sources) != 1 { + t.Fatal("identity projection must have one bounded token") + } + token, ok := sources[0].(map[string]any)["serviceAccountToken"].(map[string]any) + if !ok || token["path"] != "token" || token["expirationSeconds"] != 3600 { + t.Fatal("identity token must retain its fixed path and bounded lifetime") + } + found = true + } + if !found { + t.Fatal("snapshot requires its explicit Kubernetes identity") + } + mounts := 0 + for _, field := range []string{"initContainers", "containers"} { + for _, entry := range pod[field].([]any) { + container := entry.(map[string]any) + for _, mount := range container["volumeMounts"].([]any) { + m := mount.(map[string]any) + if m["name"] != "kubernetes-identity" { + continue + } + if container["name"] != "snapshot" || m["readOnly"] != true || m["mountPath"] != "/var/run/secrets/kubernetes.io/serviceaccount" { + t.Fatal("Kubernetes identity escaped its authenticating container") + } + mounts++ + } + } + } + if mounts != 1 { + t.Fatal("only the snapshot container may receive Kubernetes identity") + } + }) + } +} diff --git a/scripts/validate-eks-ci-role-policy/approved-surface-history.md b/scripts/validate-eks-ci-role-policy/approved-surface-history.md index 265fe949d3..494b587a49 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface-history.md +++ b/scripts/validate-eks-ci-role-policy/approved-surface-history.md @@ -3861,6 +3861,23 @@ exercise the actual KSail validator; existing producer rules stay unchanged. These offline checks do not establish production deployment; the remaining policies in #4997 still need their own migrations. +## Snapshot recovery activation (#5376) + +The main-owned retained replacement now has its two production snapshot +consumers. Its preparation marker and the corresponding Pending-readiness +exception are removed. The PVC predicate requires a non-terminating Bound claim; +the native CEL regression rejects even a stale prepared marker on a Pending claim. + +Only that readiness predicate changes in the infrastructure Kustomization. +Canonical comparison preserves every other field and health predicate, including +reconciliation waiting, immutable-conflict protection, sources and decryption. +No subject, verb, resource grant, credential or reconciliation scope is added. +The exact rendered fingerprint becomes +`f0ab6f11187056f9cb49c97d4e726df7ba0ef5c80947cbe3c0018bdcf812b77f`; +its surface-ledger entry becomes +`0cfe2fc47fa5c5ad7e0b696476d22361dc09ab095913995aca08c301156b2c3e`. +All other approvals and validation rules remain unchanged. + ## 2026-10-26 — Retire Crossview and Headlamp For #2496, on the maintainer's direction, both unused cluster UIs leave the diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index fb9aca02d4..8e8b801d7d 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -116,7 +116,7 @@ kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|bootstrap df15a74c32342 kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|flux-instance cc7711a92e331e29ddaa022dccd94ed0c8d7195f8703e20403267c73874593d3 -kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|infrastructure 348150139f81e73e5f6eea1bb8a60efd779fb4aa020b4bdf4179ffdf3fd23220 +kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|infrastructure 0cfe2fc47fa5c5ad7e0b696476d22361dc09ab095913995aca08c301156b2c3e kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|infrastructure-controllers d3cb230f3213d7a231fda9e1a700bf77e07440a0786061389357cf744e136a62 diff --git a/scripts/validate-eks-ci-role-policy/main.go b/scripts/validate-eks-ci-role-policy/main.go index 49f66ea9c9..c84fda87de 100644 --- a/scripts/validate-eks-ci-role-policy/main.go +++ b/scripts/validate-eks-ci-role-policy/main.go @@ -292,7 +292,7 @@ var expectedRenderedHashes = map[resourceIdentity]string{ {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "aws", name: "aws"}: "7bde9c682a81b752bdf9d2b14ce69ca1690008a39f2562d4887f8200447dea71", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "apps"}: "a47cc625e014f5c0202f46cd0d03c302d3d7715778dc4d5a18e22a31a0b28a12", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "bootstrap"}: "1223b3713886063661be32b3e82a8615856e3e01c526180f0d0d5b2bfade331d", - {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "infrastructure"}: "0ab2ac448ca75e30ef103d10a9747e00bdb2719998b7230bce2b9d8d8e4faa0a", + {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "infrastructure"}: "f0ab6f11187056f9cb49c97d4e726df7ba0ef5c80947cbe3c0018bdcf812b77f", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "infrastructure-controllers"}: "f0e86fa4bc64945a57bd4c4bd10ff8593134b4047b3b99d4bcbd71720dcde8ab", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "github-config", name: "github-config"}: "325a2db2626235b38b7122c10c06642697aee92f88f33a34423b6c72f3fb680a", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "unifi", name: "unifi"}: "5af50c107c3ca59ea39bf2fa334fb99b5372acc720e469d0a07f52ec97242440",