fix(backup): restore snapshot jobs and verify remote backup bytes

.github/workflows/ci.yaml

@@ -185,6 +185,7 @@ jobs:
               - 'scripts/tests/test-business-disclosure-reader.sh'
               - 'scripts/tests/test-business-staging-hold.sh'
               - 'scripts/check-business-disclosure-reader/**'
+              - 'scripts/tests/vault-backup-recovery/**'
               - 'scripts/openbao-backup-recovery/**'
               - 'scripts/run-openbao-backup-sandbox.sh'
               - 'scripts/tests/test-openbao-standby-canary.sh'
@@ -3321,6 +3322,12 @@ jobs:
           key: kubeconform-schema-cache-v1-render-regressions-${{ github.run_id }}
           restore-keys: |
             kubeconform-schema-cache-v1-
+      - name: 🔐 Prove snapshot readback with pinned static tooling
+        if: needs.changes.outputs.k8s == 'true'
+        timeout-minutes: 5
+        env:
+          VAULT_BACKUP_TOOLING_NATIVE_TEST: '1'
+        run: go test -count=1 -v ./scripts/tests/vault-backup-recovery
       - name: 🐤 Validate staged OpenBao standby repair
         if: needs.changes.outputs.k8s == 'true'
         run: |

docs/dr/openbao.md

@@ -6,7 +6,7 @@ OpenBao runs as a raft (Integrated Storage) cluster. Three artifacts make it
 recoverable:
 
 1. **Raft snapshots** — the `vault-snapshot` CronJob saves a
-   `bao operator raft snapshot` to the `vault-snapshots` PVC daily (newest 14
+   `bao operator raft snapshot` daily (newest 14
    kept) and mirrors them to the S3 backup target under `openbao-snapshots/`
    (Cloudflare R2 in prod, MinIO locally). The mirror exists because Velero's
    file-system backup only captures volumes mounted by *running* pods —
@@ -31,6 +31,36 @@ downloads only that object. Missing projected credentials or an empty or
 unreachable mirror listing leave the scratch volume empty. When recovery needs
 a snapshot and none is available, the restore guard refuses re-initialization.
 
+## Snapshot storage and verification
+
+Production's initial and scheduled snapshot jobs use the retained
+`vault-snapshots-recovery` claim. The original `vault-snapshots` claim remains
+protected and unmodified. Local deployments keep their original claim. Active
+claims must bind; the temporary preparation marker no longer bypasses readiness.
+
+The snapshot container completes before the mirror starts. The mirror copies
+without remote removal, then downloads each nonempty snapshot and compares its
+SHA-256 with the local bytes. A failed or partial GET fails the run, including a
+GET that delivers a complete body and then returns an error. Production requires
+the projected backup credentials; an empty source also fails. Fresh local
+bootstrap can report `VAULT_SNAPSHOT_REMOTE=UNSEEDED` until credentials arrive.
+
+The fixed `VAULT_SNAPSHOT_REMOTE=VERIFIED count=N` verdict requires matching
+downloaded bytes for every snapshot and successful age-based retention. Retention
+starts after upload and readback; the mirror never uses `--remove`. Credentials,
+client configuration and copied static tools use bounded memory-backed volumes.
+The existing concurrency exclusion, execution deadlines and snapshot resources
+remain in place.
+
+Only the snapshot init container receives the explicitly projected Kubernetes
+token used for its existing OpenBao login. Automatic token mounting is disabled;
+the static-tool installer and remote mirror receive no Kubernetes identity.
+
+Complete recovery acceptance with an isolated restore of the new off-cluster
+snapshot and verification of its credential records using the matching recovery
+keys. Keep #5376 open until that native restore proof is observed, before moving
+Email credentials or retiring legacy custody.
+
 ## Recovery Scenarios
 
 ### Scenario 1: Single pod restart

k8s/bases/infrastructure/vault-backup/cron-job.yaml

@@ -75,11 +75,9 @@ spec:
             app: vault-snapshot
         spec:
           serviceAccountName: vault-snapshot
-          # The token IS this pod's credential: the snapshot container reads
-          # /var/run/secrets/kubernetes.io/serviceaccount/token and presents it
-          # as the `jwt` for `auth/kubernetes/login`. Left at the mount-by-default
-          # value rather than set explicitly — an explicit `true` reads identically
-          # to Kubernetes and trips Kubescape C-0034.
+          # Only the authenticating snapshot container receives a short-lived
+          # projected token. The tool installer and remote mirror need no identity.
+          automountServiceAccountToken: false
           restartPolicy: OnFailure
           # The full restricted-PSS context is set explicitly (not left to the
           # add-security-context Kyverno mutation) so the pod stays hardened
@@ -103,8 +101,42 @@ spec:
                 secretName: vault-snapshot-r2
                 optional: true
             - name: mc-config
-              emptyDir: {}
+              emptyDir:
+                medium: Memory
+                sizeLimit: 8Mi
+            - name: mirror-tools
+              emptyDir:
+                medium: Memory
+                sizeLimit: 8Mi
+            - name: kubernetes-identity
+              projected:
+                defaultMode: 0440
+                sources:
+                  - serviceAccountToken:
+                      path: token
+                      expirationSeconds: 3600
           initContainers:
+            - name: install-mirror-tools
+              image: docker.io/library/busybox:1.38.0-musl@sha256:ea2b9914a16a4ac1981994af97b318f7c7d4db76b580c56177f08bf76f4a0be8
+              command: ["/bin/sh", "-ec", "cp /bin/busybox /tools/busybox; /tools/busybox --install -s /tools"]
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                runAsNonRoot: true
+                capabilities:
+                  drop: ["ALL"]
+                seccompProfile:
+                  type: RuntimeDefault
+              resources:
+                requests:
+                  cpu: 10m
+                  memory: 8Mi
+                limits:
+                  cpu: 100m
+                  memory: 32Mi
+              volumeMounts:
+                - name: mirror-tools
+                  mountPath: /tools
             - name: snapshot
               image: quay.io/openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806
               securityContext:
@@ -140,6 +172,9 @@ spec:
               volumeMounts:
                 - name: snapshots
                   mountPath: /snapshots
+                - name: kubernetes-identity
+                  mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+                  readOnly: true
               command:
                 - /bin/sh
                 - -ec
@@ -210,6 +245,8 @@ spec:
               env:
                 - name: MC_CONFIG_DIR
                   value: /tmp/.mc
+                - name: REQUIRE_REMOTE_SNAPSHOT
+                  value: "false"
               resources:
                 requests:
                   cpu: 10m
@@ -227,25 +264,52 @@ spec:
                   readOnly: true
                 - name: mc-config
                   mountPath: /tmp/.mc
+                - name: mirror-tools
+                  mountPath: /tools
+                  readOnly: true
               command:
-                - /bin/sh
-                - -ec
+                - /tools/sh
+                - -euc
                 - |
-                  if [ ! -f /r2/access_key_id ]; then
-                    echo "vault-snapshot-r2 Secret not synced yet — skipping the"
-                    echo "off-cluster mirror. The local PVC snapshot above still ran."
+                  set -o pipefail
+                  umask 077
+                  fail() {
+                    printf 'VAULT_SNAPSHOT_REMOTE=FAIL stage=%s\n' "$1" >&2
+                    exit 1
+                  }
+                  case "$REQUIRE_REMOTE_SNAPSHOT" in true|false) ;; *) fail configuration ;; esac
+                  [ -d "$MC_CONFIG_DIR" ] || fail configuration
+                  if [ ! -s /r2/access_key_id ] || [ ! -s /r2/secret_access_key ]; then
+                    if [ "$REQUIRE_REMOTE_SNAPSHOT" = true ]; then
+                      fail credentials
+                    fi
+                    printf '%s\n' 'VAULT_SNAPSHOT_REMOTE=UNSEEDED'
                     exit 0
                   fi
                   mc alias set backup "${r2_endpoint}" \
-                    "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)"
-                  # Copy-only mirror: NO --remove, so an empty/recreated PVC can
-                  # never wipe the off-cluster history. Remote retention is
-                  # pruned by age instead, and only after a successful upload in
-                  # the same run (this container only starts when the snapshot
-                  # initContainer succeeded), so the newest snapshot always
-                  # survives the prune.
-                  mc mirror --overwrite /snapshots/ "backup/${r2_bucket}/openbao-snapshots/"
-                  mc rm --recursive --force --older-than 14d \
-                    "backup/${r2_bucket}/openbao-snapshots/" || true
-                  echo "Mirrored snapshots:"
-                  mc ls "backup/${r2_bucket}/openbao-snapshots/"
+                    "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)" \
+                    > /dev/null 2>"$MC_CONFIG_DIR/error" || fail authentication
+                  remote="backup/${r2_bucket}/openbao-snapshots"
+                  # Copy-only: never remove remote history because a local volume is empty.
+                  mc mirror --overwrite /snapshots/ "$remote/" \
+                    > /dev/null 2>"$MC_CONFIG_DIR/error" || fail upload
+                  verified=0
+                  for snapshot in /snapshots/openbao-*.snap; do
+                    [ -e "$snapshot" ] || continue
+                    [ -f "$snapshot" ] && [ -s "$snapshot" ] && [ ! -L "$snapshot" ] || fail source
+                    name=$(/tools/basename "$snapshot") || fail source
+                    /tools/sha256sum < "$snapshot" > "$MC_CONFIG_DIR/expected.sha256" || fail local_hash
+                    # Hash an actual GET, not object metadata or an upload acknowledgement.
+                    # pipefail also rejects a GET that emits a full body before returning error.
+                    if ! mc cat "$remote/$name" 2>"$MC_CONFIG_DIR/error" |
+                        /tools/sha256sum > "$MC_CONFIG_DIR/actual.sha256"; then
+                      fail readback
+                    fi
+                    /tools/cmp -s "$MC_CONFIG_DIR/expected.sha256" "$MC_CONFIG_DIR/actual.sha256" || fail integrity
+                    verified=$((verified + 1))
+                  done
+                  [ "$verified" -gt 0 ] || fail source
+                  # Retention runs only after upload AND byte-for-byte readback succeeded.
+                  mc rm --recursive --force --older-than 14d "$remote/" \
+                    > /dev/null 2>"$MC_CONFIG_DIR/error" || fail retention
+                  printf 'VAULT_SNAPSHOT_REMOTE=VERIFIED count=%s\n' "$verified"

k8s/bases/infrastructure/vault-backup/job.yaml

@@ -69,11 +69,9 @@ spec:
         app: vault-snapshot
     spec:
       serviceAccountName: vault-snapshot
-      # The token IS this pod's credential: the snapshot container reads
-      # /var/run/secrets/kubernetes.io/serviceaccount/token and presents it as
-      # the `jwt` for `auth/kubernetes/login`. Left at the mount-by-default
-      # value rather than set explicitly — an explicit `true` reads identically
-      # to Kubernetes and trips Kubescape C-0034.
+      # Only the authenticating snapshot container receives a short-lived
+      # projected token. The tool installer and remote mirror need no identity.
+      automountServiceAccountToken: false
       restartPolicy: OnFailure
       # The full restricted-PSS context is set explicitly (not left to the
       # add-security-context Kyverno mutation) so the pod stays hardened
@@ -97,8 +95,42 @@ spec:
             secretName: vault-snapshot-r2
             optional: true
         - name: mc-config
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 8Mi
+        - name: mirror-tools
+          emptyDir:
+            medium: Memory
+            sizeLimit: 8Mi
+        - name: kubernetes-identity
+          projected:
+            defaultMode: 0440
+            sources:
+              - serviceAccountToken:
+                  path: token
+                  expirationSeconds: 3600
       initContainers:
+        - name: install-mirror-tools
+          image: docker.io/library/busybox:1.38.0-musl@sha256:ea2b9914a16a4ac1981994af97b318f7c7d4db76b580c56177f08bf76f4a0be8
+          command: ["/bin/sh", "-ec", "cp /bin/busybox /tools/busybox; /tools/busybox --install -s /tools"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            capabilities:
+              drop: ["ALL"]
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests:
+              cpu: 10m
+              memory: 8Mi
+            limits:
+              cpu: 100m
+              memory: 32Mi
+          volumeMounts:
+            - name: mirror-tools
+              mountPath: /tools
         - name: snapshot
           image: quay.io/openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806
           securityContext:
@@ -132,6 +164,9 @@ spec:
           volumeMounts:
             - name: snapshots
               mountPath: /snapshots
+            - name: kubernetes-identity
+              mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+              readOnly: true
           command:
             - /bin/sh
             - -ec
@@ -202,6 +237,8 @@ spec:
           env:
             - name: MC_CONFIG_DIR
               value: /tmp/.mc
+            - name: REQUIRE_REMOTE_SNAPSHOT
+              value: "false"
           resources:
             requests:
               cpu: 10m
@@ -219,23 +256,52 @@ spec:
               readOnly: true
             - name: mc-config
               mountPath: /tmp/.mc
+            - name: mirror-tools
+              mountPath: /tools
+              readOnly: true
           command:
-            - /bin/sh
-            - -ec
+            - /tools/sh
+            - -euc
             - |
-              if [ ! -f /r2/access_key_id ]; then
-                echo "vault-snapshot-r2 Secret not synced yet — skipping the"
-                echo "off-cluster mirror. The local PVC snapshot above still ran."
+              set -o pipefail
+              umask 077
+              fail() {
+                printf 'VAULT_SNAPSHOT_REMOTE=FAIL stage=%s\n' "$1" >&2
+                exit 1
+              }
+              case "$REQUIRE_REMOTE_SNAPSHOT" in true|false) ;; *) fail configuration ;; esac
+              [ -d "$MC_CONFIG_DIR" ] || fail configuration
+              if [ ! -s /r2/access_key_id ] || [ ! -s /r2/secret_access_key ]; then
+                if [ "$REQUIRE_REMOTE_SNAPSHOT" = true ]; then
+                  fail credentials
+                fi
+                printf '%s\n' 'VAULT_SNAPSHOT_REMOTE=UNSEEDED'
                 exit 0
               fi
               mc alias set backup "${r2_endpoint}" \
-                "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)"
-              # Copy-only mirror (no --remove) + age-based prune, same
-              # rationale as cron-job.yaml: an empty/recreated PVC can never
-              # wipe the off-cluster history, and the prune only runs after a
-              # successful upload in the same pod.
-              mc mirror --overwrite /snapshots/ "backup/${r2_bucket}/openbao-snapshots/"
-              mc rm --recursive --force --older-than 14d \
-                "backup/${r2_bucket}/openbao-snapshots/" || true
-              echo "Mirrored snapshots:"
-              mc ls "backup/${r2_bucket}/openbao-snapshots/"
+                "$(cat /r2/access_key_id)" "$(cat /r2/secret_access_key)" \
+                > /dev/null 2>"$MC_CONFIG_DIR/error" || fail authentication
+              remote="backup/${r2_bucket}/openbao-snapshots"
+              # Copy-only: never remove remote history because a local volume is empty.
+              mc mirror --overwrite /snapshots/ "$remote/" \
+                > /dev/null 2>"$MC_CONFIG_DIR/error" || fail upload
+              verified=0
+              for snapshot in /snapshots/openbao-*.snap; do
+                [ -e "$snapshot" ] || continue
+                [ -f "$snapshot" ] && [ -s "$snapshot" ] && [ ! -L "$snapshot" ] || fail source
+                name=$(/tools/basename "$snapshot") || fail source
+                /tools/sha256sum < "$snapshot" > "$MC_CONFIG_DIR/expected.sha256" || fail local_hash
+                # Hash an actual GET, not object metadata or an upload acknowledgement.
+                # pipefail also rejects a GET that emits a full body before returning error.
+                if ! mc cat "$remote/$name" 2>"$MC_CONFIG_DIR/error" |
+                    /tools/sha256sum > "$MC_CONFIG_DIR/actual.sha256"; then
+                  fail readback
+                fi
+                /tools/cmp -s "$MC_CONFIG_DIR/expected.sha256" "$MC_CONFIG_DIR/actual.sha256" || fail integrity
+                verified=$((verified + 1))
+              done
+              [ "$verified" -gt 0 ] || fail source
+              # Retention runs only after upload AND byte-for-byte readback succeeded.
+              mc rm --recursive --force --older-than 14d "$remote/" \
+                > /dev/null 2>"$MC_CONFIG_DIR/error" || fail retention
+              printf 'VAULT_SNAPSHOT_REMOTE=VERIFIED count=%s\n' "$verified"

k8s/clusters/base/flux-kustomization-infrastructure.yaml

@@ -38,21 +38,13 @@ spec:
   # KRO reports readiness via `status.state: Active|Inactive`, so teach Flux to
   # health-check RGDs by that field via a CEL expression.
   healthCheckExprs:
-    # Recovery preparation has no consumer, so hcloud WFFC remains Pending.
-    # Activation removes this marker and must wait for actual volume binding.
+    # Active snapshot storage must bind. The preparation-only Pending allowance
+    # is retired with the first consumers of its main-owned retained claim.
     - apiVersion: v1
       kind: PersistentVolumeClaim
       failed: has(status.phase) && status.phase == 'Lost'
       current: >-
-        !has(metadata.deletionTimestamp) && has(status.phase) &&
-        (status.phase == 'Bound' ||
-          (status.phase == 'Pending' &&
-           metadata.name == 'vault-snapshots-recovery' && metadata.namespace == 'openbao' &&
-           has(metadata.labels) && 'platform.devantler.tech/storage-stage' in metadata.labels &&
-           metadata.labels['platform.devantler.tech/storage-stage'] == 'prepared' &&
-           has(spec.storageClassName) && spec.storageClassName == 'hcloud' &&
-           !has(spec.volumeName) &&
-           (!has(metadata.annotations) || !('volume.kubernetes.io/selected-node' in metadata.annotations))))
+        !has(metadata.deletionTimestamp) && has(status.phase) && status.phase == 'Bound'
     - apiVersion: kro.run/v1alpha1
       kind: ResourceGraphDefinition
       current: status.state == 'Active'