feat(backup): prove recovery from the independent snapshot store

.github/workflows/verify-openbao-backup-recovery.yaml

@@ -0,0 +1,124 @@
+name: Verify OpenBao Backup Recovery
+
+"on":
+  workflow_dispatch:
+    inputs:
+      confirm:
+        description: "Type exactly: verify-openbao-backup-recovery"
+        required: true
+        type: string
+
+permissions: {}
+concurrency:
+  group: prod-deploy
+  cancel-in-progress: false
+  queue: max
+
+jobs:
+  verify:
+    name: Restore an independent backup without changing production
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment: prod
+    permissions:
+      contents: read
+    steps:
+      - name: Require main and typed confirmation
+        shell: bash
+        env:
+          CONFIRM: ${{ inputs.confirm }}
+        run: |
+          if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$CONFIRM" != verify-openbao-backup-recovery ]]; then
+            printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2
+            exit 2
+          fi
+
+      - name: Check out the confirmed revision
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Set up the repository Go version
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+          cache: false
+
+      - name: Install the repository-approved Kubernetes reader
+        shell: bash
+        run: |
+          set -euo pipefail
+          kubectl_path="$RUNNER_TEMP/openbao-recovery-kubectl"
+          curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \
+            https://dl.k8s.io/release/v1.36.2/bin/linux/amd64/kubectl -o "$kubectl_path"
+          printf '%s  %s\n' 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 "$kubectl_path" | sha256sum --check
+          mkdir -p "$RUNNER_TEMP/openbao-recovery-tools"
+          install -m 0755 "$kubectl_path" "$RUNNER_TEMP/openbao-recovery-tools/kubectl"
+          printf '%s\n' "$RUNNER_TEMP/openbao-recovery-tools" >> "$GITHUB_PATH"
+          CGO_ENABLED=0 go build -trimpath -o "$RUNNER_TEMP/openbao-recovery-tools/verifier" ./scripts/openbao-backup-recovery
+
+      - name: Read the fixed backup inputs and prove isolated recovery
+        shell: bash
+        env:
+          KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+        run: |
+          set +x
+          set -euo pipefail
+          umask 077
+          ulimit -c 0
+          fail() { printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2; exit 2; }
+          [[ $(id -u) != 0 && $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail
+          latest=$(git ls-remote --exit-code https://github.com/devantler-tech/platform.git refs/heads/main) || fail
+          [[ "$latest" == "$GITHUB_SHA"$'\trefs/heads/main' ]] || fail
+          proof_dir=$(mktemp -d /dev/shm/openbao-recovery.XXXXXXXX)
+          forward_pid=''
+          cleanup() {
+            if [[ -n "$forward_pid" ]]; then
+              kill "$forward_pid" 2>/dev/null || true
+              wait "$forward_pid" 2>/dev/null || true
+            fi
+            rm -rf "$proof_dir"
+          }
+          trap cleanup EXIT
+          binary="$RUNNER_TEMP/openbao-recovery-tools/verifier"
+          [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail
+          export KUBECONFIG="$proof_dir/kubeconfig"
+          printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG"
+          unset KUBE_CONFIG
+          if ! (
+            if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then
+              exit 1
+            fi
+          ) 2>"$proof_dir/endpoint.log"; then
+            fail
+          fi
+          unset HCLOUD_TOKEN
+          kubectl --context=admin@prod -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$proof_dir/forward.log" 2>&1 &
+          forward_pid=$!
+          ready=false
+          for ((attempt=0; attempt<20; attempt++)); do
+            kill -0 "$forward_pid" 2>/dev/null || fail
+            if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \
+                http://127.0.0.1:38200/v1/sys/health > /dev/null 2>"$proof_dir/health.log"; then
+              ready=true
+              break
+            fi
+            sleep 1
+          done
+          [[ "$ready" == true ]] || fail
+          export OPENBAO_RECOVERY_DIR="$proof_dir"
+          if ! "$binary" collect > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then fail; fi
+          [[ $(cat "$proof_dir/collection-result") == OPENBAO_BACKUP_INPUTS=PASS ]] || fail
+          kill "$forward_pid" 2>/dev/null || fail
+          wait "$forward_pid" 2>/dev/null || true
+          forward_pid=''
+          unset OPENBAO_RECOVERY_DIR KUBECONFIG
+          rm -- "$proof_dir/kubeconfig"
+          image=$("$binary" image) || fail
+          if ! docker pull "$image" > "$proof_dir/pull.log" 2>&1; then fail; fi
+          if ! bash scripts/run-openbao-backup-sandbox.sh verify "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then fail; fi
+          result=$(cat "$proof_dir/verification-result")
+          [[ "$result" == 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' ]] || fail
+          printf '%s\n' "$result"

docs/dr/openbao.md

@@ -182,6 +182,27 @@ comes first so the claims can be deleted and then recreated empty. A
 StatefulSet uses the `OnDelete` update strategy, so a restart changes the
 pod template but recreates no pod.
 
+## Prove recovery before moving Email credentials
+
+After the snapshot repair has deployed and its replacement-storage job has
+completed, dispatch **Verify OpenBao Backup Recovery** from current `main`.
+The confirmation input must be exactly `verify-openbao-backup-recovery`.
+It uses the protected production environment and the shared deployment lock.
+
+The workflow downloads a fresh, complete snapshot from the existing independent
+backup store and binds that download to the listed object. It reads the three
+original Email credential records, then ends its production connection before
+restoring in a disposable, non-root, network-disabled server. It verifies all
+record fields and a fresh unseal with the retained recovery key. Production
+storage, credentials and backup objects are never changed by this verification.
+Private inputs and diagnostic logs stay in temporary RAM storage; the workflow
+publishes only a fixed verdict and removes its owned runtime resources.
+
+Only `OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3` establishes this
+three-record proof. A missing, stale, truncated, changed or mismatched backup
+refuses success. This proof must be expanded before retiring additional records;
+it does not by itself authorize credential migration or legacy-copy removal.
+
 ## References
 
 - [OpenBao Seal/Unseal](https://openbao.org/docs/concepts/seal/)

go.mod

@@ -43,4 +43,9 @@ require (
 	google.golang.org/grpc v1.83.2 // indirect
 )
 
-require github.com/google/go-github/v88 v88.0.0
+require (
+	github.com/aws/aws-sdk-go-v2 v1.47.3
+	github.com/google/go-github/v88 v88.0.0
+)
+
+require github.com/aws/smithy-go v1.28.5 // indirect

go.sum

@@ -4,6 +4,10 @@ cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
 cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
 github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
+github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
+github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
+github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
+github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bradleyfalzon/ghinstallation/v2 v2.19.0 h1:KQfD+43pRw9NUJhGycGrFr9vF1MubZacksKol1gomFI=