diff --git a/.github/workflows/verify-openbao-backup-recovery.yaml b/.github/workflows/verify-openbao-backup-recovery.yaml
new file mode 100644
index 0000000000..68860ef3cb
--- /dev/null
+++ b/.github/workflows/verify-openbao-backup-recovery.yaml
@@ -0,0 +1,124 @@
+name: Verify OpenBao Backup Recovery
+
+"on":
+ workflow_dispatch:
+ inputs:
+ confirm:
+ description: "Type exactly: verify-openbao-backup-recovery"
+ required: true
+ type: string
+
+permissions: {}
+concurrency:
+ group: prod-deploy
+ cancel-in-progress: false
+ queue: max
+
+jobs:
+ verify:
+ name: Restore an independent backup without changing production
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ environment: prod
+ permissions:
+ contents: read
+ steps:
+ - name: Require main and typed confirmation
+ shell: bash
+ env:
+ CONFIRM: ${{ inputs.confirm }}
+ run: |
+ if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$CONFIRM" != verify-openbao-backup-recovery ]]; then
+ printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2
+ exit 2
+ fi
+
+ - name: Check out the confirmed revision
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Set up the repository Go version
+ uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+ with:
+ go-version-file: go.mod
+ cache: false
+
+ - name: Install the repository-approved Kubernetes reader
+ shell: bash
+ run: |
+ set -euo pipefail
+ kubectl_path="$RUNNER_TEMP/openbao-recovery-kubectl"
+ curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \
+ https://dl.k8s.io/release/v1.36.2/bin/linux/amd64/kubectl -o "$kubectl_path"
+ printf '%s %s\n' 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 "$kubectl_path" | sha256sum --check
+ mkdir -p "$RUNNER_TEMP/openbao-recovery-tools"
+ install -m 0755 "$kubectl_path" "$RUNNER_TEMP/openbao-recovery-tools/kubectl"
+ printf '%s\n' "$RUNNER_TEMP/openbao-recovery-tools" >> "$GITHUB_PATH"
+ CGO_ENABLED=0 go build -trimpath -o "$RUNNER_TEMP/openbao-recovery-tools/verifier" ./scripts/openbao-backup-recovery
+
+ - name: Read the fixed backup inputs and prove isolated recovery
+ shell: bash
+ env:
+ KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+ HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+ run: |
+ set +x
+ set -euo pipefail
+ umask 077
+ ulimit -c 0
+ fail() { printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2; exit 2; }
+ [[ $(id -u) != 0 && $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail
+ latest=$(git ls-remote --exit-code https://github.com/devantler-tech/platform.git refs/heads/main) || fail
+ [[ "$latest" == "$GITHUB_SHA"$'\trefs/heads/main' ]] || fail
+ proof_dir=$(mktemp -d /dev/shm/openbao-recovery.XXXXXXXX)
+ forward_pid=''
+ cleanup() {
+ if [[ -n "$forward_pid" ]]; then
+ kill "$forward_pid" 2>/dev/null || true
+ wait "$forward_pid" 2>/dev/null || true
+ fi
+ rm -rf "$proof_dir"
+ }
+ trap cleanup EXIT
+ binary="$RUNNER_TEMP/openbao-recovery-tools/verifier"
+ [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail
+ export KUBECONFIG="$proof_dir/kubeconfig"
+ printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG"
+ unset KUBE_CONFIG
+ if ! (
+ if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then
+ exit 1
+ fi
+ ) 2>"$proof_dir/endpoint.log"; then
+ fail
+ fi
+ unset HCLOUD_TOKEN
+ kubectl --context=admin@prod -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$proof_dir/forward.log" 2>&1 &
+ forward_pid=$!
+ ready=false
+ for ((attempt=0; attempt<20; attempt++)); do
+ kill -0 "$forward_pid" 2>/dev/null || fail
+ if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \
+ http://127.0.0.1:38200/v1/sys/health > /dev/null 2>"$proof_dir/health.log"; then
+ ready=true
+ break
+ fi
+ sleep 1
+ done
+ [[ "$ready" == true ]] || fail
+ export OPENBAO_RECOVERY_DIR="$proof_dir"
+ if ! "$binary" collect > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then fail; fi
+ [[ $(cat "$proof_dir/collection-result") == OPENBAO_BACKUP_INPUTS=PASS ]] || fail
+ kill "$forward_pid" 2>/dev/null || fail
+ wait "$forward_pid" 2>/dev/null || true
+ forward_pid=''
+ unset OPENBAO_RECOVERY_DIR KUBECONFIG
+ rm -- "$proof_dir/kubeconfig"
+ image=$("$binary" image) || fail
+ if ! docker pull "$image" > "$proof_dir/pull.log" 2>&1; then fail; fi
+ if ! bash scripts/run-openbao-backup-sandbox.sh verify "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then fail; fi
+ result=$(cat "$proof_dir/verification-result")
+ [[ "$result" == 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' ]] || fail
+ printf '%s\n' "$result"
diff --git a/docs/dr/openbao.md b/docs/dr/openbao.md
index 70a6410af7..33e63b5d4c 100644
--- a/docs/dr/openbao.md
+++ b/docs/dr/openbao.md
@@ -182,6 +182,27 @@ comes first so the claims can be deleted and then recreated empty. A
StatefulSet uses the `OnDelete` update strategy, so a restart changes the
pod template but recreates no pod.
+## Prove recovery before moving Email credentials
+
+After the snapshot repair has deployed and its replacement-storage job has
+completed, dispatch **Verify OpenBao Backup Recovery** from current `main`.
+The confirmation input must be exactly `verify-openbao-backup-recovery`.
+It uses the protected production environment and the shared deployment lock.
+
+The workflow downloads a fresh, complete snapshot from the existing independent
+backup store and binds that download to the listed object. It reads the three
+original Email credential records, then ends its production connection before
+restoring in a disposable, non-root, network-disabled server. It verifies all
+record fields and a fresh unseal with the retained recovery key. Production
+storage, credentials and backup objects are never changed by this verification.
+Private inputs and diagnostic logs stay in temporary RAM storage; the workflow
+publishes only a fixed verdict and removes its owned runtime resources.
+
+Only `OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3` establishes this
+three-record proof. A missing, stale, truncated, changed or mismatched backup
+refuses success. This proof must be expanded before retiring additional records;
+it does not by itself authorize credential migration or legacy-copy removal.
+
## References
- [OpenBao Seal/Unseal](https://openbao.org/docs/concepts/seal/)
diff --git a/go.mod b/go.mod
index fe1532586f..c923da7fd7 100644
--- a/go.mod
+++ b/go.mod
@@ -43,4 +43,9 @@ require (
google.golang.org/grpc v1.83.2 // indirect
)
-require github.com/google/go-github/v88 v88.0.0
+require (
+ github.com/aws/aws-sdk-go-v2 v1.47.3
+ github.com/google/go-github/v88 v88.0.0
+)
+
+require github.com/aws/smithy-go v1.28.5 // indirect
diff --git a/go.sum b/go.sum
index fd59c3605a..dd7a67c801 100644
--- a/go.sum
+++ b/go.sum
@@ -4,6 +4,10 @@ cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs=
cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
+github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
+github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
+github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k=
+github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/bradleyfalzon/ghinstallation/v2 v2.19.0 h1:KQfD+43pRw9NUJhGycGrFr9vF1MubZacksKol1gomFI=
diff --git a/scripts/openbao-backup-recovery/collector.go b/scripts/openbao-backup-recovery/collector.go
new file mode 100644
index 0000000000..0f163d8018
--- /dev/null
+++ b/scripts/openbao-backup-recovery/collector.go
@@ -0,0 +1,253 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "crypto/tls"
+ "encoding/hex"
+ "encoding/json"
+ "encoding/xml"
+ "errors"
+ "io"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+)
+
+const snapshotPrefix = "openbao-snapshots/"
+
+// recoveryHTTP refuses ambient proxies, redirects and transparent decompression.
+func recoveryHTTP() *http.Client {
+ return &http.Client{Transport: &http.Transport{Proxy: nil, DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext, ResponseHeaderTimeout: 20 * time.Second, DisableCompression: true, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}, Timeout: 90 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
+}
+
+// signedRead uses the official AWS signer and never puts credentials in a URL or process.
+func signedRead(ctx context.Context, client *http.Client, t target, path string, query url.Values, etag string, limit int64) (result []byte, headers http.Header, resultErr error) {
+ u, err := url.Parse(t.Endpoint)
+ if err != nil || !regexp.MustCompile(`^https://[a-f0-9]{32}(?:\.eu)?\.r2\.cloudflarestorage\.com$`).MatchString(t.Endpoint) || !strings.HasPrefix(path, "/"+t.Bucket) || limit <= 0 {
+ return nil, nil, errEvidence
+ }
+ u.Path = path
+ u.RawQuery = query.Encode()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+ if err != nil {
+ return nil, nil, errEvidence
+ }
+ if etag != "" {
+ req.Header.Set("If-Match", etag)
+ }
+ sum := sha256.Sum256(nil)
+ payloadHash := hex.EncodeToString(sum[:])
+ // SignHTTP uses this hash for signing but does not add S3's required header.
+ req.Header.Set("X-Amz-Content-Sha256", payloadHash)
+ if v4.NewSigner().SignHTTP(ctx, aws.Credentials{AccessKeyID: t.AccessKey, SecretAccessKey: t.SecretKey}, req, payloadHash, "s3", "auto", time.Now().UTC()) != nil {
+ return nil, nil, errEvidence
+ }
+ response, err := client.Do(req)
+ if err != nil {
+ return nil, nil, errEvidence
+ }
+ defer func() {
+ if response.Body.Close() != nil {
+ result, headers, resultErr = nil, nil, errEvidence
+ }
+ }()
+ if response.StatusCode != http.StatusOK || response.ContentLength > limit {
+ return nil, nil, errEvidence
+ }
+ b, err := io.ReadAll(io.LimitReader(response.Body, limit+1))
+ if err != nil || int64(len(b)) > limit || (response.ContentLength >= 0 && int64(len(b)) != response.ContentLength) {
+ return nil, nil, errEvidence
+ }
+ return b, response.Header, nil
+}
+
+// downloadSnapshot admits a complete bounded listing and one immutable fresh object.
+func downloadSnapshot(ctx context.Context, client *http.Client, t target, start, end time.Time) ([]byte, error) {
+ b, _, err := signedRead(ctx, client, t, "/"+t.Bucket, url.Values{"list-type": {"2"}, "prefix": {snapshotPrefix}, "max-keys": {"256"}}, "", 1<<20)
+ if err != nil {
+ return nil, errEvidence
+ }
+ var listing struct {
+ XMLName xml.Name
+ Name, Prefix, NextContinuationToken string
+ KeyCount, MaxKeys *int
+ IsTruncated *bool
+ Contents []struct {
+ Key, ETag string
+ Size int64
+ LastModified time.Time
+ }
+ }
+ decoder := xml.NewDecoder(bytes.NewReader(b))
+ if decoder.Decode(&listing) != nil {
+ return nil, errEvidence
+ }
+ if !errors.Is(decoder.Decode(new(any)), io.EOF) || listing.XMLName.Local != "ListBucketResult" || (listing.XMLName.Space != "" && listing.XMLName.Space != "http://s3.amazonaws.com/doc/2006-03-01/") || listing.Name != t.Bucket || listing.Prefix != snapshotPrefix || listing.IsTruncated == nil || *listing.IsTruncated || listing.NextContinuationToken != "" || listing.KeyCount == nil || *listing.KeyCount != len(listing.Contents) || listing.MaxKeys == nil || *listing.MaxKeys != 256 || len(listing.Contents) == 0 || len(listing.Contents) > 256 {
+ return nil, errEvidence
+ }
+ etags := map[string]string{}
+ var objects bytes.Buffer
+ encoder := json.NewEncoder(&objects)
+ for _, item := range listing.Contents {
+ if !strings.HasPrefix(item.Key, snapshotPrefix) || !regexp.MustCompile(`^"[a-fA-F0-9]{32}(?:-[1-9][0-9]{0,5})?"$`).MatchString(item.ETag) {
+ return nil, errEvidence
+ }
+ key := strings.TrimPrefix(item.Key, snapshotPrefix)
+ if err := encoder.Encode(snapshotObject{Status: "success", Type: "file", Key: key, Size: item.Size, LastModified: item.LastModified}); err != nil {
+ return nil, errEvidence
+ }
+ etags[key] = item.ETag
+ }
+ selected, err := selectSnapshot(&objects, true, start, end)
+ if err != nil {
+ return nil, errEvidence
+ }
+ b, headers, err := signedRead(ctx, client, t, "/"+t.Bucket+"/"+snapshotPrefix+selected.Key, nil, etags[selected.Key], selected.Size)
+ modified, stampErr := http.ParseTime(headers.Get("Last-Modified"))
+ if err != nil || stampErr != nil || int64(len(b)) != selected.Size || headers.Get("ETag") != etags[selected.Key] || !modified.Equal(selected.LastModified.Truncate(time.Second)) {
+ return nil, errEvidence
+ }
+ return b, nil
+}
+
+func allowedRecoveryObject(kind, namespace, name string) bool {
+ switch namespace + "/" + kind + "/" + name {
+ case "openbao/job/vault-snapshot-init", "flux-system/configmap/variables-base", "flux-system/configmap/variables-cluster", "flux-system/secret/variables-cluster", "openbao/secret/vault-snapshot-r2", "openbao/secret/openbao-unseal":
+ return true
+ default:
+ return false
+ }
+}
+
+type boundedOutput struct{ bytes.Buffer }
+
+func (b *boundedOutput) Write(p []byte) (int, error) {
+ if b.Len()+len(p) > 1<<20 {
+ return 0, errEvidence
+ }
+ return b.Buffer.Write(p)
+}
+
+// recoveryObject executes a fixed GET only, with bounded output and a fixed context.
+func recoveryObject(ctx context.Context, kind, namespace, name string) ([]byte, error) {
+ if !allowedRecoveryObject(kind, namespace, name) {
+ return nil, errEvidence
+ }
+ command := exec.CommandContext(ctx, "kubectl", "--context=admin@prod", "--request-timeout=20s", "--namespace", namespace, "get", kind, name, "--output=json")
+ var output boundedOutput
+ command.Stdout = &output
+ command.Stderr = io.Discard
+ if command.Run() != nil || output.Len() == 0 {
+ return nil, errEvidence
+ }
+ return output.Bytes(), nil
+}
+
+func recoveryDirectory(dir string) bool {
+ uid := os.Getuid()
+ if runtime.GOOS != "linux" || uid <= 0 || !strings.HasPrefix(dir, "/dev/shm/") || filepath.Clean(dir) != dir {
+ return false
+ }
+ resolvedDir, err := filepath.EvalSymlinks(dir)
+ if err != nil || resolvedDir != dir {
+ return false
+ }
+ info, err := os.Lstat(dir)
+ if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
+ return false
+ }
+ owner, ok := info.Sys().(*syscall.Stat_t)
+ if !ok || uint64(owner.Uid) != uint64(uid) {
+ return false
+ }
+ var fs syscall.Statfs_t
+ return syscall.Statfs(dir, &fs) == nil && fs.Type == 0x01021994
+}
+
+// collectRecoveryInputs reads production only; every restore mutation stays in the sandbox.
+func collectRecoveryInputs(dir string) error {
+ if !recoveryDirectory(dir) {
+ return errEvidence
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+ job, err := recoveryObject(ctx, "job", "openbao", "vault-snapshot-init")
+ if err != nil {
+ return errEvidence
+ }
+ start, end, err := freshJobWindow(job, time.Now().UTC())
+ if err != nil {
+ return errEvidence
+ }
+ api, err := newAPI("http://127.0.0.1:38200")
+ if err != nil {
+ return errEvidence
+ }
+ health, err := api.health(ctx)
+ if err != nil || !*health.Initialized || *health.Sealed {
+ return errEvidence
+ }
+ base, err := recoveryObject(ctx, "configmap", "flux-system", "variables-base")
+ if err != nil {
+ return errEvidence
+ }
+ cluster, err := recoveryObject(ctx, "configmap", "flux-system", "variables-cluster")
+ if err != nil {
+ return errEvidence
+ }
+ variables, err := recoveryObject(ctx, "secret", "flux-system", "variables-cluster")
+ if err != nil {
+ return errEvidence
+ }
+ keys, err := recoveryObject(ctx, "secret", "openbao", "vault-snapshot-r2")
+ if err != nil {
+ return errEvidence
+ }
+ target, err := backupTarget(base, cluster, variables, keys)
+ if err != nil {
+ return errEvidence
+ }
+ recovery, err := recoveryObject(ctx, "secret", "openbao", "openbao-unseal")
+ if err != nil {
+ return errEvidence
+ }
+ values, err := fixedObject(recovery, "Secret", "openbao", "openbao-unseal")
+ if err != nil {
+ return errEvidence
+ }
+ root, key := values["root-token"], values["unseal-key"]
+ if !regexp.MustCompile(`^[A-Za-z0-9._-]{20,256}$`).MatchString(root) || !regexp.MustCompile(`^[A-Za-z0-9+/=]{32,256}$`).MatchString(key) {
+ return errEvidence
+ }
+ records, err := api.credentialRecords(ctx, root)
+ if err != nil || !recordsEqual(records, records) {
+ return errEvidence
+ }
+ expected, err := json.Marshal(records)
+ if err != nil {
+ return errEvidence
+ }
+ snapshot, err := downloadSnapshot(ctx, recoveryHTTP(), target, start, end)
+ if err != nil {
+ return errEvidence
+ }
+ for name, data := range map[string][]byte{"root-token": []byte(root), "unseal-key": []byte(key), "expected.json": expected, "snapshot.snap": snapshot} {
+ if writePrivate(filepath.Join(dir, name), data) != nil {
+ return errEvidence
+ }
+ }
+ return nil
+}
diff --git a/scripts/openbao-backup-recovery/collector_test.go b/scripts/openbao-backup-recovery/collector_test.go
new file mode 100644
index 0000000000..99e43410ee
--- /dev/null
+++ b/scripts/openbao-backup-recovery/collector_test.go
@@ -0,0 +1,165 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "testing"
+ "time"
+)
+
+type failedClose struct{ io.Reader }
+
+func (failedClose) Close() error { return errors.New("synthetic incomplete response") }
+
+type fixtureTransport func(*http.Request) (*http.Response, error)
+
+func (f fixtureTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func TestIndependentSnapshotReadRequiresSignedFreshBoundedBytes(t *testing.T) {
+ now := time.Now().UTC().Truncate(time.Second)
+ name := "openbao-" + now.Format("20060102-150405") + ".snap"
+ etag := `"` + strings.Repeat("a", 32) + `"`
+ base := fmt.Sprintf(`fixture-backupsopenbao-snapshots/1256falseopenbao-snapshots/%s%s%s8`, name, now.Format(time.RFC3339), etag)
+ for _, tc := range []string{"matching", "truncated", "wrong-prefix", "duplicate", "old", "wrong-size", "changed-object", "redirect", "oversized", "trailing-xml", "close-failure"} {
+ t.Run(tc, func(t *testing.T) {
+ listing := base
+ size := int64(8)
+ gotETag := etag
+ status := http.StatusOK
+ body := "snapshot"
+ gets := 0
+ switch tc {
+ case "truncated":
+ listing = strings.Replace(listing, "false", "true", 1)
+ case "wrong-prefix":
+ listing = strings.ReplaceAll(listing, "openbao-snapshots/", "other/")
+ case "duplicate":
+ listing = strings.Replace(listing, "", listing[strings.Index(listing, ""):strings.Index(listing, "")+len("")]+"", 1)
+ case "old":
+ listing = strings.ReplaceAll(listing, now.Format(time.RFC3339), now.Add(-48*time.Hour).Format(time.RFC3339))
+ case "wrong-size":
+ size = 9
+ case "changed-object":
+ gotETag = `"` + strings.Repeat("b", 32) + `"`
+ case "redirect":
+ status = http.StatusTemporaryRedirect
+ case "oversized":
+ listing = strings.Repeat("x", (1<<20)+1)
+ case "trailing-xml":
+ listing += ""
+ }
+ client := &http.Client{Transport: fixtureTransport(func(r *http.Request) (*http.Response, error) {
+ if r.Header.Get("X-Amz-Content-Sha256") != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
+ t.Fatal("S3 read is missing the required empty-payload hash header")
+ }
+ if r.Method != http.MethodGet || r.URL.Host != strings.Repeat("c", 32)+".r2.cloudflarestorage.com" || !strings.Contains(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential="+strings.Repeat("d", 32)+"/") {
+ t.Fatal("request escaped its signed read-only origin")
+ }
+ h := make(http.Header)
+ content := body
+ n := size
+ if r.URL.Query().Get("list-type") == "2" {
+ if r.URL.Query().Get("prefix") != "openbao-snapshots/" || r.URL.Query().Get("max-keys") != "256" || r.URL.Path != "/fixture-backups" {
+ t.Fatal("unexpected list scope")
+ }
+ content = listing
+ n = int64(len(content))
+ } else {
+ gets++
+ if r.URL.Path != "/fixture-backups/openbao-snapshots/"+name || r.Header.Get("If-Match") != etag {
+ t.Fatal("object is not bound to listing")
+ }
+ h.Set("ETag", gotETag)
+ h.Set("Last-Modified", now.Format(http.TimeFormat))
+ }
+ body := io.NopCloser(strings.NewReader(content))
+ if tc == "close-failure" {
+ body = failedClose{strings.NewReader(content)}
+ }
+ return &http.Response{StatusCode: status, Header: h, Body: body, ContentLength: n, Request: r}, nil
+ })}
+ target := target{Endpoint: "https://" + strings.Repeat("c", 32) + ".r2.cloudflarestorage.com", Bucket: "fixture-backups", AccessKey: strings.Repeat("d", 32), SecretKey: strings.Repeat("e", 64)}
+ b, err := downloadSnapshot(context.Background(), client, target, now.Add(-time.Minute), now.Add(time.Minute))
+ if (err == nil) != (tc == "matching") || (err == nil && !bytes.Equal(b, []byte("snapshot"))) {
+ t.Fatal("incorrect independent snapshot verdict")
+ }
+ if tc == "matching" && gets != 1 {
+ t.Fatal("did not download the actual object")
+ }
+ if tc != "matching" && len(b) != 0 {
+ t.Fatal("refused input exposed partial content")
+ }
+ })
+ }
+}
+
+func TestIndependentProductionReaderLimitsKubernetesObjects(t *testing.T) {
+ for _, item := range []struct {
+ k, ns, n string
+ ok bool
+ }{
+ {"job", "openbao", "vault-snapshot-init", true},
+ {"configmap", "flux-system", "variables-base", true},
+ {"configmap", "flux-system", "variables-cluster", true},
+ {"secret", "flux-system", "variables-cluster", true},
+ {"secret", "openbao", "vault-snapshot-r2", true},
+ {"secret", "openbao", "openbao-unseal", true},
+ {"secret", "openbao", "other", false},
+ {"pods", "openbao", "openbao-0", false},
+ {"secret", "other", "openbao-unseal", false},
+ } {
+ if allowedRecoveryObject(item.k, item.ns, item.n) != item.ok {
+ t.Fatal("incorrect fixed Kubernetes read boundary")
+ }
+ }
+}
+
+// TestSnapshotTransportNeverForwardsSignedReads exercises real local HTTP connections.
+func TestSnapshotTransportNeverForwardsSignedReads(t *testing.T) {
+ var proxied, followed bool
+ proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { proxied = true; w.WriteHeader(http.StatusNoContent) }))
+ defer proxy.Close()
+ destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { followed = true; w.WriteHeader(http.StatusOK) }))
+ defer destination.Close()
+ source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, destination.URL, http.StatusTemporaryRedirect)
+ }))
+ defer source.Close()
+ t.Setenv("HTTP_PROXY", proxy.URL)
+ t.Setenv("HTTPS_PROXY", proxy.URL)
+ t.Setenv("NO_PROXY", "")
+ sourceURL, err := url.Parse(source.URL)
+ if err != nil {
+ t.Fatal(err)
+ }
+ client := recoveryHTTP()
+ transport := client.Transport.(*http.Transport)
+ transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) {
+ if address == "recovery-fixture.invalid:80" {
+ address = sourceURL.Host
+ }
+ return (&net.Dialer{}).DialContext(ctx, network, address)
+ }
+ request, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://recovery-fixture.invalid/", nil)
+ if err != nil {
+ t.Fatal("could not prepare local transport request")
+ }
+ response, err := client.Do(request)
+ if err != nil {
+ t.Fatal("could not exercise local transport")
+ }
+ if response.Body.Close() != nil {
+ t.Fatal("could not finish local response")
+ }
+ if response.StatusCode != http.StatusTemporaryRedirect || proxied || followed {
+ t.Fatal("transport forwarded a request through an ambient proxy or redirect")
+ }
+}
diff --git a/scripts/openbao-backup-recovery/evidence.go b/scripts/openbao-backup-recovery/evidence.go
index e3cda1e082..c8694736f3 100644
--- a/scripts/openbao-backup-recovery/evidence.go
+++ b/scripts/openbao-backup-recovery/evidence.go
@@ -104,12 +104,16 @@ func recordsEqual(expected, actual map[string]json.RawMessage) bool {
}
// privateInput refuses links, special files, shared permissions and truncated reads.
-func privateInput(path string, limit int64) ([]byte, error) {
+func privateInput(path string, limit int64) (result []byte, resultErr error) {
f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, errEvidence
}
- defer f.Close()
+ defer func() {
+ if f.Close() != nil {
+ result, resultErr = nil, errEvidence
+ }
+ }()
info, err := f.Stat()
if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() <= 0 || info.Size() > limit {
return nil, errEvidence
diff --git a/scripts/openbao-backup-recovery/main.go b/scripts/openbao-backup-recovery/main.go
index 9bf2f313dc..6067cd13e7 100644
--- a/scripts/openbao-backup-recovery/main.go
+++ b/scripts/openbao-backup-recovery/main.go
@@ -411,6 +411,14 @@ audit "file" {
}
func main() {
+ if len(os.Args) == 2 && os.Args[1] == "collect" {
+ if collectRecoveryInputs(os.Getenv("OPENBAO_RECOVERY_DIR")) != nil {
+ fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_INPUTS=UNVERIFIED")
+ os.Exit(2)
+ }
+ fmt.Println("OPENBAO_BACKUP_INPUTS=PASS")
+ return
+ }
if len(os.Args) == 2 && os.Args[1] == "image" {
fmt.Println(serverImage)
return
diff --git a/scripts/openbao-backup-recovery/workflow_runtime_test.go b/scripts/openbao-backup-recovery/workflow_runtime_test.go
new file mode 100644
index 0000000000..fab47e0585
--- /dev/null
+++ b/scripts/openbao-backup-recovery/workflow_runtime_test.go
@@ -0,0 +1,113 @@
+package main
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "gopkg.in/yaml.v3"
+)
+
+// TestRecoveryWorkflowExcludesClusterCredentials exercises the actual shell handoff.
+func TestRecoveryWorkflowExcludesClusterCredentials(t *testing.T) {
+ if os.Getuid() == 0 {
+ t.Skip("the workflow requires a non-root caller")
+ }
+ raw, err := os.ReadFile(filepath.Join("..", "..", ".github", "workflows", "verify-openbao-backup-recovery.yaml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var workflow struct {
+ Jobs map[string]struct{ Steps []struct{ Run string } }
+ }
+ if yaml.Unmarshal(raw, &workflow) != nil || len(workflow.Jobs["verify"].Steps) != 5 {
+ t.Fatal("cannot read guarded recovery workflow")
+ }
+ body := workflow.Jobs["verify"].Steps[4].Run
+ for _, tc := range []string{"matching", "collection-refused", "private-result", "changed-main"} {
+ t.Run(tc, func(t *testing.T) {
+ dir, err := filepath.EvalSymlinks(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ bin := filepath.Join(dir, "bin")
+ runner := filepath.Join(dir, "runner")
+ for _, p := range []string{bin, filepath.Join(dir, "scripts"), filepath.Join(runner, "openbao-recovery-tools")} {
+ if os.MkdirAll(p, 0700) != nil {
+ t.Fatal("cannot prepare private fixture")
+ }
+ }
+ files := map[string]string{
+ "bin/git": `#!/bin/bash
+set -eu
+[[ $1 == ls-remote ]] || exit 91
+if [[ $FIXTURE_CASE == changed-main ]]; then printf '%040d\trefs/heads/main\n' 2; else printf '%s\trefs/heads/main\n' "$GITHUB_SHA"; fi
+`,
+ "bin/findmnt": "#!/bin/sh\nprintf '%s\\n' tmpfs\n",
+ "bin/mktemp": `#!/bin/bash
+set -eu
+[[ $1 == -d && $2 == /dev/shm/openbao-recovery.XXXXXXXX ]] || exit 91
+exec /usr/bin/mktemp -d "$FIXTURE_DIR/proof.XXXXXXXX"
+`,
+ "bin/kubectl": `#!/bin/bash
+set -eu
+[[ $* == *port-forward* ]] || exit 91
+exec /bin/sleep 30
+`,
+ "bin/curl": "#!/bin/sh\nexit 0\n",
+ "bin/docker": "#!/bin/sh\n[ \"$1\" = pull ] || exit 91\n",
+ "scripts/use-prod-stable-api-endpoint.sh": "#!/bin/sh\nexit 0\n",
+ "runner/openbao-recovery-tools/verifier": `#!/bin/bash
+set -eu
+if [[ $1 == image ]]; then printf '%s\n' '` + serverImage + `'; exit 0; fi
+[[ $1 == collect ]] || exit 91
+printf '%s' "$OPENBAO_RECOVERY_DIR" > "$FIXTURE_DIR/proof-path"
+for name in root-token unseal-key expected.json snapshot.snap; do printf '%s' synthetic > "$OPENBAO_RECOVERY_DIR/$name"; done
+if [[ $FIXTURE_CASE == collection-refused ]]; then printf '%s\n' PRIVATE_SYNTHETIC_PAYLOAD; exit 91; fi
+printf '%s\n' OPENBAO_BACKUP_INPUTS=PASS
+`,
+ "scripts/run-openbao-backup-sandbox.sh": `#!/bin/bash
+set -eu
+printf '%s' reached > "$FIXTURE_DIR/sandbox-started"
+[[ $1 == verify ]] || exit 91
+if [[ -e $2/kubeconfig ]]; then printf '%s' exposed > "$FIXTURE_DIR/kubeconfig-exposed"; printf '%s\n' PRIVATE_SYNTHETIC_CLUSTER_CREDENTIAL >&2; exit 91; fi
+for name in root-token unseal-key expected.json snapshot.snap; do [[ -s $2/$name ]] || exit 91; done
+if [[ $FIXTURE_CASE == private-result ]]; then printf '%s\n' PRIVATE_SYNTHETIC_PAYLOAD; else printf '%s\n' 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3'; fi
+`,
+ }
+ for name, text := range files {
+ if os.WriteFile(filepath.Join(dir, name), []byte(text), 0700) != nil {
+ t.Fatal("cannot write private fixture tool")
+ }
+ }
+ command := exec.Command("bash", "-c", body)
+ command.Dir = dir
+ command.Env = []string{"PATH=" + bin + ":" + os.Getenv("PATH"), "RUNNER_TEMP=" + runner, "GITHUB_SHA=" + strings.Repeat("1", 40), "KUBE_CONFIG=SYNTHETIC_KUBECONFIG", "HCLOUD_TOKEN=SYNTHETIC_PROVIDER_VALUE", "FIXTURE_DIR=" + dir, "FIXTURE_CASE=" + tc}
+ out, runErr := command.CombinedOutput()
+ if (runErr == nil) != (tc == "matching") {
+ if _, err := os.Stat(filepath.Join(dir, "kubeconfig-exposed")); err == nil {
+ t.Fatal("sandbox received an unintended cluster credential")
+ }
+ t.Fatal("incorrect private handoff verdict", runErr)
+ }
+ if strings.Contains(string(out), "PRIVATE_SYNTHETIC") || strings.Contains(string(out), "SYNTHETIC_KUBECONFIG") || strings.Contains(string(out), "SYNTHETIC_PROVIDER") {
+ t.Fatal("workflow exposed private fixture content")
+ }
+ if tc == "matching" && strings.TrimSpace(string(out)) != "OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3" {
+ t.Fatal("workflow accepted an unverified result")
+ }
+ proof, readErr := os.ReadFile(filepath.Join(dir, "proof-path"))
+ if readErr == nil {
+ if _, statErr := os.Stat(string(proof)); !os.IsNotExist(statErr) {
+ t.Fatal("workflow retained its private scratch directory")
+ }
+ }
+ _, sandboxErr := os.Stat(filepath.Join(dir, "sandbox-started"))
+ if (sandboxErr == nil) != (tc == "matching" || tc == "private-result") {
+ t.Fatal("workflow reached sandbox after a refused collection or stale main")
+ }
+ })
+ }
+}
diff --git a/scripts/openbao-backup-recovery/workflow_test.go b/scripts/openbao-backup-recovery/workflow_test.go
new file mode 100644
index 0000000000..b204a40d4e
--- /dev/null
+++ b/scripts/openbao-backup-recovery/workflow_test.go
@@ -0,0 +1,84 @@
+package main
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "gopkg.in/yaml.v3"
+)
+
+func TestProductionRecoveryRefusesBranchBeforeCheckoutOrCredentials(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join("..", "..", ".github", "workflows", "verify-openbao-backup-recovery.yaml"))
+ if err != nil {
+ t.Fatal("independent backup recovery workflow is missing")
+ }
+ type step struct {
+ Name, Uses, Shell, Run string
+ Env map[string]string
+ With map[string]any
+ }
+ var workflow struct {
+ On map[string]any `yaml:"on"`
+ Permissions map[string]string
+ Concurrency struct {
+ Group string
+ Cancel bool `yaml:"cancel-in-progress"`
+ }
+ Jobs map[string]struct {
+ Environment string
+ Permissions map[string]string
+ Steps []step
+ }
+ }
+ if yaml.Unmarshal(data, &workflow) != nil || len(workflow.On) != 1 || workflow.On["workflow_dispatch"] == nil || len(workflow.Permissions) != 0 || len(workflow.Jobs) != 1 || workflow.Concurrency.Group != "prod-deploy" || workflow.Concurrency.Cancel {
+ t.Fatal("workflow execution boundary changed")
+ }
+ job, ok := workflow.Jobs["verify"]
+ if !ok || job.Environment != "prod" || len(job.Permissions) != 1 || job.Permissions["contents"] != "read" || len(job.Steps) != 5 {
+ t.Fatal("verification must use one protected read-only job")
+ }
+ guard := job.Steps[0]
+ if guard.Uses != "" || guard.Shell != "bash" || len(guard.Env) != 1 || guard.Env["CONFIRM"] != "${{ inputs.confirm }}" {
+ t.Fatal("first step must refuse before checkout and secret injection")
+ }
+ for _, tc := range []struct {
+ event, ref, repo, confirm string
+ pass bool
+ }{
+ {"workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "verify-openbao-backup-recovery", true},
+ {"pull_request", "refs/heads/main", "devantler-tech/platform", "verify-openbao-backup-recovery", false},
+ {"workflow_dispatch", "refs/heads/codex/other", "devantler-tech/platform", "verify-openbao-backup-recovery", false},
+ {"workflow_dispatch", "refs/heads/main", "other/platform", "verify-openbao-backup-recovery", false},
+ {"workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "$(exit 0)", false},
+ } {
+ command := exec.Command("bash", "-eu", "-c", guard.Run)
+ command.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=" + tc.event, "GITHUB_REF=" + tc.ref, "GITHUB_REPOSITORY=" + tc.repo, "CONFIRM=" + tc.confirm}
+ if err := command.Run(); (err == nil) != tc.pass {
+ t.Fatal("guard admitted an unconfirmed dispatch or refused main")
+ }
+ }
+ checkout := job.Steps[1]
+ if checkout.Uses != "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" || checkout.With["ref"] != "${{ github.sha }}" || checkout.With["persist-credentials"] != false {
+ t.Fatal("checkout is not pinned to the confirmed revision")
+ }
+ for _, s := range job.Steps[:4] {
+ if len(s.Env) > 1 || strings.Contains(s.Run, "secrets.") {
+ t.Fatal("credential access moved before the guarded reader")
+ }
+ }
+ run := job.Steps[4]
+ if strings.Contains(run.Run, "go build") || !strings.Contains(job.Steps[3].Run, "go build") {
+ t.Fatal("the build must run before the production credential environment is injected")
+ }
+ if len(run.Env) != 2 || run.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || run.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || strings.Contains(run.Run, "upload-artifact") {
+ t.Fatal("recovery reader credential boundary changed")
+ }
+ for _, required := range []string{"/dev/shm", "OPENBAO_RECOVERY_DIR", "service/openbao-active 38200:8200", "run-openbao-backup-sandbox.sh verify", "OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3"} {
+ if !strings.Contains(run.Run, required) {
+ t.Fatal("missing private isolated recovery boundary")
+ }
+ }
+}