From bf1f30a67e0552c8dd8081417f3d8b693a139a24 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 27 Oct 2026 23:39:23 +0100 Subject: [PATCH 1/4] feat(backup): prove recovery from the independent snapshot store --- .../verify-openbao-backup-recovery.yaml | 117 +++++++++ docs/dr/openbao.md | 21 ++ go.mod | 7 +- go.sum | 4 + scripts/openbao-backup-recovery/collector.go | 245 ++++++++++++++++++ .../openbao-backup-recovery/collector_test.go | 158 +++++++++++ scripts/openbao-backup-recovery/evidence.go | 8 +- scripts/openbao-backup-recovery/main.go | 8 + .../openbao-backup-recovery/workflow_test.go | 81 ++++++ 9 files changed, 646 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/verify-openbao-backup-recovery.yaml create mode 100644 scripts/openbao-backup-recovery/collector.go create mode 100644 scripts/openbao-backup-recovery/collector_test.go create mode 100644 scripts/openbao-backup-recovery/workflow_test.go diff --git a/.github/workflows/verify-openbao-backup-recovery.yaml b/.github/workflows/verify-openbao-backup-recovery.yaml new file mode 100644 index 0000000000..1daba75e23 --- /dev/null +++ b/.github/workflows/verify-openbao-backup-recovery.yaml @@ -0,0 +1,117 @@ +name: Verify OpenBao Backup Recovery + +"on": + workflow_dispatch: + inputs: + confirm: + description: "Type exactly: verify-openbao-backup-recovery" + required: true + type: string + +permissions: {} +concurrency: + group: prod-deploy + cancel-in-progress: false + queue: max + +jobs: + verify: + name: Restore an independent backup without changing production + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: prod + permissions: + contents: read + steps: + - name: Require main and typed confirmation + shell: bash + env: + CONFIRM: ${{ inputs.confirm }} + run: | + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/platform || "$CONFIRM" != verify-openbao-backup-recovery ]]; then + printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2 + exit 2 + fi + + - name: Check out the confirmed revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up the repository Go version + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Install the repository-approved Kubernetes reader + shell: bash + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/openbao-recovery-kubectl" + curl --disable --fail --silent --show-error --location --proto '=https' --proto-redir '=https' --max-redirs 3 --connect-timeout 10 --max-time 60 \ + https://dl.k8s.io/release/v1.36.2/bin/linux/amd64/kubectl -o "$kubectl_path" + printf '%s %s\n' 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/openbao-recovery-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/openbao-recovery-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/openbao-recovery-tools" >> "$GITHUB_PATH" + + - name: Read the fixed backup inputs and prove isolated recovery + shell: bash + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + run: | + set +x + set -euo pipefail + umask 077 + ulimit -c 0 + fail() { printf '%s\n' 'OPENBAO_BACKUP_RESTORE=UNVERIFIED' >&2; exit 2; } + [[ $(id -u) != 0 && $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || fail + latest=$(git ls-remote --exit-code https://github.com/devantler-tech/platform.git refs/heads/main) || fail + [[ "$latest" == "$GITHUB_SHA"$'\trefs/heads/main' ]] || fail + proof_dir=$(mktemp -d /dev/shm/openbao-recovery.XXXXXXXX) + forward_pid='' + cleanup() { + if [[ -n "$forward_pid" ]]; then + kill "$forward_pid" 2>/dev/null || true + wait "$forward_pid" 2>/dev/null || true + fi + rm -rf "$proof_dir" + } + trap cleanup EXIT + binary="$RUNNER_TEMP/openbao-recovery-tools/verifier" + if ! CGO_ENABLED=0 go build -trimpath -o "$binary" ./scripts/openbao-backup-recovery > "$proof_dir/build.log" 2>&1; then fail; fi + [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail + export KUBECONFIG="$proof_dir/kubeconfig" + printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" + unset KUBE_CONFIG + if ! ./scripts/use-prod-stable-api-endpoint.sh > "$proof_dir/endpoint.log" 2>&1; then fail; fi + unset HCLOUD_TOKEN + kubectl --context=admin@prod -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$proof_dir/forward.log" 2>&1 & + forward_pid=$! + ready=false + for ((attempt=0; attempt<20; attempt++)); do + kill -0 "$forward_pid" 2>/dev/null || fail + if curl --disable --silent --fail --noproxy '*' --connect-timeout 1 --max-time 2 \ + http://127.0.0.1:38200/v1/sys/health > /dev/null 2>"$proof_dir/health.log"; then + ready=true + break + fi + sleep 1 + done + [[ "$ready" == true ]] || fail + export OPENBAO_RECOVERY_DIR="$proof_dir" + if ! "$binary" collect > "$proof_dir/collection-result" 2>"$proof_dir/collection.log"; then fail; fi + [[ $(cat "$proof_dir/collection-result") == OPENBAO_BACKUP_INPUTS=PASS ]] || fail + kill "$forward_pid" 2>/dev/null || fail + wait "$forward_pid" 2>/dev/null || true + forward_pid='' + unset OPENBAO_RECOVERY_DIR KUBECONFIG + image=$("$binary" image) || fail + if ! docker pull "$image" > "$proof_dir/pull.log" 2>&1; then fail; fi + if ! bash scripts/run-openbao-backup-sandbox.sh verify "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then fail; fi + result=$(cat "$proof_dir/verification-result") + [[ "$result" == 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3' ]] || fail + printf '%s\n' "$result" diff --git a/docs/dr/openbao.md b/docs/dr/openbao.md index 70a6410af7..33e63b5d4c 100644 --- a/docs/dr/openbao.md +++ b/docs/dr/openbao.md @@ -182,6 +182,27 @@ comes first so the claims can be deleted and then recreated empty. A StatefulSet uses the `OnDelete` update strategy, so a restart changes the pod template but recreates no pod. +## Prove recovery before moving Email credentials + +After the snapshot repair has deployed and its replacement-storage job has +completed, dispatch **Verify OpenBao Backup Recovery** from current `main`. +The confirmation input must be exactly `verify-openbao-backup-recovery`. +It uses the protected production environment and the shared deployment lock. + +The workflow downloads a fresh, complete snapshot from the existing independent +backup store and binds that download to the listed object. It reads the three +original Email credential records, then ends its production connection before +restoring in a disposable, non-root, network-disabled server. It verifies all +record fields and a fresh unseal with the retained recovery key. Production +storage, credentials and backup objects are never changed by this verification. +Private inputs and diagnostic logs stay in temporary RAM storage; the workflow +publishes only a fixed verdict and removes its owned runtime resources. + +Only `OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3` establishes this +three-record proof. A missing, stale, truncated, changed or mismatched backup +refuses success. This proof must be expanded before retiring additional records; +it does not by itself authorize credential migration or legacy-copy removal. + ## References - [OpenBao Seal/Unseal](https://openbao.org/docs/concepts/seal/) diff --git a/go.mod b/go.mod index fe1532586f..c923da7fd7 100644 --- a/go.mod +++ b/go.mod @@ -43,4 +43,9 @@ require ( google.golang.org/grpc v1.83.2 // indirect ) -require github.com/google/go-github/v88 v88.0.0 +require ( + github.com/aws/aws-sdk-go-v2 v1.47.3 + github.com/google/go-github/v88 v88.0.0 +) + +require github.com/aws/smithy-go v1.28.5 // indirect diff --git a/go.sum b/go.sum index fd59c3605a..dd7a67c801 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradleyfalzon/ghinstallation/v2 v2.19.0 h1:KQfD+43pRw9NUJhGycGrFr9vF1MubZacksKol1gomFI= diff --git a/scripts/openbao-backup-recovery/collector.go b/scripts/openbao-backup-recovery/collector.go new file mode 100644 index 0000000000..678d75ee21 --- /dev/null +++ b/scripts/openbao-backup-recovery/collector.go @@ -0,0 +1,245 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/tls" + "encoding/hex" + "encoding/json" + "encoding/xml" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "syscall" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" +) + +const snapshotPrefix = "openbao-snapshots/" + +// recoveryHTTP refuses ambient proxies, redirects and transparent decompression. +func recoveryHTTP() *http.Client { + return &http.Client{Transport: &http.Transport{Proxy: nil, DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext, ResponseHeaderTimeout: 20 * time.Second, DisableCompression: true, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}, Timeout: 90 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} +} + +// signedRead uses the official AWS signer and never puts credentials in a URL or process. +func signedRead(ctx context.Context, client *http.Client, t target, path string, query url.Values, etag string, limit int64) (result []byte, headers http.Header, resultErr error) { + u, err := url.Parse(t.Endpoint) + if err != nil || !regexp.MustCompile(`^https://[a-f0-9]{32}(?:\.eu)?\.r2\.cloudflarestorage\.com$`).MatchString(t.Endpoint) || !strings.HasPrefix(path, "/"+t.Bucket) || limit <= 0 { + return nil, nil, errEvidence + } + u.Path = path + u.RawQuery = query.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, nil, errEvidence + } + if etag != "" { + req.Header.Set("If-Match", etag) + } + sum := sha256.Sum256(nil) + if v4.NewSigner().SignHTTP(ctx, aws.Credentials{AccessKeyID: t.AccessKey, SecretAccessKey: t.SecretKey}, req, hex.EncodeToString(sum[:]), "s3", "auto", time.Now().UTC()) != nil { + return nil, nil, errEvidence + } + response, err := client.Do(req) + if err != nil { + return nil, nil, errEvidence + } + defer func() { + if response.Body.Close() != nil { + result, headers, resultErr = nil, nil, errEvidence + } + }() + if response.StatusCode != http.StatusOK || response.ContentLength > limit { + return nil, nil, errEvidence + } + b, err := io.ReadAll(io.LimitReader(response.Body, limit+1)) + if err != nil || int64(len(b)) > limit || (response.ContentLength >= 0 && int64(len(b)) != response.ContentLength) { + return nil, nil, errEvidence + } + return b, response.Header, nil +} + +// downloadSnapshot admits a complete bounded listing and one immutable fresh object. +func downloadSnapshot(ctx context.Context, client *http.Client, t target, start, end time.Time) ([]byte, error) { + b, _, err := signedRead(ctx, client, t, "/"+t.Bucket, url.Values{"list-type": {"2"}, "prefix": {snapshotPrefix}, "max-keys": {"256"}}, "", 1<<20) + if err != nil { + return nil, errEvidence + } + var listing struct { + XMLName xml.Name + Name, Prefix, NextContinuationToken string + KeyCount, MaxKeys *int + IsTruncated *bool + Contents []struct { + Key, ETag string + Size int64 + LastModified time.Time + } + } + decoder := xml.NewDecoder(bytes.NewReader(b)) + if decoder.Decode(&listing) != nil || decoder.Decode(new(any)) != io.EOF || listing.XMLName.Local != "ListBucketResult" || (listing.XMLName.Space != "" && listing.XMLName.Space != "http://s3.amazonaws.com/doc/2006-03-01/") || listing.Name != t.Bucket || listing.Prefix != snapshotPrefix || listing.IsTruncated == nil || *listing.IsTruncated || listing.NextContinuationToken != "" || listing.KeyCount == nil || *listing.KeyCount != len(listing.Contents) || listing.MaxKeys == nil || *listing.MaxKeys != 256 || len(listing.Contents) == 0 || len(listing.Contents) > 256 { + return nil, errEvidence + } + etags := map[string]string{} + var objects bytes.Buffer + encoder := json.NewEncoder(&objects) + for _, item := range listing.Contents { + if !strings.HasPrefix(item.Key, snapshotPrefix) || !regexp.MustCompile(`^"[a-fA-F0-9]{32}(?:-[1-9][0-9]{0,5})?"$`).MatchString(item.ETag) { + return nil, errEvidence + } + key := strings.TrimPrefix(item.Key, snapshotPrefix) + if encoder.Encode(snapshotObject{Status: "success", Type: "file", Key: key, Size: item.Size, LastModified: item.LastModified}) != nil { + return nil, errEvidence + } + etags[key] = item.ETag + } + selected, err := selectSnapshot(&objects, true, start, end) + if err != nil { + return nil, errEvidence + } + b, headers, err := signedRead(ctx, client, t, "/"+t.Bucket+"/"+snapshotPrefix+selected.Key, nil, etags[selected.Key], selected.Size) + modified, stampErr := http.ParseTime(headers.Get("Last-Modified")) + if err != nil || stampErr != nil || int64(len(b)) != selected.Size || headers.Get("ETag") != etags[selected.Key] || !modified.Equal(selected.LastModified.Truncate(time.Second)) { + return nil, errEvidence + } + return b, nil +} + +func allowedRecoveryObject(kind, namespace, name string) bool { + switch namespace + "/" + kind + "/" + name { + case "openbao/job/vault-snapshot-init", "flux-system/configmap/variables-base", "flux-system/configmap/variables-cluster", "flux-system/secret/variables-cluster", "openbao/secret/vault-snapshot-r2", "openbao/secret/openbao-unseal": + return true + default: + return false + } +} + +type boundedOutput struct{ bytes.Buffer } + +func (b *boundedOutput) Write(p []byte) (int, error) { + if b.Len()+len(p) > 1<<20 { + return 0, errEvidence + } + return b.Buffer.Write(p) +} + +// recoveryObject executes a fixed GET only, with bounded output and a fixed context. +func recoveryObject(ctx context.Context, kind, namespace, name string) ([]byte, error) { + if !allowedRecoveryObject(kind, namespace, name) { + return nil, errEvidence + } + command := exec.CommandContext(ctx, "kubectl", "--context=admin@prod", "--request-timeout=20s", "--namespace="+namespace, "get", kind, name, "--output=json") + var output boundedOutput + command.Stdout = &output + command.Stderr = io.Discard + if command.Run() != nil || output.Len() == 0 { + return nil, errEvidence + } + return output.Bytes(), nil +} + +func recoveryDirectory(dir string) bool { + if runtime.GOOS != "linux" || os.Getuid() == 0 || !strings.HasPrefix(dir, "/dev/shm/") || filepath.Clean(dir) != dir { + return false + } + real, err := filepath.EvalSymlinks(dir) + if err != nil || real != dir { + return false + } + info, err := os.Lstat(dir) + if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 { + return false + } + owner, ok := info.Sys().(*syscall.Stat_t) + if !ok || owner.Uid != uint32(os.Getuid()) { + return false + } + var fs syscall.Statfs_t + return syscall.Statfs(dir, &fs) == nil && fs.Type == 0x01021994 +} + +// collectRecoveryInputs reads production only; every restore mutation stays in the sandbox. +func collectRecoveryInputs(dir string) error { + if !recoveryDirectory(dir) { + return errEvidence + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + job, err := recoveryObject(ctx, "job", "openbao", "vault-snapshot-init") + if err != nil { + return errEvidence + } + start, end, err := freshJobWindow(job, time.Now().UTC()) + if err != nil { + return errEvidence + } + api, err := newAPI("http://127.0.0.1:38200") + if err != nil { + return errEvidence + } + health, err := api.health(ctx) + if err != nil || !*health.Initialized || *health.Sealed { + return errEvidence + } + base, err := recoveryObject(ctx, "configmap", "flux-system", "variables-base") + if err != nil { + return errEvidence + } + cluster, err := recoveryObject(ctx, "configmap", "flux-system", "variables-cluster") + if err != nil { + return errEvidence + } + variables, err := recoveryObject(ctx, "secret", "flux-system", "variables-cluster") + if err != nil { + return errEvidence + } + keys, err := recoveryObject(ctx, "secret", "openbao", "vault-snapshot-r2") + if err != nil { + return errEvidence + } + target, err := backupTarget(base, cluster, variables, keys) + if err != nil { + return errEvidence + } + recovery, err := recoveryObject(ctx, "secret", "openbao", "openbao-unseal") + if err != nil { + return errEvidence + } + values, err := fixedObject(recovery, "Secret", "openbao", "openbao-unseal") + if err != nil { + return errEvidence + } + root, key := values["root-token"], values["unseal-key"] + if !regexp.MustCompile(`^[A-Za-z0-9._-]{20,256}$`).MatchString(root) || !regexp.MustCompile(`^[A-Za-z0-9+/=]{32,256}$`).MatchString(key) { + return errEvidence + } + records, err := api.credentialRecords(ctx, root) + if err != nil || !recordsEqual(records, records) { + return errEvidence + } + expected, err := json.Marshal(records) + if err != nil { + return errEvidence + } + snapshot, err := downloadSnapshot(ctx, recoveryHTTP(), target, start, end) + if err != nil { + return errEvidence + } + for name, data := range map[string][]byte{"root-token": []byte(root), "unseal-key": []byte(key), "expected.json": expected, "snapshot.snap": snapshot} { + if writePrivate(filepath.Join(dir, name), data) != nil { + return errEvidence + } + } + return nil +} diff --git a/scripts/openbao-backup-recovery/collector_test.go b/scripts/openbao-backup-recovery/collector_test.go new file mode 100644 index 0000000000..1b1c8912fd --- /dev/null +++ b/scripts/openbao-backup-recovery/collector_test.go @@ -0,0 +1,158 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +type failedClose struct{ io.Reader } + +func (failedClose) Close() error { return errors.New("synthetic incomplete response") } + +type fixtureTransport func(*http.Request) (*http.Response, error) + +func (f fixtureTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestIndependentSnapshotReadRequiresSignedFreshBoundedBytes(t *testing.T) { + now := time.Now().UTC().Truncate(time.Second) + name := "openbao-" + now.Format("20060102-150405") + ".snap" + etag := `"` + strings.Repeat("a", 32) + `"` + base := fmt.Sprintf(`fixture-backupsopenbao-snapshots/1256falseopenbao-snapshots/%s%s%s8`, name, now.Format(time.RFC3339), etag) + for _, tc := range []string{"matching", "truncated", "wrong-prefix", "duplicate", "old", "wrong-size", "changed-object", "redirect", "oversized", "trailing-xml", "close-failure"} { + t.Run(tc, func(t *testing.T) { + listing := base + size := int64(8) + gotETag := etag + status := http.StatusOK + body := "snapshot" + gets := 0 + switch tc { + case "truncated": + listing = strings.Replace(listing, "false", "true", 1) + case "wrong-prefix": + listing = strings.ReplaceAll(listing, "openbao-snapshots/", "other/") + case "duplicate": + listing = strings.Replace(listing, "", listing[strings.Index(listing, ""):strings.Index(listing, "")+len("")]+"", 1) + case "old": + listing = strings.ReplaceAll(listing, now.Format(time.RFC3339), now.Add(-48*time.Hour).Format(time.RFC3339)) + case "wrong-size": + size = 9 + case "changed-object": + gotETag = `"` + strings.Repeat("b", 32) + `"` + case "redirect": + status = http.StatusTemporaryRedirect + case "oversized": + listing = strings.Repeat("x", (1<<20)+1) + case "trailing-xml": + listing += "" + } + client := &http.Client{Transport: fixtureTransport(func(r *http.Request) (*http.Response, error) { + if r.Method != http.MethodGet || r.URL.Host != strings.Repeat("c", 32)+".r2.cloudflarestorage.com" || !strings.Contains(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential="+strings.Repeat("d", 32)+"/") { + t.Fatal("request escaped its signed read-only origin") + } + h := make(http.Header) + content := body + n := size + if r.URL.Query().Get("list-type") == "2" { + if r.URL.Query().Get("prefix") != "openbao-snapshots/" || r.URL.Query().Get("max-keys") != "256" || r.URL.Path != "/fixture-backups" { + t.Fatal("unexpected list scope") + } + content = listing + n = int64(len(content)) + } else { + gets++ + if r.URL.Path != "/fixture-backups/openbao-snapshots/"+name || r.Header.Get("If-Match") != etag { + t.Fatal("object is not bound to listing") + } + h.Set("ETag", gotETag) + h.Set("Last-Modified", now.Format(http.TimeFormat)) + } + body := io.NopCloser(strings.NewReader(content)) + if tc == "close-failure" { + body = failedClose{strings.NewReader(content)} + } + return &http.Response{StatusCode: status, Header: h, Body: body, ContentLength: n, Request: r}, nil + })} + target := target{Endpoint: "https://" + strings.Repeat("c", 32) + ".r2.cloudflarestorage.com", Bucket: "fixture-backups", AccessKey: strings.Repeat("d", 32), SecretKey: strings.Repeat("e", 64)} + b, err := downloadSnapshot(context.Background(), client, target, now.Add(-time.Minute), now.Add(time.Minute)) + if (err == nil) != (tc == "matching") || (err == nil && !bytes.Equal(b, []byte("snapshot"))) { + t.Fatal("incorrect independent snapshot verdict") + } + if tc == "matching" && gets != 1 { + t.Fatal("did not download the actual object") + } + if tc != "matching" && len(b) != 0 { + t.Fatal("refused input exposed partial content") + } + }) + } +} + +func TestIndependentProductionReaderLimitsKubernetesObjects(t *testing.T) { + for _, item := range []struct { + k, ns, n string + ok bool + }{ + {"job", "openbao", "vault-snapshot-init", true}, + {"configmap", "flux-system", "variables-base", true}, + {"configmap", "flux-system", "variables-cluster", true}, + {"secret", "flux-system", "variables-cluster", true}, + {"secret", "openbao", "vault-snapshot-r2", true}, + {"secret", "openbao", "openbao-unseal", true}, + {"secret", "openbao", "other", false}, + {"pods", "openbao", "openbao-0", false}, + {"secret", "other", "openbao-unseal", false}, + } { + if allowedRecoveryObject(item.k, item.ns, item.n) != item.ok { + t.Fatal("incorrect fixed Kubernetes read boundary") + } + } +} + +// TestSnapshotTransportNeverForwardsSignedReads exercises real local HTTP connections. +func TestSnapshotTransportNeverForwardsSignedReads(t *testing.T) { + var proxied, followed bool + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { proxied = true; w.WriteHeader(http.StatusNoContent) })) + defer proxy.Close() + destination := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { followed = true; w.WriteHeader(http.StatusOK) })) + defer destination.Close() + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, destination.URL, http.StatusTemporaryRedirect) + })) + defer source.Close() + t.Setenv("HTTP_PROXY", proxy.URL) + t.Setenv("HTTPS_PROXY", proxy.URL) + t.Setenv("NO_PROXY", "") + sourceURL, err := url.Parse(source.URL) + if err != nil { + t.Fatal(err) + } + client := recoveryHTTP() + transport := client.Transport.(*http.Transport) + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address == "recovery-fixture.invalid:80" { + address = sourceURL.Host + } + return (&net.Dialer{}).DialContext(ctx, network, address) + } + response, err := client.Get("http://recovery-fixture.invalid/") + if err != nil { + t.Fatal("could not exercise local transport") + } + if response.Body.Close() != nil { + t.Fatal("could not finish local response") + } + if response.StatusCode != http.StatusTemporaryRedirect || proxied || followed { + t.Fatal("transport forwarded a request through an ambient proxy or redirect") + } +} diff --git a/scripts/openbao-backup-recovery/evidence.go b/scripts/openbao-backup-recovery/evidence.go index e3cda1e082..c8694736f3 100644 --- a/scripts/openbao-backup-recovery/evidence.go +++ b/scripts/openbao-backup-recovery/evidence.go @@ -104,12 +104,16 @@ func recordsEqual(expected, actual map[string]json.RawMessage) bool { } // privateInput refuses links, special files, shared permissions and truncated reads. -func privateInput(path string, limit int64) ([]byte, error) { +func privateInput(path string, limit int64) (result []byte, resultErr error) { f, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0) if err != nil { return nil, errEvidence } - defer f.Close() + defer func() { + if f.Close() != nil { + result, resultErr = nil, errEvidence + } + }() info, err := f.Stat() if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0077 != 0 || info.Size() <= 0 || info.Size() > limit { return nil, errEvidence diff --git a/scripts/openbao-backup-recovery/main.go b/scripts/openbao-backup-recovery/main.go index 9bf2f313dc..6067cd13e7 100644 --- a/scripts/openbao-backup-recovery/main.go +++ b/scripts/openbao-backup-recovery/main.go @@ -411,6 +411,14 @@ audit "file" { } func main() { + if len(os.Args) == 2 && os.Args[1] == "collect" { + if collectRecoveryInputs(os.Getenv("OPENBAO_RECOVERY_DIR")) != nil { + fmt.Fprintln(os.Stderr, "OPENBAO_BACKUP_INPUTS=UNVERIFIED") + os.Exit(2) + } + fmt.Println("OPENBAO_BACKUP_INPUTS=PASS") + return + } if len(os.Args) == 2 && os.Args[1] == "image" { fmt.Println(serverImage) return diff --git a/scripts/openbao-backup-recovery/workflow_test.go b/scripts/openbao-backup-recovery/workflow_test.go new file mode 100644 index 0000000000..ba3946e869 --- /dev/null +++ b/scripts/openbao-backup-recovery/workflow_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestProductionRecoveryRefusesBranchBeforeCheckoutOrCredentials(t *testing.T) { + data, err := os.ReadFile(filepath.Join("..", "..", ".github", "workflows", "verify-openbao-backup-recovery.yaml")) + if err != nil { + t.Fatal("independent backup recovery workflow is missing") + } + type step struct { + Name, Uses, Shell, Run string + Env map[string]string + With map[string]any + } + var workflow struct { + On map[string]any `yaml:"on"` + Permissions map[string]string + Concurrency struct { + Group string + Cancel bool `yaml:"cancel-in-progress"` + } + Jobs map[string]struct { + Environment string + Permissions map[string]string + Steps []step + } + } + if yaml.Unmarshal(data, &workflow) != nil || len(workflow.On) != 1 || workflow.On["workflow_dispatch"] == nil || len(workflow.Permissions) != 0 || len(workflow.Jobs) != 1 || workflow.Concurrency.Group != "prod-deploy" || workflow.Concurrency.Cancel { + t.Fatal("workflow execution boundary changed") + } + job, ok := workflow.Jobs["verify"] + if !ok || job.Environment != "prod" || len(job.Permissions) != 1 || job.Permissions["contents"] != "read" || len(job.Steps) != 5 { + t.Fatal("verification must use one protected read-only job") + } + guard := job.Steps[0] + if guard.Uses != "" || guard.Shell != "bash" || len(guard.Env) != 1 || guard.Env["CONFIRM"] != "${{ inputs.confirm }}" { + t.Fatal("first step must refuse before checkout and secret injection") + } + for _, tc := range []struct { + event, ref, repo, confirm string + pass bool + }{ + {"workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "verify-openbao-backup-recovery", true}, + {"pull_request", "refs/heads/main", "devantler-tech/platform", "verify-openbao-backup-recovery", false}, + {"workflow_dispatch", "refs/heads/codex/other", "devantler-tech/platform", "verify-openbao-backup-recovery", false}, + {"workflow_dispatch", "refs/heads/main", "other/platform", "verify-openbao-backup-recovery", false}, + {"workflow_dispatch", "refs/heads/main", "devantler-tech/platform", "$(exit 0)", false}, + } { + command := exec.Command("bash", "-eu", "-c", guard.Run) + command.Env = []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=" + tc.event, "GITHUB_REF=" + tc.ref, "GITHUB_REPOSITORY=" + tc.repo, "CONFIRM=" + tc.confirm} + if err := command.Run(); (err == nil) != tc.pass { + t.Fatal("guard admitted an unconfirmed dispatch or refused main") + } + } + checkout := job.Steps[1] + if checkout.Uses != "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1" || checkout.With["ref"] != "${{ github.sha }}" || checkout.With["persist-credentials"] != false { + t.Fatal("checkout is not pinned to the confirmed revision") + } + for _, s := range job.Steps[:4] { + if len(s.Env) > 1 || strings.Contains(s.Run, "secrets.") { + t.Fatal("credential access moved before the guarded reader") + } + } + run := job.Steps[4] + if len(run.Env) != 2 || run.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || run.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || strings.Contains(run.Run, "upload-artifact") { + t.Fatal("recovery reader credential boundary changed") + } + for _, required := range []string{"/dev/shm", "OPENBAO_RECOVERY_DIR", "service/openbao-active 38200:8200", "run-openbao-backup-sandbox.sh verify", "OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3"} { + if !strings.Contains(run.Run, required) { + t.Fatal("missing private isolated recovery boundary") + } + } +} From bf9bd01e05a97bdb43b05aaf250bb5dca70b119c Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 27 Oct 2026 23:50:06 +0100 Subject: [PATCH 2/4] fix(backup): use the reviewed private endpoint invocation --- .github/workflows/verify-openbao-backup-recovery.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify-openbao-backup-recovery.yaml b/.github/workflows/verify-openbao-backup-recovery.yaml index 1daba75e23..1889b7c07f 100644 --- a/.github/workflows/verify-openbao-backup-recovery.yaml +++ b/.github/workflows/verify-openbao-backup-recovery.yaml @@ -87,7 +87,13 @@ jobs: export KUBECONFIG="$proof_dir/kubeconfig" printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" unset KUBE_CONFIG - if ! ./scripts/use-prod-stable-api-endpoint.sh > "$proof_dir/endpoint.log" 2>&1; then fail; fi + if ! ( + if ! ./scripts/use-prod-stable-api-endpoint.sh >/dev/null; then + exit 1 + fi + ) 2>"$proof_dir/endpoint.log"; then + fail + fi unset HCLOUD_TOKEN kubectl --context=admin@prod -n openbao port-forward --address=127.0.0.1 service/openbao-active 38200:8200 > "$proof_dir/forward.log" 2>&1 & forward_pid=$! From 9d78bc1a0378819f4df3a4c55ca10446f5ba1461 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 27 Oct 2026 23:57:44 +0100 Subject: [PATCH 3/4] fix(backup): authenticate snapshot reads with the required payload hash --- .github/workflows/verify-openbao-backup-recovery.yaml | 2 +- scripts/openbao-backup-recovery/collector.go | 5 ++++- scripts/openbao-backup-recovery/collector_test.go | 3 +++ scripts/openbao-backup-recovery/workflow_test.go | 3 +++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify-openbao-backup-recovery.yaml b/.github/workflows/verify-openbao-backup-recovery.yaml index 1889b7c07f..29a76ba017 100644 --- a/.github/workflows/verify-openbao-backup-recovery.yaml +++ b/.github/workflows/verify-openbao-backup-recovery.yaml @@ -56,6 +56,7 @@ jobs: mkdir -p "$RUNNER_TEMP/openbao-recovery-tools" install -m 0755 "$kubectl_path" "$RUNNER_TEMP/openbao-recovery-tools/kubectl" printf '%s\n' "$RUNNER_TEMP/openbao-recovery-tools" >> "$GITHUB_PATH" + CGO_ENABLED=0 go build -trimpath -o "$RUNNER_TEMP/openbao-recovery-tools/verifier" ./scripts/openbao-backup-recovery - name: Read the fixed backup inputs and prove isolated recovery shell: bash @@ -82,7 +83,6 @@ jobs: } trap cleanup EXIT binary="$RUNNER_TEMP/openbao-recovery-tools/verifier" - if ! CGO_ENABLED=0 go build -trimpath -o "$binary" ./scripts/openbao-backup-recovery > "$proof_dir/build.log" 2>&1; then fail; fi [[ -n "$KUBE_CONFIG" && ${#KUBE_CONFIG} -le 1048576 ]] || fail export KUBECONFIG="$proof_dir/kubeconfig" printf '%s' "$KUBE_CONFIG" > "$KUBECONFIG" diff --git a/scripts/openbao-backup-recovery/collector.go b/scripts/openbao-backup-recovery/collector.go index 678d75ee21..3f46e73ced 100644 --- a/scripts/openbao-backup-recovery/collector.go +++ b/scripts/openbao-backup-recovery/collector.go @@ -48,7 +48,10 @@ func signedRead(ctx context.Context, client *http.Client, t target, path string, req.Header.Set("If-Match", etag) } sum := sha256.Sum256(nil) - if v4.NewSigner().SignHTTP(ctx, aws.Credentials{AccessKeyID: t.AccessKey, SecretAccessKey: t.SecretKey}, req, hex.EncodeToString(sum[:]), "s3", "auto", time.Now().UTC()) != nil { + payloadHash := hex.EncodeToString(sum[:]) + // SignHTTP uses this hash for signing but does not add S3's required header. + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + if v4.NewSigner().SignHTTP(ctx, aws.Credentials{AccessKeyID: t.AccessKey, SecretAccessKey: t.SecretKey}, req, payloadHash, "s3", "auto", time.Now().UTC()) != nil { return nil, nil, errEvidence } response, err := client.Do(req) diff --git a/scripts/openbao-backup-recovery/collector_test.go b/scripts/openbao-backup-recovery/collector_test.go index 1b1c8912fd..8d16a2f169 100644 --- a/scripts/openbao-backup-recovery/collector_test.go +++ b/scripts/openbao-backup-recovery/collector_test.go @@ -57,6 +57,9 @@ func TestIndependentSnapshotReadRequiresSignedFreshBoundedBytes(t *testing.T) { listing += "" } client := &http.Client{Transport: fixtureTransport(func(r *http.Request) (*http.Response, error) { + if r.Header.Get("X-Amz-Content-Sha256") != "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" { + t.Fatal("S3 read is missing the required empty-payload hash header") + } if r.Method != http.MethodGet || r.URL.Host != strings.Repeat("c", 32)+".r2.cloudflarestorage.com" || !strings.Contains(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential="+strings.Repeat("d", 32)+"/") { t.Fatal("request escaped its signed read-only origin") } diff --git a/scripts/openbao-backup-recovery/workflow_test.go b/scripts/openbao-backup-recovery/workflow_test.go index ba3946e869..b204a40d4e 100644 --- a/scripts/openbao-backup-recovery/workflow_test.go +++ b/scripts/openbao-backup-recovery/workflow_test.go @@ -70,6 +70,9 @@ func TestProductionRecoveryRefusesBranchBeforeCheckoutOrCredentials(t *testing.T } } run := job.Steps[4] + if strings.Contains(run.Run, "go build") || !strings.Contains(job.Steps[3].Run, "go build") { + t.Fatal("the build must run before the production credential environment is injected") + } if len(run.Env) != 2 || run.Env["KUBE_CONFIG"] != "${{ secrets.KUBE_CONFIG }}" || run.Env["HCLOUD_TOKEN"] != "${{ secrets.HCLOUD_TOKEN }}" || strings.Contains(run.Run, "upload-artifact") { t.Fatal("recovery reader credential boundary changed") } From bd21fe8f20d3d0e9a609fd029cb5a608ce1a9c74 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 28 Oct 2026 00:28:02 +0100 Subject: [PATCH 4/4] fix(backup): limit isolated restore inputs --- .../verify-openbao-backup-recovery.yaml | 1 + scripts/openbao-backup-recovery/collector.go | 19 +-- .../openbao-backup-recovery/collector_test.go | 6 +- .../workflow_runtime_test.go | 113 ++++++++++++++++++ 4 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 scripts/openbao-backup-recovery/workflow_runtime_test.go diff --git a/.github/workflows/verify-openbao-backup-recovery.yaml b/.github/workflows/verify-openbao-backup-recovery.yaml index 29a76ba017..68860ef3cb 100644 --- a/.github/workflows/verify-openbao-backup-recovery.yaml +++ b/.github/workflows/verify-openbao-backup-recovery.yaml @@ -115,6 +115,7 @@ jobs: wait "$forward_pid" 2>/dev/null || true forward_pid='' unset OPENBAO_RECOVERY_DIR KUBECONFIG + rm -- "$proof_dir/kubeconfig" image=$("$binary" image) || fail if ! docker pull "$image" > "$proof_dir/pull.log" 2>&1; then fail; fi if ! bash scripts/run-openbao-backup-sandbox.sh verify "$proof_dir" "$binary" > "$proof_dir/verification-result" 2>"$proof_dir/verification.log"; then fail; fi diff --git a/scripts/openbao-backup-recovery/collector.go b/scripts/openbao-backup-recovery/collector.go index 3f46e73ced..0f163d8018 100644 --- a/scripts/openbao-backup-recovery/collector.go +++ b/scripts/openbao-backup-recovery/collector.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "encoding/xml" + "errors" "io" "net" "net/http" @@ -91,7 +92,10 @@ func downloadSnapshot(ctx context.Context, client *http.Client, t target, start, } } decoder := xml.NewDecoder(bytes.NewReader(b)) - if decoder.Decode(&listing) != nil || decoder.Decode(new(any)) != io.EOF || listing.XMLName.Local != "ListBucketResult" || (listing.XMLName.Space != "" && listing.XMLName.Space != "http://s3.amazonaws.com/doc/2006-03-01/") || listing.Name != t.Bucket || listing.Prefix != snapshotPrefix || listing.IsTruncated == nil || *listing.IsTruncated || listing.NextContinuationToken != "" || listing.KeyCount == nil || *listing.KeyCount != len(listing.Contents) || listing.MaxKeys == nil || *listing.MaxKeys != 256 || len(listing.Contents) == 0 || len(listing.Contents) > 256 { + if decoder.Decode(&listing) != nil { + return nil, errEvidence + } + if !errors.Is(decoder.Decode(new(any)), io.EOF) || listing.XMLName.Local != "ListBucketResult" || (listing.XMLName.Space != "" && listing.XMLName.Space != "http://s3.amazonaws.com/doc/2006-03-01/") || listing.Name != t.Bucket || listing.Prefix != snapshotPrefix || listing.IsTruncated == nil || *listing.IsTruncated || listing.NextContinuationToken != "" || listing.KeyCount == nil || *listing.KeyCount != len(listing.Contents) || listing.MaxKeys == nil || *listing.MaxKeys != 256 || len(listing.Contents) == 0 || len(listing.Contents) > 256 { return nil, errEvidence } etags := map[string]string{} @@ -102,7 +106,7 @@ func downloadSnapshot(ctx context.Context, client *http.Client, t target, start, return nil, errEvidence } key := strings.TrimPrefix(item.Key, snapshotPrefix) - if encoder.Encode(snapshotObject{Status: "success", Type: "file", Key: key, Size: item.Size, LastModified: item.LastModified}) != nil { + if err := encoder.Encode(snapshotObject{Status: "success", Type: "file", Key: key, Size: item.Size, LastModified: item.LastModified}); err != nil { return nil, errEvidence } etags[key] = item.ETag @@ -142,7 +146,7 @@ func recoveryObject(ctx context.Context, kind, namespace, name string) ([]byte, if !allowedRecoveryObject(kind, namespace, name) { return nil, errEvidence } - command := exec.CommandContext(ctx, "kubectl", "--context=admin@prod", "--request-timeout=20s", "--namespace="+namespace, "get", kind, name, "--output=json") + command := exec.CommandContext(ctx, "kubectl", "--context=admin@prod", "--request-timeout=20s", "--namespace", namespace, "get", kind, name, "--output=json") var output boundedOutput command.Stdout = &output command.Stderr = io.Discard @@ -153,11 +157,12 @@ func recoveryObject(ctx context.Context, kind, namespace, name string) ([]byte, } func recoveryDirectory(dir string) bool { - if runtime.GOOS != "linux" || os.Getuid() == 0 || !strings.HasPrefix(dir, "/dev/shm/") || filepath.Clean(dir) != dir { + uid := os.Getuid() + if runtime.GOOS != "linux" || uid <= 0 || !strings.HasPrefix(dir, "/dev/shm/") || filepath.Clean(dir) != dir { return false } - real, err := filepath.EvalSymlinks(dir) - if err != nil || real != dir { + resolvedDir, err := filepath.EvalSymlinks(dir) + if err != nil || resolvedDir != dir { return false } info, err := os.Lstat(dir) @@ -165,7 +170,7 @@ func recoveryDirectory(dir string) bool { return false } owner, ok := info.Sys().(*syscall.Stat_t) - if !ok || owner.Uid != uint32(os.Getuid()) { + if !ok || uint64(owner.Uid) != uint64(uid) { return false } var fs syscall.Statfs_t diff --git a/scripts/openbao-backup-recovery/collector_test.go b/scripts/openbao-backup-recovery/collector_test.go index 8d16a2f169..99e43410ee 100644 --- a/scripts/openbao-backup-recovery/collector_test.go +++ b/scripts/openbao-backup-recovery/collector_test.go @@ -148,7 +148,11 @@ func TestSnapshotTransportNeverForwardsSignedReads(t *testing.T) { } return (&net.Dialer{}).DialContext(ctx, network, address) } - response, err := client.Get("http://recovery-fixture.invalid/") + request, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://recovery-fixture.invalid/", nil) + if err != nil { + t.Fatal("could not prepare local transport request") + } + response, err := client.Do(request) if err != nil { t.Fatal("could not exercise local transport") } diff --git a/scripts/openbao-backup-recovery/workflow_runtime_test.go b/scripts/openbao-backup-recovery/workflow_runtime_test.go new file mode 100644 index 0000000000..fab47e0585 --- /dev/null +++ b/scripts/openbao-backup-recovery/workflow_runtime_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// TestRecoveryWorkflowExcludesClusterCredentials exercises the actual shell handoff. +func TestRecoveryWorkflowExcludesClusterCredentials(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("the workflow requires a non-root caller") + } + raw, err := os.ReadFile(filepath.Join("..", "..", ".github", "workflows", "verify-openbao-backup-recovery.yaml")) + if err != nil { + t.Fatal(err) + } + var workflow struct { + Jobs map[string]struct{ Steps []struct{ Run string } } + } + if yaml.Unmarshal(raw, &workflow) != nil || len(workflow.Jobs["verify"].Steps) != 5 { + t.Fatal("cannot read guarded recovery workflow") + } + body := workflow.Jobs["verify"].Steps[4].Run + for _, tc := range []string{"matching", "collection-refused", "private-result", "changed-main"} { + t.Run(tc, func(t *testing.T) { + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + bin := filepath.Join(dir, "bin") + runner := filepath.Join(dir, "runner") + for _, p := range []string{bin, filepath.Join(dir, "scripts"), filepath.Join(runner, "openbao-recovery-tools")} { + if os.MkdirAll(p, 0700) != nil { + t.Fatal("cannot prepare private fixture") + } + } + files := map[string]string{ + "bin/git": `#!/bin/bash +set -eu +[[ $1 == ls-remote ]] || exit 91 +if [[ $FIXTURE_CASE == changed-main ]]; then printf '%040d\trefs/heads/main\n' 2; else printf '%s\trefs/heads/main\n' "$GITHUB_SHA"; fi +`, + "bin/findmnt": "#!/bin/sh\nprintf '%s\\n' tmpfs\n", + "bin/mktemp": `#!/bin/bash +set -eu +[[ $1 == -d && $2 == /dev/shm/openbao-recovery.XXXXXXXX ]] || exit 91 +exec /usr/bin/mktemp -d "$FIXTURE_DIR/proof.XXXXXXXX" +`, + "bin/kubectl": `#!/bin/bash +set -eu +[[ $* == *port-forward* ]] || exit 91 +exec /bin/sleep 30 +`, + "bin/curl": "#!/bin/sh\nexit 0\n", + "bin/docker": "#!/bin/sh\n[ \"$1\" = pull ] || exit 91\n", + "scripts/use-prod-stable-api-endpoint.sh": "#!/bin/sh\nexit 0\n", + "runner/openbao-recovery-tools/verifier": `#!/bin/bash +set -eu +if [[ $1 == image ]]; then printf '%s\n' '` + serverImage + `'; exit 0; fi +[[ $1 == collect ]] || exit 91 +printf '%s' "$OPENBAO_RECOVERY_DIR" > "$FIXTURE_DIR/proof-path" +for name in root-token unseal-key expected.json snapshot.snap; do printf '%s' synthetic > "$OPENBAO_RECOVERY_DIR/$name"; done +if [[ $FIXTURE_CASE == collection-refused ]]; then printf '%s\n' PRIVATE_SYNTHETIC_PAYLOAD; exit 91; fi +printf '%s\n' OPENBAO_BACKUP_INPUTS=PASS +`, + "scripts/run-openbao-backup-sandbox.sh": `#!/bin/bash +set -eu +printf '%s' reached > "$FIXTURE_DIR/sandbox-started" +[[ $1 == verify ]] || exit 91 +if [[ -e $2/kubeconfig ]]; then printf '%s' exposed > "$FIXTURE_DIR/kubeconfig-exposed"; printf '%s\n' PRIVATE_SYNTHETIC_CLUSTER_CREDENTIAL >&2; exit 91; fi +for name in root-token unseal-key expected.json snapshot.snap; do [[ -s $2/$name ]] || exit 91; done +if [[ $FIXTURE_CASE == private-result ]]; then printf '%s\n' PRIVATE_SYNTHETIC_PAYLOAD; else printf '%s\n' 'OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3'; fi +`, + } + for name, text := range files { + if os.WriteFile(filepath.Join(dir, name), []byte(text), 0700) != nil { + t.Fatal("cannot write private fixture tool") + } + } + command := exec.Command("bash", "-c", body) + command.Dir = dir + command.Env = []string{"PATH=" + bin + ":" + os.Getenv("PATH"), "RUNNER_TEMP=" + runner, "GITHUB_SHA=" + strings.Repeat("1", 40), "KUBE_CONFIG=SYNTHETIC_KUBECONFIG", "HCLOUD_TOKEN=SYNTHETIC_PROVIDER_VALUE", "FIXTURE_DIR=" + dir, "FIXTURE_CASE=" + tc} + out, runErr := command.CombinedOutput() + if (runErr == nil) != (tc == "matching") { + if _, err := os.Stat(filepath.Join(dir, "kubeconfig-exposed")); err == nil { + t.Fatal("sandbox received an unintended cluster credential") + } + t.Fatal("incorrect private handoff verdict", runErr) + } + if strings.Contains(string(out), "PRIVATE_SYNTHETIC") || strings.Contains(string(out), "SYNTHETIC_KUBECONFIG") || strings.Contains(string(out), "SYNTHETIC_PROVIDER") { + t.Fatal("workflow exposed private fixture content") + } + if tc == "matching" && strings.TrimSpace(string(out)) != "OPENBAO_BACKUP_RESTORE=PASS records=3 version=2.6.3" { + t.Fatal("workflow accepted an unverified result") + } + proof, readErr := os.ReadFile(filepath.Join(dir, "proof-path")) + if readErr == nil { + if _, statErr := os.Stat(string(proof)); !os.IsNotExist(statErr) { + t.Fatal("workflow retained its private scratch directory") + } + } + _, sandboxErr := os.Stat(filepath.Join(dir, "sandbox-started")) + if (sandboxErr == nil) != (tc == "matching" || tc == "private-result") { + t.Fatal("workflow reached sandbox after a refused collection or stale main") + } + }) + } +}