Add resilient S3 content storage

.env.example

@@ -8,6 +8,15 @@ HOSTING_DOMAIN=http://localhost:5173
 DATA_DIR=./data
 JSON_BODY_LIMIT=80mb
 
+# Durable media defaults to the local DATA_DIR. Production replicas can share
+# an S3-compatible backend while retaining DATA_DIR only for conversion work.
+STORAGE_BACKEND=filesystem
+# S3_ENDPOINT=http://garage.object-store-system.svc.cluster.local:3900
+# S3_REGION=garage
+# S3_BUCKET=papertrade
+# S3_ACCESS_KEY_ID=
+# S3_SECRET_ACCESS_KEY=
+
 SQL_CLIENT=mysql2
 SQL_DATABASE_HOST=127.0.0.1
 SQL_DATABASE_PORT=3306

.github/security/trivy-critical-allowlist.json

@@ -3,71 +3,71 @@
     "vulnerability": "CVE-2026-58016",
     "package": "libglib2.0-0t64",
     "installed_version": "2.84.4-3~deb13u3",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "Debian Trixie marks the GDBus introspection XML denial of service as minor/no-DSA and has no Trixie fix; PaperTrade does not parse caller-supplied D-Bus introspection XML.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-58016"
   },
   {
     "vulnerability": "CVE-2026-34873",
     "package": "libmbedcrypto16",
     "installed_version": "3.6.5-0.1~deb13u1",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "Debian Trixie marks this TLS 1.3 resumption issue as minor/no-DSA and has no Trixie fix; the runtime carries the crypto library transitively and does not use Mbed TLS for its public HTTPS termination.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-34873"
   },
   {
     "vulnerability": "CVE-2026-34875",
     "package": "libmbedcrypto16",
     "installed_version": "3.6.5-0.1~deb13u1",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "Debian Trixie marks the FFDH export overflow as minor/no-DSA and has no Trixie fix; PaperTrade does not expose an Mbed TLS FFDH key-export operation.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-34875"
   },
   {
     "vulnerability": "CVE-2026-6653",
     "package": "libxml2",
     "installed_version": "2.12.7+dfsg+really2.9.14-2.1+deb13u3",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "Debian Trixie marks the crafted-XML denial of service as minor/no-DSA and has no Trixie fix. Document conversion can process untrusted input, so this remains a short exception that must be removed as soon as Debian publishes a fix.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-6653"
   },
   {
     "vulnerability": "CVE-2026-13221",
     "package": "perl-base",
     "installed_version": "5.40.1-6",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "Trixie has no fixed Perl package; PaperTrade does not construct attacker-controlled Perl regular expressions with more than 65,535 literal branches.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-13221"
   },
   {
     "vulnerability": "CVE-2026-42496",
     "package": "perl-base",
     "installed_version": "5.40.1-6",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "Debian postponed the Trixie Archive::Tar symlink fix while upstream regressions are resolved; the application does not extract caller-supplied archives through Perl Archive::Tar.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-42496"
   },
   {
     "vulnerability": "CVE-2026-8376",
     "package": "perl-base",
     "installed_version": "5.40.1-6",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "The overflow requires a 32-bit Perl build and an attacker-controlled regular expression; the production image is Linux/amd64. Debian classifies it as minor/no-DSA with no current Trixie fix.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-8376"
   },
   {
     "vulnerability": "CVE-2026-54058",
     "package": "python3-pil",
     "installed_version": "11.1.0-5+deb13u4",
-    "reviewed_on": "2026-08-24",
-    "expires": "2026-09-07",
+    "reviewed_on": "2026-09-10",
+    "expires": "2026-09-17",
     "rationale": "Debian Trixie marks the McIDAS AREA mmap disclosure/denial of service as minor/no-DSA and has no Trixie fix. The affected decoder is not a supported PaperTrade input path, but conversion handles untrusted files, so the exception is deliberately short.",
     "source": "https://security-tracker.debian.org/tracker/CVE-2026-54058"
   }

README.md

@@ -90,6 +90,13 @@ docker build -f Dockerfile.runtime-base -t papertrade-runtime-base:local .
 docker build -t papertrade:local .
 ```
 
+Production instances can set `STORAGE_BACKEND=s3` and the `S3_*` variables in
+`.env.example` so publication pages, source files, avatars, and appearance
+assets are shared across replicas. Document conversion still uses `DATA_DIR`
+as local scratch space. Each processed publication version is written to a new
+object prefix before its database rows are updated, so a failed replacement
+cannot overwrite the previously published files.
+
 ## Project Layout
 
 ```text

docs/devops.md

@@ -31,6 +31,22 @@ docker build -t papertrade:local .
 Production builds pass `RUNTIME_BASE_IMAGE` explicitly, so the public Dockerfile
 default remains usable for local builders without changing the cluster workflow.
 
+## Durable content storage
+
+Production runs two replicas on separate nodes. Both use the internal S3 API
+for publication files, rendered pages, avatars, and appearance assets. The
+`papertrade-s3-credentials` Secret supplies a bucket-specific access key;
+database, wallet, and S3 credentials are never stored in this repository.
+`/data/papertrade` is an `emptyDir` scratch filesystem used only for upload,
+conversion, rendering, and OCR work.
+
+The deployment has a PDB with `minAvailable: 1` and hard hostname anti-affinity.
+Before node maintenance, require two Ready endpoints on separate nodes and a
+healthy four-member object store. Existing filesystem content must be copied
+from a read-only source mount without delete or sync semantics before enabling
+the S3-backed deployment. Retain the source PVC and PV through cutover
+validation and the first verified off-site backup.
+
 ## Build Cache
 
 `scripts/k8s/build-local-image.sh` runs Kaniko in the cluster and enables a

infra/kubernetes/base/configmap.yaml

@@ -16,6 +16,10 @@ data:
   HOSTING_DOMAIN: "https://papertrade.metanet.app"
   WALLET_STORAGE_URL: "https://storage.babbage.systems"
   DATA_DIR: "/data/papertrade"
+  STORAGE_BACKEND: "s3"
+  S3_ENDPOINT: "http://garage.object-store-system.svc.cluster.local:3900"
+  S3_REGION: "garage"
+  S3_BUCKET: "papertrade"
   SQL_DATABASE_HOST: "shared-mysql-haproxy.cars-operator-system.svc.cluster.local"
   SQL_DATABASE_PORT: "3306"
   SQL_DATABASE_DB_NAME: "papertrade"

infra/kubernetes/base/kustomization.yaml

@@ -4,10 +4,9 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - configmap.yaml
-  - pvc.yaml
   - service.yaml
   - workload.yaml
-  - ingress.yaml
+  - pdb.yaml
 
 commonLabels:
   app.kubernetes.io/part-of: papertrade

infra/kubernetes/base/pdb.yaml

@@ -0,0 +1,13 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: papertrade
+  labels:
+    app.kubernetes.io/name: papertrade
+    app.kubernetes.io/component: api
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: papertrade
+      app.kubernetes.io/component: api

infra/kubernetes/base/service.yaml

@@ -10,6 +10,7 @@ spec:
   selector:
     app.kubernetes.io/name: papertrade
     app.kubernetes.io/component: api
+    storage.papertrade.metanet.app/backend: s3
   ports:
     - name: http
       port: 8080

infra/kubernetes/base/workload.yaml

@@ -6,7 +6,8 @@ metadata:
     app.kubernetes.io/name: papertrade
     app.kubernetes.io/component: api
 spec:
-  replicas: 1
+  replicas: 2
+  minReadySeconds: 10
   strategy:
     type: RollingUpdate
     rollingUpdate:
@@ -21,18 +22,16 @@ spec:
       labels:
         app.kubernetes.io/name: papertrade
         app.kubernetes.io/component: api
+        storage.papertrade.metanet.app/backend: s3
     spec:
-      # PaperTrade's content PVC is ReadWriteOnce. Keep the surge pod on the
-      # current pod's node so both revisions can mount it during a zero-downtime
-      # rollout without attempting a cross-node Longhorn attachment.
       affinity:
-        podAffinity:
+        podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
+            - topologyKey: kubernetes.io/hostname
+              labelSelector:
                 matchLabels:
                   app.kubernetes.io/name: papertrade
                   app.kubernetes.io/component: api
-              topologyKey: kubernetes.io/hostname
       containers:
         - name: papertrade
           image: papertrade:local
@@ -45,6 +44,17 @@ spec:
                 name: papertrade-config
             - secretRef:
                 name: papertrade-secrets
+          env:
+            - name: S3_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: papertrade-s3-credentials
+                  key: access-key
+            - name: S3_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: papertrade-s3-credentials
+                  key: secret-key
           volumeMounts:
             - name: papertrade-data
               mountPath: /data/papertrade
@@ -79,5 +89,5 @@ spec:
               memory: 6Gi
       volumes:
         - name: papertrade-data
-          persistentVolumeClaim:
-            claimName: papertrade-data
+          emptyDir:
+            sizeLimit: 10Gi