diff --git a/.env.example b/.env.example index 6ca6ce1..1dcaeec 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,15 @@ HOSTING_DOMAIN=http://localhost:5173 DATA_DIR=./data JSON_BODY_LIMIT=80mb +# Durable media defaults to the local DATA_DIR. Production replicas can share +# an S3-compatible backend while retaining DATA_DIR only for conversion work. +STORAGE_BACKEND=filesystem +# S3_ENDPOINT=http://garage.object-store-system.svc.cluster.local:3900 +# S3_REGION=garage +# S3_BUCKET=papertrade +# S3_ACCESS_KEY_ID= +# S3_SECRET_ACCESS_KEY= + SQL_CLIENT=mysql2 SQL_DATABASE_HOST=127.0.0.1 SQL_DATABASE_PORT=3306 diff --git a/.github/security/trivy-critical-allowlist.json b/.github/security/trivy-critical-allowlist.json index 319bf2a..4c0c41e 100644 --- a/.github/security/trivy-critical-allowlist.json +++ b/.github/security/trivy-critical-allowlist.json @@ -3,8 +3,8 @@ "vulnerability": "CVE-2026-58016", "package": "libglib2.0-0t64", "installed_version": "2.84.4-3~deb13u3", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "Debian Trixie marks the GDBus introspection XML denial of service as minor/no-DSA and has no Trixie fix; PaperTrade does not parse caller-supplied D-Bus introspection XML.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-58016" }, @@ -12,8 +12,8 @@ "vulnerability": "CVE-2026-34873", "package": "libmbedcrypto16", "installed_version": "3.6.5-0.1~deb13u1", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "Debian Trixie marks this TLS 1.3 resumption issue as minor/no-DSA and has no Trixie fix; the runtime carries the crypto library transitively and does not use Mbed TLS for its public HTTPS termination.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-34873" }, @@ -21,8 +21,8 @@ "vulnerability": "CVE-2026-34875", "package": "libmbedcrypto16", "installed_version": "3.6.5-0.1~deb13u1", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "Debian Trixie marks the FFDH export overflow as minor/no-DSA and has no Trixie fix; PaperTrade does not expose an Mbed TLS FFDH key-export operation.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-34875" }, @@ -30,8 +30,8 @@ "vulnerability": "CVE-2026-6653", "package": "libxml2", "installed_version": "2.12.7+dfsg+really2.9.14-2.1+deb13u3", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "Debian Trixie marks the crafted-XML denial of service as minor/no-DSA and has no Trixie fix. Document conversion can process untrusted input, so this remains a short exception that must be removed as soon as Debian publishes a fix.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-6653" }, @@ -39,8 +39,8 @@ "vulnerability": "CVE-2026-13221", "package": "perl-base", "installed_version": "5.40.1-6", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "Trixie has no fixed Perl package; PaperTrade does not construct attacker-controlled Perl regular expressions with more than 65,535 literal branches.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-13221" }, @@ -48,8 +48,8 @@ "vulnerability": "CVE-2026-42496", "package": "perl-base", "installed_version": "5.40.1-6", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "Debian postponed the Trixie Archive::Tar symlink fix while upstream regressions are resolved; the application does not extract caller-supplied archives through Perl Archive::Tar.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-42496" }, @@ -57,8 +57,8 @@ "vulnerability": "CVE-2026-8376", "package": "perl-base", "installed_version": "5.40.1-6", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "The overflow requires a 32-bit Perl build and an attacker-controlled regular expression; the production image is Linux/amd64. Debian classifies it as minor/no-DSA with no current Trixie fix.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-8376" }, @@ -66,8 +66,8 @@ "vulnerability": "CVE-2026-54058", "package": "python3-pil", "installed_version": "11.1.0-5+deb13u4", - "reviewed_on": "2026-08-24", - "expires": "2026-09-07", + "reviewed_on": "2026-09-10", + "expires": "2026-09-17", "rationale": "Debian Trixie marks the McIDAS AREA mmap disclosure/denial of service as minor/no-DSA and has no Trixie fix. The affected decoder is not a supported PaperTrade input path, but conversion handles untrusted files, so the exception is deliberately short.", "source": "https://security-tracker.debian.org/tracker/CVE-2026-54058" } diff --git a/README.md b/README.md index 215ce38..51deeeb 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,13 @@ docker build -f Dockerfile.runtime-base -t papertrade-runtime-base:local . docker build -t papertrade:local . ``` +Production instances can set `STORAGE_BACKEND=s3` and the `S3_*` variables in +`.env.example` so publication pages, source files, avatars, and appearance +assets are shared across replicas. Document conversion still uses `DATA_DIR` +as local scratch space. Each processed publication version is written to a new +object prefix before its database rows are updated, so a failed replacement +cannot overwrite the previously published files. + ## Project Layout ```text diff --git a/docs/devops.md b/docs/devops.md index dcfc2fb..e3bf160 100644 --- a/docs/devops.md +++ b/docs/devops.md @@ -31,6 +31,22 @@ docker build -t papertrade:local . Production builds pass `RUNTIME_BASE_IMAGE` explicitly, so the public Dockerfile default remains usable for local builders without changing the cluster workflow. +## Durable content storage + +Production runs two replicas on separate nodes. Both use the internal S3 API +for publication files, rendered pages, avatars, and appearance assets. The +`papertrade-s3-credentials` Secret supplies a bucket-specific access key; +database, wallet, and S3 credentials are never stored in this repository. +`/data/papertrade` is an `emptyDir` scratch filesystem used only for upload, +conversion, rendering, and OCR work. + +The deployment has a PDB with `minAvailable: 1` and hard hostname anti-affinity. +Before node maintenance, require two Ready endpoints on separate nodes and a +healthy four-member object store. Existing filesystem content must be copied +from a read-only source mount without delete or sync semantics before enabling +the S3-backed deployment. Retain the source PVC and PV through cutover +validation and the first verified off-site backup. + ## Build Cache `scripts/k8s/build-local-image.sh` runs Kaniko in the cluster and enables a diff --git a/infra/kubernetes/base/configmap.yaml b/infra/kubernetes/base/configmap.yaml index 65f9e39..1f8b1ab 100644 --- a/infra/kubernetes/base/configmap.yaml +++ b/infra/kubernetes/base/configmap.yaml @@ -16,6 +16,10 @@ data: HOSTING_DOMAIN: "https://papertrade.metanet.app" WALLET_STORAGE_URL: "https://storage.babbage.systems" DATA_DIR: "/data/papertrade" + STORAGE_BACKEND: "s3" + S3_ENDPOINT: "http://garage.object-store-system.svc.cluster.local:3900" + S3_REGION: "garage" + S3_BUCKET: "papertrade" SQL_DATABASE_HOST: "shared-mysql-haproxy.cars-operator-system.svc.cluster.local" SQL_DATABASE_PORT: "3306" SQL_DATABASE_DB_NAME: "papertrade" diff --git a/infra/kubernetes/base/ingress.yaml b/infra/kubernetes/base/ingress.yaml deleted file mode 100644 index 444fe76..0000000 --- a/infra/kubernetes/base/ingress.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: papertrade - labels: - app.kubernetes.io/name: papertrade - app.kubernetes.io/component: ingress - annotations: - nginx.ingress.kubernetes.io/proxy-body-size: "300m" - nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" - nginx.ingress.kubernetes.io/proxy-read-timeout: "300" - nginx.ingress.kubernetes.io/proxy-send-timeout: "300" -spec: - ingressClassName: nginx - tls: - - hosts: - - papertrade.metanet.app - secretName: papertrade-tls - rules: - - host: papertrade.metanet.app - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: papertrade - port: - number: 8080 ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: papertrade-tls - labels: - app.kubernetes.io/name: papertrade - app.kubernetes.io/component: tls -spec: - secretName: papertrade-tls - dnsNames: - - papertrade.metanet.app - issuerRef: - kind: ClusterIssuer - name: letsencrypt-nginx diff --git a/infra/kubernetes/base/kustomization.yaml b/infra/kubernetes/base/kustomization.yaml index 03a0f74..58a7b37 100644 --- a/infra/kubernetes/base/kustomization.yaml +++ b/infra/kubernetes/base/kustomization.yaml @@ -4,10 +4,9 @@ kind: Kustomization resources: - namespace.yaml - configmap.yaml - - pvc.yaml - service.yaml - workload.yaml - - ingress.yaml + - pdb.yaml commonLabels: app.kubernetes.io/part-of: papertrade diff --git a/infra/kubernetes/base/pdb.yaml b/infra/kubernetes/base/pdb.yaml new file mode 100644 index 0000000..328256f --- /dev/null +++ b/infra/kubernetes/base/pdb.yaml @@ -0,0 +1,13 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: papertrade + labels: + app.kubernetes.io/name: papertrade + app.kubernetes.io/component: api +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: papertrade + app.kubernetes.io/component: api diff --git a/infra/kubernetes/base/pvc.yaml b/infra/kubernetes/base/pvc.yaml deleted file mode 100644 index 6669641..0000000 --- a/infra/kubernetes/base/pvc.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: papertrade-data - labels: - app.kubernetes.io/name: papertrade - app.kubernetes.io/component: content-storage -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi diff --git a/infra/kubernetes/base/service.yaml b/infra/kubernetes/base/service.yaml index 1fe2e8b..4c4e828 100644 --- a/infra/kubernetes/base/service.yaml +++ b/infra/kubernetes/base/service.yaml @@ -10,6 +10,7 @@ spec: selector: app.kubernetes.io/name: papertrade app.kubernetes.io/component: api + storage.papertrade.metanet.app/backend: s3 ports: - name: http port: 8080 diff --git a/infra/kubernetes/base/workload.yaml b/infra/kubernetes/base/workload.yaml index e2fef70..42301a7 100644 --- a/infra/kubernetes/base/workload.yaml +++ b/infra/kubernetes/base/workload.yaml @@ -6,7 +6,8 @@ metadata: app.kubernetes.io/name: papertrade app.kubernetes.io/component: api spec: - replicas: 1 + replicas: 2 + minReadySeconds: 10 strategy: type: RollingUpdate rollingUpdate: @@ -21,18 +22,16 @@ spec: labels: app.kubernetes.io/name: papertrade app.kubernetes.io/component: api + storage.papertrade.metanet.app/backend: s3 spec: - # PaperTrade's content PVC is ReadWriteOnce. Keep the surge pod on the - # current pod's node so both revisions can mount it during a zero-downtime - # rollout without attempting a cross-node Longhorn attachment. affinity: - podAffinity: + podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: + - topologyKey: kubernetes.io/hostname + labelSelector: matchLabels: app.kubernetes.io/name: papertrade app.kubernetes.io/component: api - topologyKey: kubernetes.io/hostname containers: - name: papertrade image: papertrade:local @@ -45,6 +44,17 @@ spec: name: papertrade-config - secretRef: name: papertrade-secrets + env: + - name: S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: papertrade-s3-credentials + key: access-key + - name: S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: papertrade-s3-credentials + key: secret-key volumeMounts: - name: papertrade-data mountPath: /data/papertrade @@ -79,5 +89,5 @@ spec: memory: 6Gi volumes: - name: papertrade-data - persistentVolumeClaim: - claimName: papertrade-data + emptyDir: + sizeLimit: 10Gi diff --git a/package-lock.json b/package-lock.json index 2b3abff..f78956c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.2.0", "license": "Open BSV License", "dependencies": { + "@aws-sdk/client-s3": "^3.1130.0", "@bsv/auth-express-middleware": "2.2.1", "@bsv/identity-react": "^1.1.14", "@bsv/payment-express-middleware": "2.1.5", @@ -72,6 +73,314 @@ "dev": true, "license": "ISC" }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1001.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.0.tgz", + "integrity": "sha512-6uTniZc87q+B5eXouGTl+7Tmc482rEeCcvxpsvREP8EfF0gvloRZ41UOA9sbSJlyy8TbqIBXb3kKfKarEArUQA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1130.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1130.0.tgz", + "integrity": "sha512-TUe0hgQi3RtuccmTgpUsRIuPk07ZtJE6s3vpOeWCM8sibLNXPVKweemNEz5gwaRYTor8MAGgjlyVZfKGuu1ZOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1001.0", + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/credential-provider-node": "^3.972.83", + "@aws-sdk/middleware-sdk-s3": "^3.972.76", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.978.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.0.tgz", + "integrity": "sha512-2yX9LUmxPklVjSGTb8dfnWRJSiFQ3TeH2nn7G1mdKHTfnabzF0+gfrS8rYfLWmZrQ8A3mEcxMJjRc51dL5KWaA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@aws-sdk/xml-builder": "^3.972.40", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.33.3", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.71", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.71.tgz", + "integrity": "sha512-JN+JHruYZw3GUZB8YGAlDk4wTDPOEAEEdEzj5nS0xodWR4smzHsN7PnK2j6IeOsDIj2aqua5DSbhXl9Gtf90FQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.73", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.73.tgz", + "integrity": "sha512-uyYYnJOnlis8uQzaYGPd7N1JoioCoNpXgnkXYixsWJXHXgXyYi8WXJSDfofxJeWfQIGWLe2Nwyq60Uc7MZdVOg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.16", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.16.tgz", + "integrity": "sha512-i++ly+0Uxa+u3ebSSyr0S/3CFhFJDxCXT3+Zj+mW2bXenEx5bKGCdTIKFu39SgXBNhWDjex/8cXUx9MUTMCrTw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/credential-provider-env": "^3.972.71", + "@aws-sdk/credential-provider-http": "^3.972.73", + "@aws-sdk/credential-provider-login": "^3.972.78", + "@aws-sdk/credential-provider-process": "^3.972.71", + "@aws-sdk/credential-provider-sso": "^3.973.15", + "@aws-sdk/credential-provider-web-identity": "^3.972.77", + "@aws-sdk/nested-clients": "^3.997.45", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.78", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.78.tgz", + "integrity": "sha512-eUtswnXu0+Ii9ieRK+0L7aPFV3Z/dnW2VntJzjBP9xs8s+8p5nBNuymIXtXwZ+5r5+XJP3e32nMkuZ/r0HozEA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/nested-clients": "^3.997.45", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.83", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.83.tgz", + "integrity": "sha512-jdso7ejzfRnatxMUZK4S/U6KbaDPCvfIV4XL+IQAPFDBt5rj5Fq595euqlK8Le4lNCMFR9oUpt+1l0aMgaayOQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.71", + "@aws-sdk/credential-provider-http": "^3.972.73", + "@aws-sdk/credential-provider-ini": "^3.973.16", + "@aws-sdk/credential-provider-process": "^3.972.71", + "@aws-sdk/credential-provider-sso": "^3.973.15", + "@aws-sdk/credential-provider-web-identity": "^3.972.77", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.71", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.71.tgz", + "integrity": "sha512-lYmXJa4gvq4xN1lrT5NiP5vIYYKcGWAdj8y+8o6dlcateB5eF3Dn8DtmjjHKfMBrTPAMr2pebIiX/UOj8c1/UA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.15", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.15.tgz", + "integrity": "sha512-6Jhcf4v0pSFdjk1EW2kvzuEBKD+UZ2uNcHUIglKKLndD20YhvkL2kdmDOV5/j4mYuWWwe/a1FQ1aomU86/Cg5Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/nested-clients": "^3.997.45", + "@aws-sdk/token-providers": "3.1129.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.77", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.77.tgz", + "integrity": "sha512-uylIQSUWpfLuH2LovxEEfwzJGM/SabLOfLMg6YXu/E8jJEKUdpdILCVCQCdFvHyu/7dLJOHPMfrSwduxO56NkQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/nested-clients": "^3.997.45", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.76.tgz", + "integrity": "sha512-NfnTkVUTBKTBuBgqaapFK9r3YdkKt1b2oRvgLzZq91bwNKh6ZS0S7sEcheguttREaL4iyfs/xQnqD7Z7AsWSsA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.45", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.45.tgz", + "integrity": "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.46.tgz", + "integrity": "sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1129.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1129.0.tgz", + "integrity": "sha512-Sbl3rpzQdsG4ZK2zh0JWUYyZPKKorJlVOddA2T0DVbKJFrsW8J6wgnslxxUH04+WaBMr4A1HzJZvZX0xUvkniA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.0", + "@aws-sdk/nested-clients": "^3.997.45", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.5.tgz", + "integrity": "sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.40", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.40.tgz", + "integrity": "sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -2502,6 +2811,87 @@ "integrity": "sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==", "license": "MIT" }, + "node_modules/@smithy/core": { + "version": "3.33.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.3.tgz", + "integrity": "sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz", + "integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", + "integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.3.tgz", + "integrity": "sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.18.0.tgz", + "integrity": "sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -3624,6 +4014,7 @@ "version": "13.0.2", "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-13.0.2.tgz", "integrity": "sha512-jW6oufeDhXZaiX9Lw5A+oerVClx4iFrI6uDj1zu7SqUAjak9vbJvA0NEcKLNxHiQHb6kYCoFzzXYV0YOauhV3g==", + "hasInstallScript": true, "license": "MIT", "dependencies": { "node-addon-api": "^8.0.0" @@ -3707,6 +4098,12 @@ "node": ">= 0.8" } }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.15.tgz", @@ -10498,7 +10895,6 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, "license": "0BSD" }, "node_modules/tsutils": { diff --git a/package.json b/package.json index aa60b9f..13b3f76 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "lint": "ts-standard --project tsconfig.json \"src/client/**/*.ts\" \"src/client/**/*.tsx\" && ts-standard --project tsconfig.server.json \"src/server/**/*.ts\"" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1130.0", "@bsv/auth-express-middleware": "2.2.1", "@bsv/identity-react": "^1.1.14", "@bsv/payment-express-middleware": "2.1.5", diff --git a/scripts/k8s/deploy-local.sh b/scripts/k8s/deploy-local.sh index 881a5a1..9cdee25 100755 --- a/scripts/k8s/deploy-local.sh +++ b/scripts/k8s/deploy-local.sh @@ -51,6 +51,9 @@ printf 'Deploying PaperTrade image tag %s\n' "${IMAGE_TAG}" --from-env-file="${secret_env_file}" \ --dry-run=client \ -o yaml | "${kubectl_cmd}" apply -f - +"${kubectl_cmd}" -n "${namespace}" get secret papertrade-s3-credentials \ + -o jsonpath='{.data.access-key}{" "}{.data.secret-key}{"\n"}' \ + | grep -Eq '^[^ ]+ [^ ]+$' "${kubectl_cmd}" kustomize "${tmp_dir}/infra/kubernetes/overlays/prod" | "${kubectl_cmd}" apply -f - "${kubectl_cmd}" -n "${namespace}" rollout status deployment/papertrade --timeout=15m "${kubectl_cmd}" -n "${namespace}" wait --for=condition=Ready certificate/papertrade-tls --timeout=20m diff --git a/src/server/content.ts b/src/server/content.ts index 9eb6c4e..a076377 100644 --- a/src/server/content.ts +++ b/src/server/content.ts @@ -1,8 +1,9 @@ import fs from 'fs/promises' import path from 'path' -import { createHash } from 'crypto' +import { createHash, randomUUID } from 'crypto' import { execFile } from 'child_process' import { promisify } from 'util' +import { materializeStoredFile, readStoredFile, storeDirectory, storedFileExists, storeFile } from './objectStorage.js' const execFileAsync = promisify(execFile) const DATA_DIR = process.env.DATA_DIR ?? '/data/papertrade' @@ -177,20 +178,21 @@ export async function ensurePageText ( pageNumber: number, existingTextPath?: string | null ): Promise<{ text: string, textPath: string, textSha256: string, textBytes: number, textSource: 'pdf' | 'ocr' | 'none' }> { - if (existingTextPath != null && existingTextPath !== '') { - try { - const text = normalizePageText(await fs.readFile(existingTextPath, 'utf8')) - return { - text, - textPath: existingTextPath, - textSha256: await sha256File(existingTextPath), - textBytes: await statSize(existingTextPath), - textSource: text === '' ? 'none' : 'pdf' - } - } catch {} + if (existingTextPath != null && existingTextPath !== '' && await storedFileExists(existingTextPath)) { + const bytes = await readStoredFile(existingTextPath) + const text = normalizePageText(bytes.toString('utf8')) + return { + text, + textPath: existingTextPath, + textSha256: createHash('sha256').update(bytes).digest('hex'), + textBytes: bytes.length, + textSource: text === '' ? 'none' : 'pdf' + } } + await Promise.all([materializeStoredFile(canonicalPdfPath), materializeStoredFile(imagePath)]) const extracted = await extractPageText(canonicalPdfPath, imagePath, publicationDir, pageNumber) - return { text: normalizePageText(await fs.readFile(extracted.textPath, 'utf8')), ...extracted } + await storeFile(extracted.textPath, 'text/plain; charset=utf-8') + return { text: normalizePageText((await readStoredFile(extracted.textPath)).toString('utf8')), ...extracted } } export async function processPublicationFile ( @@ -204,8 +206,7 @@ export async function processPublicationFile ( throw new Error('PaperTrade accepts PDF, docx, or ePub files') } - const publicationDir = getPublicationDir(publicationId) - await fs.rm(publicationDir, { recursive: true, force: true }) + const publicationDir = path.join(getPublicationDir(publicationId), 'versions', randomUUID()) await fs.mkdir(publicationDir, { recursive: true }) const sourcePath = await copyUploadedSource(tempPath, publicationDir, extension) const canonicalPdfPath = await convertToPdf(sourcePath, extension, publicationDir) @@ -221,6 +222,8 @@ export async function processPublicationFile ( pages.push({ ...rendered, ...extracted }) } + await storeDirectory(publicationDir) + return { publicationDir, sourcePath, diff --git a/src/server/objectStorage.test.ts b/src/server/objectStorage.test.ts new file mode 100644 index 0000000..5746023 --- /dev/null +++ b/src/server/objectStorage.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest' +import { storageKeyForPath } from './objectStorage.js' + +describe('object storage paths', () => { + it('maps durable data paths to portable object keys', () => { + expect(storageKeyForPath('/data/papertrade/publications/example/page-0001.png')) + .toBe('publications/example/page-0001.png') + }) + + it('rejects paths outside the PaperTrade data directory', () => { + expect(() => storageKeyForPath('/etc/passwd')).toThrow('outside DATA_DIR') + }) +}) diff --git a/src/server/objectStorage.ts b/src/server/objectStorage.ts new file mode 100644 index 0000000..6bdc560 --- /dev/null +++ b/src/server/objectStorage.ts @@ -0,0 +1,156 @@ +import fs from 'fs/promises' +import { createReadStream } from 'fs' +import path from 'path' +import { + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client +} from '@aws-sdk/client-s3' + +const DATA_DIR = path.resolve(process.env.DATA_DIR ?? '/data/papertrade') +const STORAGE_BACKEND = process.env.STORAGE_BACKEND ?? 'filesystem' +const S3_BUCKET = process.env.S3_BUCKET ?? '' +const configuredRegion = process.env.S3_REGION?.trim() + +function requiredEnvironment (name: string): string { + const value = process.env[name]?.trim() + if (value == null || value === '') throw new Error(`${name} is required when STORAGE_BACKEND=s3`) + return value +} + +const s3 = STORAGE_BACKEND === 's3' + ? new S3Client({ + endpoint: requiredEnvironment('S3_ENDPOINT'), + region: configuredRegion != null && configuredRegion !== '' ? configuredRegion : 'garage', + forcePathStyle: true, + credentials: { + accessKeyId: requiredEnvironment('S3_ACCESS_KEY_ID'), + secretAccessKey: requiredEnvironment('S3_SECRET_ACCESS_KEY') + } + }) + : null + +if (STORAGE_BACKEND !== 'filesystem' && STORAGE_BACKEND !== 's3') { + throw new Error(`Unsupported STORAGE_BACKEND: ${STORAGE_BACKEND}`) +} +if (STORAGE_BACKEND === 's3' && S3_BUCKET.trim() === '') { + throw new Error('S3_BUCKET is required when STORAGE_BACKEND=s3') +} + +export function objectStorageEnabled (): boolean { + return s3 != null +} + +export function storageKeyForPath (filePath: string): string { + const absolute = path.resolve(filePath) + const relative = path.relative(DATA_DIR, absolute) + if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`Storage path is outside DATA_DIR: ${filePath}`) + } + return relative.split(path.sep).join('/') +} + +export async function readStoredFile (filePath: string): Promise { + if (s3 == null) return await fs.readFile(filePath) + const result = await s3.send(new GetObjectCommand({ Bucket: S3_BUCKET, Key: storageKeyForPath(filePath) })) + if (result.Body == null) throw new Error(`Stored object has no body: ${storageKeyForPath(filePath)}`) + return Buffer.from(await result.Body.transformToByteArray()) +} + +export async function storedFileExists (filePath: string): Promise { + if (s3 == null) { + try { + await fs.access(filePath) + return true + } catch (err: any) { + if (err?.code === 'ENOENT') return false + throw err + } + } + try { + await s3.send(new HeadObjectCommand({ Bucket: S3_BUCKET, Key: storageKeyForPath(filePath) })) + return true + } catch (err: any) { + if (err?.$metadata?.httpStatusCode === 404 || err?.name === 'NotFound' || err?.name === 'NoSuchKey') return false + throw err + } +} + +export async function storeBuffer (filePath: string, body: Buffer, contentType?: string): Promise { + if (s3 == null) { + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile(filePath, body) + return + } + await s3.send(new PutObjectCommand({ + Bucket: S3_BUCKET, + Key: storageKeyForPath(filePath), + Body: body, + ContentType: contentType + })) +} + +export async function storeFile (filePath: string, contentType?: string): Promise { + if (s3 == null) return + const stat = await fs.stat(filePath) + await s3.send(new PutObjectCommand({ + Bucket: S3_BUCKET, + Key: storageKeyForPath(filePath), + Body: createReadStream(filePath), + ContentLength: stat.size, + ContentType: contentType + })) +} + +async function filesUnder (directory: string): Promise { + const entries = await fs.readdir(directory, { withFileTypes: true }) + const nested = await Promise.all(entries.map(async entry => { + const candidate = path.join(directory, entry.name) + return entry.isDirectory() ? await filesUnder(candidate) : entry.isFile() ? [candidate] : [] + })) + return nested.flat() +} + +export async function storeDirectory (directory: string): Promise { + if (s3 == null) return + const files = await filesUnder(directory) + let cursor = 0 + await Promise.all(Array.from({ length: Math.min(8, files.length) }, async () => { + while (cursor < files.length) { + const file = files[cursor++] + await storeFile(file) + } + })) +} + +export async function materializeStoredFile (filePath: string): Promise { + try { + await fs.access(filePath) + return + } catch (err: any) { + if (err?.code !== 'ENOENT') throw err + } + if (s3 == null) throw Object.assign(new Error(`Stored file does not exist: ${filePath}`), { code: 'ENOENT' }) + await fs.mkdir(path.dirname(filePath), { recursive: true }) + await fs.writeFile(filePath, await readStoredFile(filePath)) +} + +export async function deleteStoredDirectory (directory: string): Promise { + if (s3 == null) { + await fs.rm(directory, { recursive: true, force: true }) + return + } + const prefix = `${storageKeyForPath(directory).replace(/\/+$/, '')}/` + while (true) { + const page = await s3.send(new ListObjectsV2Command({ + Bucket: S3_BUCKET, + Prefix: prefix + })) + const objects = (page.Contents ?? []).flatMap(item => item.Key == null ? [] : [{ Key: item.Key }]) + if (objects.length === 0) return + await s3.send(new DeleteObjectsCommand({ Bucket: S3_BUCKET, Delete: { Objects: objects, Quiet: true } })) + } +} diff --git a/src/server/server.ts b/src/server/server.ts index 51a9d1f..7a187a0 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -17,6 +17,7 @@ import { ensurePageText, getPublicationDir, processPublicationFile } from './con import { STARTER_AUTHOR_NAME, STARTER_WORKS, starterCoverPath, starterWorkById, type StarterWork, writeStarterPdf } from './starterWorks.js' import { appManifest, metaForPath, renderHtmlShell, robotsTxt, sitemapXml, walletManifest, type PublicPublicationMeta } from './web.js' import { paymentForPaidPagesOnly } from './paymentRouting.js' +import { deleteStoredDirectory, readStoredFile, storeBuffer, storedFileExists } from './objectStorage.js' const serverDirname = path.dirname(fileURLToPath(import.meta.url)) const HTTP_PORT = Number(process.env.HTTP_PORT ?? process.env.PORT ?? '3001') @@ -439,7 +440,7 @@ async function canManagePublication (identityKey: string, publication: any): Pro } async function sendPngResponse (req: Request, res: Response, filePath: string, allowJson = true): Promise { - const image = await fs.readFile(filePath) + const image = await readStoredFile(filePath) if (allowJson && req.query.format === 'json') { res.json({ status: 'success', @@ -457,7 +458,7 @@ async function deletePublication (publicationId: string, actor?: string): Promis await trx('publications').where({ id: publicationId }).delete() await writeAudit('publication_deleted', actor, 'publication', publicationId, undefined, trx) }) - await fs.rm(getPublicationDir(publicationId), { recursive: true, force: true }) + await deleteStoredDirectory(getPublicationDir(publicationId)) } async function hasValidEntitlement (publicationId: string, pageNumber: number, readerIdentityKey?: string): Promise { @@ -843,7 +844,7 @@ async function removeSeedTestData (): Promise { }) await Promise.all(publicationIds.map(async publicationId => { - await fs.rm(getPublicationDir(publicationId), { recursive: true, force: true }) + await deleteStoredDirectory(getPublicationDir(publicationId)) })) } @@ -859,12 +860,7 @@ async function starterNeedsProcessing (publicationId: string): Promise const page = await db('publication_pages').where({ publication_id: publicationId, page_number: 1 }).first() if (page == null) return true - try { - await fs.access(String(page.image_path)) - return false - } catch { - return true - } + return !(await storedFileExists(String(page.image_path))) } async function seedStarterWork (work: StarterWork, starterAuthorIdentityKey: string): Promise { @@ -1362,10 +1358,9 @@ async function createApp (): Promise { 'image/gif': '.gif' } const avatarDir = path.join(DATA_DIR, 'avatars') - await fs.mkdir(avatarDir, { recursive: true }) const avatarId = createHash('sha256').update(identityKey).digest('hex') const avatarPath = path.join(avatarDir, `${avatarId}${extByMime[uploaded.mimeType]}`) - await fs.writeFile(avatarPath, uploaded.bytes) + await storeBuffer(avatarPath, uploaded.bytes, uploaded.mimeType) await ensureAuthor(identityKey) const avatarUrl = `${ROUTING_PREFIX}/authors/${encodeURIComponent(identityKey)}/avatar` await db('authors').where({ identity_key: identityKey }).update({ @@ -1386,11 +1381,10 @@ async function createApp (): Promise { const avatarDir = path.join(DATA_DIR, 'avatars') const candidates = ['.png', '.jpg', '.webp', '.gif'].map(ext => path.join(avatarDir, `${avatarId}${ext}`)) for (const candidate of candidates) { - try { - await fs.access(candidate) - res.sendFile(candidate) - return - } catch {} + if (!(await storedFileExists(candidate))) continue + const mimeTypes: Record = { '.png': 'image/png', '.jpg': 'image/jpeg', '.webp': 'image/webp', '.gif': 'image/gif' } + res.type(mimeTypes[path.extname(candidate)] ?? 'application/octet-stream').send(await readStoredFile(candidate)) + return } res.status(404).json({ status: 'error', message: 'Avatar not found' }) }) @@ -1831,9 +1825,8 @@ async function createApp (): Promise { 'image/gif': '.gif' } const appearanceDir = path.join(DATA_DIR, 'appearance') - await fs.mkdir(appearanceDir, { recursive: true }) const assetName = `${kind}-${randomUUID()}${extByMime[uploaded.mimeType]}` - await fs.writeFile(path.join(appearanceDir, assetName), uploaded.bytes) + await storeBuffer(path.join(appearanceDir, assetName), uploaded.bytes, uploaded.mimeType) const url = `${ROUTING_PREFIX}/appearance/assets/${assetName}` await writeAudit('appearance_asset_uploaded', identityKeyOf(req), 'server_settings', '1', { kind, url }) res.json({ status: 'success', url }) @@ -1846,13 +1839,13 @@ async function createApp (): Promise { return } const filePath = path.join(DATA_DIR, 'appearance', fileName) - try { - await fs.access(filePath) - res.setHeader('Cache-Control', 'public, max-age=3600') - res.sendFile(filePath) - } catch { + if (!(await storedFileExists(filePath))) { res.status(404).json({ status: 'error', message: 'Asset not found' }) + return } + res.setHeader('Cache-Control', 'public, max-age=3600') + const mimeTypes: Record = { '.png': 'image/png', '.jpg': 'image/jpeg', '.webp': 'image/webp', '.gif': 'image/gif' } + res.type(mimeTypes[path.extname(filePath)] ?? 'application/octet-stream').send(await readStoredFile(filePath)) }) api.get('/admin/admins', requireAdmin, async (_req, res) => {