From d10c9c3e68c376c8c197bf5f4b0c6f7af633f561 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Tue, 18 Aug 2026 06:54:59 +0000 Subject: [PATCH 1/2] fix(uploads): accept underscore-led keys, bound bucket names, dedup only live objects Three independent static-deploy upload defects: - Custom keys rejected a leading underscore, so every Next.js export failed at its first _next/static/** file. Validation moves to a shared module and admits the underscore while keeping traversal, leading-slash, NUL and length guards. - Physical S3 bucket names were prefix-key-databaseId, which overran the 63-char limit and could carry illegal characters. Names are now sanitized, truncated to a readable head and suffixed with a digest of the full logical identity. Existing buckets are untouched: physical_name is authoritative once stored. - Deduplication treated any row matching a content hash as a live object, so an upload that was only requested, or was rejected or expired, could be reported as already stored while no bytes existed. Dedup now requires a confirmed status; a stale row is deleted and the upload restarted. --- .../__tests__/custom-key.test.ts | 61 +++++++++++ .../__tests__/managed-upload.test.ts | 84 +++++++++++++++ .../src/custom-key.ts | 38 +++++++ .../src/file-lifecycle.ts | 29 +++++ .../src/index.ts | 1 + .../src/managed-upload.ts | 30 +++++- .../src/plugin.ts | 101 ++++++++++-------- .../src/storage-module-cache.ts | 4 + .../src/types.ts | 6 ++ .../constructive-preset-bucket-wiring.test.ts | 14 ++- .../__tests__/presigned-url-resolver.test.ts | 57 ++++++++-- .../src/presigned-url-resolver.ts | 67 ++++++++++-- .../__tests__/upload.integration.test.ts | 17 ++- 13 files changed, 432 insertions(+), 77 deletions(-) create mode 100644 graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts create mode 100644 graphile/graphile-presigned-url-plugin/src/custom-key.ts create mode 100644 graphile/graphile-presigned-url-plugin/src/file-lifecycle.ts diff --git a/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts new file mode 100644 index 0000000000..087d8a4385 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts @@ -0,0 +1,61 @@ +/** + * Unit tests for custom object-key validation. + * + * The rule being pinned is containment, not aesthetics: a key may look however a + * build tool wants it to as long as it cannot escape the bucket's namespace or + * mean something different to S3 than to the gateway serving it. A static export + * puts its immutable assets under `_next/static/**`, so a leading underscore must + * be accepted while the traversal guards stay in force. + */ + +import { validateCustomKey } from '../src/custom-key'; + +describe('validateCustomKey', () => { + it.each([ + '_next/static/chunks/main-abc123.js', + '_next/static/css/app.css', + '_headers', + 'index.html', + 'assets/img/logo.svg', + 'docs/v1.2.3/guide.pdf', + 'a', + 'my-file_name.v2.tar.gz', + ])('accepts %s', (key) => { + expect(validateCustomKey(key)).toBeNull(); + }); + + it('accepts a key at the length limit and rejects one past it', () => { + expect(validateCustomKey('a'.repeat(1024))).toBeNull(); + expect(validateCustomKey('a'.repeat(1025))).toMatch(/INVALID_KEY_LENGTH/); + }); + + it('rejects an empty key', () => { + expect(validateCustomKey('')).toMatch(/INVALID_KEY_LENGTH/); + }); + + it.each(['../etc/passwd', 'assets/../../secret', '_next/../..'])( + 'rejects path traversal in %s', + (key) => { + expect(validateCustomKey(key)).toMatch(/path traversal/); + }, + ); + + it('rejects a leading slash', () => { + expect(validateCustomKey('/_next/static/main.js')).toMatch(/leading slash/); + }); + + it('rejects NUL bytes', () => { + expect(validateCustomKey('index.html\0.png')).toMatch(/null bytes/); + }); + + it.each([ + '-leading-hyphen.js', + '.leading-dot.js', + 'has space.js', + 'has:colon.js', + 'has?query=1', + 'emoji-🚀.png', + ])('rejects %s', (key) => { + expect(validateCustomKey(key)).toMatch(/^INVALID_KEY:/); + }); +}); diff --git a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts index 56d1c6fed6..cb83ea5b48 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/managed-upload.test.ts @@ -471,6 +471,90 @@ describe('finalizeStagedUpload', () => { expect(s3.client.send).toHaveBeenCalledTimes(1); }); + // With the confirm-upload lifecycle, a row is a claim on bytes rather than + // proof of them, so only a confirmed row may absorb an upload. + describe('with the confirm-upload lifecycle', () => { + const lifecycleTarget = { + ...target, + storageConfig: { ...storageConfig(), hasConfirmUpload: true } as StorageModuleConfig, + }; + + const existingRow = (status: string) => ({ + id: FILE_ID, + key: staged.contentHash, + mime_type: 'image/png', + size: 16, + filename: 'original.png', + status, + }); + + it.each(['uploaded', 'processed'])('deduplicates against a %s row', async (status) => { + const { finalizeStagedUpload } = await import('../src/managed-upload'); + const db = fakeDb([ + SET_CONFIG, + { match: /SELECT id, key, mime_type/, rows: () => [existingRow(status)] }, + ]); + + const { projection, deduplicated } = await finalizeStagedUpload({ + target: lifecycleTarget, withPgClient: db.withPgClient, pgSettings: null, staged, + }); + + expect(deduplicated).toBe(true); + expect(projection.id).toBe(FILE_ID); + expect(db.queries.some((q) => /INSERT|DELETE/.test(q.text))).toBe(false); + }); + + it.each(['requested', 'rejected', 'expired'])( + 'drops the %s row and uploads afresh instead of reporting a dedup hit', + async (status) => { + const { finalizeStagedUpload } = await import('../src/managed-upload'); + const NEW_FILE_ID = '55555555-5555-5555-5555-555555555555'; + const db = fakeDb([ + SET_CONFIG, + { match: /SELECT id, key, mime_type/, rows: () => [existingRow(status)] }, + { match: /DELETE FROM storage_public\.app_files/, rows: () => [] }, + { match: /INSERT INTO storage_public\.app_files/, rows: () => [{ id: NEW_FILE_ID }] }, + ]); + + const { projection, deduplicated } = await finalizeStagedUpload({ + target: lifecycleTarget, withPgClient: db.withPgClient, pgSettings: null, staged, + }); + + expect(deduplicated).toBe(false); + // The caller is handed the row that actually names the promoted bytes. + expect(projection.id).toBe(NEW_FILE_ID); + const del = db.queries.find((q) => /DELETE/.test(q.text)); + expect(del?.values).toEqual([FILE_ID]); + // Promote to the content key, then drop the staged object. + expect(s3.client.send).toHaveBeenCalledTimes(2); + }, + ); + + it('asks for the status column only when the module has one', async () => { + const { finalizeStagedUpload } = await import('../src/managed-upload'); + const withLifecycle = fakeDb([ + SET_CONFIG, + { match: /SELECT id, key, mime_type/, rows: () => [existingRow('uploaded')] }, + ]); + await finalizeStagedUpload({ + target: lifecycleTarget, withPgClient: withLifecycle.withPgClient, pgSettings: null, staged, + }); + expect(withLifecycle.queries.find((q) => /SELECT id, key/.test(q.text))!.text).toContain('status'); + + const without = fakeDb([ + SET_CONFIG, + { + match: /SELECT id, key, mime_type/, + rows: () => [{ id: FILE_ID, key: staged.contentHash, mime_type: 'image/png', size: 16, filename: null }], + }, + ]); + await finalizeStagedUpload({ + target, withPgClient: without.withPgClient, pgSettings: null, staged, + }); + expect(without.queries.find((q) => /SELECT id, key/.test(q.text))!.text).not.toContain('status'); + }); + }); + it('abandons both keys when the files row cannot be inserted', async () => { const { finalizeStagedUpload } = await import('../src/managed-upload'); const db = fakeDb([ diff --git a/graphile/graphile-presigned-url-plugin/src/custom-key.ts b/graphile/graphile-presigned-url-plugin/src/custom-key.ts new file mode 100644 index 0000000000..161ae0c321 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/custom-key.ts @@ -0,0 +1,38 @@ +/** + * Validation for a caller-supplied ("custom") object key. + * + * A custom key is the one place a client names an S3 object directly, so what is + * enforced here is containment: the key must land inside the bucket's namespace + * and mean the same thing to S3 as it does to the gateway that later serves it. + */ + +const MAX_CUSTOM_KEY_LENGTH = 1024; + +/** + * The key alphabet. A leading underscore is legal — a static export puts its + * hashed assets under `_next/static/**` — and containment is enforced by the + * `..`, leading-slash and NUL checks below rather than by the first character. + */ +const CUSTOM_KEY_REGEX = /^[a-zA-Z0-9_][a-zA-Z0-9_.\-/]*$/; + +/** + * Returns an error string describing why `key` is unusable, or null if it is fine. + */ +export function validateCustomKey(key: string): string | null { + if (key.length === 0 || key.length > MAX_CUSTOM_KEY_LENGTH) { + return 'INVALID_KEY_LENGTH: must be 1-1024 characters'; + } + if (key.includes('..')) { + return 'INVALID_KEY: path traversal (..) not allowed'; + } + if (key.startsWith('/')) { + return 'INVALID_KEY: leading slash not allowed'; + } + if (key.includes('\0')) { + return 'INVALID_KEY: null bytes not allowed'; + } + if (!CUSTOM_KEY_REGEX.test(key)) { + return 'INVALID_KEY: must start with alphanumeric or underscore and contain only alphanumeric, dots, hyphens, underscores, and slashes'; + } + return null; +} diff --git a/graphile/graphile-presigned-url-plugin/src/file-lifecycle.ts b/graphile/graphile-presigned-url-plugin/src/file-lifecycle.ts new file mode 100644 index 0000000000..b78dc8c737 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/file-lifecycle.ts @@ -0,0 +1,29 @@ +import type { StorageModuleConfig } from './types'; + +/** + * The statuses in which a files row stands for bytes a reader can actually GET. + * A `requested` row is a claim, not an object — its presigned PUT may never have + * run — and `rejected`/`expired` are settled failures. + */ +export const LIVE_FILE_STATUSES = ['uploaded', 'processed']; + +/** + * The `status` column, when the module has one, for splicing into a select list. + */ +export function statusSelectFragment(storageConfig: StorageModuleConfig): string { + return storageConfig.hasConfirmUpload ? ', status' : ''; +} + +/** + * Whether an existing row may be handed back as a dedup hit. + * + * Modules without the confirm-upload lifecycle have no `status` column, so there + * is nothing to read and every row is presumed live, as before. + */ +export function isLiveFileRow( + storageConfig: StorageModuleConfig, + row: { status?: string } +): boolean { + if (!storageConfig.hasConfirmUpload) return true; + return LIVE_FILE_STATUSES.includes(row.status as string); +} diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index bba54f44a2..4968f2da03 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -33,6 +33,7 @@ export { type ConfirmUploadInput, type ConfirmUploadVerdict, } from './confirm-upload'; +export { validateCustomKey } from './custom-key'; export type { ResolvedBucketCoordinate } from './default-bucket'; export { resolveDefaultBucket } from './default-bucket'; export { createDownloadUrlPlugin } from './download-url-field'; diff --git a/graphile/graphile-presigned-url-plugin/src/managed-upload.ts b/graphile/graphile-presigned-url-plugin/src/managed-upload.ts index 26f730baeb..5b9fe69c89 100644 --- a/graphile/graphile-presigned-url-plugin/src/managed-upload.ts +++ b/graphile/graphile-presigned-url-plugin/src/managed-upload.ts @@ -21,6 +21,7 @@ import { Logger } from '@pgpmjs/logger'; import { resolveDefaultBucket } from './default-bucket'; +import { isLiveFileRow, statusSelectFragment } from './file-lifecycle'; import { type FileRefFieldBinding, getFileRefFieldBinding } from './file-ref-registry'; import { provisionAndRecordPhysicalBucket, resolveS3ForDatabase } from './physical-bucket'; import { type WithPgClient, withRequestPgClient } from './request-pg-client'; @@ -289,18 +290,41 @@ export async function finalizeStagedUpload(args: { const existing = await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { const result = await pgClient.query({ - text: `SELECT id, key, mime_type, size, filename + text: `SELECT id, key, mime_type, size, filename${statusSelectFragment(storageConfig)} FROM ${storageConfig.filesQualifiedName} WHERE content_hash = $1 AND bucket_id = $2 LIMIT 1`, values: [staged.contentHash, bucket.id], }); return result.rows[0] as - | { id: string; key: string; mime_type: string; size: number; filename: string | null } + | { + id: string; + key: string; + mime_type: string; + size: number; + filename: string | null; + status?: string; + } | undefined; }); - if (existing) { + // Only a row that already stands for stored bytes may absorb this upload. One + // that never received them is dropped, and the staged object is promoted as a + // fresh file below — which is also what keeps the insert possible, since the + // final key is the content hash and (bucket_id, key) is unique. The GC job the + // delete enqueues re-takes the reference count when it runs, by which point + // the replacement row exists, so it no-ops. + if (existing && !isLiveFileRow(storageConfig, existing)) { + log.info( + `Restarting upload of hash ${staged.contentHash}: file ${existing.id} is ${existing.status}, so it carries no bytes` + ); + await withRequestPgClient(withPgClient, pgSettings, async (pgClient) => { + await pgClient.query({ + text: `DELETE FROM ${storageConfig.filesQualifiedName} WHERE id = $1`, + values: [existing.id], + }); + }); + } else if (existing) { log.info(`Dedup hit: file ${existing.id} already carries hash ${staged.contentHash}`); await deleteS3Object(s3, staged.stagingKey); diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index 00b9b54de4..ee56554a30 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -24,7 +24,9 @@ import { access, context as grafastContext, lambda, object } from 'grafast'; import type { GraphileConfig } from 'graphile-config'; import { checkTypeAgreement } from 'mime-bytes'; +import { validateCustomKey } from './custom-key'; import { resolveDefaultBucket } from './default-bucket'; +import { isLiveFileRow, statusSelectFragment } from './file-lifecycle'; import { buildFileProjection, type FileProjection } from './managed-upload'; import { provisionAndRecordPhysicalBucket, resolveS3ForDatabase } from './physical-bucket'; import { withRequestPgClient } from './request-pg-client'; @@ -38,9 +40,7 @@ const log = new Logger('graphile-presigned-url:plugin'); const MAX_CONTENT_HASH_LENGTH = 128; const MAX_CONTENT_TYPE_LENGTH = 255; -const MAX_CUSTOM_KEY_LENGTH = 1024; const SHA256_HEX_REGEX = /^[a-f0-9]{64}$/; -const CUSTOM_KEY_REGEX = /^[a-zA-Z0-9][a-zA-Z0-9_.\-/]*$/; // --- Helpers --- @@ -52,25 +52,6 @@ function buildS3Key(contentHash: string): string { return contentHash; } -function validateCustomKey(key: string): string | null { - if (key.length === 0 || key.length > MAX_CUSTOM_KEY_LENGTH) { - return 'INVALID_KEY_LENGTH: must be 1-1024 characters'; - } - if (key.includes('..')) { - return 'INVALID_KEY: path traversal (..) not allowed'; - } - if (key.startsWith('/')) { - return 'INVALID_KEY: leading slash not allowed'; - } - if (key.includes('\0')) { - return 'INVALID_KEY: null bytes not allowed'; - } - if (!CUSTOM_KEY_REGEX.test(key)) { - return 'INVALID_KEY: must start with alphanumeric and contain only alphanumeric, dots, hyphens, underscores, and slashes'; - } - return null; -} - function derivePathFromKey(key: string): string | null { const lastSlash = key.lastIndexOf('/'); if (lastSlash <= 0) return null; @@ -717,9 +698,21 @@ async function processSingleFile( // Dedup / versioning check let previousVersionId: string | null = null; + // A row whose bytes never landed must not be reported as a dedup hit: the + // caller would store a reference to an object that is not in S3. Such a row is + // dropped instead, and the upload proceeds as a fresh one below — the insert is + // what enqueues the confirm-upload job, so restarting the lifecycle is the only + // way the retry can ever leave `requested`. Dropping it is also what keeps the + // retry insertable at all for a content-addressed key, where the key *is* the + // hash and a second row would collide on (bucket_id, key). The GC job the + // delete enqueues re-takes the reference count when it runs (≥5s later), by + // which point the replacement row exists, so it no-ops. + const statusColumn = statusSelectFragment(storageConfig); + let staleFileId: string | null = null; + if (isCustomKey) { const existingResult = await txClient.query({ - text: `SELECT id, content_hash + text: `SELECT id, content_hash${statusColumn} FROM ${storageConfig.filesQualifiedName} WHERE key = $1 AND bucket_id = $2 @@ -731,23 +724,28 @@ async function processSingleFile( if (existingResult.rows.length > 0) { const existing = existingResult.rows[0]; if (existing.content_hash === contentHash) { - log.info(`Dedup hit (custom key): file ${existing.id} for key ${s3Key}`); - return { - uploadUrl: null as string | null, - fileId: existing.id as string, - key: s3Key, - deduplicated: true, - expiresAt: null as string | null, - previousVersionId: null as string | null, - file: projectFile(existing.id as string, s3Key), - }; + if (isLiveFileRow(storageConfig, existing)) { + log.info(`Dedup hit (custom key): file ${existing.id} for key ${s3Key}`); + return { + uploadUrl: null as string | null, + fileId: existing.id as string, + key: s3Key, + deduplicated: true, + expiresAt: null as string | null, + previousVersionId: null as string | null, + file: projectFile(existing.id as string, s3Key), + }; + } + staleFileId = existing.id as string; + log.info(`Restarting upload of key ${s3Key}: file ${staleFileId} is ${existing.status}, so it carries no bytes`); + } else { + previousVersionId = existing.id; + log.info(`Versioning: new version of key ${s3Key}, previous=${previousVersionId}`); } - previousVersionId = existing.id; - log.info(`Versioning: new version of key ${s3Key}, previous=${previousVersionId}`); } } else { const dedupResult = await txClient.query({ - text: `SELECT id + text: `SELECT id${statusColumn} FROM ${storageConfig.filesQualifiedName} WHERE content_hash = $1 AND bucket_id = $2 @@ -757,20 +755,31 @@ async function processSingleFile( if (dedupResult.rows.length > 0) { const existingFile = dedupResult.rows[0]; - log.info(`Dedup hit: file ${existingFile.id} for hash ${contentHash}`); - - return { - uploadUrl: null as string | null, - fileId: existingFile.id as string, - key: s3Key, - deduplicated: true, - expiresAt: null as string | null, - previousVersionId: null as string | null, - file: projectFile(existingFile.id as string, s3Key), - }; + if (isLiveFileRow(storageConfig, existingFile)) { + log.info(`Dedup hit: file ${existingFile.id} for hash ${contentHash}`); + + return { + uploadUrl: null as string | null, + fileId: existingFile.id as string, + key: s3Key, + deduplicated: true, + expiresAt: null as string | null, + previousVersionId: null as string | null, + file: projectFile(existingFile.id as string, s3Key), + }; + } + staleFileId = existingFile.id as string; + log.info(`Restarting upload of hash ${contentHash}: file ${staleFileId} is ${existingFile.status}, so it carries no bytes`); } } + if (staleFileId !== null) { + await txClient.query({ + text: `DELETE FROM ${storageConfig.filesQualifiedName} WHERE id = $1`, + values: [staleFileId], + }); + } + // Auto-derive ltree path from custom key directory (only when has_path_shares) const derivedPath = isCustomKey && storageConfig.hasPathShares ? derivePathFromKey(s3Key) : null; diff --git a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts index 12e48b6781..c5f062412f 100644 --- a/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts +++ b/graphile/graphile-presigned-url-plugin/src/storage-module-cache.ts @@ -63,6 +63,7 @@ const APP_STORAGE_MODULE_QUERY = ` sm.max_bulk_files, sm.max_bulk_total_size, sm.has_path_shares, + sm.has_confirm_upload, NULL AS entity_schema, NULL AS entity_table FROM metaschema_modules_public.storage_module sm @@ -102,6 +103,7 @@ const ALL_STORAGE_MODULES_QUERY = ` sm.max_bulk_files, sm.max_bulk_total_size, sm.has_path_shares, + sm.has_confirm_upload, es.schema_name AS entity_schema, et.name AS entity_table FROM metaschema_modules_public.storage_module sm @@ -134,6 +136,7 @@ interface StorageModuleRow { max_bulk_files: number | null; max_bulk_total_size: number | null; has_path_shares: boolean; + has_confirm_upload: boolean; entity_schema: string | null; entity_table: string | null; } @@ -165,6 +168,7 @@ function buildConfig(row: StorageModuleRow): StorageModuleConfig { maxFilenameLength: row.max_filename_length ?? DEFAULT_MAX_FILENAME_LENGTH, cacheTtlSeconds, hasPathShares: row.has_path_shares ?? false, + hasConfirmUpload: row.has_confirm_upload ?? false, maxBulkFiles: row.max_bulk_files ?? DEFAULT_MAX_BULK_FILES, maxBulkTotalSize: row.max_bulk_total_size ?? DEFAULT_MAX_BULK_TOTAL_SIZE, }; diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index 0eaf03244c..930b7a67f1 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -72,6 +72,12 @@ export interface StorageModuleConfig { cacheTtlSeconds: number; /** Whether this storage module uses ltree path + path shares (determines if path column exists on files) */ hasPathShares: boolean; + /** + * Whether the files table carries the confirm-upload lifecycle (`status`, + * `promoted_at`). Only then can a row be told apart from the bytes it claims: + * without it every row is treated as live, because there is nothing to read. + */ + hasConfirmUpload: boolean; // --- Bulk upload limits --- diff --git a/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts b/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts index e045370cd4..6fcb4cd6a9 100644 --- a/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts +++ b/graphile/graphile-settings/__tests__/constructive-preset-bucket-wiring.test.ts @@ -52,13 +52,21 @@ describe('ConstructivePreset bucket-provisioner wiring', () => { expect(typeof options.resolveBucketName).toBe('function'); }); - it('the wired resolver mints the tenant-aware {prefix}-{bucketKey}-{databaseId} name', () => { + it('the wired resolver mints the tenant-aware {prefix}-{bucketKey}-{digest} name', () => { createConstructivePreset(); const { resolveBucketName } = captured.bucketProvisionerOptions; // provisioner plugin signature: (bucketKey, databaseId) - expect(resolveBucketName('public', DATABASE_ID)).toBe(`${PREFIX}-public-${DATABASE_ID}`); - expect(resolveBucketName('private', DATABASE_ID)).toBe(`${PREFIX}-private-${DATABASE_ID}`); + expect(resolveBucketName('public', DATABASE_ID)).toMatch( + new RegExp(`^${PREFIX}-public-[a-f0-9]{12}$`), + ); + expect(resolveBucketName('private', DATABASE_ID)).toMatch( + new RegExp(`^${PREFIX}-private-[a-f0-9]{12}$`), + ); + // The digest is what carries the tenant, so two databases cannot collide. + expect(resolveBucketName('public', DATABASE_ID)).not.toBe( + resolveBucketName('public', '11111111-2222-3333-4444-555555555555'), + ); }); it('disables auto-provision-on-create so buckets are minted lazily / explicitly', () => { diff --git a/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts b/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts index ac316f4511..1ae9521894 100644 --- a/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts +++ b/graphile/graphile-settings/__tests__/presigned-url-resolver.test.ts @@ -1,13 +1,16 @@ /** * Unit tests for the bucket-name resolvers. * - * The presigned (lazy) upload path and the bucket-provisioner (eager) path - * must mint the *same* physical S3 bucket name for a given (database, bucket - * key) pair — `{prefix}-{bucketKey}-{databaseId}` — so a bucket's physical - * coordinate is identical regardless of which path first provisions it. + * The presigned (lazy) upload path and the bucket-provisioner (eager) path must + * mint the *same* physical S3 bucket name for a given (database, bucket key) + * pair — `{prefix}-{bucketKey}-{digest}` — so a bucket's physical coordinate is + * identical regardless of which path first provisions it. * * The two plugins declare their resolver with opposite argument order, so the - * equality below also guards against re-introducing an argument-order bug. + * equality below also guards against re-introducing an argument-order bug. The + * remaining tests pin the properties S3 enforces on a bucket name: bounded + * length, a restricted alphabet, and — because the name is truncated — a tail + * that still separates identities the readable part can no longer distinguish. */ interface CdnOptions { @@ -26,15 +29,16 @@ async function loadResolverModule(cdn: CdnOptions | undefined) { const PREFIX = 'test-bucket'; const DATABASE_ID = '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9'; +const S3_BUCKET_NAME = /^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/; describe('bucket-name resolvers', () => { - it('presigned resolver mints {prefix}-{bucketKey}-{databaseId}', async () => { + it('presigned resolver mints {prefix}-{bucketKey}-{digest}', async () => { const { createBucketNameResolver } = await loadResolverModule({ bucketName: PREFIX }); const resolve = createBucketNameResolver(); // presigned plugin signature: (databaseId, bucketKey) - expect(resolve(DATABASE_ID, 'public')).toBe(`${PREFIX}-public-${DATABASE_ID}`); - expect(resolve(DATABASE_ID, 'private')).toBe(`${PREFIX}-private-${DATABASE_ID}`); + expect(resolve(DATABASE_ID, 'public')).toMatch(/^test-bucket-public-[a-f0-9]{12}$/); + expect(resolve(DATABASE_ID, 'private')).toMatch(/^test-bucket-private-[a-f0-9]{12}$/); }); it('provisioner resolver mints the identical name despite opposite arg order', async () => { @@ -47,10 +51,45 @@ describe('bucket-name resolvers', () => { for (const key of ['public', 'private', 'temp', 'custom-cdn']) { // presigned: (databaseId, bucketKey) — provisioner: (bucketKey, databaseId) expect(provisioner(key, DATABASE_ID)).toBe(presigned(DATABASE_ID, key)); - expect(provisioner(key, DATABASE_ID)).toBe(`${PREFIX}-${key}-${DATABASE_ID}`); } }); + it('names are stable across calls and resolver instances', async () => { + const { createBucketNameResolver } = await loadResolverModule({ bucketName: PREFIX }); + + const first = createBucketNameResolver(); + const second = createBucketNameResolver(); + + expect(first(DATABASE_ID, 'public')).toBe(first(DATABASE_ID, 'public')); + expect(second(DATABASE_ID, 'public')).toBe(first(DATABASE_ID, 'public')); + }); + + it('stays inside S3 length and alphabet limits for oversized, mixed-case inputs', async () => { + const { createBucketNameResolver } = await loadResolverModule({ + bucketName: 'Some_Very.Long CDN Prefix That Nobody Would Choose', + }); + const resolve = createBucketNameResolver(); + + const name = resolve(DATABASE_ID, 'Marketing_Site/Assets — 2024'.repeat(5)); + + expect(name.length).toBeLessThanOrEqual(63); + expect(name.length).toBeGreaterThanOrEqual(3); + expect(name).toMatch(S3_BUCKET_NAME); + }); + + it('separates identities that survive truncation identically', async () => { + const { createBucketNameResolver } = await loadResolverModule({ bucketName: PREFIX }); + const resolve = createBucketNameResolver(); + + const shared = 'a'.repeat(80); + // Same truncated prefix, different full keys. + expect(resolve(DATABASE_ID, `${shared}-one`)).not.toBe(resolve(DATABASE_ID, `${shared}-two`)); + // Same key, different tenant. + expect(resolve(DATABASE_ID, 'public')).not.toBe( + resolve('11111111-2222-3333-4444-555555555555', 'public'), + ); + }); + it('presigned resolver throws (no default bucket name) when the prefix is missing', async () => { const { createBucketNameResolver } = await loadResolverModule({}); expect(() => createBucketNameResolver()).toThrow(/CDN_BUCKET_NAME/); diff --git a/graphile/graphile-settings/src/presigned-url-resolver.ts b/graphile/graphile-settings/src/presigned-url-resolver.ts index 850a6a6517..cd6eb22c02 100644 --- a/graphile/graphile-settings/src/presigned-url-resolver.ts +++ b/graphile/graphile-settings/src/presigned-url-resolver.ts @@ -15,6 +15,7 @@ import { BucketProvisioner } from '@constructive-io/bucket-provisioner'; import { getEnvOptions } from '@constructive-io/graphql-env'; import { createS3Client } from '@constructive-io/s3-utils'; import { Logger } from '@pgpmjs/logger'; +import { createHash } from 'crypto'; import type { BucketNameResolver as ProvisionerBucketNameResolver } from 'graphile-bucket-provisioner-plugin'; import type { BucketNameResolver, EnsureBucketProvisioned,S3Config } from 'graphile-presigned-url-plugin'; @@ -108,14 +109,66 @@ function getBucketNamePrefix(): string { return prefix; } +/** S3's hard ceiling on a bucket name. */ +const MAX_BUCKET_NAME_LENGTH = 63; +/** S3's floor, which a degenerate prefix/key could otherwise fall under. */ +const MIN_BUCKET_NAME_LENGTH = 3; +/** Hex characters of the identity digest kept as the uniqueness tail. */ +const IDENTITY_DIGEST_LENGTH = 12; +/** Readable budget: how much of the name the prefix and key may each occupy. */ +const PREFIX_BUDGET = 20; +const BUCKET_KEY_BUDGET = 63 - IDENTITY_DIGEST_LENGTH - PREFIX_BUDGET - 3; + +/** + * Reduce a component to the S3 bucket-name alphabet: lowercase, `[a-z0-9-]`, + * with runs of separators collapsed and no leading or trailing hyphen. + * + * Dots are legal in a bucket name but deliberately dropped — a dotted name + * cannot be used with virtual-hosted-style HTTPS, because the wildcard + * certificate does not match a further label. + */ +function sanitizeBucketNameComponent(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + /** - * The single physical-bucket naming policy: `{prefix}-{bucketKey}-{databaseId}` - * (e.g., "myapp-public-abc123def456"). Both the presigned-upload (lazy) path - * and the bucket-provisioner (eager) path derive names from this one function - * so a bucket's physical name is identical regardless of which path mints it. + * The single physical-bucket naming policy: + * `{prefix}-{bucketKey}-{digest}` (e.g. `myapp-public-3f9c1a2b7e04`). + * + * Both the presigned-upload (lazy) path and the bucket-provisioner (eager) path + * derive names from this one function, so a bucket's physical name is identical + * regardless of which path mints it. + * + * The name is bounded and S3-legal by construction: the prefix and key are + * sanitized to `[a-z0-9-]` and truncated to a readable budget, and the tail is a + * digest of the *untruncated* identity — so two buckets whose keys agree only + * past the truncation point, or the same key in two databases, still get distinct + * names. Names remain stable for a given (prefix, databaseId, bucketKey) because + * nothing here reads the clock or a counter; and an already-provisioned bucket + * never consults this function at all, since `platform_buckets.physical_name` is + * authoritative once recorded. */ function mintPhysicalBucketName(prefix: string, databaseId: string, bucketKey: string): string { - return `${prefix}-${bucketKey}-${databaseId}`; + const identity = `${prefix}/${databaseId}/${bucketKey}`; + const digest = createHash('sha256').update(identity).digest('hex').slice(0, IDENTITY_DIGEST_LENGTH); + + const safePrefix = sanitizeBucketNameComponent(prefix).slice(0, PREFIX_BUDGET).replace(/-+$/, ''); + const safeKey = sanitizeBucketNameComponent(bucketKey).slice(0, BUCKET_KEY_BUDGET).replace(/-+$/, ''); + + const name = [safePrefix, safeKey, digest].filter((part) => part.length > 0).join('-'); + + // The digest alone already satisfies both bounds, so this is only reachable + // when both readable components sanitize away to nothing. + if (name.length < MIN_BUCKET_NAME_LENGTH || name.length > MAX_BUCKET_NAME_LENGTH) { + throw new Error( + `[presigned-url-resolver] Cannot mint a legal S3 bucket name for key "${bucketKey}": got "${name}"`, + ); + } + + return name; } /** @@ -123,7 +176,7 @@ function mintPhysicalBucketName(prefix: string, databaseId: string, bucketKey: s * URL plugin (argument order: `(databaseId, bucketKey)`). * * Uses CDN_BUCKET_NAME as a prefix. For each (database, bucketKey) pair, the - * S3 bucket name becomes `{prefix}-{bucketKey}-{databaseId}`. + * S3 bucket name becomes `{prefix}-{bucketKey}-{digest}`. * * This aligns with the bucket provisioner plugin which creates separate * S3 buckets per logical bucket key. @@ -139,7 +192,7 @@ export function createBucketNameResolver(): BucketNameResolver { * (argument order: `(bucketKey, databaseId)`). * * Produces the exact same physical name as createBucketNameResolver() - * (`{prefix}-{bucketKey}-{databaseId}`) so the eager `provisionBucket` + * (`{prefix}-{bucketKey}-{digest}`) so the eager `provisionBucket` * mutation mints the identical tenant-aware name that the lazy first-upload * path would. Throws on a missing prefix — no default bucket name. */ diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts index 5e824de462..5a710dccb0 100644 --- a/graphql/server-test/__tests__/upload.integration.test.ts +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -600,7 +600,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { ); }; - it('mints {prefix}-{key}-{databaseId} and records it, matching what the lazy path would mint', async () => { + it('mints {prefix}-{key}-{digest} and records it, matching what the lazy path would mint', async () => { // 1. Derive the naming prefix from a bucket the LAZY path provisions. const lazyKey = 'eager-lazy'; await seedBucket(lazyKey); @@ -623,11 +623,11 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { // The lazy path records the exact bucket the presigned PUT targets. expect(bucketFromPresignedUrl(lazyUrl)).toBe(lazyPhysical); - // The shared convention: {prefix}-{key}-{databaseId}. - const suffix = `-${lazyKey}-${aliceDatabaseId}`; - expect(lazyPhysical!.endsWith(suffix)).toBe(true); - const prefix = lazyPhysical!.slice(0, -suffix.length); - expect(prefix.length).toBeGreaterThan(0); + // The shared convention: {prefix}-{key}-{digest}, bounded to S3's 63 chars. + const lazyMatch = new RegExp(`^(.+)-${lazyKey}-[a-f0-9]{12}$`).exec(lazyPhysical!); + expect(lazyMatch).not.toBeNull(); + expect(lazyPhysical!.length).toBeLessThanOrEqual(63); + const prefix = lazyMatch![1]; // 2. EAGER path: a fresh bucket row, provisioned via the mutation. const eagerKey = 'eager-prov'; @@ -642,12 +642,11 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(payload.error).toBeNull(); expect(payload.success).toBe(true); - const expected = `${prefix}-${eagerKey}-${aliceDatabaseId}`; // Eager mints the tenant-aware name — NOT the bare logical key. - expect(payload.bucketName).toBe(expected); + expect(payload.bucketName).toMatch(new RegExp(`^${prefix}-${eagerKey}-[a-f0-9]{12}$`)); expect(payload.bucketName).not.toBe(eagerKey); // ...and persists it on the row (physical_name IS NULL-guarded record). - expect(await physicalNameFor(eagerKey)).toBe(expected); + expect(await physicalNameFor(eagerKey)).toBe(payload.bucketName); }); it('does not clobber a physical_name recorded by a prior provision', async () => { From b116a8181cb18d604e24149eb2d59a12798c648f Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Tue, 18 Aug 2026 07:18:14 +0000 Subject: [PATCH 2/2] test(uploads): assert digest-suffixed physical bucket names --- .../server-test/__tests__/upload.integration.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts index 5a710dccb0..b7f516a669 100644 --- a/graphql/server-test/__tests__/upload.integration.test.ts +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -506,9 +506,11 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { const stored = await physicalNameFor('public'); expect(stored).toBeTruthy(); - // Matches the resolver contract: {prefix}-{bucketKey}-{databaseId} + // Matches the resolver contract: {prefix}-{bucketKey}-{digest}, bounded + // to S3's 63-character limit. expect(stored).toContain('public'); - expect(stored).toContain(aliceDatabaseId); + expect(stored).toMatch(/-public-[a-f0-9]{12}$/); + expect(stored!.length).toBeLessThanOrEqual(63); // ...and is exactly the bucket the presigned PUT targets. expect(bucketFromPresignedUrl(payload.uploadUrl)).toBe(stored); }); @@ -571,7 +573,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { // 1c. Eager provisioning via the provisionBucket mutation (Alice) // // The explicit provisionBucket mutation must mint the SAME tenant-aware - // physical name the lazy first-upload path would (`{prefix}-{key}-{db}`) and + // physical name the lazy first-upload path would (`{prefix}-{key}-{digest}`) and // persist it on the bucket row — never the bare logical key. This is the // regression guard for BucketProvisionerPreset being wired without a // resolveBucketName. @@ -732,7 +734,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(physical).toBeTruthy(); // Lazy mints the same tenant-aware name the presigned PUT targets. expect(bucketFromPresignedUrl(uploadUrl)).toBe(physical); - expect(physical!.endsWith(`-${key}-${aliceDatabaseId}`)).toBe(true); + expect(physical).toMatch(new RegExp(`-${key}-[a-f0-9]{12}$`)); }); });