From 7f11a11dcd1db2b1f98c798ad95927fdc871d730 Mon Sep 17 00:00:00 2001 From: zetazzz Date: Mon, 17 Aug 2026 16:54:43 +0800 Subject: [PATCH 1/5] Add exact PostgreSQL pool identities and leases --- pnpm-lock.yaml | 3 - postgres/pg-cache/package.json | 1 - .../pg-cache/src/__tests__/driver.test.ts | 265 ++++++++- postgres/pg-cache/src/__tests__/lru.test.ts | 240 +++++++- postgres/pg-cache/src/driver.ts | 19 +- postgres/pg-cache/src/index.ts | 28 +- postgres/pg-cache/src/lru.ts | 554 +++++++++++++++--- postgres/pg-cache/src/pg.ts | 373 +++++++++++- postgres/pg-env/src/pg-config.ts | 31 +- 9 files changed, 1369 insertions(+), 145 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e766da0896..cae3372a75 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3360,9 +3360,6 @@ importers: '@pgpmjs/types': specifier: workspace:^ version: link:../../pgpm/types/dist - lru-cache: - specifier: ^11.2.7 - version: 11.2.7 pg: specifier: ^8.21.0 version: 8.21.0 diff --git a/postgres/pg-cache/package.json b/postgres/pg-cache/package.json index d85d357cb8..a55b6669cf 100644 --- a/postgres/pg-cache/package.json +++ b/postgres/pg-cache/package.json @@ -32,7 +32,6 @@ "12factor-env": "workspace:^", "@pgpmjs/logger": "workspace:^", "@pgpmjs/types": "workspace:^", - "lru-cache": "^11.2.7", "pg": "^8.21.0", "pg-env": "workspace:^" }, diff --git a/postgres/pg-cache/src/__tests__/driver.test.ts b/postgres/pg-cache/src/__tests__/driver.test.ts index 105e3a669a..6613c2311e 100644 --- a/postgres/pg-cache/src/__tests__/driver.test.ts +++ b/postgres/pg-cache/src/__tests__/driver.test.ts @@ -3,25 +3,39 @@ // lets an alternate backend (e.g. PGlite) plug in without any change to pgpm / // pgsql-* — and guarantees the default path is untouched when nothing registers. -import { randomUUID } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; import pg from 'pg'; import { + acquirePgPool, defaultPgPoolFactory, getActivePgPoolFactory, getPgPool, + getPgPoolConfig, + getPgPoolDriverIdentity, + getPgPoolIdentity, hasPgPoolFactory, PgPoolFactory, - registerPgPoolFactory + registerPgPoolFactory, } from '../index'; import { pgCache } from '../lru'; const createMockPool = (): pg.Pool => - ({ query: jest.fn(), connect: jest.fn(), end: jest.fn(async () => {}) } as unknown as pg.Pool); + ({ + query: jest.fn(), + connect: jest.fn(), + end: jest.fn(async () => {}), + }) as unknown as pg.Pool; const freshConfig = () => { const database = `seam_${randomUUID()}`; - return { database, host: 'localhost', port: 5432, user: 'postgres', password: 'x' }; + return { + database, + host: 'localhost', + port: 5432, + user: 'postgres', + password: 'x', + }; }; describe('pg-cache pool-factory seam', () => { @@ -57,10 +71,10 @@ describe('pg-cache pool-factory seam', () => { expect(pool).toBe(mock); expect(pool.connect).toBe(alternateConnect); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); }); - it('caches by database: a second call reuses the pool and does not re-invoke the factory', () => { + it('caches by exact identity: an identical call reuses the pool', () => { const cfg = freshConfig(); const factory = jest.fn(() => createMockPool()); registerPgPoolFactory(factory); @@ -71,7 +85,210 @@ describe('pg-cache pool-factory seam', () => { expect(first).toBe(second); expect(factory).toHaveBeenCalledTimes(1); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); + }); + + it('leases the exact identity with idempotent release', () => { + const cfg = freshConfig(); + const mock = createMockPool(); + const factory = jest.fn(() => mock); + registerPgPoolFactory(factory); + + const first = acquirePgPool(cfg, { purpose: 'runtime' }); + const second = acquirePgPool(cfg, { purpose: 'runtime' }); + + expect(first.identity).toBe(getPgPoolIdentity(cfg, { purpose: 'runtime' })); + expect(first.pool).toBe(mock); + expect(second.pool).toBe(mock); + expect(factory).toHaveBeenCalledTimes(1); + expect(pgCache.getStats().activeLeases).toBeGreaterThanOrEqual(2); + + first.release(); + first.release(); + second.release(); + pgCache.delete(first.identity); + }); + + it('separates credentials and purpose for one physical database', () => { + const cfg = freshConfig(); + const factory = jest.fn(() => createMockPool()); + registerPgPoolFactory(factory); + + const control = getPgPool(cfg, { purpose: 'control' }); + const runtime = getPgPool( + { ...cfg, user: 'runtime', password: 'runtime-secret' }, + { purpose: 'runtime' } + ); + const notification = getPgPool( + { ...cfg, user: 'runtime', password: 'runtime-secret' }, + { purpose: 'notification' } + ); + + expect(control).not.toBe(runtime); + expect(runtime).not.toBe(notification); + expect(factory).toHaveBeenCalledTimes(3); + + pgCache.delete(getPgPoolIdentity(cfg, { purpose: 'control' })); + pgCache.delete( + getPgPoolIdentity( + { ...cfg, user: 'runtime', password: 'runtime-secret' }, + { purpose: 'runtime' } + ) + ); + pgCache.delete( + getPgPoolIdentity( + { ...cfg, user: 'runtime', password: 'runtime-secret' }, + { purpose: 'notification' } + ) + ); + }); + + it('normalizes maxUses into the exact pool identity', () => { + const cfg = freshConfig(); + const unlimited = getPgPoolIdentity(cfg); + const explicitUnlimited = getPgPoolIdentity({ + ...cfg, + pool: { maxUses: 0 }, + }); + const singleUse = getPgPoolIdentity({ ...cfg, pool: { maxUses: 1 } }); + + expect(explicitUnlimited).toBe(unlimited); + expect(singleUse).not.toBe(unlimited); + }); + + it('uses a process-keyed identity instead of an offline password verifier', () => { + const cfg = { + ...freshConfig(), + pool: { + max: 3, + idleTimeoutMillis: 1234, + connectionTimeoutMillis: 5678, + allowExitOnIdle: true, + }, + }; + const identity = getPgPoolIdentity(cfg, { purpose: 'runtime' }); + const unkeyedInput = JSON.stringify({ + version: 1, + driver: getPgPoolDriverIdentity(), + host: cfg.host, + port: cfg.port, + database: cfg.database, + user: cfg.user, + password: cfg.password, + ssl: null, + pool: { + max: 3, + maxUses: null, + idleTimeoutMillis: 1234, + connectionTimeoutMillis: 5678, + allowExitOnIdle: true, + }, + purpose: 'runtime', + checkout: 'registered-factory-owned-v1', + }); + const offlineDigest = `pg:v1:${createHash('sha256') + .update(unkeyedInput) + .digest('hex')}`; + + expect(getPgPoolIdentity(cfg, { purpose: 'runtime' })).toBe(identity); + expect(identity).toMatch(/^pg:v1:[a-f0-9]{64}$/); + expect(identity).not.toBe(offlineDigest); + }); + + it('rejects credential callbacks and noncanonical identity inputs', () => { + const cfg = freshConfig(); + const accessorSsl = {} as Record; + Object.defineProperty(accessorSsl, 'ca', { get: () => 'dynamic-ca' }); + + expect(() => + getPgPoolIdentity({ + ...cfg, + password: (async () => 'secret') as unknown as string, + }) + ).toThrow('pg.password must be a string'); + expect(() => + getPgPoolIdentity({ + ...cfg, + port: '5432' as unknown as number, + }) + ).toThrow('pg.port must be a safe integer'); + expect(() => + getPgPoolIdentity(cfg, { + purpose: {} as unknown as string, + }) + ).toThrow('pg pool purpose must be a non-empty string'); + expect(() => + getPgPoolIdentity({ + ...cfg, + ssl: accessorSsl as never, + }) + ).toThrow('pg.ssl.ca must be a data property'); + }); + + it('canonicalizes TLS data but separates distinct trust contracts', () => { + const cfg = freshConfig(); + const verified = getPgPoolIdentity({ + ...cfg, + ssl: { + ca: 'tenant-ca', + rejectUnauthorized: true, + servername: 'db.internal', + }, + }); + const reordered = getPgPoolIdentity({ + ...cfg, + ssl: { + servername: 'db.internal', + rejectUnauthorized: true, + ca: 'tenant-ca', + }, + }); + const insecure = getPgPoolIdentity({ + ...cfg, + ssl: { + ca: 'tenant-ca', + rejectUnauthorized: false, + servername: 'db.internal', + }, + }); + + expect(reordered).toBe(verified); + expect(insecure).not.toBe(verified); + expect(() => + getPgPoolIdentity({ + ...cfg, + ssl: { checkServerIdentity: (): undefined => undefined } as any, + }) + ).toThrow('must contain only deterministic data values'); + }); + + it('never discloses credentials in an identity', () => { + const cfg = { ...freshConfig(), password: 'top-secret-password' }; + const identity = getPgPoolIdentity(cfg, { purpose: 'runtime' }); + + expect(identity).toMatch(/^pg:v1:[a-f0-9]{64}$/); + expect(identity).not.toContain(cfg.user); + expect(identity).not.toContain(cfg.password); + }); + + it('parses and validates maxUses before constructing a pool', () => { + const previous = process.env.PG_POOL_MAX_USES; + try { + process.env.PG_POOL_MAX_USES = '0'; + expect(getPgPoolConfig().maxUses).toBeUndefined(); + process.env.PG_POOL_MAX_USES = '17'; + expect(getPgPoolConfig().maxUses).toBe(17); + process.env.PG_POOL_MAX_USES = '1e2'; + expect(() => getPgPoolConfig()).toThrow( + 'PG_POOL_MAX_USES must be 0 or a positive safe integer' + ); + expect(() => getPgPoolConfig({ maxUses: -1 })).toThrow( + 'pool.maxUses must be 0 or a positive safe integer' + ); + } finally { + if (previous === undefined) delete process.env.PG_POOL_MAX_USES; + else process.env.PG_POOL_MAX_USES = previous; + } }); it('falls back to defaultPgPoolFactory when nothing is registered', () => { @@ -80,7 +297,7 @@ describe('pg-cache pool-factory seam', () => { // query runs, so this is safe without a live server. const pool = getPgPool(cfg); expect(pool).toBeInstanceOf(pg.Pool); - pgCache.delete(cfg.database); + pgCache.delete(getPgPoolIdentity(cfg)); }); it('defaultPgPoolFactory returns a pg.Pool', () => { @@ -88,4 +305,36 @@ describe('pg-cache pool-factory seam', () => { expect(pool).toBeInstanceOf(pg.Pool); return pool.end(); }); + + it('passes exact credentials, pool limits, and TLS fields to node-postgres', async () => { + const ssl = { + ca: 'tenant-ca', + cert: 'runtime-cert', + key: 'runtime-key', + rejectUnauthorized: true, + servername: 'db.internal', + minVersion: 'TLSv1.2' as const, + }; + const cfg = { + ...freshConfig(), + user: 'runtime@tenant', + password: 'x@evil.example/other?sslmode=require', + database: 'tenant/database', + ssl, + pool: { maxUses: 1 }, + }; + const pool = defaultPgPoolFactory(cfg); + const options = (pool as pg.Pool & { options: pg.PoolConfig }).options; + + expect(options).toMatchObject({ + host: cfg.host, + port: cfg.port, + database: cfg.database, + user: cfg.user, + password: cfg.password, + maxUses: 1, + ssl, + }); + await pool.end(); + }); }); diff --git a/postgres/pg-cache/src/__tests__/lru.test.ts b/postgres/pg-cache/src/__tests__/lru.test.ts index a68afa616f..c853641424 100644 --- a/postgres/pg-cache/src/__tests__/lru.test.ts +++ b/postgres/pg-cache/src/__tests__/lru.test.ts @@ -1,22 +1,37 @@ -// Guards against the pg-cache close() resource leak fixed in feat/observability. -// -// Previously, close() reset this.closed = false after shutdown, allowing -// set() to silently accept new pools that were never cleaned up. The module- -// level closePromise also reset to null, enabling double-shutdown. -// -// These tests lock the fix: close() is final, set() rejects, and repeated -// close() calls are idempotent. See pg-cache-close-leak.md for full details. - import pg from 'pg'; -import { PgPoolCacheManager } from '../lru'; +import { + DEFAULT_PG_CACHE_MAX, + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY, + PG_CACHE_OPERATIONAL_RESERVE, + PgPoolCacheManager, + PgPoolCapacityError, +} from '../lru'; + +describe('process lifecycle ownership', () => { + it('does not install process signal handlers from a library import', () => { + const beforeSigterm = process.listenerCount('SIGTERM'); + const beforeSigint = process.listenerCount('SIGINT'); + + jest.isolateModules(() => { + jest.requireActual('../lru'); + }); + + expect(process.listenerCount('SIGTERM')).toBe(beforeSigterm); + expect(process.listenerCount('SIGINT')).toBe(beforeSigint); + }); +}); // Minimal mock — we only need pool.end() and pool.ended const createMockPool = (): pg.Pool => { let ended = false; return { - get ended() { return ended; }, - end: jest.fn(async () => { ended = true; }), + get ended() { + return ended; + }, + end: jest.fn(async () => { + ended = true; + }), } as unknown as pg.Pool; }; @@ -29,7 +44,11 @@ describe('PgPoolCacheManager', () => { afterEach(async () => { // Ensure all pools are cleaned up even if a test fails mid-way - try { await cache.close(); } catch { /* already closed */ } + try { + await cache.close(); + } catch { + /* already closed */ + } }); it('stores and retrieves a pool', () => { @@ -45,8 +64,11 @@ describe('PgPoolCacheManager', () => { }); describe('configuration', () => { - it('uses env-var defaults (max=50) when no overrides given', () => { - expect(cache.config.max).toBe(50); + it('reserves two identities per supported Graphile contract plus operations', () => { + expect(DEFAULT_PG_CACHE_MAX).toBe( + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY * 2 + PG_CACHE_OPERATIONAL_RESERVE + ); + expect(cache.config.max).toBe(2064); }); it('accepts constructor overrides', () => { @@ -90,6 +112,171 @@ describe('PgPoolCacheManager', () => { }); }); + describe('leases and fail-closed admission', () => { + it('keeps database-name deletion as a migration alias for opaque keys', async () => { + const pool = createMockPool(); + cache.set('pg:v1:opaque', pool); + cache.registerAlias('tenant_a', 'pg:v1:opaque'); + + cache.delete('tenant_a'); + await cache.waitForDisposals(); + + expect(cache.has('pg:v1:opaque')).toBe(false); + expect(pool.end).toHaveBeenCalledTimes(1); + }); + + it('counts an existing exact identity as zero new slots', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const pool = createMockPool(); + const factory = jest.fn(() => pool); + + const first = small.acquire('runtime-a', factory); + const second = small.acquire('runtime-a', factory); + + expect(first.pool).toBe(pool); + expect(second.pool).toBe(pool); + expect(factory).toHaveBeenCalledTimes(1); + expect(small.getStats()).toMatchObject({ + size: 1, + leasedPools: 1, + activeLeases: 2, + leasesAcquired: 2, + }); + + first.release(); + first.release(); + expect(small.getStats().activeLeases).toBe(1); + second.release(); + await small.close(); + }); + + it('refuses before constructing or ending when every slot is leased', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const firstPool = createMockPool(); + const first = small.acquire('runtime-a', () => firstPool); + const rejectedFactory = jest.fn(() => createMockPool()); + + let capacityError: PgPoolCapacityError | undefined; + try { + small.acquire('runtime-b', rejectedFactory); + } catch (error) { + capacityError = error as PgPoolCapacityError; + } + + expect(capacityError).toBeInstanceOf(PgPoolCapacityError); + expect(capacityError).toMatchObject({ + code: 'PG_POOL_CAPACITY', + retryAfterSeconds: 15, + max: 1, + size: 1, + leased: 1, + }); + expect(rejectedFactory).not.toHaveBeenCalled(); + expect(firstPool.end).not.toHaveBeenCalled(); + expect(small.getStats().capacityRefusals).toBe(1); + + first.release(); + await small.close(); + }); + + it('evicts only the least-recent zero-lease identity', async () => { + const small = new PgPoolCacheManager({ max: 2 }); + const leasedPool = createMockPool(); + const idlePool = createMockPool(); + const replacementPool = createMockPool(); + const lease = small.acquire('leased', () => leasedPool); + small.set('idle', idlePool); + + small.set('replacement', replacementPool); + await small.waitForDisposals(); + + expect(small.has('leased')).toBe(true); + expect(leasedPool.end).not.toHaveBeenCalled(); + expect(small.has('idle')).toBe(false); + expect(idlePool.end).toHaveBeenCalledTimes(1); + expect(small.has('replacement')).toBe(true); + + lease.release(); + await small.close(); + }); + + it('keeps an expired leased identity until release', async () => { + jest.useFakeTimers(); + const small = new PgPoolCacheManager({ max: 1, ttl: 50 }); + const pool = createMockPool(); + const lease = small.acquire('runtime', () => pool); + try { + jest.advanceTimersByTime(51); + expect(small.has('runtime')).toBe(true); + expect(pool.end).not.toHaveBeenCalled(); + + lease.release(); + await small.waitForDisposals(); + + expect(small.has('runtime')).toBe(false); + expect(pool.end).toHaveBeenCalledTimes(1); + expect(small.getStats().ttlExpirations).toBe(1); + } finally { + jest.useRealTimers(); + await small.close(); + } + }); + + it('deterministically gives the final slot to the first synchronous acquisition', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const firstFactory = jest.fn(() => createMockPool()); + const secondFactory = jest.fn(() => createMockPool()); + + const outcomes = await Promise.allSettled([ + Promise.resolve().then(() => small.acquire('first', firstFactory)), + Promise.resolve().then(() => small.acquire('second', secondFactory)), + ]); + + expect(outcomes[0].status).toBe('fulfilled'); + expect(outcomes[1].status).toBe('rejected'); + expect((outcomes[1] as PromiseRejectedResult).reason).toBeInstanceOf( + PgPoolCapacityError + ); + expect(firstFactory).toHaveBeenCalledTimes(1); + expect(secondFactory).not.toHaveBeenCalled(); + + if (outcomes[0].status === 'fulfilled') outcomes[0].value.release(); + await small.close(); + }); + + it('rolls back its reservation if pool construction fails', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const retained = createMockPool(); + small.set('retained', retained); + + expect(() => + small.acquire('broken', () => { + throw new Error('factory failed'); + }) + ).toThrow('factory failed'); + + expect(small.has('retained')).toBe(true); + expect(retained.end).not.toHaveBeenCalled(); + expect(small.getStats()).toMatchObject({ size: 1, reservations: 0 }); + await small.close(); + }); + + it('does not end a physical pool retained under another exact identity', async () => { + const small = new PgPoolCacheManager({ max: 1 }); + const sharedPool = createMockPool(); + + small.set('identity-a', sharedPool); + small.set('identity-b', sharedPool); + await small.waitForDisposals(); + expect(sharedPool.end).not.toHaveBeenCalled(); + + small.delete('identity-b'); + await small.waitForDisposals(); + expect(sharedPool.end).toHaveBeenCalledTimes(1); + await small.close(); + }); + }); + describe('close() lifecycle', () => { it('set() after close() succeeds (cache re-opens for restart)', async () => { const pool1 = createMockPool(); @@ -122,6 +309,29 @@ describe('PgPoolCacheManager', () => { expect(pool.end).toHaveBeenCalledTimes(1); }); + it('makes concurrent close callers await the same pool teardown', async () => { + let finishEnd!: () => void; + const ended = new Promise((resolve) => { + finishEnd = resolve; + }); + const pool = createMockPool(); + (pool.end as jest.Mock).mockImplementation(() => ended); + cache.set('key1', pool); + + const first = cache.close(); + const second = cache.close(); + let secondSettled = false; + void second.then(() => { + secondSettled = true; + }); + await Promise.resolve(); + + expect(secondSettled).toBe(false); + finishEnd(); + await Promise.all([first, second]); + expect(pool.end).toHaveBeenCalledTimes(1); + }); + it('close() disposes all pools', async () => { const pool1 = createMockPool(); const pool2 = createMockPool(); diff --git a/postgres/pg-cache/src/driver.ts b/postgres/pg-cache/src/driver.ts index 9a6c22ffb7..b110a1bcab 100644 --- a/postgres/pg-cache/src/driver.ts +++ b/postgres/pg-cache/src/driver.ts @@ -14,10 +14,14 @@ import type { PgConfig, PgPoolConfig } from 'pg-env'; * `end()` (plus an `ended` flag for disposal), so a factory may return anything * implementing that subset — `QueryablePool`. A real `pg.Pool` structurally * satisfies it, so the default path is unchanged and fully backward-compatible. + * The default TCP factory owns the checkout sanitation contract. Registered + * factories retain their existing behavior and advertise a separate driver + * identity because they own their own checkout semantics. */ export interface QueryableClient { query(text: string, values?: any[]): Promise; - release(...args: any[]): void; + /** A truthy error argument must permanently discard this client. */ + release(error?: Error | boolean): void; } export interface QueryablePool { @@ -27,10 +31,16 @@ export interface QueryablePool { } export type PgPoolFactory = ( - config: Partial & { pool?: PgPoolConfig } + config: Partial & { pool?: PgPoolConfig }, + options?: PgPoolFactoryOptions ) => pg.Pool | QueryablePool; +export interface PgPoolFactoryOptions { + purpose: string; +} + let activeFactory: PgPoolFactory | undefined; +let driverGeneration = 0; /** * Register the factory `getPgPool` uses to build new pools. Pass `undefined` @@ -42,6 +52,7 @@ let activeFactory: PgPoolFactory | undefined; */ export const registerPgPoolFactory = (factory: PgPoolFactory | undefined): void => { activeFactory = factory; + driverGeneration++; }; /** The currently-registered factory, or `undefined` when using the default. */ @@ -49,3 +60,7 @@ export const getActivePgPoolFactory = (): PgPoolFactory | undefined => activeFac /** Whether a non-default pool factory is currently registered. */ export const hasPgPoolFactory = (): boolean => activeFactory !== undefined; + +/** Stable until the active factory registration changes. */ +export const getPgPoolDriverIdentity = (): string => + activeFactory ? `registered:${driverGeneration}` : 'node-postgres'; diff --git a/postgres/pg-cache/src/index.ts b/postgres/pg-cache/src/index.ts index 286748297a..369a4c039d 100644 --- a/postgres/pg-cache/src/index.ts +++ b/postgres/pg-cache/src/index.ts @@ -1,23 +1,39 @@ // Main exports from pg-cache package export { getActivePgPoolFactory, + getPgPoolDriverIdentity, hasPgPoolFactory, registerPgPoolFactory } from './driver'; export { close, + DEFAULT_PG_CACHE_MAX, getPgCacheConfig, - pgCache, - PgPoolCacheManager, + getPgCacheStats, + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY, + PG_CACHE_OPERATIONAL_RESERVE, + PG_POOL_CAPACITY_ERROR_CODE, + pgCache, + PgPoolCacheManager, + PgPoolCapacityError, teardownPgPools } from './lru'; export { + acquirePgPool, buildConnectionString, defaultPgPoolFactory, + getPgDatabaseTargetIdentity, getPgPool, - getPgPoolConfig + getPgPoolConfig, + getPgPoolIdentity } from './pg'; - // Re-export types -export type { PgPoolFactory, QueryableClient, QueryablePool } from './driver'; -export type { PgCacheConfig, PoolCleanupCallback } from './lru'; \ No newline at end of file +export type { PgPoolFactory, PgPoolFactoryOptions, QueryableClient, QueryablePool } from './driver'; +export type { + PgCacheConfig, + PgPoolCacheStats, + PgPoolDisposalReason, + PgPoolLease, + PoolCleanupCallback, +} from './lru'; +export type { GetPgPoolOptions } from './pg'; diff --git a/postgres/pg-cache/src/lru.ts b/postgres/pg-cache/src/lru.ts index 633dc6388e..38fe32d0e5 100644 --- a/postgres/pg-cache/src/lru.ts +++ b/postgres/pg-cache/src/lru.ts @@ -1,6 +1,5 @@ import { Logger } from '@pgpmjs/logger'; import { parseEnvNumber } from '12factor-env'; -import { LRUCache } from 'lru-cache'; import pg from 'pg'; const log = new Logger('pg-cache'); @@ -9,58 +8,136 @@ const ONE_HOUR_IN_MS = 1000 * 60 * 60; const ONE_DAY = ONE_HOUR_IN_MS * 24; const ONE_YEAR = ONE_DAY * 366; -// Kubernetes sends only SIGTERM on pod shutdown -const SYS_EVENTS = ['SIGTERM']; +// One runtime and one control identity per database-per-tenant Graphile +// contract, plus room for routing, diagnostics, listeners, and build overlap. +export const PG_CACHE_GRAPHILE_CONTRACT_CAPACITY = 1024; +export const PG_CACHE_OPERATIONAL_RESERVE = 16; +export const DEFAULT_PG_CACHE_MAX = + PG_CACHE_GRAPHILE_CONTRACT_CAPACITY * 2 + PG_CACHE_OPERATIONAL_RESERVE; type PgPoolKey = string; +type PoolFactory = () => pg.Pool; -// Cleanup callback type - called when a pg pool is disposed +export type PgPoolDisposalReason = + 'capacity' | 'ttl' | 'delete' | 'clear' | 'close' | 'replace'; + +// Called only when an identity is actually removed from the registry. export type PoolCleanupCallback = (pgPoolKey: string) => void; -// --- Cache Configuration --- +export interface PgPoolLease { + pool: pg.Pool; + identity: string; + /** Idempotently release this exact ownership claim. */ + release(): void; +} export interface PgCacheConfig { - /** Maximum number of pools in the LRU cache (env: PG_CACHE_MAX, default: 50) */ + /** Maximum number of lazy pool identities retained by this process. */ max: number; - /** TTL for cached pools in ms (default: ONE_YEAR) */ + /** Idle identity TTL in milliseconds. Leased identities never expire. */ ttl: number; } -/** - * Read cache configuration from environment variables. - * - * Supports: - * - PG_CACHE_MAX: Maximum number of pools (default: 50) - * - PG_CACHE_TTL_MS: TTL in milliseconds (default: ONE_YEAR) - */ +export interface PgPoolCacheStats { + size: number; + max: number; + ttl: number; + leasedPools: number; + idlePools: number; + activeLeases: number; + reservations: number; + pendingDisposals: number; + hits: number; + misses: number; + poolsCreated: number; + leasesAcquired: number; + leasesReleased: number; + capacityEvictions: number; + ttlExpirations: number; + capacityRefusals: number; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; +} + +interface PgPoolCacheCounters { + hits: number; + misses: number; + poolsCreated: number; + leasesAcquired: number; + leasesReleased: number; + capacityEvictions: number; + ttlExpirations: number; + capacityRefusals: number; + disposalsStarted: number; + disposalsCompleted: number; + disposalFailures: number; +} + +interface SlotReservation { + key: PgPoolKey; + victims: ManagedPgPool[]; +} + +export const PG_POOL_CAPACITY_ERROR_CODE = 'PG_POOL_CAPACITY'; + +/** Fail-closed pool admission error suitable for a stable HTTP 503 mapping. */ +export class PgPoolCapacityError extends Error { + readonly code = PG_POOL_CAPACITY_ERROR_CODE; + readonly retryAfterSeconds = 15; + + constructor( + readonly max: number, + readonly size: number, + readonly leased: number + ) { + super( + `PostgreSQL pool capacity exhausted: ${size}/${max} identities are retained ` + + `and ${leased} are leased` + ); + this.name = 'PgPoolCapacityError'; + } +} + +/** Read cache configuration without allocating any pools or connections. */ export function getPgCacheConfig(): PgCacheConfig { return { - max: parseEnvNumber(process.env.PG_CACHE_MAX) ?? 50, + max: parseEnvNumber(process.env.PG_CACHE_MAX) ?? DEFAULT_PG_CACHE_MAX, ttl: parseEnvNumber(process.env.PG_CACHE_TTL_MS) ?? ONE_YEAR, }; } class ManagedPgPool { public isDisposed = false; + public leaseCount = 0; + public lastAccessOrder = 0; + public expiresAt = 0; private disposePromise: Promise | null = null; - constructor(public readonly pool: pg.Pool, public readonly key: string) {} + constructor( + public readonly pool: pg.Pool, + public readonly key: string + ) {} + + touch(order: number, now: number, ttl: number): void { + this.lastAccessOrder = order; + this.expiresAt = now + ttl; + } + + isExpired(now: number): boolean { + return now >= this.expiresAt; + } async dispose(): Promise { if (this.isDisposed) return this.disposePromise; this.isDisposed = true; this.disposePromise = (async () => { - try { - if (!this.pool.ended) { - await this.pool.end(); - log.success(`pg.Pool ${this.key} ended.`); - } else { - log.info(`pg.Pool ${this.key} already ended.`); - } - } catch (err) { - log.error(`Error ending pg.Pool ${this.key}: ${(err as Error).message}`); - throw err; + if (!this.pool.ended) { + await this.pool.end(); + log.success(`pg.Pool ${this.key} ended.`); + } else { + log.info(`pg.Pool ${this.key} already ended.`); } })(); @@ -68,37 +145,75 @@ class ManagedPgPool { } } +/** + * A lease-aware, lazy pool registry. + * + * JavaScript executes acquisition synchronously, including slot reservation and + * factory invocation. Two callers therefore cannot both claim the final slot. + * Pools may finish ending asynchronously after a zero-lease identity is removed. + */ export class PgPoolCacheManager { - private cleanupTasks: Promise[] = []; + private readonly records = new Map(); + private readonly cleanupTasks = new Set>(); + private readonly cleanupCallbacks = new Set(); + private readonly aliasKeys = new Map>(); + private readonly keyAliases = new Map>(); + private readonly reservedKeys = new Set(); + private reservations = 0; + private accessOrder = 0; private closed = false; - private cleanupCallbacks: Set = new Set(); + private closePromise: Promise | null = null; readonly config: PgCacheConfig; - private readonly pgCache: LRUCache; + private readonly counters: PgPoolCacheCounters = { + hits: 0, + misses: 0, + poolsCreated: 0, + leasesAcquired: 0, + leasesReleased: 0, + capacityEvictions: 0, + ttlExpirations: 0, + capacityRefusals: 0, + disposalsStarted: 0, + disposalsCompleted: 0, + disposalFailures: 0, + }; constructor(config?: Partial) { const defaults = getPgCacheConfig(); this.config = { ...defaults, ...config }; + if (!Number.isSafeInteger(this.config.max) || this.config.max <= 0) { + throw new Error('pg-cache max must be a positive safe integer'); + } + if (!Number.isFinite(this.config.ttl) || this.config.ttl <= 0) { + throw new Error('pg-cache ttl must be a positive number'); + } + } - this.pgCache = new LRUCache({ - max: this.config.max, - ttl: this.config.ttl, - updateAgeOnGet: true, - dispose: (managedPool, key, reason) => { - log.debug(`Disposing pg pool [${key}] (${reason})`); - this.notifyCleanup(key); - this.disposePool(managedPool); - } - }); + get size(): number { + return this.records.size; } - // Register a cleanup callback to be called when pools are disposed registerCleanupCallback(callback: PoolCleanupCallback): () => void { this.cleanupCallbacks.add(callback); - // Return unregister function - return () => { - this.cleanupCallbacks.delete(callback); - }; + return () => this.cleanupCallbacks.delete(callback); + } + + /** Preserve database-name cleanup for callers migrating to opaque keys. */ + registerAlias(alias: string, key: PgPoolKey): void { + if (!this.records.has(key)) return; + let keys = this.aliasKeys.get(alias); + if (!keys) { + keys = new Set(); + this.aliasKeys.set(alias, keys); + } + keys.add(key); + let aliases = this.keyAliases.get(key); + if (!aliases) { + aliases = new Set(); + this.keyAliases.set(key, aliases); + } + aliases.add(alias); } get(key: PgPoolKey): pg.Pool | undefined { @@ -106,51 +221,335 @@ export class PgPoolCacheManager { log.warn(`Cache is closed, ignoring get(${key})`); return undefined; } - return this.pgCache.get(key)?.pool; + const managedPool = this.getLiveRecord(key, true); + if (!managedPool) { + this.counters.misses++; + return undefined; + } + this.counters.hits++; + return managedPool.pool; } has(key: PgPoolKey): boolean { - return this.pgCache.has(key); + if (this.closed) return false; + return Boolean(this.getLiveRecord(key, false)); } + /** + * Legacy direct insertion. Prefer getOrCreate/acquire so capacity is checked + * before the caller constructs a pool. + */ set(key: PgPoolKey, pool: pg.Pool): void { - if (this.closed) throw new Error(`Cannot add to cache after it has been closed (key: ${key})`); - this.pgCache.set(key, new ManagedPgPool(pool, key)); + this.assertOpen(key); + const existing = this.records.get(key); + if (existing?.pool === pool) { + this.touch(existing); + return; + } + if (existing?.leaseCount) { + throw new Error(`Cannot replace leased pg pool identity ${key}`); + } + if (existing) this.removeRecord(existing, 'replace'); + + const reservation = this.reserveSlot(key); + this.commitReservation(reservation, pool, 0); } + /** Atomically capacity-check, synchronously construct, and cache an idle pool. */ + getOrCreate(key: PgPoolKey, factory: PoolFactory): pg.Pool { + this.assertOpen(key); + const existing = this.getLiveRecord(key, true); + if (existing) { + this.counters.hits++; + return existing.pool; + } + + this.counters.misses++; + return this.createWithReservation(key, factory, 0).pool; + } + + /** + * Atomically get/create and lease an exact identity. A leased identity cannot + * be selected by capacity or TTL eviction until every lease is released. + */ + acquire(key: PgPoolKey, factory: PoolFactory): PgPoolLease { + this.assertOpen(key); + let managedPool = this.getLiveRecord(key, true); + if (managedPool) { + this.counters.hits++; + managedPool.leaseCount++; + } else { + this.counters.misses++; + managedPool = this.createWithReservation(key, factory, 1); + } + this.counters.leasesAcquired++; + return this.makeLease(managedPool); + } + + /** Explicit deletion never interrupts a lease; callers may retry after release. */ delete(key: PgPoolKey): void { - const managedPool = this.pgCache.get(key); - const existed = this.pgCache.delete(key); - if (!existed && managedPool) { - this.notifyCleanup(key); - this.disposePool(managedPool); + const managedPool = this.records.get(key); + if (managedPool) { + if (managedPool.leaseCount === 0) + this.removeRecord(managedPool, 'delete'); + return; + } + for (const aliasedKey of [...(this.aliasKeys.get(key) ?? [])]) { + const aliasedPool = this.records.get(aliasedKey); + if (aliasedPool && aliasedPool.leaseCount === 0) { + this.removeRecord(aliasedPool, 'delete'); + } } } + /** Clear every currently unleased identity. */ clear(): void { - const entries = [...this.pgCache.entries()]; - this.pgCache.clear(); - for (const [key, managedPool] of entries) { - this.notifyCleanup(key); - this.disposePool(managedPool); + for (const managedPool of [...this.records.values()]) { + if (managedPool.leaseCount === 0) this.removeRecord(managedPool, 'clear'); } } async close(): Promise { - if (this.closed) return; + if (this.closePromise) return this.closePromise; this.closed = true; - this.clear(); - await this.waitForDisposals(); - // Re-open the cache so it can accept new entries if the process - // survives the shutdown signal (e.g. during provisioning or restart). - this.closed = false; + this.closePromise = (async () => { + try { + // Explicit process teardown is the only operation that may override leases. + for (const managedPool of [...this.records.values()]) { + this.removeRecord(managedPool, 'close'); + } + await this.waitForDisposals(); + } finally { + // Preserve the established restart/provisioning behavior. + this.closed = false; + this.closePromise = null; + } + })(); + return this.closePromise; } async waitForDisposals(): Promise { - if (this.cleanupTasks.length === 0) return; - const tasks = [...this.cleanupTasks]; - this.cleanupTasks = []; - await Promise.allSettled(tasks); + while (this.cleanupTasks.size > 0) { + await Promise.allSettled([...this.cleanupTasks]); + } + } + + getStats(): PgPoolCacheStats { + let leasedPools = 0; + let activeLeases = 0; + for (const managedPool of this.records.values()) { + if (managedPool.leaseCount > 0) leasedPools++; + activeLeases += managedPool.leaseCount; + } + return { + size: this.records.size, + max: this.config.max, + ttl: this.config.ttl, + leasedPools, + idlePools: this.records.size - leasedPools, + activeLeases, + reservations: this.reservations, + pendingDisposals: this.cleanupTasks.size, + ...this.counters, + }; + } + + private assertOpen(key: PgPoolKey): void { + if (this.closed) { + throw new Error( + `Cannot access pg cache while it is closed (key: ${key})` + ); + } + } + + private touch(managedPool: ManagedPgPool): void { + managedPool.touch(++this.accessOrder, Date.now(), this.config.ttl); + } + + private getLiveRecord( + key: PgPoolKey, + updateAge: boolean + ): ManagedPgPool | undefined { + const managedPool = this.records.get(key); + if (!managedPool) return undefined; + if (managedPool.leaseCount === 0 && managedPool.isExpired(Date.now())) { + this.removeRecord(managedPool, 'ttl'); + return undefined; + } + if (updateAge) this.touch(managedPool); + return managedPool; + } + + private idleRecordsByAge(): ManagedPgPool[] { + return [...this.records.values()] + .filter((managedPool) => managedPool.leaseCount === 0) + .sort((a, b) => a.lastAccessOrder - b.lastAccessOrder); + } + + private reserveSlot(key: PgPoolKey): SlotReservation { + if (this.reservedKeys.has(key)) { + throw new Error(`Re-entrant pg pool acquisition for identity ${key}`); + } + + const overflow = Math.max( + 0, + this.records.size + this.reservations + 1 - this.config.max + ); + const candidates = this.idleRecordsByAge(); + if (candidates.length < overflow) { + this.counters.capacityRefusals++; + throw new PgPoolCapacityError( + this.config.max, + this.records.size + this.reservations, + this.countLeasedPools() + ); + } + + const victims = candidates.slice(0, overflow); + for (const victim of victims) this.records.delete(victim.key); + this.reservations++; + this.reservedKeys.add(key); + return { key, victims }; + } + + private rollbackReservation(reservation: SlotReservation): void { + this.reservations = Math.max(0, this.reservations - 1); + this.reservedKeys.delete(reservation.key); + for (const victim of reservation.victims) { + this.records.set(victim.key, victim); + } + } + + private commitReservation( + reservation: SlotReservation, + pool: pg.Pool, + leaseCount: number + ): ManagedPgPool { + const managedPool = new ManagedPgPool(pool, reservation.key); + managedPool.leaseCount = leaseCount; + this.touch(managedPool); + this.records.set(reservation.key, managedPool); + this.reservations = Math.max(0, this.reservations - 1); + this.reservedKeys.delete(reservation.key); + this.counters.poolsCreated++; + + for (const victim of reservation.victims) { + this.counters.capacityEvictions++; + this.disposeRemovedRecord(victim); + } + return managedPool; + } + + private createWithReservation( + key: PgPoolKey, + factory: PoolFactory, + leaseCount: number + ): ManagedPgPool { + const reservation = this.reserveSlot(key); + let pool: pg.Pool; + try { + pool = factory(); + } catch (error) { + this.rollbackReservation(reservation); + throw error; + } + return this.commitReservation(reservation, pool, leaseCount); + } + + private makeLease(managedPool: ManagedPgPool): PgPoolLease { + let released = false; + return { + pool: managedPool.pool, + identity: managedPool.key, + release: () => { + if (released) return; + released = true; + this.counters.leasesReleased++; + managedPool.leaseCount = Math.max(0, managedPool.leaseCount - 1); + + // close() may already have detached this record. + if (this.records.get(managedPool.key) !== managedPool) return; + if (managedPool.leaseCount > 0) return; + if (managedPool.isExpired(Date.now())) { + this.removeRecord(managedPool, 'ttl'); + return; + } + this.enforceCapacity(); + }, + }; + } + + private enforceCapacity(): void { + while (this.records.size > this.config.max) { + const victim = this.idleRecordsByAge()[0]; + if (!victim) return; + this.removeRecord(victim, 'capacity'); + } + } + + private countLeasedPools(): number { + let leased = 0; + for (const managedPool of this.records.values()) { + if (managedPool.leaseCount > 0) leased++; + } + return leased; + } + + private removeRecord( + managedPool: ManagedPgPool, + reason: PgPoolDisposalReason + ): void { + if (this.records.get(managedPool.key) !== managedPool) return; + this.records.delete(managedPool.key); + if (reason === 'capacity') this.counters.capacityEvictions++; + if (reason === 'ttl') this.counters.ttlExpirations++; + this.disposeRemovedRecord(managedPool); + } + + private disposeRemovedRecord(managedPool: ManagedPgPool): void { + const cleanupKeys = [ + managedPool.key, + ...this.unregisterAliases(managedPool.key), + ]; + for (const cleanupKey of cleanupKeys) this.notifyCleanup(cleanupKey); + + // Alternate drivers may intentionally return one physical pool for multiple + // exact identities. Never end it while another retained identity owns it. + if ( + [...this.records.values()].some( + (entry) => entry.pool === managedPool.pool + ) + ) { + return; + } + if (managedPool.isDisposed) return; + + this.counters.disposalsStarted++; + let task: Promise; + task = managedPool + .dispose() + .then(() => { + this.counters.disposalsCompleted++; + }) + .catch((error) => { + this.counters.disposalFailures++; + log.error( + `Error ending pg.Pool ${managedPool.key}: ${(error as Error).message}` + ); + }) + .finally(() => this.cleanupTasks.delete(task)); + this.cleanupTasks.add(task); + } + + private unregisterAliases(key: PgPoolKey): string[] { + const aliases = [...(this.keyAliases.get(key) ?? [])]; + this.keyAliases.delete(key); + for (const alias of aliases) { + const keys = this.aliasKeys.get(alias); + keys?.delete(key); + if (keys?.size === 0) this.aliasKeys.delete(alias); + } + return aliases; } private notifyCleanup(pgPoolKey: string): void { @@ -162,17 +561,14 @@ export class PgPoolCacheManager { } }); } - - private disposePool(managedPool: ManagedPgPool): void { - if (managedPool.isDisposed) return; - const task = managedPool.dispose(); - this.cleanupTasks.push(task); - } } -// Create the singleton instance +// Process-wide registry. Its large capacity is only a key limit; pools and +// PostgreSQL connections remain lazily allocated on first use. export const pgCache = new PgPoolCacheManager(); +export const getPgCacheStats = (): PgPoolCacheStats => pgCache.getStats(); + // --- Graceful Shutdown --- const closePromise: { promise: Promise | null } = { promise: null }; @@ -185,7 +581,6 @@ export const close = async (verbose = false): Promise => { await pgCache.close(); if (verbose) log.success('PG cache disposed.'); } finally { - // Reset so close() can be called again if the process survives. closePromise.promise = null; } })(); @@ -193,13 +588,6 @@ export const close = async (verbose = false): Promise => { return closePromise.promise; }; -SYS_EVENTS.forEach(event => { - process.on(event, () => { - log.info(`Received ${event}`); - close(); - }); -}); - export const teardownPgPools = async (verbose = false): Promise => { return close(verbose); }; diff --git a/postgres/pg-cache/src/pg.ts b/postgres/pg-cache/src/pg.ts index 920a900ddd..67d1102dbd 100644 --- a/postgres/pg-cache/src/pg.ts +++ b/postgres/pg-cache/src/pg.ts @@ -1,50 +1,329 @@ +import { createHash, createHmac, randomBytes } from 'node:crypto'; + import { Logger } from '@pgpmjs/logger'; import { parseEnvNumber } from '12factor-env'; import pg from 'pg'; import { getPgEnvOptions, PgConfig, PgPoolConfig } from 'pg-env'; -import { getActivePgPoolFactory, PgPoolFactory } from './driver'; -import { pgCache } from './lru'; +import { + getActivePgPoolFactory, + getPgPoolDriverIdentity, + PgPoolFactory, +} from './driver'; +import { pgCache, type PgPoolLease } from './lru'; import { installCheckoutSanitizer } from './sanitizer'; const log = new Logger('pg-cache'); +export interface GetPgPoolOptions { + /** Separates pools used by different trust boundaries. */ + purpose?: string; +} + +const normalizePoolOptions = ( + options: GetPgPoolOptions = {} +): Required => { + const purpose = options.purpose ?? 'default'; + if (typeof purpose !== 'string' || purpose.length === 0) { + throw new TypeError('pg pool purpose must be a non-empty string'); + } + return { purpose }; +}; + +// Pool identities may appear in diagnostics and cache lifecycle logs. A plain +// digest over a known connection shape could act as an offline password +// verifier, so identities are keyed and intentionally process-local. +const pgIdentityHmacKey = randomBytes(32); + +const hmacIdentity = (prefix: string, identity: string): string => + `${prefix}:${createHmac('sha256', pgIdentityHmacKey) + .update(identity) + .digest('hex')}`; + +const requireIdentityString = (value: unknown, path: string): string => { + if (typeof value !== 'string') { + throw new TypeError(`${path} must be a string`); + } + return value; +}; + +const requireIdentityInteger = ( + value: unknown, + path: string, + minimum: number, + maximum = Number.MAX_SAFE_INTEGER +): number => { + if ( + typeof value !== 'number' || + !Number.isSafeInteger(value) || + value < minimum || + value > maximum + ) { + throw new TypeError( + `${path} must be a safe integer between ${minimum} and ${maximum}` + ); + } + return value; +}; + +const canonicalizeIdentityValue = ( + value: unknown, + path: string, + ancestors = new Set() +): unknown => { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' + ) { + return value; + } + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new TypeError(`${path} must not contain a non-finite number`); + } + return Object.is(value, -0) ? ['number', '-0'] : value; + } + if (Buffer.isBuffer(value)) { + return ['buffer-sha256', createHash('sha256').update(value).digest('hex')]; + } + if (Array.isArray(value)) { + if (ancestors.has(value)) throw new TypeError(`${path} must not be cyclic`); + const ownKeys = Reflect.ownKeys(value); + if ( + ownKeys.some((key) => typeof key !== 'string') || + ownKeys.some( + (key) => key !== 'length' && !/^(?:0|[1-9]\d*)$/.test(key as string) + ) || + value.some( + (_entry, index) => !Object.prototype.hasOwnProperty.call(value, index) + ) || + Object.keys(value).length !== value.length + ) { + throw new TypeError( + `${path} must be a dense array without custom properties` + ); + } + ancestors.add(value); + const result = value.map((entry, index) => { + if (entry === undefined) { + throw new TypeError(`${path}[${index}] must not be undefined`); + } + return canonicalizeIdentityValue(entry, `${path}[${index}]`, ancestors); + }); + ancestors.delete(value); + return ['array', result]; + } + if (typeof value === 'object') { + const record = value as Record; + const prototype = Object.getPrototypeOf(record); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError(`${path} must contain only data values`); + } + if (ancestors.has(record)) + throw new TypeError(`${path} must not be cyclic`); + ancestors.add(record); + const result: Array<[string, unknown]> = []; + const ownKeys = Reflect.ownKeys(record); + if (ownKeys.some((key) => typeof key !== 'string')) { + throw new TypeError(`${path} must not contain symbol properties`); + } + for (const key of (ownKeys as string[]).sort()) { + const descriptor = Object.getOwnPropertyDescriptor(record, key); + if (!descriptor || !('value' in descriptor)) { + throw new TypeError(`${path}.${key} must be a data property`); + } + const entry = descriptor.value; + if (entry === undefined) { + throw new TypeError(`${path}.${key} must not be undefined`); + } + result.push([ + key, + canonicalizeIdentityValue(entry, `${path}.${key}`, ancestors), + ]); + } + ancestors.delete(record); + return ['object', result]; + } + throw new TypeError(`${path} must contain only deterministic data values`); +}; + export const buildConnectionString = ( user: string, password: string, host: string, port: string | number, database: string -): string => - `postgres://${user}:${password}@${host}:${port}/${database}`; +): string => { + const encodedHost = + host.includes(':') && !host.startsWith('[') + ? `[${host}]` + : encodeURIComponent(host); + return ( + `postgres://${encodeURIComponent(user)}:${encodeURIComponent(password)}` + + `@${encodedHost}:${port}/${encodeURIComponent(database)}` + ); +}; /** * Read per-pool configuration from environment variables. * * Supports: * - PG_POOL_MAX: Maximum clients per pool (default: 5) + * - PG_POOL_MAX_USES: Retire a client after this many checkouts (0/unset: unlimited) * - PG_POOL_IDLE_TIMEOUT_MS: Close idle clients after ms (default: 30000) * - PG_POOL_CONNECTION_TIMEOUT_MS: Fail connect() after ms (default: 5000) */ +const normalizeMaxUses = ( + value: number | string | undefined, + source: 'pool.maxUses' | 'PG_POOL_MAX_USES' +): number | undefined => { + if (value === undefined || value === '') return undefined; + if (typeof value !== 'number' && typeof value !== 'string') { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + if (typeof value === 'string' && !/^(?:0|[1-9]\d*)$/.test(value)) { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + const parsed = typeof value === 'number' ? value : Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 0) { + throw new TypeError(`${source} must be 0 or a positive safe integer`); + } + return parsed === 0 ? undefined : parsed; +}; + export function getPgPoolConfig(overrides?: PgPoolConfig): pg.PoolConfig { - return { + const maxUses = + overrides?.maxUses !== undefined + ? normalizeMaxUses(overrides.maxUses, 'pool.maxUses') + : normalizeMaxUses(process.env.PG_POOL_MAX_USES, 'PG_POOL_MAX_USES'); + const pool = { max: overrides?.max ?? parseEnvNumber(process.env.PG_POOL_MAX) ?? 5, - idleTimeoutMillis: overrides?.idleTimeoutMillis ?? parseEnvNumber(process.env.PG_POOL_IDLE_TIMEOUT_MS) ?? 30000, - connectionTimeoutMillis: overrides?.connectionTimeoutMillis ?? parseEnvNumber(process.env.PG_POOL_CONNECTION_TIMEOUT_MS) ?? 5000, - ...(overrides?.allowExitOnIdle !== undefined && { allowExitOnIdle: overrides.allowExitOnIdle }), + ...(maxUses !== undefined && { maxUses }), + idleTimeoutMillis: + overrides?.idleTimeoutMillis ?? + parseEnvNumber(process.env.PG_POOL_IDLE_TIMEOUT_MS) ?? + 30000, + connectionTimeoutMillis: + overrides?.connectionTimeoutMillis ?? + parseEnvNumber(process.env.PG_POOL_CONNECTION_TIMEOUT_MS) ?? + 5000, + ...(overrides?.allowExitOnIdle !== undefined && { + allowExitOnIdle: overrides.allowExitOnIdle, + }), + }; + requireIdentityInteger(pool.max, 'pool.max', 1); + if (pool.maxUses !== undefined) { + requireIdentityInteger(pool.maxUses, 'pool.maxUses', 1); + } + requireIdentityInteger(pool.idleTimeoutMillis, 'pool.idleTimeoutMillis', 0); + requireIdentityInteger( + pool.connectionTimeoutMillis, + 'pool.connectionTimeoutMillis', + 0 + ); + if ( + pool.allowExitOnIdle !== undefined && + typeof pool.allowExitOnIdle !== 'boolean' + ) { + throw new TypeError('pool.allowExitOnIdle must be a boolean'); + } + return pool; +} + +const normalizeIdentityConfig = ( + pgConfig: Partial & { pool?: PgPoolConfig } +): { config: PgConfig; ssl: unknown } => { + const config = getPgEnvOptions(pgConfig); + requireIdentityString(config.host, 'pg.host'); + requireIdentityInteger(config.port, 'pg.port', 1, 65_535); + requireIdentityString(config.database, 'pg.database'); + requireIdentityString(config.user, 'pg.user'); + // node-postgres accepts password callbacks at runtime; captured credentials + // cannot be represented exactly, so the shared cache rejects them. + requireIdentityString(config.password, 'pg.password'); + return { + config, + ssl: canonicalizeIdentityValue(config.ssl ?? null, 'pg.ssl'), }; +}; + +/** Opaque identity for the complete connection and reuse contract. */ +export function getPgPoolIdentity( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): string { + const { config, ssl } = normalizeIdentityConfig(pgConfig); + const pool = getPgPoolConfig(pgConfig.pool); + const normalizedOptions = normalizePoolOptions(options); + const driver = requireIdentityString( + getPgPoolDriverIdentity(), + 'pg driver identity' + ); + const identity = JSON.stringify({ + version: 1, + driver, + host: config.host, + port: config.port, + database: config.database, + user: config.user, + password: config.password, + ssl, + pool: { + max: pool.max, + maxUses: pool.maxUses ?? null, + idleTimeoutMillis: pool.idleTimeoutMillis, + connectionTimeoutMillis: pool.connectionTimeoutMillis, + allowExitOnIdle: pool.allowExitOnIdle ?? false, + }, + purpose: normalizedOptions.purpose, + checkout: getActivePgPoolFactory() + ? 'registered-factory-owned-v1' + : 'discard-all-v1', + }); + return hmacIdentity('pg:v1', identity); +} + +/** Opaque identity for a physical database, excluding login and pool policy. */ +export function getPgDatabaseTargetIdentity( + pgConfig: Partial +): string { + const { config } = normalizeIdentityConfig(pgConfig); + const driver = requireIdentityString( + getPgPoolDriverIdentity(), + 'pg driver identity' + ); + const identity = JSON.stringify({ + version: 1, + driver, + host: config.host, + port: config.port, + database: config.database, + }); + return hmacIdentity('pg-target:v1', identity); } /** * Default pool factory: builds a real `pg.Pool` over TCP. This is the behavior * used whenever no alternate driver is registered (see `./driver`). */ -export const defaultPgPoolFactory: PgPoolFactory = (pgConfig): pg.Pool => { - const config = getPgEnvOptions(pgConfig); - const { user, password, host, port, database } = config; - const connectionString = buildConnectionString(user, password, host, port, database); +export const defaultPgPoolFactory: PgPoolFactory = ( + pgConfig, + options +): pg.Pool => { + const { config } = normalizeIdentityConfig(pgConfig); + normalizePoolOptions(options); + const { user, password, host, port, database, ssl } = config; const poolConfig = getPgPoolConfig(pgConfig.pool); - const pgPool = new pg.Pool({ connectionString, ...poolConfig }); + const pgPool = new pg.Pool({ + host, + port, + database, + user, + password, + ...(ssl !== undefined && { ssl }), + ...poolConfig, + }); /** * IMPORTANT: Pool-level error handler for idle connection errors. @@ -90,31 +369,73 @@ export const defaultPgPoolFactory: PgPoolFactory = (pgConfig): pg.Pool => { pgPool.on('error', (err: Error & { code?: string }) => { if (err.code === '57P01') { // Expected during database cleanup - log at debug level - log.debug(`Pool ${database} connection terminated (expected during cleanup): ${err.message}`); + log.debug( + `Pool ${database} connection terminated (expected during cleanup): ${err.message}` + ); } else { // Unexpected pool error - log at error level for visibility // Note: This does NOT swallow query errors - those still throw via Promise rejection - log.error(`Pool ${database} unexpected idle connection error [${err.code || 'unknown'}]: ${err.message}`); + log.error( + `Pool ${database} unexpected idle connection error [${err.code || 'unknown'}]: ${err.message}` + ); } }); return installCheckoutSanitizer(pgPool); }; -export const getPgPool = (pgConfig: Partial & { pool?: PgPoolConfig }): pg.Pool => { - const config = getPgEnvOptions(pgConfig); - const { database } = config; - if (pgCache.has(database)) { - const cached = pgCache.get(database); - if (cached) return cached; - } - +const createPgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: Required +): pg.Pool => { // Route through the registered driver (default = pg.Pool over TCP). A custom // factory may return any QueryablePool (e.g. an in-process PGlite pool); it is // treated as a pg.Pool since that is the only surface consumers use. const factory = getActivePgPoolFactory() ?? defaultPgPoolFactory; - const pgPool = factory(pgConfig) as pg.Pool; + return factory(pgConfig, options) as pg.Pool; +}; + +const getPgPoolWithOptions = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): pg.Pool => { + const normalizedOptions = normalizePoolOptions(options); + const identity = getPgPoolIdentity(pgConfig, normalizedOptions); + const pool = pgCache.getOrCreate(identity, () => + createPgPool(pgConfig, normalizedOptions) + ); + pgCache.registerAlias( + normalizeIdentityConfig(pgConfig).config.database, + identity + ); + return pool; +}; + +/** Compatibility API with an optional exact-purpose reuse boundary. */ +export function getPgPool( + pgConfig: Partial & { pool?: PgPoolConfig }, + options?: GetPgPoolOptions +): pg.Pool; +export function getPgPool( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): pg.Pool { + return getPgPoolWithOptions(pgConfig, options); +} - pgCache.set(database, pgPool); - return pgPool; +/** Acquire an idempotently releasable ownership claim over one exact pool. */ +export const acquirePgPool = ( + pgConfig: Partial & { pool?: PgPoolConfig }, + options: GetPgPoolOptions = {} +): PgPoolLease => { + const normalizedOptions = normalizePoolOptions(options); + const identity = getPgPoolIdentity(pgConfig, normalizedOptions); + const lease = pgCache.acquire(identity, () => + createPgPool(pgConfig, normalizedOptions) + ); + pgCache.registerAlias( + normalizeIdentityConfig(pgConfig).config.database, + identity + ); + return lease; }; diff --git a/postgres/pg-env/src/pg-config.ts b/postgres/pg-env/src/pg-config.ts index 7ed78ce5cd..673a89cde3 100644 --- a/postgres/pg-env/src/pg-config.ts +++ b/postgres/pg-env/src/pg-config.ts @@ -1,9 +1,36 @@ +import type { SecureVersion } from 'node:tls'; + +/** + * Serializable TLS options supported by the shared PostgreSQL connection + * contract. Keeping this surface data-only is intentional: pool identities + * must account for every TLS input, which callback and socket objects cannot + * do deterministically. + */ +export interface PgSslOptions { + ca?: string | Buffer | Array; + cert?: string | Buffer | Array; + key?: + | string + | Buffer + | Array; + passphrase?: string; + rejectUnauthorized?: boolean; + servername?: string; + minVersion?: SecureVersion; + maxVersion?: SecureVersion; + ciphers?: string; +} + +export type PgSslConfig = boolean | PgSslOptions; + export interface PgConfig { host: string; port: number; user: string; password: string; database: string; + /** TLS settings passed directly to node-postgres. */ + ssl?: PgSslConfig; } /** @@ -15,6 +42,8 @@ export interface PgConfig { export interface PgPoolConfig { /** Maximum number of clients in the pool (env: PG_POOL_MAX, default: 5) */ max?: number; + /** Retire a client after this many checkouts (env: PG_POOL_MAX_USES, 0/unset: unlimited) */ + maxUses?: number; /** Close idle clients after this many ms (env: PG_POOL_IDLE_TIMEOUT_MS, default: 30000) */ idleTimeoutMillis?: number; /** Reject pool.connect() after this many ms (env: PG_POOL_CONNECTION_TIMEOUT_MS, default: 5000) */ @@ -29,4 +58,4 @@ export const defaultPgConfig: PgConfig = { user: 'postgres', password: 'password', database: 'postgres' -}; \ No newline at end of file +}; From 04476a636866b44a800e3ef2d76c3d8621192756 Mon Sep 17 00:00:00 2001 From: zetazzz Date: Mon, 17 Aug 2026 16:54:56 +0800 Subject: [PATCH 2/5] Add optional runtime PostgreSQL credential contracts --- graphql/env/src/__tests__/runtime-pg.test.ts | 63 ++++ graphql/env/src/env.ts | 9 + graphql/env/src/merge.ts | 4 + graphql/types/src/constructive.ts | 26 +- graphql/types/src/index.ts | 5 +- .../__tests__/context-pg-settings.test.ts | 1 + .../__tests__/context-pool-leases.test.ts | 249 +++++++++++++++ packages/express-context/package.json | 1 + packages/express-context/src/context.ts | 292 +++++++++++++++++- packages/express-context/src/index.ts | 2 +- packages/express-context/src/types.ts | 2 + pnpm-lock.yaml | 3 + 12 files changed, 640 insertions(+), 17 deletions(-) create mode 100644 graphql/env/src/__tests__/runtime-pg.test.ts create mode 100644 packages/express-context/__tests__/context-pool-leases.test.ts diff --git a/graphql/env/src/__tests__/runtime-pg.test.ts b/graphql/env/src/__tests__/runtime-pg.test.ts new file mode 100644 index 0000000000..6c738f5976 --- /dev/null +++ b/graphql/env/src/__tests__/runtime-pg.test.ts @@ -0,0 +1,63 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { getGraphQLEnvVars } from '../env'; +import { getEnvOptions } from '../merge'; + +describe('GraphQL runtime PostgreSQL environment', () => { + it('maps dedicated runtime credentials without changing control-plane pg', () => { + const result = getGraphQLEnvVars({ + GRAPHQL_RUNTIME_PGUSER: 'graphql_runtime', + GRAPHQL_RUNTIME_PGPASSWORD: 'runtime-secret', + }); + + expect(result.runtimePg).toEqual({ + user: 'graphql_runtime', + password: 'runtime-secret', + }); + expect(result.pg).toBeUndefined(); + }); + + it('does not create a runtime override when both variables are absent', () => { + expect(getGraphQLEnvVars({}).runtimePg).toBeUndefined(); + }); + + it('merges static runtime config, env password, and exact route identity', () => { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'graphql-runtime-pg-')); + const identity = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public'], + roles: ['anonymous', 'authenticated'], + }; + fs.writeFileSync( + path.join(cwd, 'pgpm.json'), + JSON.stringify({ + runtimePg: { + host: 'runtime.internal', + database: 'tenant_a', + user: 'tenant_runtime', + password: 'config-secret', + }, + runtimePgStaticIdentity: identity, + }) + ); + + try { + const result = getEnvOptions({}, cwd, { + GRAPHQL_RUNTIME_PGPASSWORD: 'env-secret', + }); + expect(result.runtimePg).toMatchObject({ + host: 'runtime.internal', + database: 'tenant_a', + user: 'tenant_runtime', + password: 'env-secret', + }); + expect(result.runtimePgStaticIdentity).toEqual(identity); + } finally { + fs.rmSync(cwd, { recursive: true, force: true }); + } + }); +}); diff --git a/graphql/env/src/env.ts b/graphql/env/src/env.ts index 014924ef24..d459d759ae 100644 --- a/graphql/env/src/env.ts +++ b/graphql/env/src/env.ts @@ -8,6 +8,9 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial const { GRAPHILE_SCHEMA, + GRAPHQL_RUNTIME_PGUSER, + GRAPHQL_RUNTIME_PGPASSWORD, + FEATURES_SIMPLE_INFLECTION, FEATURES_OPPOSITE_BASE_NAMES, FEATURES_POSTGIS, @@ -47,6 +50,12 @@ export const getGraphQLEnvVars = (env: NodeJS.ProcessEnv = process.env): Partial ); return { + ...((GRAPHQL_RUNTIME_PGUSER || GRAPHQL_RUNTIME_PGPASSWORD) && { + runtimePg: { + ...(GRAPHQL_RUNTIME_PGUSER && { user: GRAPHQL_RUNTIME_PGUSER }), + ...(GRAPHQL_RUNTIME_PGPASSWORD && { password: GRAPHQL_RUNTIME_PGPASSWORD }) + } + }), graphile: { ...(GRAPHILE_SCHEMA && { schema: GRAPHILE_SCHEMA.includes(',') diff --git a/graphql/env/src/merge.ts b/graphql/env/src/merge.ts index 15f1402c53..977c6c57f3 100644 --- a/graphql/env/src/merge.ts +++ b/graphql/env/src/merge.ts @@ -44,6 +44,10 @@ export const getEnvOptions = ( ...(configOptions.graphile && { graphile: configOptions.graphile }), ...(configOptions.features && { features: configOptions.features }), ...(configOptions.api && { api: configOptions.api }), + ...(configOptions.runtimePg && { runtimePg: configOptions.runtimePg }), + ...(configOptions.runtimePgStaticIdentity && { + runtimePgStaticIdentity: configOptions.runtimePgStaticIdentity + }), ...(configOptions.sms && { sms: configOptions.sms }), }, graphqlEnvOptions, diff --git a/graphql/types/src/constructive.ts b/graphql/types/src/constructive.ts index 485a4f4a59..4c6eba42f0 100644 --- a/graphql/types/src/constructive.ts +++ b/graphql/types/src/constructive.ts @@ -7,7 +7,7 @@ import { PgTestConnectionOptions, ServerOptions} from '@pgpmjs/types'; import deepmerge from 'deepmerge'; -import { PgConfig } from 'pg-env'; +import type { PgConfig, PgPoolConfig } from 'pg-env'; import { apiDefaults, @@ -19,6 +19,24 @@ import { import { LlmOptions } from './llm'; import { SmsOptions } from './sms'; +/** Credential-free route facts used to resolve one runtime login. */ +export interface RuntimePgResolverInput { + databaseId: string; + databaseName: string; + apiId: string; + /** Physical schemas in Graphile exposure order. */ + schemas: readonly string[]; + /** Request roles in `[anonymous, authenticated]` order. */ + roles: readonly [anonymous: string, authenticated: string]; +} + +export type RuntimePgConfig = Partial & { pool?: PgPoolConfig }; + +/** Resolve a least-privilege login from credential-free exact route facts. */ +export type RuntimePgResolver = ( + input: Readonly +) => RuntimePgConfig | Promise; + /** * GraphQL-specific options for Constructive */ @@ -40,6 +58,12 @@ export interface ConstructiveOptions extends PgpmOptions, ConstructiveGraphQLOpt db?: Partial; /** PostgreSQL connection configuration */ pg?: Partial; + /** Static least-privilege tenant execution login. */ + runtimePg?: RuntimePgConfig; + /** Exact route authorized to use the static runtime login. */ + runtimePgStaticIdentity?: RuntimePgResolverInput; + /** Per-route least-privilege tenant execution login resolver. */ + runtimePgResolver?: RuntimePgResolver; /** PostGraphile/Graphile configuration */ graphile?: GraphileOptions; /** HTTP server configuration */ diff --git a/graphql/types/src/index.ts b/graphql/types/src/index.ts index 895604e137..3e66b2d520 100644 --- a/graphql/types/src/index.ts +++ b/graphql/types/src/index.ts @@ -12,7 +12,10 @@ export { constructiveDefaults, constructiveGraphqlDefaults, ConstructiveGraphQLOptions, - ConstructiveOptions} from './constructive'; + ConstructiveOptions, + RuntimePgConfig, + RuntimePgResolver, + RuntimePgResolverInput} from './constructive'; // Export GraphQL adapter types export { diff --git a/packages/express-context/__tests__/context-pg-settings.test.ts b/packages/express-context/__tests__/context-pg-settings.test.ts index a24a21cc0b..24f7c6b22e 100644 --- a/packages/express-context/__tests__/context-pg-settings.test.ts +++ b/packages/express-context/__tests__/context-pg-settings.test.ts @@ -4,6 +4,7 @@ import { buildContext } from '../src/context'; jest.mock('pg-cache', () => ({ getPgPool: jest.fn(() => ({ query: jest.fn(), connect: jest.fn() })), + getPgPoolIdentity: jest.fn(() => 'pg:v1:test'), })); describe('buildContext pgSettings forwarding', () => { diff --git a/packages/express-context/__tests__/context-pool-leases.test.ts b/packages/express-context/__tests__/context-pool-leases.test.ts new file mode 100644 index 0000000000..5c5a1d7c75 --- /dev/null +++ b/packages/express-context/__tests__/context-pool-leases.test.ts @@ -0,0 +1,249 @@ +import { EventEmitter } from 'node:events'; + +import type { Request, Response } from 'express'; +import type { Pool } from 'pg'; +import { acquirePgPool, getPgPool, getPgPoolIdentity } from 'pg-cache'; + +import { buildContext, createContextMiddleware } from '../src/context'; +import type { ApiStructure } from '../src/types'; + +jest.mock('pg-cache', () => ({ + acquirePgPool: jest.fn(), + getPgPool: jest.fn(), + getPgPoolIdentity: jest.fn(() => 'pg:v1:test'), +})); + +const mockedAcquire = acquirePgPool as jest.MockedFunction< + typeof acquirePgPool +>; +const mockedGet = getPgPool as jest.MockedFunction; +const mockedIdentity = getPgPoolIdentity as jest.MockedFunction< + typeof getPgPoolIdentity +>; + +const api: ApiStructure = { + apiId: 'api-a', + databaseId: 'database-a', + dbname: 'tenant_a', + anonRole: 'anonymous', + roleName: 'authenticated', + schema: ['tenant_a_public'], + domains: [], + isPublic: false, +}; + +const runtimeRoute = { + databaseId: 'database-a', + databaseName: 'tenant_a', + apiId: 'api-a', + schemas: ['tenant_a_public'], + roles: ['anonymous', 'authenticated'] as [string, string], +}; + +const makeRequest = (): Request => + Object.assign(new EventEmitter(), { + api, + requestId: 'request-a', + get: jest.fn((): undefined => undefined), + aborted: false, + socket: { destroyed: false }, + }) as unknown as Request; + +const makePool = (): Pool => ({ query: jest.fn() }) as unknown as Pool; + +const makeResponse = (): Response => { + const response = new EventEmitter() as EventEmitter & { + destroyed: boolean; + writableEnded: boolean; + }; + response.destroyed = false; + response.writableEnded = false; + return response as unknown as Response; +}; + +let leaseSequence = 0; +const leaseFor = (pool: Pool) => ({ + pool, + identity: `pg:v1:lease-${++leaseSequence}`, + release: jest.fn(), +}); + +describe('context PostgreSQL identities and lifetimes', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedIdentity.mockReturnValue('pg:v1:test'); + }); + + it('preserves the unconfigured single-login path for direct callers', () => { + const pool = makePool(); + mockedGet.mockReturnValue(pool); + + const context = buildContext(makeRequest(), { pg: { user: 'control' } }); + + expect(context?.pool).toBe(pool); + expect(mockedGet).toHaveBeenCalledWith( + { user: 'control', database: 'tenant_a' }, + { purpose: 'runtime' } + ); + expect(mockedAcquire).not.toHaveBeenCalled(); + }); + + it('pins runtime and control pools until response completion', () => { + const leases = [ + leaseFor(makePool()), + leaseFor(makePool()), + leaseFor(makePool()), + ]; + mockedAcquire + .mockReturnValueOnce(leases[0]) + .mockReturnValueOnce(leases[1]) + .mockReturnValueOnce(leases[2]); + const response = makeResponse(); + const next = jest.fn(); + + createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any, + })(makeRequest(), response, next); + + expect(next).toHaveBeenCalledWith(); + expect(mockedAcquire).toHaveBeenCalledTimes(3); + response.emit('finish'); + response.emit('close'); + for (const lease of leases) expect(lease.release).toHaveBeenCalledTimes(1); + }); + + it('binds a static runtime login to one exact route', () => { + const runtime = leaseFor(makePool()); + runtime.identity = 'pg:v1:test'; + mockedAcquire.mockReturnValue(runtime); + const next = jest.fn(); + + createContextMiddleware({ + pg: { host: 'control.internal' }, + runtimePg: { + database: 'tenant_a', + user: 'runtime', + password: 'runtime-secret', + }, + runtimePgStaticIdentity: runtimeRoute, + })(makeRequest(), makeResponse(), next); + + expect(next).toHaveBeenCalledWith(); + expect(mockedAcquire).toHaveBeenCalledWith( + { + host: 'control.internal', + database: 'tenant_a', + user: 'runtime', + password: 'runtime-secret', + }, + { purpose: 'runtime' } + ); + }); + + it('rejects a static login when the route identity differs', () => { + const next = jest.fn(); + + createContextMiddleware({ + runtimePg: { + database: 'tenant_a', + user: 'runtime', + password: 'secret', + }, + runtimePgStaticIdentity: { ...runtimeRoute, apiId: 'api-b' }, + })(makeRequest(), makeResponse(), next); + + expect(next.mock.calls[0][0]).toEqual( + expect.objectContaining({ + message: expect.stringContaining('identity does not match'), + }) + ); + expect(mockedAcquire).not.toHaveBeenCalled(); + }); + + it('passes frozen credential-free facts to the runtime resolver', () => { + const lease = leaseFor(makePool()); + lease.identity = 'pg:v1:test'; + mockedAcquire.mockReturnValue(lease); + const resolver = jest.fn((input) => { + expect(Object.isFrozen(input)).toBe(true); + expect(Object.isFrozen(input.schemas)).toBe(true); + expect(Object.isFrozen(input.roles)).toBe(true); + expect(input).not.toHaveProperty('password'); + return { + host: 'runtime.internal', + database: input.databaseName, + user: 'runtime', + password: 'resolver-secret', + }; + }); + const request = makeRequest(); + const next = jest.fn(); + + createContextMiddleware({ runtimePgResolver: resolver })( + request, + makeResponse(), + next + ); + + expect(resolver).toHaveBeenCalledWith(runtimeRoute); + expect(next).toHaveBeenCalledWith(); + expect(request.constructive?.runtimePoolIdentity).toBe('pg:v1:test'); + }); + + it('does not fall back when the runtime resolver rejects', async () => { + const failure = new Error('resolver unavailable'); + const next = jest.fn(); + + createContextMiddleware({ + runtimePgResolver: async () => { + throw failure; + }, + })(makeRequest(), makeResponse(), next); + await new Promise((resolve) => setImmediate(resolve)); + + expect(next).toHaveBeenCalledWith(failure); + expect(mockedAcquire).not.toHaveBeenCalled(); + expect(mockedGet).not.toHaveBeenCalled(); + }); + + it.each([ + [ + { database: 'tenant_b', user: 'runtime', password: 'secret' }, + 'does not match', + ], + [{ database: 'tenant_a', user: 'runtime' }, 'explicit password'], + ])('fails closed for an invalid resolved login', (resolved, message) => { + const next = jest.fn(); + + createContextMiddleware({ runtimePgResolver: () => resolved })( + makeRequest(), + makeResponse(), + next + ); + + expect(next.mock.calls[0][0]).toEqual( + expect.objectContaining({ + message: expect.stringContaining(message), + }) + ); + expect(mockedAcquire).not.toHaveBeenCalled(); + }); + + it('releases earlier leases when a later control acquisition fails', () => { + const first = leaseFor(makePool()); + const failure = new Error('capacity'); + mockedAcquire.mockReturnValueOnce(first).mockImplementationOnce(() => { + throw failure; + }); + const next = jest.fn(); + + createContextMiddleware({ + pg: { database: 'routing' }, + loaders: { resolve: jest.fn() } as any, + })(makeRequest(), makeResponse(), next); + + expect(first.release).toHaveBeenCalledTimes(1); + expect(next).toHaveBeenCalledWith(failure); + }); +}); diff --git a/packages/express-context/package.json b/packages/express-context/package.json index 56602d5a48..754fe2925b 100644 --- a/packages/express-context/package.json +++ b/packages/express-context/package.json @@ -29,6 +29,7 @@ "test:watch": "jest --watch" }, "dependencies": { + "@constructive-io/graphql-types": "workspace:^", "@constructive-io/url-domains": "workspace:^", "@pgpmjs/env": "workspace:^", "@pgpmjs/logger": "workspace:^", diff --git a/packages/express-context/src/context.ts b/packages/express-context/src/context.ts index 87ece44dca..2138360d74 100644 --- a/packages/express-context/src/context.ts +++ b/packages/express-context/src/context.ts @@ -14,10 +14,21 @@ * route handler can use for tenant-scoped database operations. */ +import type { + RuntimePgConfig, + RuntimePgResolver, + RuntimePgResolverInput, +} from '@constructive-io/graphql-types'; import type { PgpmOptions } from '@pgpmjs/types'; import type { NextFunction, Request, RequestHandler, Response } from 'express'; import type { Pool } from 'pg'; -import { getPgPool } from 'pg-cache'; +import { + acquirePgPool, + getPgPool, + getPgPoolIdentity, + type GetPgPoolOptions, + type PgPoolLease, +} from 'pg-cache'; import type { BillingClient } from './billing-client'; import { createBillingClient } from './billing-client'; @@ -25,11 +36,32 @@ import type { LoaderRegistry } from './loaders/registry'; import type { LoaderContext } from './loaders/types'; import { withPgClient as withPgClientFn } from './pg-client'; import { buildPgSettings } from './pg-settings'; -import type { BillingConfig, BuiltinModuleMap, ConstructiveContext, InferenceLogConfig, LlmConfig } from './types'; +import type { + ApiStructure, + BillingConfig, + BuiltinModuleMap, + ConstructiveContext, + InferenceLogConfig, + LlmConfig, +} from './types'; + +type PoolConfig = Parameters[0]; + +/** Secret-bearing config paired with its process-local opaque identity. */ +export interface RuntimePgPoolResolution { + pgConfig: PoolConfig; + poolIdentity: string; +} export interface ContextMiddlewareOptions { /** Base PG options for pool creation (host, port, user, password) */ pg?: PgpmOptions['pg']; + /** Static least-privilege tenant execution login. */ + runtimePg?: RuntimePgConfig; + /** Exact route authorized to use `runtimePg`. */ + runtimePgStaticIdentity?: RuntimePgResolverInput; + /** Resolve one least-privilege login from credential-free route facts. */ + runtimePgResolver?: RuntimePgResolver; /** Module loader registry for per-database cached lookups */ loaders?: LoaderRegistry; /** Routing-plane schema loaders query (defaults to routing_public) */ @@ -38,6 +70,138 @@ export interface ContextMiddlewareOptions { dependencySchemas?: readonly string[]; } +interface ResolvedPool { + pool: Pool; + identity: string; +} + +const resolvePool = ( + config: PoolConfig, + options: GetPgPoolOptions, + leases?: PgPoolLease[] +): ResolvedPool => { + if (!leases) { + return { + pool: getPgPool(config, options), + identity: getPgPoolIdentity(config, options), + }; + } + const lease = acquirePgPool(config, options); + leases.push(lease); + return { pool: lease.pool, identity: lease.identity }; +}; + +const requireRouteFact = (value: unknown, name: string): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`Runtime PostgreSQL route requires ${name}`); + } + return value; +}; + +const runtimeRouteInput = ( + api: ApiStructure +): Readonly => + Object.freeze({ + databaseId: requireRouteFact(api.databaseId, 'databaseId'), + databaseName: requireRouteFact(api.dbname, 'databaseName'), + apiId: requireRouteFact(api.apiId, 'apiId'), + schemas: Object.freeze([...api.schema]), + roles: Object.freeze([ + requireRouteFact(api.anonRole, 'anonymous role'), + requireRouteFact(api.roleName, 'authenticated role'), + ]) as readonly [string, string], + }); + +const sameRuntimeRoute = ( + left: Readonly, + right: Readonly +): boolean => + left.databaseId === right.databaseId && + left.databaseName === right.databaseName && + left.apiId === right.apiId && + left.schemas.length === right.schemas.length && + left.schemas.every((schema, index) => schema === right.schemas[index]) && + left.roles[0] === right.roles[0] && + left.roles[1] === right.roles[1]; + +const requireExplicitRuntimeConfig = ( + candidate: RuntimePgConfig, + route: Readonly, + controlPg: PgpmOptions['pg'] | undefined +): PoolConfig => { + if (!candidate || typeof candidate !== 'object') { + throw new Error( + 'Runtime PostgreSQL resolver returned an invalid configuration' + ); + } + for (const field of ['database', 'user', 'password'] as const) { + if ( + typeof candidate[field] !== 'string' || + candidate[field]!.length === 0 + ) { + throw new Error( + `Runtime PostgreSQL configuration requires explicit ${field}` + ); + } + } + if (candidate.database !== route.databaseName) { + throw new Error( + 'Runtime PostgreSQL database does not match the resolved route' + ); + } + return { ...controlPg, ...candidate }; +}; + +const makeRuntimeResolution = ( + candidate: RuntimePgConfig, + route: Readonly, + controlPg: PgpmOptions['pg'] | undefined +): Readonly => { + const pgConfig = requireExplicitRuntimeConfig(candidate, route, controlPg); + return Object.freeze({ + pgConfig, + poolIdentity: getPgPoolIdentity(pgConfig, { purpose: 'runtime' }), + }); +}; + +const isPromiseLike = (value: T | Promise): value is Promise => + typeof (value as Promise)?.then === 'function'; + +const resolveConfiguredRuntime = ( + api: ApiStructure, + opts: ContextMiddlewareOptions +): + | Readonly + | Promise> + | undefined => { + if (opts.runtimePgResolver && opts.runtimePg) { + throw new Error( + 'Configure either runtimePgResolver or runtimePg, not both' + ); + } + if (!opts.runtimePgResolver && !opts.runtimePg) return undefined; + const route = runtimeRouteInput(api); + if (opts.runtimePgResolver) { + const candidate = opts.runtimePgResolver(route); + return isPromiseLike(candidate) + ? Promise.resolve(candidate).then((resolved) => + makeRuntimeResolution(resolved, route, opts.pg) + ) + : makeRuntimeResolution(candidate, route, opts.pg); + } + if (!opts.runtimePgStaticIdentity) { + throw new Error( + 'Static runtime PostgreSQL requires runtimePgStaticIdentity' + ); + } + if (!sameRuntimeRoute(route, opts.runtimePgStaticIdentity)) { + throw new Error( + 'Static runtime PostgreSQL identity does not match the resolved route' + ); + } + return makeRuntimeResolution(opts.runtimePg, route, opts.pg); +}; + /** * Create a `useModule` function bound to the given loader context. * @@ -67,7 +231,11 @@ function createUseModule( */ export function buildContext( req: Request, - opts: ContextMiddlewareOptions = {} + opts: ContextMiddlewareOptions = {}, + /** Internal request lifetime; omitted for backwards-compatible direct use. */ + poolLeases?: PgPoolLease[], + /** Internal async resolver result; raw credentials never enter the request. */ + suppliedRuntimeResolution?: Readonly ): ConstructiveContext | null { const api = req.api; if (!api) return null; @@ -86,19 +254,53 @@ export function buildContext( dependencySchemas: opts.dependencySchemas, }); - const tenantPool: Pool = getPgPool({ + let runtimeResolution = suppliedRuntimeResolution; + if (!runtimeResolution) { + if (opts.runtimePgResolver) { + throw new Error( + 'runtimePgResolver must be used through createContextMiddleware' + ); + } + const configured = resolveConfiguredRuntime(api, opts); + runtimeResolution = configured as + Readonly | undefined; + } + const runtimeConfig = runtimeResolution?.pgConfig ?? { ...opts.pg, - database: api.dbname - }); + database: api.dbname, + }; + const runtimePool = resolvePool( + runtimeConfig, + { purpose: 'runtime' }, + poolLeases + ); + if ( + runtimeResolution && + runtimePool.identity !== runtimeResolution.poolIdentity + ) { + throw new Error( + 'Resolved runtime PostgreSQL pool identity changed before acquisition' + ); + } + const tenantPool = runtimePool.pool; // Build loader context (if registry provided and databaseId known) let loaderCtx: LoaderContext | null = null; if (opts.loaders && api.databaseId) { - const routingPool: Pool = getPgPool(opts.pg); + const routingPool = resolvePool( + opts.pg ?? {}, + { purpose: 'routing-request-control' }, + poolLeases + ); + const controlTenantPool = resolvePool( + { ...opts.pg, database: api.dbname }, + { purpose: 'tenant-request-control' }, + poolLeases + ); loaderCtx = { - routingPool, + routingPool: routingPool.pool, routingSchema: opts.routingSchema, - tenantPool, + tenantPool: controlTenantPool.pool, databaseId: api.databaseId, apiId: api.apiId, dbname: api.dbname @@ -122,6 +324,7 @@ export function buildContext( userId: token?.user_id ?? null, requestId, pool: tenantPool, + runtimePoolIdentity: runtimePool.identity, withPgClient, useModule, async useBilling() { @@ -187,11 +390,72 @@ export function buildContext( export function createContextMiddleware( opts: ContextMiddlewareOptions = {} ): RequestHandler { - return (req: Request, _res: Response, next: NextFunction): void => { - const ctx = buildContext(req, opts); - if (ctx) { - req.constructive = ctx; + return (req: Request, res: Response, next: NextFunction): void => { + const requestEnded = (): boolean => + Boolean( + req.aborted || + req.socket?.destroyed || + res.destroyed || + res.writableEnded + ); + if (requestEnded()) return; + + const finish = ( + runtimeResolution?: Readonly + ): void => { + if (requestEnded()) return; + const leases: PgPoolLease[] = []; + let released = false; + const releaseLeases = (): void => { + if (released) return; + released = true; + req.removeListener('aborted', releaseLeases); + res.removeListener('finish', releaseLeases); + res.removeListener('close', releaseLeases); + for (const lease of leases.reverse()) lease.release(); + }; + + try { + const ctx = buildContext(req, opts, leases, runtimeResolution); + if (!ctx) { + releaseLeases(); + next(); + return; + } + req.constructive = ctx; + req.once('aborted', releaseLeases); + res.once('finish', releaseLeases); + res.once('close', releaseLeases); + if (requestEnded()) { + releaseLeases(); + return; + } + next(); + } catch (error) { + releaseLeases(); + next(error); + } + }; + + try { + const api = req.api; + if (!api) { + next(); + return; + } + const resolution = resolveConfiguredRuntime(api, opts); + if (isPromiseLike(resolution)) { + void resolution.then( + (resolved) => finish(resolved), + (error) => { + if (!requestEnded()) next(error); + } + ); + } else { + finish(resolution); + } + } catch (error) { + next(error); } - next(); }; } diff --git a/packages/express-context/src/index.ts b/packages/express-context/src/index.ts index d02109b158..b2d77f294c 100644 --- a/packages/express-context/src/index.ts +++ b/packages/express-context/src/index.ts @@ -89,7 +89,7 @@ export { withPgClient } from './pg-client'; export { requestIdMiddleware } from './request-id'; // Context middleware -export type { ContextMiddlewareOptions } from './context'; +export type { ContextMiddlewareOptions, RuntimePgPoolResolution } from './context'; export { buildContext, createContextMiddleware } from './context'; // Module loaders diff --git a/packages/express-context/src/types.ts b/packages/express-context/src/types.ts index fd6258357c..0f2f8666ac 100644 --- a/packages/express-context/src/types.ts +++ b/packages/express-context/src/types.ts @@ -286,6 +286,8 @@ export interface ConstructiveContext { requestId: string; /** Tenant database connection pool */ pool: Pool; + /** Opaque identity of the exact runtime connection contract. */ + runtimePoolIdentity: string; /** Execute a function within a tenant-scoped RLS transaction */ withPgClient: WithPgClient; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cae3372a75..9244e5dcff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2512,6 +2512,9 @@ importers: packages/express-context: dependencies: + '@constructive-io/graphql-types': + specifier: workspace:^ + version: link:../../graphql/types/dist '@constructive-io/url-domains': specifier: workspace:^ version: link:../url-domains/dist From 4fe869b119ed0c6d934464344516137aadee8fbb Mon Sep 17 00:00:00 2001 From: zetazzz Date: Mon, 17 Aug 2026 16:55:13 +0800 Subject: [PATCH 3/5] Add notification listener role attestation --- .../src/__tests__/notification-role.test.ts | 309 +++++++++++++ postgres/pg-cache/src/index.ts | 19 + postgres/pg-cache/src/notification-role.ts | 436 ++++++++++++++++++ 3 files changed, 764 insertions(+) create mode 100644 postgres/pg-cache/src/__tests__/notification-role.test.ts create mode 100644 postgres/pg-cache/src/notification-role.ts diff --git a/postgres/pg-cache/src/__tests__/notification-role.test.ts b/postgres/pg-cache/src/__tests__/notification-role.test.ts new file mode 100644 index 0000000000..27541defd6 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-role.test.ts @@ -0,0 +1,309 @@ +import type { Pool } from 'pg'; + +import { + assertPgNotificationRole, + assertPgNotificationRoleClient, + auditPgNotificationRole, + auditPgNotificationRoleClient, + normalizePgNotificationRoleContracts, + PG_NOTIFICATION_ROLE_AUDIT_SQL, + PG_NOTIFICATION_ROLE_AUDIT_VERSION, + type PgNotificationRoleClient, + PgNotificationRoleContractError, + type PgNotificationRoleViolationCode, + UnsafePgNotificationRoleError, +} from '../notification-role'; + +const contract = { + role: 'tenant_001_notification', + database: 'tenant_001', +}; + +const safeRow = { + expected_role: contract.role, + session_role: contract.role, + active_role: contract.role, + active_database: contract.database, + rolcanlogin: true, + rolinherit: false, + rolsuper: false, + rolbypassrls: false, + rolcreaterole: false, + rolcreatedb: false, + rolreplication: false, + membership_count: 0, + target_database_exists: true, + target_connect: true, + other_database_connect_count: 0, + target_database_owner: false, + target_database_create: false, + target_database_temp: false, + schema_owner_count: 0, + schema_create_count: 0, + schema_usage_count: 0, + relation_privilege_count: 0, + function_privilege_count: 0, + sequence_privilege_count: 0, +}; + +const poolWithRow = (row: Record | undefined) => { + const client = { + query: jest.fn(async (query: string) => + query === PG_NOTIFICATION_ROLE_AUDIT_SQL + ? { rows: row ? [row] : [] } + : { rows: [] } + ), + release: jest.fn(), + }; + return { + pool: { connect: jest.fn(async () => client) } as unknown as Pool, + client, + }; +}; + +describe('PostgreSQL notification-role audit', () => { + it('returns a frozen credential-free attestation for an exact safe login', async () => { + const { pool, client } = poolWithRow(safeRow); + const audit = await assertPgNotificationRole(pool, { + ...contract, + password: 'must-not-escape', + } as typeof contract); + + expect(audit).toEqual({ + version: PG_NOTIFICATION_ROLE_AUDIT_VERSION, + ...contract, + safe: true, + violations: [], + }); + expect(Object.isFrozen(audit)).toBe(true); + expect(Object.isFrozen(audit.violations)).toBe(true); + expect(JSON.stringify(audit)).not.toContain('must-not-escape'); + expect(client.query).toHaveBeenNthCalledWith(1, 'BEGIN READ ONLY'); + expect(client.query).toHaveBeenNthCalledWith(2, 'SET LOCAL jit TO off'); + expect(client.query).toHaveBeenNthCalledWith( + 3, + PG_NOTIFICATION_ROLE_AUDIT_SQL, + [contract.role, contract.database] + ); + expect(client.query).toHaveBeenNthCalledWith(4, 'COMMIT'); + expect(client.release).toHaveBeenCalledWith(false); + }); + + it('audits an already-owned listener client without releasing it', async () => { + const { client } = poolWithRow(safeRow); + + await expect( + assertPgNotificationRoleClient( + client as unknown as PgNotificationRoleClient, + contract + ) + ).resolves.toMatchObject({ ...contract, safe: true }); + expect(client.release).not.toHaveBeenCalled(); + }); + + it('rolls back a failed pinned-client audit without taking ownership of release', async () => { + const failure = new Error('catalog unavailable'); + const client = { + query: jest + .fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ rows: [] }), + release: jest.fn(), + }; + + await expect(auditPgNotificationRoleClient(client, contract)).rejects.toBe( + failure + ); + expect(client.query).toHaveBeenNthCalledWith(4, 'ROLLBACK'); + expect(client.release).not.toHaveBeenCalled(); + }); + + it('reports both audit and rollback failures without releasing the pinned client', async () => { + const auditFailure = new Error('catalog unavailable'); + const rollbackFailure = new Error('rollback unavailable'); + const client = { + query: jest + .fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(auditFailure) + .mockRejectedValueOnce(rollbackFailure), + release: jest.fn(), + }; + + const rejected = auditPgNotificationRoleClient(client, contract); + await expect(rejected).rejects.toBeInstanceOf(AggregateError); + await expect(rejected).rejects.toMatchObject({ + cause: auditFailure, + errors: [auditFailure, rollbackFailure], + }); + expect(client.query).toHaveBeenNthCalledWith(4, 'ROLLBACK'); + expect(client.release).not.toHaveBeenCalled(); + }); + + it.each<[keyof typeof safeRow, unknown, PgNotificationRoleViolationCode]>([ + ['session_role', 'different_login', 'LOGIN_ROLE_MISMATCH'], + ['active_role', 'set_role_target', 'CURRENT_ROLE_MISMATCH'], + ['active_database', 'different_database', 'DATABASE_MISMATCH'], + ['rolcanlogin', false, 'LOGIN_REQUIRED'], + ['rolinherit', true, 'NOINHERIT_REQUIRED'], + ['rolsuper', true, 'SUPERUSER'], + ['rolbypassrls', true, 'BYPASSRLS'], + ['rolcreaterole', true, 'CREATEROLE'], + ['rolcreatedb', true, 'CREATEDB'], + ['rolreplication', true, 'REPLICATION'], + ['membership_count', 1, 'ROLE_MEMBERSHIP'], + ['target_database_exists', false, 'TARGET_DATABASE_MISSING'], + ['target_connect', false, 'TARGET_CONNECT_REQUIRED'], + ['other_database_connect_count', 1, 'CROSS_DATABASE_CONNECT'], + ['target_database_owner', true, 'DATABASE_OWNER'], + ['target_database_create', true, 'DATABASE_CREATE'], + ['target_database_temp', true, 'DATABASE_TEMP'], + ['schema_owner_count', 1, 'SCHEMA_OWNER'], + ['schema_create_count', 1, 'SCHEMA_CREATE'], + ['schema_usage_count', 1, 'SCHEMA_USAGE'], + ['relation_privilege_count', 1, 'RELATION_PRIVILEGE'], + ['function_privilege_count', 1, 'FUNCTION_PRIVILEGE'], + ['sequence_privilege_count', 1, 'SEQUENCE_PRIVILEGE'], + ])( + 'maps %s to its stable violation code', + async (field, unsafeValue, code) => { + const { pool } = poolWithRow({ ...safeRow, [field]: unsafeValue }); + const audit = await auditPgNotificationRole(pool, contract); + + expect(audit.safe).toBe(false); + expect(audit.violations).toContain(code); + await expect( + assertPgNotificationRole( + poolWithRow({ ...safeRow, [field]: unsafeValue }).pool, + contract + ) + ).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_ROLE_UNSAFE', + audit: expect.objectContaining({ + violations: expect.arrayContaining([code]), + }), + }); + } + ); + + it('fails closed when the catalog audit returns no role row', async () => { + const { pool } = poolWithRow(undefined); + const audit = await auditPgNotificationRole(pool, contract); + + expect(audit).toMatchObject({ + safe: false, + violations: ['AUDIT_NO_RESULT'], + }); + await expect( + assertPgNotificationRole(poolWithRow(undefined).pool, contract) + ).rejects.toBeInstanceOf(UnsafePgNotificationRoleError); + }); + + it('rolls back and destroys the client when the catalog query fails', async () => { + const failure = new Error('catalog unavailable'); + const client = { + query: jest + .fn() + .mockResolvedValueOnce({ rows: [] }) + .mockResolvedValueOnce({ rows: [] }) + .mockRejectedValueOnce(failure) + .mockResolvedValueOnce({ rows: [] }), + release: jest.fn(), + }; + const pool = { connect: jest.fn(async () => client) } as unknown as Pool; + + await expect(auditPgNotificationRole(pool, contract)).rejects.toBe(failure); + expect(client.query).toHaveBeenNthCalledWith(4, 'ROLLBACK'); + expect(client.release).toHaveBeenCalledWith(true); + }); + + it('audits exact database scope, membership edges, and every prohibited ACL class', () => { + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'membership.member = r.oid OR membership.roleid = r.oid' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'database_record.datname <> $2::text' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'CONNECT'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'CREATE'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'TEMP'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'schema_record.nspowner = r.oid' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("'USAGE'"); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_table_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_any_column_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_function_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain( + 'pg_catalog.has_sequence_privilege' + ); + expect(PG_NOTIFICATION_ROLE_AUDIT_SQL).toContain("n.nspname !~ '^pg_'"); + }); +}); + +describe('notification-role fleet contract', () => { + it('collapses exact generation duplicates and returns a deterministic frozen mapping', () => { + const normalized = normalizePgNotificationRoleContracts([ + { role: 'notify_b', database: 'tenant_b' }, + { ...contract }, + { ...contract }, + ]); + + expect(normalized).toEqual([ + contract, + { role: 'notify_b', database: 'tenant_b' }, + ]); + expect(Object.isFrozen(normalized)).toBe(true); + expect(normalized.every(Object.isFrozen)).toBe(true); + }); + + it('rejects multiple logins for one database and one login spanning databases', () => { + expect(() => + normalizePgNotificationRoleContracts([ + contract, + { role: 'another_notification', database: contract.database }, + ]) + ).toThrow('maps to multiple login roles'); + expect(() => + normalizePgNotificationRoleContracts([ + contract, + { role: contract.role, database: 'tenant_002' }, + ]) + ).toThrow('maps to multiple databases'); + }); + + const malformedContracts: Array<{ + contracts: readonly { role: string; database: string }[]; + }> = [ + { contracts: [] }, + { contracts: [{ role: '', database: 'tenant_001' }] }, + { contracts: [{ role: 'notify', database: '' }] }, + { contracts: [{ role: 'n'.repeat(64), database: 'tenant_001' }] }, + { + contracts: [ + { + role: 'notify', + database: `bad${String.fromCharCode(0xd800)}`, + }, + ], + }, + ]; + + it.each(malformedContracts)( + 'rejects malformed contract input', + ({ contracts }) => { + expect(() => normalizePgNotificationRoleContracts(contracts)).toThrow( + PgNotificationRoleContractError + ); + } + ); +}); diff --git a/postgres/pg-cache/src/index.ts b/postgres/pg-cache/src/index.ts index 369a4c039d..77bbc13908 100644 --- a/postgres/pg-cache/src/index.ts +++ b/postgres/pg-cache/src/index.ts @@ -18,6 +18,19 @@ export { PgPoolCapacityError, teardownPgPools } from './lru'; +export { + assertPgNotificationRole, + assertPgNotificationRoleClient, + auditPgNotificationRole, + auditPgNotificationRoleClient, + normalizePgNotificationRoleContracts, + PG_NOTIFICATION_ROLE_AUDIT_SQL, + PG_NOTIFICATION_ROLE_AUDIT_VERSION, + PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE, + PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE, + PgNotificationRoleContractError, + UnsafePgNotificationRoleError, +} from './notification-role'; export { acquirePgPool, buildConnectionString, @@ -36,4 +49,10 @@ export type { PgPoolLease, PoolCleanupCallback, } from './lru'; +export type { + PgNotificationRoleAudit, + PgNotificationRoleClient, + PgNotificationRoleContract, + PgNotificationRoleViolationCode, +} from './notification-role'; export type { GetPgPoolOptions } from './pg'; diff --git a/postgres/pg-cache/src/notification-role.ts b/postgres/pg-cache/src/notification-role.ts new file mode 100644 index 0000000000..5468c20c80 --- /dev/null +++ b/postgres/pg-cache/src/notification-role.ts @@ -0,0 +1,436 @@ +import type { Pool, PoolClient, QueryResult } from 'pg'; + +export const PG_NOTIFICATION_ROLE_AUDIT_VERSION = 'pg-notification-role:v1'; +export const PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE = + 'PG_NOTIFICATION_ROLE_UNSAFE'; +export const PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE = + 'PG_NOTIFICATION_ROLE_CONTRACT_INVALID'; + +export type PgNotificationRoleViolationCode = + | 'LOGIN_ROLE_MISMATCH' + | 'CURRENT_ROLE_MISMATCH' + | 'DATABASE_MISMATCH' + | 'LOGIN_REQUIRED' + | 'NOINHERIT_REQUIRED' + | 'SUPERUSER' + | 'BYPASSRLS' + | 'CREATEROLE' + | 'CREATEDB' + | 'REPLICATION' + | 'ROLE_MEMBERSHIP' + | 'TARGET_DATABASE_MISSING' + | 'TARGET_CONNECT_REQUIRED' + | 'CROSS_DATABASE_CONNECT' + | 'DATABASE_OWNER' + | 'DATABASE_CREATE' + | 'DATABASE_TEMP' + | 'SCHEMA_OWNER' + | 'SCHEMA_CREATE' + | 'SCHEMA_USAGE' + | 'RELATION_PRIVILEGE' + | 'FUNCTION_PRIVILEGE' + | 'SEQUENCE_PRIVILEGE' + | 'AUDIT_NO_RESULT'; + +/** Credential-free identity expected from one dedicated listener login. */ +export interface PgNotificationRoleContract { + role: string; + database: string; +} + +/** Safe to persist in diagnostics: connection secrets/config are never copied. */ +export interface PgNotificationRoleAudit { + version: typeof PG_NOTIFICATION_ROLE_AUDIT_VERSION; + role: string; + database: string; + safe: boolean; + violations: readonly PgNotificationRoleViolationCode[]; +} + +/** Catalog-query capability used by the broker's pinned LISTEN client. */ +export type PgNotificationRoleClient = Pick; + +interface PgNotificationRoleAuditRow { + expected_role: string; + session_role: string; + active_role: string; + active_database: string; + rolcanlogin: boolean; + rolinherit: boolean; + rolsuper: boolean; + rolbypassrls: boolean; + rolcreaterole: boolean; + rolcreatedb: boolean; + rolreplication: boolean; + membership_count: number; + target_database_exists: boolean; + target_connect: boolean; + other_database_connect_count: number; + target_database_owner: boolean; + target_database_create: boolean; + target_database_temp: boolean; + schema_owner_count: number; + schema_create_count: number; + schema_usage_count: number; + relation_privilege_count: number; + function_privilege_count: number; + sequence_privilege_count: number; +} + +/** + * Audit only the session login's effective privileges. PostgreSQL system + * schemas/objects are excluded because ordinary logins necessarily use the + * catalog; every non-system schema and object remains in scope. + */ +export const PG_NOTIFICATION_ROLE_AUDIT_SQL = ` +WITH login_role AS MATERIALIZED ( + SELECT r.oid, r.rolname, r.rolcanlogin, r.rolinherit, r.rolsuper, + r.rolbypassrls, r.rolcreaterole, r.rolcreatedb, r.rolreplication + FROM pg_catalog.pg_roles r + WHERE r.rolname = session_user +), target_database AS MATERIALIZED ( + SELECT d.oid, d.datname, d.datdba + FROM pg_catalog.pg_database d + WHERE d.datname = $2::text +), application_schemas AS MATERIALIZED ( + SELECT n.oid, n.nspname, n.nspowner + FROM pg_catalog.pg_namespace n + WHERE n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_' +) +SELECT $1::text AS expected_role, + session_user AS session_role, + current_user AS active_role, + pg_catalog.current_database() AS active_database, + r.rolcanlogin, + r.rolinherit, + r.rolsuper, + r.rolbypassrls, + r.rolcreaterole, + r.rolcreatedb, + r.rolreplication, + ( + SELECT count(*)::int + FROM pg_catalog.pg_auth_members membership + WHERE membership.member = r.oid OR membership.roleid = r.oid + ) AS membership_count, + (target.oid IS NOT NULL) AS target_database_exists, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'CONNECT'), + false + ) AS target_connect, + ( + SELECT count(*)::int + FROM pg_catalog.pg_database database_record + WHERE database_record.datname <> $2::text + AND pg_catalog.has_database_privilege( + r.rolname, + database_record.oid, + 'CONNECT' + ) + ) AS other_database_connect_count, + COALESCE(target.datdba = r.oid, false) AS target_database_owner, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'CREATE'), + false + ) AS target_database_create, + COALESCE( + pg_catalog.has_database_privilege(r.rolname, target.oid, 'TEMP'), + false + ) AS target_database_temp, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE schema_record.nspowner = r.oid + ) AS schema_owner_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE pg_catalog.has_schema_privilege( + r.rolname, + schema_record.oid, + 'CREATE' + ) + ) AS schema_create_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + WHERE pg_catalog.has_schema_privilege( + r.rolname, + schema_record.oid, + 'USAGE' + ) + ) AS schema_usage_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_class relation + ON relation.relnamespace = schema_record.oid + WHERE CASE WHEN relation.relkind IN ('r', 'p', 'v', 'm', 'f') + THEN pg_catalog.has_table_privilege( + r.rolname, + relation.oid, + 'SELECT,INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER' + ) + OR pg_catalog.has_any_column_privilege( + r.rolname, + relation.oid, + 'SELECT,INSERT,UPDATE,REFERENCES' + ) + ELSE false + END + ) AS relation_privilege_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_proc routine + ON routine.pronamespace = schema_record.oid + WHERE pg_catalog.has_function_privilege( + r.rolname, + routine.oid, + 'EXECUTE' + ) + ) AS function_privilege_count, + ( + SELECT count(*)::int + FROM application_schemas schema_record + INNER JOIN pg_catalog.pg_class sequence_record + ON sequence_record.relnamespace = schema_record.oid + WHERE CASE WHEN sequence_record.relkind = 'S' + THEN pg_catalog.has_sequence_privilege( + r.rolname, + sequence_record.oid, + 'USAGE,SELECT,UPDATE' + ) + ELSE false + END + ) AS sequence_privilege_count +FROM login_role r +LEFT JOIN target_database target ON true +`; + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +const assertIdentifier = ( + kind: 'role' | 'database', + value: unknown +): string => { + if (typeof value !== 'string' || value.length === 0) { + throw new PgNotificationRoleContractError( + `${kind} must be a non-empty string` + ); + } + if (value.includes('\0') || containsUnpairedSurrogate(value)) { + throw new PgNotificationRoleContractError( + `${kind} is not a valid PostgreSQL name` + ); + } + const bytes = Buffer.byteLength(value, 'utf8'); + if (bytes > 63) { + throw new PgNotificationRoleContractError( + `${kind} is ${bytes} UTF-8 bytes; PostgreSQL allows at most 63` + ); + } + return value; +}; + +const normalizeContract = ( + contract: PgNotificationRoleContract +): Readonly => + Object.freeze({ + role: assertIdentifier('role', contract?.role), + database: assertIdentifier('database', contract?.database), + }); + +export class PgNotificationRoleContractError extends Error { + readonly code = PG_NOTIFICATION_ROLE_CONTRACT_ERROR_CODE; + + constructor(reason: string) { + super(`Invalid PostgreSQL notification-role contract: ${reason}`); + this.name = 'PgNotificationRoleContractError'; + } +} + +export class UnsafePgNotificationRoleError extends Error { + readonly code = PG_NOTIFICATION_ROLE_UNSAFE_ERROR_CODE; + + constructor(readonly audit: PgNotificationRoleAudit) { + super( + `PostgreSQL notification role ${JSON.stringify(audit.role)} for database ` + + `${JSON.stringify(audit.database)} is unsafe: ${audit.violations.join(',')}` + ); + this.name = 'UnsafePgNotificationRoleError'; + } +} + +/** + * Enforce a one-to-one role/database mapping without accepting connection + * config. Exact duplicate pairs are collapsed for multi-generation reuse. + */ +export const normalizePgNotificationRoleContracts = ( + contracts: readonly PgNotificationRoleContract[] +): readonly Readonly[] => { + if (!Array.isArray(contracts) || contracts.length === 0) { + throw new PgNotificationRoleContractError( + 'at least one role/database pair is required' + ); + } + const byDatabase = new Map(); + const byRole = new Map(); + const unique = new Map>(); + for (const candidate of contracts) { + const contract = normalizeContract(candidate); + const databaseRole = byDatabase.get(contract.database); + if (databaseRole && databaseRole !== contract.role) { + throw new PgNotificationRoleContractError( + `database ${JSON.stringify(contract.database)} maps to multiple login roles` + ); + } + const roleDatabase = byRole.get(contract.role); + if (roleDatabase && roleDatabase !== contract.database) { + throw new PgNotificationRoleContractError( + `login role ${JSON.stringify(contract.role)} maps to multiple databases` + ); + } + byDatabase.set(contract.database, contract.role); + byRole.set(contract.role, contract.database); + unique.set(`${contract.database}\0${contract.role}`, contract); + } + return Object.freeze( + [...unique.values()].sort((left, right) => { + if (left.database !== right.database) { + return left.database < right.database ? -1 : 1; + } + if (left.role === right.role) return 0; + return left.role < right.role ? -1 : 1; + }) + ); +}; + +const violationCodes = ( + row: PgNotificationRoleAuditRow | undefined, + contract: Readonly +): PgNotificationRoleViolationCode[] => { + if (!row) return ['AUDIT_NO_RESULT']; + const violations: PgNotificationRoleViolationCode[] = []; + if (row.session_role !== contract.role) + violations.push('LOGIN_ROLE_MISMATCH'); + if (row.active_role !== row.session_role) + violations.push('CURRENT_ROLE_MISMATCH'); + if (row.active_database !== contract.database) + violations.push('DATABASE_MISMATCH'); + if (!row.rolcanlogin) violations.push('LOGIN_REQUIRED'); + if (row.rolinherit) violations.push('NOINHERIT_REQUIRED'); + if (row.rolsuper) violations.push('SUPERUSER'); + if (row.rolbypassrls) violations.push('BYPASSRLS'); + if (row.rolcreaterole) violations.push('CREATEROLE'); + if (row.rolcreatedb) violations.push('CREATEDB'); + if (row.rolreplication) violations.push('REPLICATION'); + if (row.membership_count > 0) violations.push('ROLE_MEMBERSHIP'); + if (!row.target_database_exists) violations.push('TARGET_DATABASE_MISSING'); + if (!row.target_connect) violations.push('TARGET_CONNECT_REQUIRED'); + if (row.other_database_connect_count > 0) + violations.push('CROSS_DATABASE_CONNECT'); + if (row.target_database_owner) violations.push('DATABASE_OWNER'); + if (row.target_database_create) violations.push('DATABASE_CREATE'); + if (row.target_database_temp) violations.push('DATABASE_TEMP'); + if (row.schema_owner_count > 0) violations.push('SCHEMA_OWNER'); + if (row.schema_create_count > 0) violations.push('SCHEMA_CREATE'); + if (row.schema_usage_count > 0) violations.push('SCHEMA_USAGE'); + if (row.relation_privilege_count > 0) violations.push('RELATION_PRIVILEGE'); + if (row.function_privilege_count > 0) violations.push('FUNCTION_PRIVILEGE'); + if (row.sequence_privilege_count > 0) violations.push('SEQUENCE_PRIVILEGE'); + return violations; +}; + +/** Execute one fresh audit on an already-owned client without releasing it. */ +export const auditPgNotificationRoleClient = async ( + client: PgNotificationRoleClient, + candidate: PgNotificationRoleContract +): Promise => { + const contract = normalizeContract(candidate); + let inTransaction = false; + let result: QueryResult; + try { + await client.query('BEGIN READ ONLY'); + inTransaction = true; + await client.query('SET LOCAL jit TO off'); + result = await client.query( + PG_NOTIFICATION_ROLE_AUDIT_SQL, + [contract.role, contract.database] + ); + await client.query('COMMIT'); + inTransaction = false; + } catch (error) { + if (inTransaction) { + try { + await client.query('ROLLBACK'); + } catch (rollbackError) { + throw new AggregateError( + [error, rollbackError], + 'PostgreSQL notification role audit and rollback both failed', + { cause: error } + ); + } + } + throw error; + } + + const violations = Object.freeze(violationCodes(result.rows[0], contract)); + return Object.freeze({ + version: PG_NOTIFICATION_ROLE_AUDIT_VERSION, + role: contract.role, + database: contract.database, + safe: violations.length === 0, + violations, + }); +}; + +/** Execute one fresh, read-only catalog audit. Successful results are not cached. */ +export const auditPgNotificationRole = async ( + pool: Pool, + candidate: PgNotificationRoleContract +): Promise => { + const client: PoolClient = await pool.connect(); + let destroyClient = false; + try { + return await auditPgNotificationRoleClient(client, candidate); + } catch (error) { + destroyClient = true; + throw error; + } finally { + client.release(destroyClient); + } +}; + +/** Fail closed on a pinned client without exposing general query access. */ +export const assertPgNotificationRoleClient = async ( + client: PgNotificationRoleClient, + contract: PgNotificationRoleContract +): Promise => { + const audit = await auditPgNotificationRoleClient(client, contract); + if (!audit.safe) throw new UnsafePgNotificationRoleError(audit); + return audit; +}; + +/** Fail closed with a stable code while retaining a credential-free audit. */ +export const assertPgNotificationRole = async ( + pool: Pool, + contract: PgNotificationRoleContract +): Promise => { + const audit = await auditPgNotificationRole(pool, contract); + if (!audit.safe) throw new UnsafePgNotificationRoleError(audit); + return audit; +}; From 6bfcf1909ad534fcd6f72762de44711f694e1703 Mon Sep 17 00:00:00 2001 From: zetazzz Date: Mon, 17 Aug 2026 16:55:30 +0800 Subject: [PATCH 4/5] Add the PostgreSQL notification broker lifecycle --- .../src/__tests__/notification-broker.test.ts | 970 ++++++++++++++ postgres/pg-cache/src/index.ts | 29 + postgres/pg-cache/src/notification-broker.ts | 1178 +++++++++++++++++ 3 files changed, 2177 insertions(+) create mode 100644 postgres/pg-cache/src/__tests__/notification-broker.test.ts create mode 100644 postgres/pg-cache/src/notification-broker.ts diff --git a/postgres/pg-cache/src/__tests__/notification-broker.test.ts b/postgres/pg-cache/src/__tests__/notification-broker.test.ts new file mode 100644 index 0000000000..3d26f5d962 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-broker.test.ts @@ -0,0 +1,970 @@ +import { EventEmitter } from 'node:events'; + +import { + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + getPgNotificationBrokerIdentity, + getPgNotificationDatabaseIdentity, + PgNotificationBrokerFailedError, + PgNotificationBrokerRegistry, + PgNotificationConnectionSource, + PgNotificationOperationTimeoutError, + PgNotificationQueueOverflowError, + PgNotificationTopicError, +} from '../notification-broker'; +import { + PG_NOTIFICATION_ROLE_AUDIT_SQL, + UnsafePgNotificationRoleError, +} from '../notification-role'; + +const roleContract = { + role: 'tenant_a_notify', + database: 'tenant_a', +}; + +const safeRoleAuditRow = { + expected_role: roleContract.role, + session_role: roleContract.role, + active_role: roleContract.role, + active_database: roleContract.database, + rolcanlogin: true, + rolinherit: false, + rolsuper: false, + rolbypassrls: false, + rolcreaterole: false, + rolcreatedb: false, + rolreplication: false, + membership_count: 0, + target_database_exists: true, + target_connect: true, + other_database_connect_count: 0, + target_database_owner: false, + target_database_create: false, + target_database_temp: false, + schema_owner_count: 0, + schema_create_count: 0, + schema_usage_count: 0, + relation_privilege_count: 0, + function_privilege_count: 0, + sequence_privilege_count: 0, +}; + +class MockNotificationClient extends EventEmitter { + readonly queries: string[] = []; + roleAuditRow: Record | undefined = safeRoleAuditRow; + readonly query = jest.fn( + async (text: string, _values?: readonly unknown[]): Promise => { + this.queries.push(text); + if (text === PG_NOTIFICATION_ROLE_AUDIT_SQL) { + return { rows: this.roleAuditRow ? [this.roleAuditRow] : [] }; + } + return { rows: [] }; + } + ); + readonly release = jest.fn( + async (_error?: Error | boolean): Promise => {} + ); + + notification(channel: string, payload?: string): void { + this.emit('notification', { channel, payload }); + } +} + +const createSource = (client = new MockNotificationClient()) => { + const source: PgNotificationConnectionSource = { + connect: jest.fn(async () => client), + release: jest.fn(async () => {}), + }; + return { client, source }; +}; + +const deferred = () => { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +const flushMicrotasks = async (): Promise => { + for (let index = 0; index < 20; index++) await Promise.resolve(); +}; + +describe('PgNotificationBrokerRegistry', () => { + it('shares one dedicated listener and reference-counts exact topics', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + + const [first, second] = await Promise.all([ + registry.acquireForTests('opaque-a', sourceFactory, [ + 'tenant.a', + 'shared', + ]), + registry.acquireForTests('opaque-a', sourceFactory, [ + 'shared', + 'tenant.b', + ]), + ]); + + expect(sourceFactory).toHaveBeenCalledTimes(1); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.queries).toEqual([ + 'LISTEN "tenant.a"', + 'LISTEN "shared"', + 'LISTEN "tenant.b"', + ]); + expect(registry.stats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 2, + topics: 3, + }); + + await first.release(); + expect(client.queries).toContain('UNLISTEN "tenant.a"'); + expect(client.queries).not.toContain('UNLISTEN "shared"'); + expect(client.release).not.toHaveBeenCalled(); + + await second.release(); + expect(client.queries.slice(-2)).toEqual([ + 'UNLISTEN "shared"', + 'UNLISTEN "tenant.b"', + ]); + expect(client.release).toHaveBeenCalledWith(true); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + leases: 0, + topics: 0, + }); + }); + + it('audits three generations on the one pinned listener before admission', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + + const first = await registry.acquireAttestedForTests( + 'opaque-attested', + sourceFactory, + ['tenant.a'], + roleContract + ); + const second = await registry.acquireAttestedForTests( + 'opaque-attested', + sourceFactory, + ['tenant.b'], + roleContract + ); + const third = await registry.acquireAttestedForTests( + 'opaque-attested', + sourceFactory, + ['tenant.c'], + roleContract + ); + + expect(sourceFactory).toHaveBeenCalledTimes(1); + expect(source.connect).toHaveBeenCalledTimes(1); + expect( + client.queries.filter((query) => query === PG_NOTIFICATION_ROLE_AUDIT_SQL) + ).toHaveLength(3); + expect(client.queries).toEqual([ + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.a"', + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.b"', + 'BEGIN READ ONLY', + 'SET LOCAL jit TO off', + PG_NOTIFICATION_ROLE_AUDIT_SQL, + 'COMMIT', + 'LISTEN "tenant.c"', + ]); + expect(first.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(second.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(third.roleAudit).toMatchObject({ ...roleContract, safe: true }); + expect(registry.stats()).toMatchObject({ + listenerConnections: 1, + leases: 3, + roleAuditAttempts: 3, + roleAuditFailures: 0, + }); + + await Promise.all([first.release(), second.release(), third.release()]); + }); + + it('serializes concurrent admission audits without another connection', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const firstCatalogAudit = deferred(); + let catalogAuditsStarted = 0; + let activeCatalogAudits = 0; + let peakCatalogAudits = 0; + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text === PG_NOTIFICATION_ROLE_AUDIT_SQL) { + catalogAuditsStarted++; + activeCatalogAudits++; + peakCatalogAudits = Math.max(peakCatalogAudits, activeCatalogAudits); + if (catalogAuditsStarted === 1) await firstCatalogAudit.promise; + activeCatalogAudits--; + return { rows: [safeRoleAuditRow] }; + } + return { rows: [] }; + }); + + const acquisitions = [ + registry.acquireAttestedForTests( + 'opaque-attested', + () => source, + ['a'], + roleContract + ), + registry.acquireAttestedForTests( + 'opaque-attested', + () => source, + ['b'], + roleContract + ), + registry.acquireAttestedForTests( + 'opaque-attested', + () => source, + ['c'], + roleContract + ), + ]; + await flushMicrotasks(); + expect(catalogAuditsStarted).toBe(1); + expect(source.connect).toHaveBeenCalledTimes(1); + + firstCatalogAudit.resolve(); + const leases = await Promise.all(acquisitions); + expect(catalogAuditsStarted).toBe(3); + expect(peakCatalogAudits).toBe(1); + expect(source.connect).toHaveBeenCalledTimes(1); + await Promise.all(leases.map((lease) => lease.release())); + }); + + it('bounds a never-resolving admission audit and destroys its client', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text === 'BEGIN READ ONLY') return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const acquiring = registry.acquireAttestedForTests( + 'opaque-timeout', + () => source, + ['a'], + roleContract + ); + const rejected = expect(acquiring).rejects.toBeInstanceOf( + PgNotificationOperationTimeoutError + ); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.queries).toEqual(['BEGIN READ ONLY']); + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + fatalFailures: 1, + roleAuditAttempts: 1, + roleAuditFailures: 1, + }); + } finally { + jest.useRealTimers(); + } + }); + + it('revalidates on the pinned listener and fails every lease closed on drift', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const first = await registry.acquireAttestedForTests( + 'opaque-attested', + () => source, + ['a'], + roleContract + ); + const second = await registry.acquireAttestedForTests( + 'opaque-attested', + () => source, + ['b'], + roleContract + ); + + await expect(first.revalidateRole()).resolves.toMatchObject({ safe: true }); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + roleAuditAttempts: 3, + roleAuditFailures: 0, + }); + + const firstNext = first.subscribe('a').next(); + const secondNext = second.subscribe('b').next(); + client.roleAuditRow = { ...safeRoleAuditRow, rolsuper: true }; + await expect(second.revalidateRole()).rejects.toBeInstanceOf( + UnsafePgNotificationRoleError + ); + await expect(firstNext).rejects.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect(secondNext).rejects.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect(first.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect(second.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(registry.stats()).toMatchObject({ + listenerConnections: 0, + leases: 2, + fatalFailures: 1, + roleAuditAttempts: 4, + roleAuditFailures: 1, + }); + + await Promise.all([first.release(), second.release()]); + }); + + it('bounds a never-resolving TTL role refresh on the pinned listener', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + const lease = await registry.acquireAttestedForTests( + 'opaque-refresh-timeout', + () => source, + ['a'], + roleContract + ); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text === 'BEGIN READ ONLY') return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const refreshing = lease.revalidateRole(); + const rejected = expect(refreshing).rejects.toBeInstanceOf( + PgNotificationOperationTimeoutError + ); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + await expect(lease.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(registry.stats()).toMatchObject({ + listenerConnections: 0, + leases: 1, + fatalFailures: 1, + roleAuditAttempts: 2, + roleAuditFailures: 1, + }); + await lease.release(); + } finally { + jest.useRealTimers(); + } + }); + + it('uses exact topic equality for prefix and quoted-identifier channels', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const hostileButValid = 'tenant"; UNLISTEN *;--'; + const lease = await registry.acquireForTests('opaque-a', () => source, [ + 'tenant', + 'tenant.longer', + hostileButValid, + ]); + const exact = lease.subscribe('tenant'); + const longer = lease.subscribe('tenant.longer'); + const hostile = lease.subscribe(hostileButValid); + + expect(() => lease.subscribe('ten')).toThrow(PgNotificationTopicError); + expect(client.queries).toContain('LISTEN "tenant""; UNLISTEN *;--"'); + + client.notification('ten', 'wrong-prefix'); + client.notification('tenant.longer', 'longer'); + client.notification(hostileButValid, 'quoted'); + client.notification('tenant', 'exact'); + + await expect(exact.next()).resolves.toEqual({ + done: false, + value: 'exact', + }); + await expect(longer.next()).resolves.toEqual({ + done: false, + value: 'longer', + }); + await expect(hostile.next()).resolves.toEqual({ + done: false, + value: 'quoted', + }); + expect(registry.stats().ignoredNotifications).toBe(1); + await lease.release(); + }); + + it('rejects channels PostgreSQL would truncate, including multi-byte Unicode', async () => { + const registry = new PgNotificationBrokerRegistry(); + const { source } = createSource(); + + const ascii63 = 'a'.repeat(63); + const unicode63 = '界'.repeat(21); + const lease = await registry.acquireForTests('opaque-a', () => source, [ + ascii63, + unicode63, + ]); + expect(lease.topics).toEqual([ascii63, unicode63]); + + await expect( + registry.acquireForTests('opaque-b', () => source, ['a'.repeat(64)]) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await expect( + registry.acquireForTests('opaque-b', () => source, ['界'.repeat(22)]) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await expect( + registry.acquireForTests('opaque-b', () => source, [ + `bad${String.fromCharCode(0xd800)}`, + ]) + ).rejects.toBeInstanceOf(PgNotificationTopicError); + await lease.release(); + }); + + it('does not normalize canonically equivalent Unicode topics', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const composed = 'réaltime'; + const decomposed = 're\u0301altime'; + const lease = await registry.acquireForTests('opaque-a', () => source, [ + composed, + decomposed, + ]); + const composedStream = lease.subscribe(composed); + const decomposedStream = lease.subscribe(decomposed); + + client.notification(composed, 'composed-only'); + client.notification(decomposed, 'decomposed-only'); + + await expect(composedStream.next()).resolves.toMatchObject({ + value: 'composed-only', + }); + await expect(decomposedStream.next()).resolves.toMatchObject({ + value: 'decomposed-only', + }); + await lease.release(); + }); + + it('fans out only to subscribers for the exact allowed topic', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const first = await registry.acquireForTests('opaque-a', () => source, [ + 'a', + ]); + const second = await registry.acquireForTests('opaque-a', () => source, [ + 'a', + 'b', + ]); + const firstA = first.subscribe('a'); + const secondA = second.subscribe('a'); + const secondB = second.subscribe('b'); + + client.notification('a', 'for-a'); + client.notification('b', 'for-b'); + + await expect(firstA.next()).resolves.toMatchObject({ value: 'for-a' }); + await expect(secondA.next()).resolves.toMatchObject({ value: 'for-a' }); + await expect(secondB.next()).resolves.toMatchObject({ value: 'for-b' }); + await Promise.all([first.release(), second.release()]); + }); + + it('fails only the slow subscriber when its bounded queue overflows', async () => { + const registry = new PgNotificationBrokerRegistry(1); + const { client, source } = createSource(); + const lease = await registry.acquireForTests('opaque-a', () => source, [ + 'events', + ]); + const slow = lease.subscribe('events'); + const fast = lease.subscribe('events'); + + const fastFirst = fast.next(); + client.notification('events', 'one'); + const fastSecond = fast.next(); + client.notification('events', 'two'); + + await expect(fastFirst).resolves.toMatchObject({ value: 'one' }); + await expect(fastSecond).resolves.toMatchObject({ value: 'two' }); + await expect(slow.next()).rejects.toBeInstanceOf( + PgNotificationQueueOverflowError + ); + expect(registry.stats()).toMatchObject({ + subscribers: 1, + queueOverflows: 1, + }); + await lease.release(); + }); + + it('fails every active subscriber and never silently reconnects', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const sourceFactory = jest.fn(() => source); + const first = await registry.acquireForTests('opaque-a', sourceFactory, [ + 'a', + ]); + const second = await registry.acquireForTests('opaque-a', sourceFactory, [ + 'b', + ]); + const firstNext = first.subscribe('a').next(); + const secondNext = second.subscribe('b').next(); + + client.emit('error', new Error('socket lost')); + + await expect(firstNext).rejects.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect(secondNext).rejects.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect(first.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await expect( + registry.acquireForTests('opaque-a', sourceFactory, ['a']) + ).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + expect(source.connect).toHaveBeenCalledTimes(1); + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + + await Promise.all([first.release(), second.release()]); + const replacement = createSource(); + const explicitReplacement = await registry.acquireForTests( + 'opaque-a', + () => replacement.source, + ['a'] + ); + expect(replacement.source.connect).toHaveBeenCalledTimes(1); + await explicitReplacement.release(); + }); + + it('bounds a never-resolving LISTEN and fails admission closed', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text.startsWith('LISTEN')) return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const acquiring = registry.acquireForTests( + 'opaque-listen-timeout', + () => source, + ['a'] + ); + const rejected = expect(acquiring).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_BROKER_FAILED', + cause: { code: 'PG_NOTIFICATION_OPERATION_TIMEOUT' }, + }); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + fatalFailures: 1, + }); + } finally { + jest.useRealTimers(); + } + }); + + it('fails closed when a listener emits a malformed notification', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const lease = await registry.acquireForTests('opaque-a', () => source, [ + 'a', + ]); + const next = lease.subscribe('a').next(); + + client.emit('notification', { channel: 'a', payload: { hostile: true } }); + + await expect(next).rejects.toBeInstanceOf(PgNotificationBrokerFailedError); + await expect(lease.terminated).resolves.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + await lease.release(); + }); + + it('makes double release idempotent and awaits UNLISTEN plus both releases', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const unlisten = deferred(); + const clientReleased = deferred(); + const sourceReleased = deferred(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) await unlisten.promise; + return { rows: [] }; + }); + client.release.mockImplementation(async () => clientReleased.promise); + (source.release as jest.Mock).mockImplementation( + async () => sourceReleased.promise + ); + const lease = await registry.acquireForTests('opaque-a', () => source, [ + 'a', + ]); + + const firstRelease = lease.release(); + const secondRelease = lease.release(); + expect(firstRelease).toBe(secondRelease); + await flushMicrotasks(); + expect(client.queries).toContain('UNLISTEN "a"'); + + let settled = false; + void firstRelease.then(() => { + settled = true; + }); + unlisten.resolve(); + await flushMicrotasks(); + expect(settled).toBe(false); + clientReleased.resolve(); + await flushMicrotasks(); + expect(settled).toBe(false); + sourceReleased.resolve(); + await firstRelease; + await expect(lease.terminated).resolves.toBeNull(); + expect(settled).toBe(true); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('serializes a final release against a concurrent new acquisition', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const firstSource = createSource(); + const unlisten = deferred(); + firstSource.client.query.mockImplementation(async (text: string) => { + firstSource.client.queries.push(text); + if (text.startsWith('UNLISTEN')) await unlisten.promise; + return { rows: [] }; + }); + const first = await registry.acquireForTests( + 'opaque-a', + () => firstSource.source, + ['a'] + ); + const releasing = first.release(); + await flushMicrotasks(); + + const secondSource = createSource(); + const acquiring = registry.acquireForTests( + 'opaque-a', + () => secondSource.source, + ['a'] + ); + await flushMicrotasks(); + expect(secondSource.source.connect).not.toHaveBeenCalled(); + + unlisten.resolve(); + await releasing; + const second = await acquiring; + expect(secondSource.source.connect).toHaveBeenCalledTimes(1); + await second.release(); + }); + + it('makes concurrent registry close calls await the same teardown', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { source } = createSource(); + const sourceReleased = deferred(); + (source.release as jest.Mock).mockImplementation( + async () => sourceReleased.promise + ); + await registry.acquireForTests('opaque-a', () => source, ['a']); + + const firstClose = registry.close(); + const secondClose = registry.close(); + let firstSettled = false; + let secondSettled = false; + void firstClose.then(() => { + firstSettled = true; + }); + void secondClose.then(() => { + secondSettled = true; + }); + await flushMicrotasks(); + + expect(firstSettled).toBe(false); + expect(secondSettled).toBe(false); + sourceReleased.resolve(); + await Promise.all([firstClose, secondClose]); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); + + it('bounds a never-resolving UNLISTEN so teardown cannot hang', async () => { + jest.useFakeTimers(); + try { + const registry = new PgNotificationBrokerRegistry(4, 25); + const { client, source } = createSource(); + await registry.acquireForTests('opaque-unlisten-timeout', () => source, [ + 'a', + ]); + client.query.mockImplementation((text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) return new Promise(() => undefined); + return Promise.resolve({ rows: [] }); + }); + + const closing = registry.close(); + const rejected = expect(closing).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_BROKER_FAILED', + cause: { code: 'PG_NOTIFICATION_OPERATION_TIMEOUT' }, + }); + await flushMicrotasks(); + await jest.advanceTimersByTimeAsync(25); + await rejected; + + expect(client.release).toHaveBeenCalledWith( + expect.any(PgNotificationBrokerFailedError) + ); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + fatalFailures: 1, + }); + } finally { + jest.useRealTimers(); + } + }); + + it('drains an in-flight acquisition before registry close resolves', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const connected = deferred(); + const sourceReleased = deferred(); + (source.connect as jest.Mock).mockImplementation( + async () => connected.promise + ); + (source.release as jest.Mock).mockImplementation( + async () => sourceReleased.promise + ); + const acquiring = registry.acquireForTests('opaque-a', () => source, ['a']); + await flushMicrotasks(); + expect(source.connect).toHaveBeenCalledTimes(1); + + const closing = registry.close(); + let closeSettled = false; + void closing.then(() => { + closeSettled = true; + }); + connected.resolve(client); + await flushMicrotasks(); + const closeSettledBeforeSourceRelease = closeSettled; + const issuedListenDuringClose = client.queries.includes('LISTEN "a"'); + sourceReleased.resolve(); + await expect(acquiring).rejects.toThrow( + 'PostgreSQL notification broker registry is closed' + ); + await closing; + expect(closeSettledBeforeSourceRelease).toBe(false); + expect(issuedListenDuringClose).toBe(false); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); + + it('UNLISTENs a provisional topic when close races an in-flight LISTEN', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + const listened = deferred(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text === 'LISTEN "a"') await listened.promise; + return { rows: [] }; + }); + const acquiring = registry.acquireForTests('opaque-a', () => source, ['a']); + await flushMicrotasks(); + expect(client.queries).toEqual(['LISTEN "a"']); + + const closing = registry.close(); + listened.resolve(); + await expect(acquiring).rejects.toThrow( + 'PostgreSQL notification broker registry is closed' + ); + await closing; + + expect(client.queries).toEqual(['LISTEN "a"', 'UNLISTEN *']); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + }); + + it('reports a failed UNLISTEN only after finishing registry teardown', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + client.query.mockImplementation(async (text: string) => { + client.queries.push(text); + if (text.startsWith('UNLISTEN')) throw new Error('unlisten failed'); + return { rows: [] }; + }); + await registry.acquireForTests('opaque-a', () => source, ['a']); + + await expect(registry.close()).rejects.toBeInstanceOf( + PgNotificationBrokerFailedError + ); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); + + it('aggregates client and pool-lease cleanup failures after teardown attempts both', async () => { + const registry = new PgNotificationBrokerRegistry(4); + const { client, source } = createSource(); + client.release.mockRejectedValue(new Error('client release failed')); + (source.release as jest.Mock).mockRejectedValue( + new Error('source release failed') + ); + await registry.acquireForTests('opaque-cleanup', () => source, ['a']); + + const closing = registry.close(); + await expect(closing).rejects.toMatchObject({ + code: 'PG_NOTIFICATION_BROKER_FAILED', + cause: expect.any(AggregateError), + }); + expect(client.release).toHaveBeenCalledTimes(1); + expect(source.release).toHaveBeenCalledTimes(1); + expect(registry.stats()).toMatchObject({ brokers: 0, leases: 0 }); + }); +}); + +describe('getPgNotificationBrokerIdentity', () => { + const baseConfig = { + host: 'db.internal', + port: 5432, + database: 'customer', + user: 'listener', + password: 'secret', + }; + + it('is versioned, opaque, stable, and includes the canonical SSL contract', () => { + const first = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { + connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + }, + }); + const reordered = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { ca: 'ca-one', rejectUnauthorized: true }, + pool: { + connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + }, + }); + const changedTls = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: false, ca: 'ca-one' }, + pool: { + connectionTimeoutMillis: DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + }, + }); + const changedDeadline = getPgNotificationBrokerIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { + connectionTimeoutMillis: + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + 1, + }, + }); + + expect(first).toBe(reordered); + expect(first).not.toBe(changedTls); + expect(first).not.toBe(changedDeadline); + expect(first).toMatch(/^pg-notification-broker:v1:pg:v1:[a-f0-9]{64}$/); + expect(first).not.toContain('listener'); + expect(first).not.toContain('secret'); + }); + + it.each([0, -1, 1.5, 2_147_483_648])( + 'rejects invalid notification operation timeout %p before identity publication', + (connectionTimeoutMillis) => { + expect(() => + getPgNotificationBrokerIdentity({ + ...baseConfig, + pool: { connectionTimeoutMillis }, + }) + ).toThrow('notification operation timeout'); + } + ); + + it('uses one credential-free identity for the same physical database target', () => { + const first = getPgNotificationDatabaseIdentity({ + ...baseConfig, + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + pool: { max: 2 }, + }); + const rotated = getPgNotificationDatabaseIdentity({ + ...baseConfig, + user: 'rotated-listener', + password: 'rotated-secret', + ssl: { ca: 'different-ca', rejectUnauthorized: false }, + pool: { max: 20, idleTimeoutMillis: 99_000 }, + }); + const otherHost = getPgNotificationDatabaseIdentity({ + ...baseConfig, + host: 'other-db.internal', + }); + const otherPort = getPgNotificationDatabaseIdentity({ + ...baseConfig, + port: 5433, + }); + const otherDatabase = getPgNotificationDatabaseIdentity({ + ...baseConfig, + database: 'other-customer', + ssl: { rejectUnauthorized: true, ca: 'ca-one' }, + }); + + expect(first).toBe(rotated); + expect(first).not.toBe(otherHost); + expect(first).not.toBe(otherPort); + expect(first).not.toBe(otherDatabase); + expect(first).toMatch( + /^pg-notification-database:v1:pg-target:v1:[a-f0-9]{64}$/ + ); + expect(first).not.toContain('listener'); + expect(first).not.toContain('secret'); + }); +}); diff --git a/postgres/pg-cache/src/index.ts b/postgres/pg-cache/src/index.ts index 77bbc13908..b7a5853489 100644 --- a/postgres/pg-cache/src/index.ts +++ b/postgres/pg-cache/src/index.ts @@ -18,6 +18,28 @@ export { PgPoolCapacityError, teardownPgPools } from './lru'; +export { + acquirePgNotificationBroker, + assertValidPgNotificationTopic, + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS, + getPgNotificationBrokerIdentity, + getPgNotificationBrokerStats, + getPgNotificationDatabaseIdentity, + PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE, + PG_NOTIFICATION_BROKER_IDENTITY_VERSION, + PG_NOTIFICATION_DATABASE_IDENTITY_VERSION, + PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE, + PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE, + PG_NOTIFICATION_QUEUE_CAPACITY, + PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE, + PG_NOTIFICATION_TOPIC_ERROR_CODE, + PgNotificationBrokerFailedError, + PgNotificationLeaseReleasedError, + PgNotificationOperationTimeoutError, + PgNotificationQueueOverflowError, + PgNotificationTopicError, + teardownPgNotificationBrokers, +} from './notification-broker'; export { assertPgNotificationRole, assertPgNotificationRoleClient, @@ -49,6 +71,13 @@ export type { PgPoolLease, PoolCleanupCallback, } from './lru'; +export type { + AcquirePgNotificationBrokerOptions, + PgAttestedNotificationBrokerLease, + PgNotificationBrokerLease, + PgNotificationBrokerStats, + PgNotificationListenerConfig, +} from './notification-broker'; export type { PgNotificationRoleAudit, PgNotificationRoleClient, diff --git a/postgres/pg-cache/src/notification-broker.ts b/postgres/pg-cache/src/notification-broker.ts new file mode 100644 index 0000000000..682c2ce7ce --- /dev/null +++ b/postgres/pg-cache/src/notification-broker.ts @@ -0,0 +1,1178 @@ +import type { PgConfig, PgPoolConfig } from 'pg-env'; + +import { + assertPgNotificationRoleClient, + type PgNotificationRoleAudit, + type PgNotificationRoleClient, + type PgNotificationRoleContract, +} from './notification-role'; +import { + acquirePgPool, + getPgDatabaseTargetIdentity, + getPgPoolConfig, + getPgPoolIdentity, +} from './pg'; + +export const PG_NOTIFICATION_BROKER_IDENTITY_VERSION = + 'pg-notification-broker:v1'; +export const PG_NOTIFICATION_DATABASE_IDENTITY_VERSION = + 'pg-notification-database:v1'; +export const PG_NOTIFICATION_QUEUE_CAPACITY = 256; +export const DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS = 5_000; + +export const PG_NOTIFICATION_TOPIC_ERROR_CODE = 'PG_NOTIFICATION_TOPIC_INVALID'; +export const PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE = + 'PG_NOTIFICATION_BROKER_FAILED'; +export const PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE = + 'PG_NOTIFICATION_QUEUE_OVERFLOW'; +export const PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE = + 'PG_NOTIFICATION_LEASE_RELEASED'; +export const PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE = + 'PG_NOTIFICATION_OPERATION_TIMEOUT'; + +type PromiseOrDirect = T | Promise; + +export interface PgNotification { + channel: string; + payload?: string; +} + +export interface PgNotificationClient { + query(text: string, values?: readonly unknown[]): Promise; + on(event: string, listener: (...args: any[]) => void): unknown; + off(event: string, listener: (...args: any[]) => void): unknown; + release(error?: Error | boolean): PromiseOrDirect; +} + +export interface PgNotificationConnectionSource { + connect(): Promise; + release(): PromiseOrDirect; +} + +export interface PgNotificationBrokerLease { + /** Versioned digest of the complete listener connection contract. */ + readonly identity: string; + /** Frozen, exact PostgreSQL channels this lease may subscribe to. */ + readonly topics: readonly string[]; + /** Resolves on fatal broker failure or with null after graceful release. */ + readonly terminated: Promise; + subscribe(topic: string): AsyncIterableIterator; + /** Idempotent and awaited through UNLISTEN and connection release. */ + release(): Promise; +} + +/** + * A production lease whose login was audited on the same pinned PostgreSQL + * client before admission. Arbitrary SQL and the client itself stay private. + */ +export interface PgAttestedNotificationBrokerLease extends PgNotificationBrokerLease { + readonly roleAudit: PgNotificationRoleAudit; + revalidateRole(): Promise; +} + +export interface AcquirePgNotificationBrokerOptions { + /** Every channel this generation may observe. Prefix matching is never used. */ + topics: readonly string[]; +} + +export type PgNotificationListenerConfig = PgConfig & { pool?: PgPoolConfig }; + +export interface PgNotificationBrokerStats { + brokers: number; + listenerConnections: number; + leases: number; + topics: number; + subscribers: number; + acquisitions: number; + releases: number; + notifications: number; + ignoredNotifications: number; + queueOverflows: number; + fatalFailures: number; + roleAuditAttempts: number; + roleAuditFailures: number; +} + +type PgNotificationBrokerSnapshot = Pick< + PgNotificationBrokerStats, + 'listenerConnections' | 'leases' | 'topics' | 'subscribers' +>; + +interface MutableBrokerCounters { + acquisitions: number; + releases: number; + notifications: number; + ignoredNotifications: number; + queueOverflows: number; + fatalFailures: number; + roleAuditAttempts: number; + roleAuditFailures: number; +} + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +const deferred = (): Deferred => { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +}; + +export class PgNotificationTopicError extends Error { + readonly code = PG_NOTIFICATION_TOPIC_ERROR_CODE; + + constructor( + readonly topic: unknown, + reason: string + ) { + super(`Invalid PostgreSQL notification topic: ${reason}`); + this.name = 'PgNotificationTopicError'; + } +} + +export class PgNotificationBrokerFailedError extends Error { + readonly code = PG_NOTIFICATION_BROKER_FAILED_ERROR_CODE; + + constructor(reason: unknown) { + const cause = reason instanceof Error ? reason : new Error(String(reason)); + super( + 'PostgreSQL notification broker failed; all subscribers were terminated', + { + cause, + } + ); + this.name = 'PgNotificationBrokerFailedError'; + } +} + +export class PgNotificationQueueOverflowError extends Error { + readonly code = PG_NOTIFICATION_QUEUE_OVERFLOW_ERROR_CODE; + + constructor( + readonly topic: string, + readonly capacity: number + ) { + super( + `PostgreSQL notification subscriber queue for ${JSON.stringify(topic)} ` + + `exceeded its fixed capacity of ${capacity}` + ); + this.name = 'PgNotificationQueueOverflowError'; + } +} + +export class PgNotificationLeaseReleasedError extends Error { + readonly code = PG_NOTIFICATION_LEASE_RELEASED_ERROR_CODE; + + constructor() { + super('PostgreSQL notification broker lease has been released'); + this.name = 'PgNotificationLeaseReleasedError'; + } +} + +export class PgNotificationOperationTimeoutError extends Error { + readonly code = PG_NOTIFICATION_OPERATION_TIMEOUT_ERROR_CODE; + + constructor( + readonly operation: 'role-audit' | 'listen' | 'unlisten', + readonly timeoutMs: number + ) { + super( + `PostgreSQL notification ${operation} exceeded its fixed ${timeoutMs}ms deadline` + ); + this.name = 'PgNotificationOperationTimeoutError'; + } +} + +class BrokerClosedError extends Error {} + +const containsUnpairedSurrogate = (value: string): boolean => { + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (!(next >= 0xdc00 && next <= 0xdfff)) return true; + index++; + } else if (code >= 0xdc00 && code <= 0xdfff) { + return true; + } + } + return false; +}; + +/** + * PostgreSQL identifiers are limited to 63 UTF-8 bytes. PostgreSQL truncates + * longer identifiers, so accepting them here could collapse distinct tenant + * topics onto one physical LISTEN channel. + */ +export function assertValidPgNotificationTopic( + topic: unknown +): asserts topic is string { + if (typeof topic !== 'string') { + throw new PgNotificationTopicError(topic, 'the topic must be a string'); + } + if (topic.length === 0) { + throw new PgNotificationTopicError(topic, 'the topic must not be empty'); + } + if (topic.includes('\0')) { + throw new PgNotificationTopicError(topic, 'NUL bytes are not allowed'); + } + if (containsUnpairedSurrogate(topic)) { + throw new PgNotificationTopicError( + topic, + 'unpaired UTF-16 surrogates are not allowed' + ); + } + const bytes = Buffer.byteLength(topic, 'utf8'); + if (bytes > 63) { + throw new PgNotificationTopicError( + topic, + `the UTF-8 encoding is ${bytes} bytes; PostgreSQL allows at most 63` + ); + } +} + +const normalizeTopics = (topics: readonly string[]): readonly string[] => { + if (!Array.isArray(topics) || topics.length === 0) { + throw new PgNotificationTopicError( + topics, + 'at least one exact topic is required' + ); + } + for (const topic of topics) assertValidPgNotificationTopic(topic); + return Object.freeze([...new Set(topics)]); +}; + +const quoteIdentifier = (identifier: string): string => + `"${identifier.replace(/"/g, '""')}"`; + +class BoundedNotificationQueue implements AsyncIterableIterator { + private readonly buffered: string[] = []; + private readonly waiting: Deferred>[] = []; + private terminal: 'open' | 'complete' | 'failed' = 'open'; + private failure: Error | null = null; + + constructor( + private readonly topic: string, + private readonly capacity: number, + private readonly onClose: () => void, + private readonly onOverflow: () => void + ) {} + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise> { + const buffered = this.buffered.shift(); + if (buffered !== undefined) { + return Promise.resolve({ done: false, value: buffered }); + } + if (this.terminal === 'failed') return Promise.reject(this.failure); + if (this.terminal === 'complete') { + return Promise.resolve({ done: true, value: undefined }); + } + + const result = deferred>(); + this.waiting.push(result); + return result.promise; + } + + return(value?: unknown): Promise> { + this.complete(); + return Promise.resolve({ done: true, value: value as string }); + } + + throw(error?: unknown): Promise> { + const failure = error instanceof Error ? error : new Error(String(error)); + this.fail(failure); + return Promise.reject(failure); + } + + push(payload: string): void { + if (this.terminal !== 'open') return; + const waiter = this.waiting.shift(); + if (waiter) { + waiter.resolve({ done: false, value: payload }); + return; + } + if (this.buffered.length >= this.capacity) { + this.onOverflow(); + this.fail( + new PgNotificationQueueOverflowError(this.topic, this.capacity) + ); + return; + } + this.buffered.push(payload); + } + + complete(): void { + if (this.terminal !== 'open') return; + this.terminal = 'complete'; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) { + waiter.resolve({ done: true, value: undefined }); + } + this.onClose(); + } + + fail(error: Error): void { + if (this.terminal !== 'open') return; + this.terminal = 'failed'; + this.failure = error; + this.buffered.length = 0; + for (const waiter of this.waiting.splice(0)) waiter.reject(error); + this.onClose(); + } +} + +type BrokerState = 'new' | 'active' | 'failed' | 'closing' | 'closed'; +type ConnectionSourceFactory = + () => PromiseOrDirect; +type NotificationOperation = PgNotificationOperationTimeoutError['operation']; + +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +const assertNotificationOperationTimeoutMs = (timeoutMs: number): number => { + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs <= 0 || + timeoutMs > MAX_TIMER_DELAY_MS + ) { + throw new TypeError( + 'PostgreSQL notification operation timeout must be an integer ' + + `between 1 and ${MAX_TIMER_DELAY_MS}` + ); + } + return timeoutMs; +}; + +const getNotificationOperationTimeoutMs = ( + listenerPgConfig: PgNotificationListenerConfig +): number => { + // Preserve this API's narrower deadline contract and stable error before the + // generic pool validator runs as part of identity construction. + const configured = listenerPgConfig.pool?.connectionTimeoutMillis; + return assertNotificationOperationTimeoutMs( + configured ?? + getPgPoolConfig(listenerPgConfig.pool).connectionTimeoutMillis ?? + DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + ); +}; + +class NotificationBrokerLease implements PgAttestedNotificationBrokerLease { + readonly topics: readonly string[]; + readonly terminated: Promise; + private readonly allowedTopics: ReadonlySet; + private readonly termination = + deferred(); + private readonly queues = new Map>(); + private audit: PgNotificationRoleAudit | null = null; + private released = false; + private releasePromise: Promise | null = null; + + constructor( + readonly identity: string, + topics: readonly string[], + private readonly broker: NotificationBrokerRecord, + private readonly queueCapacity: number, + private readonly counters: MutableBrokerCounters, + readonly roleContract: Readonly | null + ) { + this.topics = topics; + this.terminated = this.termination.promise; + this.allowedTopics = new Set(topics); + } + + get subscriberCount(): number { + let count = 0; + for (const topicQueues of this.queues.values()) count += topicQueues.size; + return count; + } + + get isReleased(): boolean { + return this.released; + } + + get roleAudit(): PgNotificationRoleAudit { + if (!this.audit) { + throw new Error( + 'PostgreSQL notification broker lease is not role-attested' + ); + } + return this.audit; + } + + setRoleAudit(audit: PgNotificationRoleAudit): void { + this.audit = audit; + } + + revalidateRole(): Promise { + if (this.released) + return Promise.reject(new PgNotificationLeaseReleasedError()); + if (!this.roleContract) { + return Promise.reject( + new Error('PostgreSQL notification broker lease is not role-attested') + ); + } + return this.broker.revalidateLeaseRole(this); + } + + subscribe(topic: string): AsyncIterableIterator { + if (this.released) throw new PgNotificationLeaseReleasedError(); + this.broker.assertAvailable(); + if (!this.allowedTopics.has(topic)) { + throw new PgNotificationTopicError( + topic, + "the topic is not in this lease's exact allowlist" + ); + } + + let topicQueues = this.queues.get(topic); + if (!topicQueues) { + topicQueues = new Set(); + this.queues.set(topic, topicQueues); + } + let queue!: BoundedNotificationQueue; + queue = new BoundedNotificationQueue( + topic, + this.queueCapacity, + () => { + topicQueues!.delete(queue); + if (topicQueues!.size === 0) this.queues.delete(topic); + }, + () => { + this.counters.queueOverflows++; + } + ); + topicQueues.add(queue); + return queue; + } + + dispatch(topic: string, payload: string): void { + const queues = this.queues.get(topic); + if (!queues) return; + for (const queue of [...queues]) queue.push(payload); + } + + fail(error: PgNotificationBrokerFailedError): void { + this.termination.resolve(error); + for (const queues of [...this.queues.values()]) { + for (const queue of [...queues]) queue.fail(error); + } + } + + release(): Promise { + if (this.releasePromise) return this.releasePromise; + this.released = true; + for (const queues of [...this.queues.values()]) { + for (const queue of [...queues]) queue.complete(); + } + this.releasePromise = this.broker.releaseLease(this); + void this.releasePromise.then( + () => this.termination.resolve(null), + (error) => + this.termination.resolve( + error instanceof PgNotificationBrokerFailedError + ? error + : new PgNotificationBrokerFailedError(error) + ) + ); + return this.releasePromise; + } +} + +class NotificationBrokerRecord { + private state: BrokerState = 'new'; + private acceptingLeases = true; + private operation: Promise = Promise.resolve(); + private source: PgNotificationConnectionSource | null = null; + private client: PgNotificationClient | null = null; + private clientCleanup: Promise | null = null; + private sourceCleanup: Promise | null = null; + private fatalError: PgNotificationBrokerFailedError | null = null; + private readonly leases = new Set(); + private readonly topicReferences = new Map(); + /** Includes provisional LISTENs whose lease admission has not committed yet. */ + private readonly listenedTopics = new Set(); + + private readonly onNotification = (notification: PgNotification): void => { + if (this.state !== 'active') return; + if ( + !notification || + typeof notification.channel !== 'string' || + (notification.payload !== undefined && + typeof notification.payload !== 'string') + ) { + this.markFailed( + new Error('PostgreSQL listener emitted a malformed notification') + ); + return; + } + if (!this.topicReferences.has(notification.channel)) { + this.counters.ignoredNotifications++; + return; + } + this.counters.notifications++; + const payload = notification.payload ?? ''; + for (const lease of [...this.leases]) { + lease.dispatch(notification.channel, payload); + } + }; + + private readonly onClientError = (error: unknown): void => { + this.markFailed(error); + }; + + private readonly onClientEnd = (): void => { + this.markFailed( + new Error('PostgreSQL notification listener connection ended') + ); + }; + + constructor( + readonly identity: string, + private readonly sourcePromise: Promise, + private readonly queueCapacity: number, + private readonly operationTimeoutMs: number, + private readonly counters: MutableBrokerCounters, + private readonly onTerminal: (record: NotificationBrokerRecord) => void + ) {} + + get snapshot(): PgNotificationBrokerSnapshot { + let subscribers = 0; + for (const lease of this.leases) subscribers += lease.subscriberCount; + return { + listenerConnections: this.client ? 1 : 0, + leases: this.leases.size, + topics: this.topicReferences.size, + subscribers, + }; + } + + assertAvailable(): void { + if (this.state === 'failed') throw this.fatalError!; + if (this.state !== 'active') throw new PgNotificationLeaseReleasedError(); + } + + async acquire( + topics: readonly string[], + roleContract: Readonly | null = null + ): Promise { + const lease = new NotificationBrokerLease( + this.identity, + topics, + this, + this.queueCapacity, + this.counters, + roleContract + ); + await this.enqueue(async () => { + if (!this.acceptingLeases) throw new BrokerClosedError(); + if (this.state === 'failed') throw this.fatalError!; + if (this.state === 'closing' || this.state === 'closed') { + throw new BrokerClosedError(); + } + const client = await this.ensureClient(); + if (!this.acceptingLeases) throw new BrokerClosedError(); + if (roleContract) { + lease.setRoleAudit(await this.auditRole(client, roleContract)); + } + if (!this.acceptingLeases) throw new BrokerClosedError(); + for (const topic of topics) { + if ((this.topicReferences.get(topic) ?? 0) === 0) { + await this.executeListenerQuery( + client, + `LISTEN ${quoteIdentifier(topic)}` + ); + this.listenedTopics.add(topic); + } + } + if (!this.acceptingLeases || this.state !== 'active') { + if (this.fatalError) throw this.fatalError; + throw new BrokerClosedError(); + } + for (const topic of topics) { + this.topicReferences.set( + topic, + (this.topicReferences.get(topic) ?? 0) + 1 + ); + } + this.leases.add(lease); + this.counters.acquisitions++; + }); + return lease; + } + + async revalidateLeaseRole( + lease: NotificationBrokerLease + ): Promise { + return this.enqueue(async () => { + if (lease.isReleased || !this.leases.has(lease)) { + throw new PgNotificationLeaseReleasedError(); + } + if (this.state === 'failed') throw this.fatalError!; + if (this.state !== 'active' || !this.client || !lease.roleContract) { + throw new PgNotificationLeaseReleasedError(); + } + const audit = await this.auditRole(this.client, lease.roleContract); + lease.setRoleAudit(audit); + return audit; + }); + } + + async releaseLease(lease: NotificationBrokerLease): Promise { + return this.enqueue(async () => { + if (!this.leases.delete(lease)) return; + this.counters.releases++; + + const topicsToUnlisten: string[] = []; + for (const topic of lease.topics) { + const next = (this.topicReferences.get(topic) ?? 0) - 1; + if (next <= 0) { + this.topicReferences.delete(topic); + topicsToUnlisten.push(topic); + } else { + this.topicReferences.set(topic, next); + } + } + + let releaseError: Error | null = null; + if (this.state === 'active' && this.client) { + for (const topic of topicsToUnlisten) { + try { + await this.executeListenerQuery( + this.client, + `UNLISTEN ${quoteIdentifier(topic)}` + ); + this.listenedTopics.delete(topic); + } catch (error) { + releaseError = + this.fatalError ?? new PgNotificationBrokerFailedError(error); + break; + } + } + } + + if (this.leases.size === 0) await this.closeUnused(); + if (releaseError) throw releaseError; + }); + } + + async closeAll(): Promise { + this.acceptingLeases = false; + // Cross the serialized-operation barrier before snapshotting leases. This + // either rejects an acquisition already waiting on connect/LISTEN or makes + // its completed lease visible to the release snapshot below. + await this.enqueue((): void => undefined); + const releases = [...this.leases].map((lease) => lease.release()); + const releaseResults = await Promise.allSettled(releases); + const closeErrors = releaseResults + .filter( + (result): result is PromiseRejectedResult => + result.status === 'rejected' + ) + .map((result) => result.reason); + try { + await this.enqueue(() => this.closeUnused()); + } catch (error) { + closeErrors.push(error); + } + if (closeErrors.length === 1) throw closeErrors[0]; + if (closeErrors.length > 1) { + throw new PgNotificationBrokerFailedError( + new AggregateError( + closeErrors, + 'Multiple PostgreSQL notification broker close operations failed' + ) + ); + } + } + + async closeIfUnused(): Promise { + await this.enqueue(() => this.closeUnused()); + } + + private enqueue(operation: () => PromiseOrDirect): Promise { + const pending = this.operation.then(operation, operation); + this.operation = pending.then( + (): void => undefined, + (): void => undefined + ); + return pending; + } + + private async ensureClient(): Promise { + if (this.client) return this.client; + try { + this.source = await this.sourcePromise; + if (this.fatalError) throw this.fatalError; + const client = await this.source.connect(); + this.client = client; + client.on('notification', this.onNotification); + client.on('error', this.onClientError); + client.on('end', this.onClientEnd); + this.state = 'active'; + return client; + } catch (error) { + this.markFailed(error); + await this.awaitFailedClientCleanup(); + throw this.fatalError!; + } + } + + private async executeListenerQuery( + client: PgNotificationClient, + text: string + ): Promise { + try { + await this.runWithOperationDeadline( + text.startsWith('UNLISTEN') ? 'unlisten' : 'listen', + () => client.query(text) + ); + if (this.state === 'failed') throw this.fatalError!; + } catch (error) { + this.markFailed(error); + await this.awaitFailedClientCleanup(); + throw this.fatalError!; + } + } + + private async auditRole( + client: PgNotificationClient, + contract: Readonly + ): Promise { + this.counters.roleAuditAttempts++; + try { + const audit = await this.runWithOperationDeadline('role-audit', () => + assertPgNotificationRoleClient( + client as unknown as PgNotificationRoleClient, + contract + ) + ); + if (this.state === 'failed') throw this.fatalError!; + return audit; + } catch (error) { + this.counters.roleAuditFailures++; + this.markFailed(error); + await this.awaitFailedClientCleanup(); + // Preserve the stable unsafe-role error for startup and attestation + // diagnostics. Active leases separately observe the broker-failed latch. + throw error; + } + } + + private async runWithOperationDeadline( + operation: NotificationOperation, + task: () => PromiseOrDirect + ): Promise { + let timer: ReturnType | null = null; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new PgNotificationOperationTimeoutError( + operation, + this.operationTimeoutMs + ); + // Latch failure and start client destruction at the exact deadline. The + // driver promise remains observed below, so a later rejection is safe. + this.markFailed(error); + reject(error); + }, this.operationTimeoutMs); + timer.unref?.(); + }); + // Promise.race installs a rejection handler on the driver query. If the + // deadline wins, destroying the client may settle that abandoned query + // later without producing an unhandled rejection. + const operationPromise = Promise.resolve().then(task); + try { + return await Promise.race([operationPromise, timeout]); + } finally { + if (timer) clearTimeout(timer); + } + } + + private markFailed(reason: unknown): void { + if ( + this.state === 'failed' || + this.state === 'closing' || + this.state === 'closed' + ) + return; + this.state = 'failed'; + this.fatalError = + reason instanceof PgNotificationBrokerFailedError + ? reason + : new PgNotificationBrokerFailedError(reason); + this.counters.fatalFailures++; + for (const lease of [...this.leases]) lease.fail(this.fatalError); + + const client = this.client; + this.client = null; + if (client) { + this.clientCleanup = this.releaseClient(client, this.fatalError); + // Event-driven failures may not have an immediate waiter. Keep the + // cleanup rejection observed; release/teardown still await and report it. + void this.clientCleanup.catch(() => {}); + } + } + + private async awaitFailedClientCleanup(): Promise { + try { + if (this.clientCleanup) await this.clientCleanup; + } catch (cleanupError) { + throw new PgNotificationBrokerFailedError( + new AggregateError( + [this.fatalError, cleanupError], + 'PostgreSQL notification failure cleanup did not complete safely', + { cause: this.fatalError ?? undefined } + ) + ); + } + } + + private async releaseClient( + client: PgNotificationClient, + error?: Error, + destroy = false + ): Promise { + client.off('notification', this.onNotification); + client.off('end', this.onClientEnd); + try { + await client.release(error ?? (destroy ? true : undefined)); + } catch (releaseError) { + if (error) { + throw new AggregateError( + [error, releaseError], + 'PostgreSQL notification failure and client cleanup both failed', + { cause: error } + ); + } + throw releaseError; + } finally { + client.off('error', this.onClientError); + } + } + + private async closeUnused(): Promise { + if (this.leases.size > 0 || this.state === 'closed') return; + + const cleanupErrors: unknown[] = []; + if (this.client && this.listenedTopics.size > 0) { + try { + // This also covers a shutdown racing between a successful LISTEN and + // lease admission, where no committed topic reference exists yet. + await this.executeListenerQuery(this.client, 'UNLISTEN *'); + this.listenedTopics.clear(); + } catch (error) { + cleanupErrors.push(error); + } + } + if (this.state !== 'failed') this.state = 'closing'; + + const client = this.client; + this.client = null; + if (client) { + const releaseError = + cleanupErrors.length > 0 + ? new PgNotificationBrokerFailedError(cleanupErrors[0]) + : undefined; + // Once the last exact-generation lease is gone, retaining an idle + // listener backend only delays PostgreSQL memory reclamation. Destroy it + // after UNLISTEN; the identity-only pool can create a fresh client later. + this.clientCleanup = this.releaseClient(client, releaseError, true); + } + try { + if (this.clientCleanup) await this.clientCleanup; + } catch (error) { + cleanupErrors.push(error); + } + + if (this.source && !this.sourceCleanup) { + const source = this.source; + this.source = null; + this.sourceCleanup = Promise.resolve(source.release()); + } + try { + if (this.sourceCleanup) await this.sourceCleanup; + } catch (error) { + cleanupErrors.push(error); + } + + this.state = 'closed'; + this.onTerminal(this); + if (cleanupErrors.length === 1) { + const [error] = cleanupErrors; + throw error instanceof PgNotificationBrokerFailedError + ? error + : new PgNotificationBrokerFailedError(error); + } + if (cleanupErrors.length > 1) { + throw new PgNotificationBrokerFailedError( + new AggregateError( + cleanupErrors, + 'Multiple PostgreSQL notification cleanup operations failed' + ) + ); + } + } +} + +/** + * Registry implementation exposed for deterministic unit tests. Production + * callers must use acquirePgNotificationBroker so identity and pool ownership + * always come from the canonical PgConfig path. + * + * @internal + */ +export class PgNotificationBrokerRegistry { + private readonly records = new Map(); + private closed = false; + private closePromise: Promise | null = null; + private readonly counters: MutableBrokerCounters = { + acquisitions: 0, + releases: 0, + notifications: 0, + ignoredNotifications: 0, + queueOverflows: 0, + fatalFailures: 0, + roleAuditAttempts: 0, + roleAuditFailures: 0, + }; + + constructor( + private readonly queueCapacity = PG_NOTIFICATION_QUEUE_CAPACITY, + private readonly defaultOperationTimeoutMs = DEFAULT_PG_NOTIFICATION_OPERATION_TIMEOUT_MS + ) { + if (!Number.isSafeInteger(queueCapacity) || queueCapacity <= 0) { + throw new Error( + 'PostgreSQL notification queue capacity must be a positive safe integer' + ); + } + assertNotificationOperationTimeoutMs(defaultOperationTimeoutMs); + } + + async acquireForTests( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + operationTimeoutMs = this.defaultOperationTimeoutMs + ): Promise { + return this.acquireInternal( + identity, + sourceFactory, + topics, + null, + operationTimeoutMs + ); + } + + /** @internal Exercise production attestation without constructing PgConfig. */ + async acquireAttestedForTests( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + roleContract: PgNotificationRoleContract, + operationTimeoutMs = this.defaultOperationTimeoutMs + ): Promise { + return this.acquireInternal( + identity, + sourceFactory, + topics, + roleContract, + operationTimeoutMs + ); + } + + private async acquireInternal( + identity: string, + sourceFactory: ConnectionSourceFactory, + topics: readonly string[], + roleContract: PgNotificationRoleContract | null, + operationTimeoutMs: number + ): Promise { + if (this.closed) + throw new Error('PostgreSQL notification broker registry is closed'); + if (typeof identity !== 'string' || identity.length === 0) { + throw new Error( + 'PostgreSQL notification broker identity must be a non-empty string' + ); + } + const normalizedTopics = normalizeTopics(topics); + const normalizedOperationTimeoutMs = + assertNotificationOperationTimeoutMs(operationTimeoutMs); + + for (;;) { + let record = this.records.get(identity); + if (!record) { + const sourcePromise = Promise.resolve(sourceFactory()); + // Acquisition consumes this immediately, but guard the small interval + // before its serialized operation attaches a rejection handler. + void sourcePromise.catch(() => {}); + record = new NotificationBrokerRecord( + identity, + sourcePromise, + this.queueCapacity, + normalizedOperationTimeoutMs, + this.counters, + (terminal) => { + if (this.records.get(identity) === terminal) + this.records.delete(identity); + } + ); + this.records.set(identity, record); + } + try { + const lease = await record.acquire(normalizedTopics, roleContract); + if (this.closed) { + await lease.release(); + throw new Error('PostgreSQL notification broker registry is closed'); + } + return lease; + } catch (error) { + if (error instanceof BrokerClosedError && !this.closed) continue; + // A failed broker remains pinned until every existing owner explicitly + // releases it. This prevents an acquisition attempt from silently + // replacing a listener after a possible notification gap. + await record.closeIfUnused(); + if (error instanceof BrokerClosedError && this.closed) { + throw new Error('PostgreSQL notification broker registry is closed'); + } + throw error; + } + } + } + + stats(): PgNotificationBrokerStats { + let listenerConnections = 0; + let leases = 0; + let topics = 0; + let subscribers = 0; + for (const record of this.records.values()) { + const snapshot = record.snapshot; + listenerConnections += snapshot.listenerConnections; + leases += snapshot.leases; + topics += snapshot.topics; + subscribers += snapshot.subscribers; + } + return { + brokers: this.records.size, + listenerConnections, + leases, + topics, + subscribers, + ...this.counters, + }; + } + + close(): Promise { + if (this.closePromise) return this.closePromise; + this.closed = true; + this.closePromise = (async () => { + const closeResults = await Promise.allSettled( + [...this.records.values()].map((record) => record.closeAll()) + ); + this.records.clear(); + const closeErrors = closeResults + .filter( + (result): result is PromiseRejectedResult => + result.status === 'rejected' + ) + .map((result) => result.reason); + if (closeErrors.length === 1) throw closeErrors[0]; + if (closeErrors.length > 1) { + throw new PgNotificationBrokerFailedError( + new AggregateError( + closeErrors, + 'Multiple PostgreSQL notification registries failed to close' + ) + ); + } + })(); + return this.closePromise; + } +} + +let brokerRegistry = new PgNotificationBrokerRegistry(); +let brokerTeardownTail: Promise = Promise.resolve(); + +/** Opaque identity over the complete canonical listener pool contract. */ +export const getPgNotificationBrokerIdentity = ( + listenerPgConfig: PgNotificationListenerConfig +): string => { + // The operation deadline is represented by the pool connection timeout in + // the identity below. Validate it before publishing an apparently usable key. + getNotificationOperationTimeoutMs(listenerPgConfig); + const poolIdentity = getPgPoolIdentity(listenerPgConfig, { + purpose: 'notification-broker', + }); + return `${PG_NOTIFICATION_BROKER_IDENTITY_VERSION}:${poolIdentity}`; +}; + +/** + * Opaque identity for one physical database target, deliberately excluding + * credentials, TLS policy, pool sizing, and checkout behavior. Those inputs + * split listener pools, but must not let two active listener contracts silently + * fragment one database's broker. + */ +export const getPgNotificationDatabaseIdentity = ( + listenerPgConfig: PgNotificationListenerConfig +): string => { + const targetIdentity = getPgDatabaseTargetIdentity(listenerPgConfig); + return `${PG_NOTIFICATION_DATABASE_IDENTITY_VERSION}:${targetIdentity}`; +}; + +/** + * Acquire a generation lease over one process-local listener. The supplied + * config must name the dedicated least-privilege notification login; this API + * never falls back to a request runtime or control-plane credential. + */ +export const acquirePgNotificationBroker = async ( + listenerPgConfig: PgNotificationListenerConfig, + options: AcquirePgNotificationBrokerOptions +): Promise => { + const operationTimeoutMs = + getNotificationOperationTimeoutMs(listenerPgConfig); + const identity = getPgNotificationBrokerIdentity(listenerPgConfig); + return brokerRegistry.acquireAttestedForTests( + identity, + () => { + const poolLease = acquirePgPool(listenerPgConfig, { + purpose: 'notification-broker', + }); + return { + connect: () => + poolLease.pool.connect() as Promise, + release: () => poolLease.release(), + }; + }, + options.topics, + { + role: listenerPgConfig.user, + database: listenerPgConfig.database, + }, + operationTimeoutMs + ); +}; + +export const getPgNotificationBrokerStats = (): PgNotificationBrokerStats => + brokerRegistry.stats(); + +/** Await every UNLISTEN and checked-out connection release, then reset. */ +export const teardownPgNotificationBrokers = (): Promise => { + const closing = brokerRegistry; + brokerRegistry = new PgNotificationBrokerRegistry(); + const teardown = brokerTeardownTail.then(() => closing.close()); + // A later teardown must wait until this registry has fully drained even when + // this caller observes a cleanup failure. + brokerTeardownTail = teardown.then( + (): void => undefined, + (): void => undefined + ); + return teardown; +}; From 1326b6de036f401dcde0b1a35ca37de7536cb82e Mon Sep 17 00:00:00 2001 From: zetazzz Date: Mon, 17 Aug 2026 16:55:37 +0800 Subject: [PATCH 5/5] Add PostgreSQL notification integration coverage --- .../notification-broker.integration.test.ts | 230 ++++++++++++++++++ .../notification-role.integration.test.ts | 132 ++++++++++ 2 files changed, 362 insertions(+) create mode 100644 postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts create mode 100644 postgres/pg-cache/src/__tests__/notification-role.integration.test.ts diff --git a/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts b/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts new file mode 100644 index 0000000000..850f43b688 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-broker.integration.test.ts @@ -0,0 +1,230 @@ +import type pg from 'pg'; +import { getPgEnvOptions, type PgConfig } from 'pg-env'; + +import { teardownPgPools } from '../lru'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + PgNotificationTopicError, + teardownPgNotificationBrokers, +} from '../notification-broker'; +import { defaultPgPoolFactory, getPgPool } from '../pg'; + +// Production acquisition always audits the login on its pinned listener, so +// this test requires the dedicated least-privilege notification fixture. +const describeWithPostgres = + process.env.PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithPostgres('notification broker against PostgreSQL', () => { + let observerPool: pg.Pool; + let listenerPgConfig: PgConfig & { pool: { max: number } }; + + beforeAll(() => { + listenerPgConfig = { + ...getPgEnvOptions(), + pool: { max: 1 }, + }; + observerPool = defaultPgPoolFactory( + { ...listenerPgConfig, pool: { max: 1 } }, + { purpose: 'notification-broker-integration-observer' } + ) as pg.Pool; + }); + + afterAll(async () => { + await teardownPgNotificationBrokers(); + await teardownPgPools(); + await observerPool?.end(); + }); + + it('shares one LISTEN backend across three isolated generation leases and releases it', async () => { + const nonce = `${process.pid.toString(36)}_${Date.now().toString(36)}`; + const topics = [ + `pg_cache_it_${nonce}_a`, + `pg_cache_it_${nonce}_b`, + `pg_cache_it_${nonce}_c`, + ]; + const listenQueries = topics.map((topic) => `LISTEN "${topic}"`); + + const first = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[0]], + }); + const second = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[1]], + }); + const third = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [topics[2]], + }); + const brokerPool = getPgPool(listenerPgConfig, { + purpose: 'notification-broker', + }); + + expect( + new Set([first.identity, second.identity, third.identity]).size + ).toBe(1); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 3, + topics: 3, + }); + expect(brokerPool.totalCount).toBe(1); + expect(brokerPool.idleCount).toBe(0); + + const activeListeners = await observerPool.query<{ + pid: number; + query: string; + }>( + ` + SELECT pid, query + FROM pg_stat_activity + WHERE datname = current_database() + AND usename = current_user + AND pid <> pg_backend_pid() + AND query = ANY($1::text[]) + `, + [listenQueries] + ); + expect(activeListeners.rows).toEqual([ + { pid: expect.any(Number), query: listenQueries[2] }, + ]); + const listenerPid = activeListeners.rows[0].pid; + + expect(() => first.subscribe(topics[1])).toThrow(PgNotificationTopicError); + const firstStream = first.subscribe(topics[0]); + const secondStream = second.subscribe(topics[1]); + const thirdStream = third.subscribe(topics[2]); + let firstResolved = false; + let thirdResolved = false; + const firstNext = firstStream.next().then((result) => { + firstResolved = true; + return result; + }); + const secondNext = secondStream.next(); + const thirdNext = thirdStream.next().then((result) => { + thirdResolved = true; + return result; + }); + + await observerPool.query('SELECT pg_notify($1, $2)', [ + topics[1], + 'for-second', + ]); + await expect(secondNext).resolves.toEqual({ + done: false, + value: 'for-second', + }); + // Delivery to every lease happens synchronously inside one notification + // callback, so these flags prove the second topic did not reach its peers. + expect(firstResolved).toBe(false); + expect(thirdResolved).toBe(false); + + await observerPool.query('SELECT pg_notify($1, $2)', [ + topics[0], + 'for-first', + ]); + await observerPool.query('SELECT pg_notify($1, $2)', [ + topics[2], + 'for-third', + ]); + await expect(firstNext).resolves.toEqual({ + done: false, + value: 'for-first', + }); + await expect(thirdNext).resolves.toEqual({ + done: false, + value: 'for-third', + }); + + await second.release(); + await first.release(); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 1, + topics: 1, + }); + expect(brokerPool.idleCount).toBe(0); + + await third.release(); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + topics: 0, + }); + expect(brokerPool.totalCount).toBe(0); + expect(brokerPool.idleCount).toBe(0); + + let releasedListenerRows: Array<{ pid: number }> = []; + for (let attempt = 0; attempt < 50; attempt++) { + const releasedListener = await observerPool.query<{ pid: number }>( + ` + SELECT pid + FROM pg_stat_activity + WHERE pid = $1 + `, + [listenerPid] + ); + releasedListenerRows = releasedListener.rows; + if (releasedListenerRows.length === 0) break; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(releasedListenerRows).toEqual([]); + }); + + it('replaces a released generation with a new backend and no old topic owner', async () => { + const nonce = `${process.pid.toString(36)}_${Date.now().toString(36)}`; + const oldTopic = `pg_cache_generation_${nonce}_old`; + const newTopic = `pg_cache_generation_${nonce}_new`; + + const oldLease = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [oldTopic], + }); + const oldListener = await observerPool.query<{ pid: number }>( + ` + SELECT pid + FROM pg_stat_activity + WHERE datname = current_database() + AND usename = current_user + AND pid <> pg_backend_pid() + AND query = $1 + `, + [`LISTEN "${oldTopic}"`] + ); + expect(oldListener.rows).toHaveLength(1); + const oldPid = oldListener.rows[0].pid; + + await oldLease.release(); + const newLease = await acquirePgNotificationBroker(listenerPgConfig, { + topics: [newTopic], + }); + const newListener = await observerPool.query<{ pid: number }>( + ` + SELECT pid + FROM pg_stat_activity + WHERE datname = current_database() + AND usename = current_user + AND pid <> pg_backend_pid() + AND query = $1 + `, + [`LISTEN "${newTopic}"`] + ); + expect(newListener.rows).toHaveLength(1); + expect(newListener.rows[0].pid).not.toBe(oldPid); + + const next = newLease.subscribe(newTopic).next(); + await observerPool.query('SELECT pg_notify($1, $2)', [oldTopic, 'stale']); + await observerPool.query('SELECT pg_notify($1, $2)', [newTopic, 'current']); + await expect(next).resolves.toEqual({ done: false, value: 'current' }); + + await newLease.release(); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + topics: 0, + }); + }); +}); diff --git a/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts b/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts new file mode 100644 index 0000000000..94a89f9657 --- /dev/null +++ b/postgres/pg-cache/src/__tests__/notification-role.integration.test.ts @@ -0,0 +1,132 @@ +import type pg from 'pg'; +import { getPgEnvOptions } from 'pg-env'; + +import { teardownPgPools } from '../lru'; +import { + acquirePgNotificationBroker, + getPgNotificationBrokerStats, + teardownPgNotificationBrokers, +} from '../notification-broker'; +import { + assertPgNotificationRole, + auditPgNotificationRole, +} from '../notification-role'; +import { defaultPgPoolFactory, getPgPool } from '../pg'; + +const describeWithNotificationRole = + process.env.PG_CACHE_RUN_NOTIFICATION_ROLE_INTEGRATION === '1' + ? describe + : describe.skip; + +describeWithNotificationRole( + 'dedicated notification role against PostgreSQL', + () => { + const pgConfig = getPgEnvOptions(); + let pool: pg.Pool; + + beforeAll(() => { + pool = defaultPgPoolFactory( + { ...pgConfig, pool: { max: 1 } }, + { purpose: 'notification-role-integration' } + ) as pg.Pool; + }); + + afterAll(async () => { + await teardownPgNotificationBrokers(); + await teardownPgPools(); + await pool?.end(); + }); + + it('accepts only the exact credential-free role/database contract', async () => { + const audit = await assertPgNotificationRole(pool, { + role: pgConfig.user, + database: pgConfig.database, + }); + + expect(audit).toMatchObject({ + role: pgConfig.user, + database: pgConfig.database, + safe: true, + violations: [], + }); + expect(Object.keys(audit).sort()).toEqual([ + 'database', + 'role', + 'safe', + 'version', + 'violations', + ]); + expect(audit).not.toHaveProperty('password'); + expect(audit).not.toHaveProperty('host'); + + const wrongRole = await auditPgNotificationRole(pool, { + role: `wrong_${process.pid}`, + database: pgConfig.database, + }); + expect(wrongRole).toMatchObject({ + safe: false, + violations: expect.arrayContaining(['LOGIN_ROLE_MISMATCH']), + }); + + const wrongDatabase = await auditPgNotificationRole(pool, { + role: pgConfig.user, + database: `wrong_${process.pid}`, + }); + expect(wrongDatabase).toMatchObject({ + safe: false, + violations: expect.arrayContaining([ + 'DATABASE_MISMATCH', + 'TARGET_DATABASE_MISSING', + 'TARGET_CONNECT_REQUIRED', + 'CROSS_DATABASE_CONNECT', + ]), + }); + }); + + it('retains enough privilege for isolated LISTEN and NOTIFY delivery', async () => { + const nonce = `${process.pid}_${Date.now().toString(36)}`; + const topics = [0, 1, 2].map( + (index) => `notify_role_it_${nonce}_${index}` + ); + const listenerConfig = { ...pgConfig, pool: { max: 1 } }; + const statsBefore = getPgNotificationBrokerStats(); + const [first, second, third] = await Promise.all( + topics.map((topic) => + acquirePgNotificationBroker(listenerConfig, { topics: [topic] }) + ) + ); + const brokerPool = getPgPool(listenerConfig, { + purpose: 'notification-broker', + }); + await first.revalidateRole(); + const next = second.subscribe(topics[1]).next(); + + await pool.query('SELECT pg_notify($1, $2)', [ + topics[1], + 'safe-listener', + ]); + await expect(next).resolves.toEqual({ + done: false, + value: 'safe-listener', + }); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 1, + listenerConnections: 1, + leases: 3, + topics: 3, + roleAuditAttempts: statsBefore.roleAuditAttempts + 4, + roleAuditFailures: statsBefore.roleAuditFailures, + }); + expect(brokerPool.totalCount).toBe(1); + expect(brokerPool.idleCount).toBe(0); + + await Promise.all([first.release(), second.release(), third.release()]); + expect(getPgNotificationBrokerStats()).toMatchObject({ + brokers: 0, + listenerConnections: 0, + leases: 0, + topics: 0, + }); + }); + } +);