PgBouncer connection pooling and horizontal scaling

.dockerignore

@@ -0,0 +1,10 @@
+node_modules
+dist
+.kits
+.git
+.env
+.env.*
+*.log
+test
+docs
+examples

.env.example

@@ -45,8 +45,21 @@ DB_BACKEND=pg
 # [required in production] [secret] Connection string for the backend above.
 #   PostgreSQL: postgres://USER:PASSWORD@HOST:5432/mgpl
 #   MongoDB:    mongodb://USER:PASSWORD@HOST:27017/mgpl?replicaSet=rs0   (a replica set is required: transactions)
+#   Through PgBouncer: postgres://USER:PASSWORD@PGBOUNCER_HOST:6432/mgpl   (transaction pooling is supported)
 # URL-encode special characters in the password (@ → %40, : → %3A, / → %2F).
 DATABASE_URL=postgres://postgres:postgres@localhost:5432/mgpl
+# [optional] Connections this process keeps open (the Sequelize pool for pg, the MongoDB driver's pool for mongo),
+# per API server and per worker. Behind PgBouncer they are cheap client connections: keep
+# (API replicas + workers) × DB_POOL_MAX under PGBOUNCER_MAX_CLIENT_CONN. Without PgBouncer, keep it under
+# PostgreSQL's max_connections instead.
+# DB_POOL_MAX=10
+# DB_POOL_MIN=0
+# [optional] Close a pooled connection after this long idle (ms); fail a query that waited this long for one (ms).
+# DB_POOL_IDLE_MS=10000
+# DB_POOL_ACQUIRE_MS=30000
+# [optional] Create missing tables (pg) or indexes (mongo) on start. With several replicas set it to false on
+# all of them and run `npm run db:sync` (node dist/sync.js) once per release instead (docs/17-scaling.md).
+# DB_SYNC=true
 
 
 # ── Redis ────────────────────────────────────────────────────────────────────────────────────────────
@@ -146,9 +159,32 @@ PASSWORD_RESET_URL=http://localhost:3000/reset-password
 # SEED_OWNER_PASSWORD=
 
 
+# ── Scaling: graceful shutdown, replicas, PgBouncer ──────────────────────────────────────────────────
+# Running several API replicas: REDIS_URL and the same JWT_SECRET on every replica (docs/17-scaling.md).
+# [optional] On SIGTERM, GET /health answers 503 for this long while requests are still served, so the load
+# balancer takes the replica out first (ms). Default 5000 in production, 0 otherwise.
+# SHUTDOWN_DRAIN_MS=5000
+# [optional] Then the server stops accepting connections and finishes the requests in flight; it exits
+# anyway this long after the signal (ms). The worker uses it too, for the jobs in hand.
+# SHUTDOWN_TIMEOUT_MS=30000
+#
+# Read only by docker-compose.yml (Docker Compose reads this file for them), not by the API:
+# API_REPLICAS=2                    api containers behind nginx (or `docker compose up -d --scale api=N`)
+# HTTP_PORT=8080                    nginx's port on the host
+# POSTGRES_PASSWORD=postgres        [secret] URL-safe characters only: it goes into connection URLs
+# POSTGRES_MAX_CONNECTIONS=100      PostgreSQL's limit; PgBouncer's server pool must fit under it
+# PGBOUNCER_POOL_MODE=transaction   transaction (recommended; `npm test` runs the api under it) | session
+# PGBOUNCER_DEFAULT_POOL_SIZE=20    server connections PgBouncer opens to PostgreSQL for the database
+# PGBOUNCER_MIN_POOL_SIZE=0
+# PGBOUNCER_RESERVE_POOL_SIZE=5     extra server connections when clients have waited 5 s
+# PGBOUNCER_MAX_CLIENT_CONN=500     client connections in all: at least (API_REPLICAS + workers) × DB_POOL_MAX
+# PGBOUNCER_MAX_DB_CONNECTIONS=50   hard cap on server connections: below POSTGRES_MAX_CONNECTIONS
+
+
 # ── Development and test tooling (not read by the running API) ──────────────────────────────────────
 # NODE_AUTH_TOKEN=          [secret] export it in your shell for `npm ci` (npm does not read .env): GitHub token, read:packages
 # KITS_SRC=..               `npm run kits`: build the kits from local checkouts instead of kits.json refs
 # TEST_BACKENDS=pg,mongo    `npm test`: which databases to run (pg only when MongoDB binaries can't download)
 # MONGOMS_SYSTEM_BINARY=    `npm test`: path to a local mongod
+# PGBOUNCER_BIN=pgbouncer   `npm test`: the pgbouncer binary for the [pgbouncer] and [replicas] checks (skipped without it)
 # DEBUG=1                   `npm test`: print resolver errors

.github/workflows/ci.yml

@@ -37,10 +37,14 @@ jobs:
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: npm run typecheck
-      - name: End-to-end tests (GraphQL over HTTP, tokens, worker on Redis, client SDK) on PostgreSQL and MongoDB
+      # the [pgbouncer] and [replicas] checks run the API through PgBouncer in transaction mode
+      - name: Install PgBouncer
+        run: sudo apt-get update -q && sudo apt-get install -y -q pgbouncer
+      - name: End-to-end tests (GraphQL over HTTP, tokens, worker on Redis, client SDK) on PostgreSQL, PostgreSQL through PgBouncer and MongoDB
         run: npm test
         env:
           REDIS_URL: redis://localhost:6379
+          PGBOUNCER_BIN: /usr/sbin/pgbouncer
       - run: npm run build
       - name: schema.graphql is up to date (run `npm run schema` and commit if this fails)
         run: npm run schema && git diff --exit-code -- schema.graphql

AGENTS.md

@@ -9,6 +9,8 @@ the kits, used as internal libraries. Read [docs/README.md](docs/README.md) for
 ```
 src/main.ts            start (config → container → server)
 src/worker.ts          start the worker (config → container → BullMQ workers; needs REDIS_URL)
+src/sync.ts            create missing tables / indexes once, then exit (`npm run db:sync`; replicas run DB_SYNC=false)
+src/lifecycle.ts       readiness while draining, graceful shutdown on SIGTERM
 src/server.ts          Fastify + Yoga + GET /health (no other routes); security-kit headers and client IP
 src/container.ts       createDb (posting: financePosting) + createAuth (JWT) + job queue + services, once per process
 src/jobs/              queues (BullMQ or in-process), job payloads, processors, webhook mailer
@@ -22,6 +24,7 @@ test/run.ts            end-to-end over HTTP on PostgreSQL + MongoDB, plus docs/e
 docs/                  A-to-Z docs (Mermaid), examples/*.graphql, LOCAL-AGENT-BRIEF.md
 schema.graphql         printed schema (generated)
 kits.json              kit refs for `npm run kits` (building unreleased kit branches from source)
+Dockerfile, docker-compose.yml, deploy/   image; PostgreSQL ← PgBouncer ← N api replicas ← nginx, Redis, worker (docs/17-scaling.md)
 ```
 
 ## Rules
@@ -52,7 +55,12 @@ kits.json              kit refs for `npm run kits` (building unreleased kit bran
     signals are collected by security-kit in the client; `src/graphql/device.ts` maps DeviceInput to auth-kit.
     Security headers and the client IP come from security-kit's `fastifySecurity` (never Fastify's own
     `req.ip` with a trust-all proxy setting: the left X-Forwarded-For entry is client-controlled).
-14. Never commit patient data, signatures, secrets or `.env` files. Test fixtures use fictional names.
+14. **Replicas.** Nothing per-user lives in process memory when `REDIS_URL` is set: sessions, refresh tokens,
+    login limits and jobs are in Redis, data behind PgBouncer. Keep it that way (no module-level caches of
+    users or sessions), and keep the db layer free of session-level PostgreSQL features (named prepared
+    statements, `SET` without `LOCAL`, advisory locks held across transactions, `LISTEN`, temp tables):
+    PgBouncer runs in transaction mode and `npm test` runs the suite through it.
+15. Never commit patient data, signatures, secrets or `.env` files. Test fixtures use fictional names.
 
 ## Commands
 

Dockerfile

@@ -0,0 +1,33 @@
+# syntax=docker/dockerfile:1
+# The api image: `node dist/main.js` (API), `node dist/worker.js` (worker), `node dist/sync.js` (create tables).
+# The kits come from GitHub Packages, so the build needs a token with read:packages, passed as a secret
+# (never a build arg, which would stay in the image history):
+#   NODE_AUTH_TOKEN=… docker compose build          (docker-compose.yml passes it as the npm_token secret)
+#   docker build --secret id=npm_token,env=NODE_AUTH_TOKEN -t mgpl-api .
+# report-kit's resvg is a native module: build on the platform the image runs on (linux/amd64 or arm64).
+
+FROM node:22-bookworm-slim AS build
+WORKDIR /app
+COPY package.json package-lock.json .npmrc ./
+RUN --mount=type=secret,id=npm_token \
+    echo "//npm.pkg.github.com/:_authToken=$(cat /run/secrets/npm_token)" > /root/.npmrc \
+    && npm ci --no-audit --no-fund \
+    && rm /root/.npmrc
+COPY tsconfig.json tsup.config.ts ./
+COPY src ./src
+RUN npm run build && npm prune --omit=dev
+
+FROM node:22-bookworm-slim
+ENV NODE_ENV=production
+WORKDIR /app
+COPY --from=build /app/package.json ./
+COPY --from=build /app/node_modules ./node_modules
+COPY --from=build /app/dist ./dist
+USER node
+EXPOSE 4000
+# Docker's own check (compose shows "healthy"): readiness, so a replica that lost its database reads unhealthy.
+HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=3 \
+  CMD ["node", "-e", "fetch('http://127.0.0.1:'+(process.env.PORT||4000)+'/health').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"]
+# node is PID 1 and handles SIGTERM itself (src/lifecycle.ts): drain, finish requests, close, exit.
+STOPSIGNAL SIGTERM
+CMD ["node", "dist/main.js"]

README.md

@@ -7,23 +7,28 @@
 **The MGPL Diagnostics backend: one GraphQL endpoint for the whole lab.**
 
 Sign-in, patients, bookings, samples, results, reports, billing, books, GST, receivables, settlements,
-cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL or MongoDB.
+cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL (through PgBouncer) or MongoDB,
+Redis. Runs as any number of replicas behind a load balancer.
 
 [![Endpoint](https://img.shields.io/badge/endpoint-POST%20%2Fgraphql-14264A)](#)
 [![Databases](https://img.shields.io/badge/db-PostgreSQL%20%7C%20MongoDB-14264A)](#)
+[![Pooling](https://img.shields.io/badge/pool-PgBouncer-14264A)](docs/17-scaling.md)
 [![License](https://img.shields.io/badge/license-All%20rights%20reserved-E9A23B)](LICENSE)
 
-[Docs](docs/README.md) · [Architecture](docs/02-architecture.md) · [Getting started](docs/03-getting-started.md) · [API reference](docs/04-api-reference.md) · [Client](docs/14-client.md)
+[Docs](docs/README.md) · [Architecture](docs/02-architecture.md) · [Getting started](docs/03-getting-started.md) · [API reference](docs/04-api-reference.md) · [Client](docs/14-client.md) · [Scaling](docs/17-scaling.md)
 
 </div>
 
 ---
 
 ```mermaid
 flowchart LR
-  APP["Apps<br/>@mgpl-diagnostics/client"] -- "POST /graphql" --> API["api<br/>controllers → services"]
+  APP["Apps<br/>@mgpl-diagnostics/client"] -- "POST /graphql" --> LB["load balancer<br/>(nginx)"]
+  LB --> API["api × N replicas<br/>controllers → services"]
   API --> K["auth-kit · db-kit · catalog-kit<br/>report-kit · invoice-kit · ledger-kit · finance-kit"]
-  K --> DB[("PostgreSQL / MongoDB")]
+  K --> PGB["PgBouncer<br/>transaction pooling"]
+  PGB --> PG[("PostgreSQL")]
+  K --> MG[("or MongoDB")]
   K -.-> R[("Redis")]
   R -. "BullMQ jobs" .-> W["worker<br/>emails · documents"]
   W --> K
@@ -40,6 +45,10 @@ flowchart LR
   device; logout and disabling still apply at once. See [docs/06](docs/06-auth-and-permissions.md).
 - **A worker on Redis + BullMQ** for reset emails and background documents (`npm run worker`); without
   Redis the same jobs run in the API process. See [docs/16-worker.md](docs/16-worker.md).
+- **Horizontal scaling through PgBouncer.** Run N identical replicas behind a load balancer: sessions,
+  login limits and jobs live in Redis, and PostgreSQL is reached through PgBouncer in transaction mode
+  (the whole test suite runs through it). `GET /health` is the readiness check, SIGTERM drains gracefully,
+  and `docker compose up --scale api=N` runs the whole stack. See [docs/17-scaling.md](docs/17-scaling.md).
 - **Layers.** `schema/` (SDL) → `controllers/` (permissions, mapping) → `services/` (kit calls) → kits.
 - **Safe by construction.** The organisation and user come from the session; every record is checked
   against them and against its branch; unknown errors are masked; depth and list limits.
@@ -73,7 +82,7 @@ const hits = await mgpl.catalog.search("cbc");
 
 Everything from A to Z is in [docs/](docs/README.md), with Mermaid diagrams: overview, architecture,
 getting started, the generated API reference, errors, auth and permissions, domain flows, documents,
-configuration, data and money, testing, deployment, adding a feature, client, glossary, and runnable
+configuration, data and money, testing, deployment, adding a feature, client, glossary, worker, scaling, and runnable
 [examples](docs/examples/). What has to happen outside this repo (releases, org settings, deployment) is
 in [docs/LOCAL-AGENT-BRIEF.md](docs/LOCAL-AGENT-BRIEF.md).
 
@@ -97,6 +106,8 @@ npm run llms                # llms-full.txt
 | [auth-kit](https://github.com/mgpl-diagnostics/auth-kit) · [db-kit](https://github.com/mgpl-diagnostics/db-kit) · [catalog-kit](https://github.com/mgpl-diagnostics/catalog-kit) | auth, data, test catalog |
 | [report-kit](https://github.com/mgpl-diagnostics/report-kit) · [invoice-kit](https://github.com/mgpl-diagnostics/invoice-kit) | report and invoice rendering |
 | [ledger-kit](https://github.com/mgpl-diagnostics/ledger-kit) · [finance-kit](https://github.com/mgpl-diagnostics/finance-kit) | books, GST and finance |
+| [security-kit](https://github.com/mgpl-diagnostics/security-kit) · [mail-kit](https://github.com/mgpl-diagnostics/mail-kit) · [worker-kit](https://github.com/mgpl-diagnostics/worker-kit) | security headers, client IP and device signals; mail; background jobs |
+| [pgbouncer-kit](https://github.com/mgpl-diagnostics/pgbouncer-kit) | PgBouncer setup (config, userlist, compose, TLS) for the connection pooler in front of PostgreSQL |
 | [report-studio](https://github.com/mgpl-diagnostics/report-studio) | Next.js reference app |
 
 <sub>© MGPL Diagnostics (Maa Gayatri Path Lab). All rights reserved; see [LICENSE](LICENSE). ledger-kit is our own software, not affiliated with Tally Solutions.</sub>

deploy/nginx.conf

@@ -0,0 +1,31 @@
+# The load balancer in docker-compose.yml: spreads requests over every `api` replica.
+# Docker's DNS (127.0.0.11) answers "api" with one address per replica; resolving through a variable makes
+# nginx re-resolve every 10 s, so `docker compose up --scale api=N` needs no restart here.
+# TLS: terminate it here (listen 443 ssl + certificates) or in front of it (a CDN or a cloud load balancer);
+# then count that proxy too in TRUST_PROXY.
+events {}
+
+http {
+  resolver 127.0.0.11 valid=10s ipv6=off;
+  access_log off;
+  # the api's BODY_LIMIT_BYTES (2 MB) plus room
+  client_max_body_size 3m;
+
+  server {
+    listen 80;
+
+    location / {
+      set $api http://api:4000;
+      proxy_pass $api;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      proxy_set_header Host $host;
+      # one trusted hop: TRUST_PROXY=1 on the api reads the right-most entry, the address nginx saw
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      # a replica that is shutting down answers 503: try the next one (nginx never retries a POST that reached a replica)
+      proxy_next_upstream error timeout http_502 http_503;
+      proxy_read_timeout 120s;
+    }
+  }
+}