From 2763d2c18c779ddc1b06b20dfaa52216a6cf0f19 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 21:13:21 +0000 Subject: [PATCH 1/3] feat: PgBouncer connection pooling and horizontal scaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - DATABASE_URL may point at PgBouncer; transaction pooling verified: npm test runs the whole PostgreSQL suite again through PgBouncer, then two replicas sharing PgBouncer and Redis (sessions, refresh, logout, login limits, concurrent invoices, shutdown) - DB_POOL_MAX/MIN/IDLE_MS/ACQUIRE_MS size the Sequelize / MongoDB pools; DB_SYNC and node dist/sync.js (npm run db:sync) so replicas don't race to create tables - GET /health is readiness (database, Redis, draining → 503); /health?live=1 is liveness - graceful shutdown on SIGTERM: drain (SHUTDOWN_DRAIN_MS), finish requests, close, hard exit after SHUTDOWN_TIMEOUT_MS; the worker gets the same timeout - Dockerfile, docker-compose.yml (PostgreSQL, PgBouncer, Redis, db-sync, N api replicas, worker, nginx), deploy/nginx.conf; CI installs pgbouncer - docs/17-scaling.md, .env.example, docs 09/11/12, AGENTS.md Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dcqyj37vsGrmgdnp7DLbkn --- .dockerignore | 10 + .env.example | 36 +++ .github/workflows/ci.yml | 6 +- AGENTS.md | 10 +- Dockerfile | 33 +++ deploy/nginx.conf | 31 +++ docker-compose.yml | 135 ++++++++++ docs/09-configuration.md | 14 +- docs/11-testing.md | 2 + docs/12-deployment.md | 12 +- docs/17-scaling.md | 152 ++++++++++++ docs/README.md | 2 + llms-full.txt | 518 +++++++++++++++++++++++++++++++++++++-- package.json | 1 + src/config/env.ts | 30 ++- src/container.ts | 8 +- src/index.ts | 1 + src/lifecycle.ts | 57 +++++ src/main.ts | 11 +- src/server.ts | 23 +- src/sync.ts | 18 ++ src/worker.ts | 3 +- test/run.ts | 147 ++++++++++- tsup.config.ts | 2 +- 24 files changed, 1211 insertions(+), 51 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 deploy/nginx.conf create mode 100644 docker-compose.yml create mode 100644 docs/17-scaling.md create mode 100644 src/lifecycle.ts create mode 100644 src/sync.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b5e15b7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +node_modules +dist +.kits +.git +.env +.env.* +*.log +test +docs +examples diff --git a/.env.example b/.env.example index 9b37918..4bd1efc 100644 --- a/.env.example +++ b/.env.example @@ -45,8 +45,21 @@ DB_BACKEND=pg # [required in production] [secret] Connection string for the backend above. # PostgreSQL: postgres://USER:PASSWORD@HOST:5432/mgpl # MongoDB: mongodb://USER:PASSWORD@HOST:27017/mgpl?replicaSet=rs0 (a replica set is required: transactions) +# Through PgBouncer: postgres://USER:PASSWORD@PGBOUNCER_HOST:6432/mgpl (transaction pooling is supported) # URL-encode special characters in the password (@ → %40, : → %3A, / → %2F). DATABASE_URL=postgres://postgres:postgres@localhost:5432/mgpl +# [optional] Connections this process keeps open (the Sequelize pool for pg, the MongoDB driver's pool for mongo), +# per API server and per worker. Behind PgBouncer they are cheap client connections: keep +# (API replicas + workers) × DB_POOL_MAX under PGBOUNCER_MAX_CLIENT_CONN. Without PgBouncer, keep it under +# PostgreSQL's max_connections instead. +# DB_POOL_MAX=10 +# DB_POOL_MIN=0 +# [optional] Close a pooled connection after this long idle (ms); fail a query that waited this long for one (ms). +# DB_POOL_IDLE_MS=10000 +# DB_POOL_ACQUIRE_MS=30000 +# [optional] Create missing tables (pg) or indexes (mongo) on start. With several replicas set it to false on +# all of them and run `npm run db:sync` (node dist/sync.js) once per release instead (docs/17-scaling.md). +# DB_SYNC=true # ── Redis ──────────────────────────────────────────────────────────────────────────────────────────── @@ -146,9 +159,32 @@ PASSWORD_RESET_URL=http://localhost:3000/reset-password # SEED_OWNER_PASSWORD= +# ── Scaling: graceful shutdown, replicas, PgBouncer ────────────────────────────────────────────────── +# Running several API replicas: REDIS_URL and the same JWT_SECRET on every replica (docs/17-scaling.md). +# [optional] On SIGTERM, GET /health answers 503 for this long while requests are still served, so the load +# balancer takes the replica out first (ms). Default 5000 in production, 0 otherwise. +# SHUTDOWN_DRAIN_MS=5000 +# [optional] Then the server stops accepting connections and finishes the requests in flight; it exits +# anyway this long after the signal (ms). The worker uses it too, for the jobs in hand. +# SHUTDOWN_TIMEOUT_MS=30000 +# +# Read only by docker-compose.yml (Docker Compose reads this file for them), not by the API: +# API_REPLICAS=2 api containers behind nginx (or `docker compose up -d --scale api=N`) +# HTTP_PORT=8080 nginx's port on the host +# POSTGRES_PASSWORD=postgres [secret] URL-safe characters only: it goes into connection URLs +# POSTGRES_MAX_CONNECTIONS=100 PostgreSQL's limit; PgBouncer's server pool must fit under it +# PGBOUNCER_POOL_MODE=transaction transaction (recommended; `npm test` runs the api under it) | session +# PGBOUNCER_DEFAULT_POOL_SIZE=20 server connections PgBouncer opens to PostgreSQL for the database +# PGBOUNCER_MIN_POOL_SIZE=0 +# PGBOUNCER_RESERVE_POOL_SIZE=5 extra server connections when clients have waited 5 s +# PGBOUNCER_MAX_CLIENT_CONN=500 client connections in all: at least (API_REPLICAS + workers) × DB_POOL_MAX +# PGBOUNCER_MAX_DB_CONNECTIONS=50 hard cap on server connections: below POSTGRES_MAX_CONNECTIONS + + # ── Development and test tooling (not read by the running API) ────────────────────────────────────── # NODE_AUTH_TOKEN= [secret] export it in your shell for `npm ci` (npm does not read .env): GitHub token, read:packages # KITS_SRC=.. `npm run kits`: build the kits from local checkouts instead of kits.json refs # TEST_BACKENDS=pg,mongo `npm test`: which databases to run (pg only when MongoDB binaries can't download) # MONGOMS_SYSTEM_BINARY= `npm test`: path to a local mongod +# PGBOUNCER_BIN=pgbouncer `npm test`: the pgbouncer binary for the [pgbouncer] and [replicas] checks (skipped without it) # DEBUG=1 `npm test`: print resolver errors diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32d0fb3..da2763b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,10 +37,14 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: npm run typecheck - - name: End-to-end tests (GraphQL over HTTP, tokens, worker on Redis, client SDK) on PostgreSQL and MongoDB + # the [pgbouncer] and [replicas] checks run the API through PgBouncer in transaction mode + - name: Install PgBouncer + run: sudo apt-get update -q && sudo apt-get install -y -q pgbouncer + - name: End-to-end tests (GraphQL over HTTP, tokens, worker on Redis, client SDK) on PostgreSQL, PostgreSQL through PgBouncer and MongoDB run: npm test env: REDIS_URL: redis://localhost:6379 + PGBOUNCER_BIN: /usr/sbin/pgbouncer - run: npm run build - name: schema.graphql is up to date (run `npm run schema` and commit if this fails) run: npm run schema && git diff --exit-code -- schema.graphql diff --git a/AGENTS.md b/AGENTS.md index 02725d5..0d71cec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,8 @@ the kits, used as internal libraries. Read [docs/README.md](docs/README.md) for ``` src/main.ts start (config → container → server) src/worker.ts start the worker (config → container → BullMQ workers; needs REDIS_URL) +src/sync.ts create missing tables / indexes once, then exit (`npm run db:sync`; replicas run DB_SYNC=false) +src/lifecycle.ts readiness while draining, graceful shutdown on SIGTERM src/server.ts Fastify + Yoga + GET /health (no other routes); security-kit headers and client IP src/container.ts createDb (posting: financePosting) + createAuth (JWT) + job queue + services, once per process src/jobs/ queues (BullMQ or in-process), job payloads, processors, webhook mailer @@ -22,6 +24,7 @@ test/run.ts end-to-end over HTTP on PostgreSQL + MongoDB, plus docs/e docs/ A-to-Z docs (Mermaid), examples/*.graphql, LOCAL-AGENT-BRIEF.md schema.graphql printed schema (generated) kits.json kit refs for `npm run kits` (building unreleased kit branches from source) +Dockerfile, docker-compose.yml, deploy/ image; PostgreSQL ← PgBouncer ← N api replicas ← nginx, Redis, worker (docs/17-scaling.md) ``` ## Rules @@ -52,7 +55,12 @@ kits.json kit refs for `npm run kits` (building unreleased kit bran signals are collected by security-kit in the client; `src/graphql/device.ts` maps DeviceInput to auth-kit. Security headers and the client IP come from security-kit's `fastifySecurity` (never Fastify's own `req.ip` with a trust-all proxy setting: the left X-Forwarded-For entry is client-controlled). -14. Never commit patient data, signatures, secrets or `.env` files. Test fixtures use fictional names. +14. **Replicas.** Nothing per-user lives in process memory when `REDIS_URL` is set: sessions, refresh tokens, + login limits and jobs are in Redis, data behind PgBouncer. Keep it that way (no module-level caches of + users or sessions), and keep the db layer free of session-level PostgreSQL features (named prepared + statements, `SET` without `LOCAL`, advisory locks held across transactions, `LISTEN`, temp tables): + PgBouncer runs in transaction mode and `npm test` runs the suite through it. +15. Never commit patient data, signatures, secrets or `.env` files. Test fixtures use fictional names. ## Commands diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..45a8b34 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1 +# The api image: `node dist/main.js` (API), `node dist/worker.js` (worker), `node dist/sync.js` (create tables). +# The kits come from GitHub Packages, so the build needs a token with read:packages, passed as a secret +# (never a build arg, which would stay in the image history): +# NODE_AUTH_TOKEN=… docker compose build (docker-compose.yml passes it as the npm_token secret) +# docker build --secret id=npm_token,env=NODE_AUTH_TOKEN -t mgpl-api . +# report-kit's resvg is a native module: build on the platform the image runs on (linux/amd64 or arm64). + +FROM node:22-bookworm-slim AS build +WORKDIR /app +COPY package.json package-lock.json .npmrc ./ +RUN --mount=type=secret,id=npm_token \ + echo "//npm.pkg.github.com/:_authToken=$(cat /run/secrets/npm_token)" > /root/.npmrc \ + && npm ci --no-audit --no-fund \ + && rm /root/.npmrc +COPY tsconfig.json tsup.config.ts ./ +COPY src ./src +RUN npm run build && npm prune --omit=dev + +FROM node:22-bookworm-slim +ENV NODE_ENV=production +WORKDIR /app +COPY --from=build /app/package.json ./ +COPY --from=build /app/node_modules ./node_modules +COPY --from=build /app/dist ./dist +USER node +EXPOSE 4000 +# Docker's own check (compose shows "healthy"): readiness, so a replica that lost its database reads unhealthy. +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=3 \ + CMD ["node", "-e", "fetch('http://127.0.0.1:'+(process.env.PORT||4000)+'/health').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"] +# node is PID 1 and handles SIGTERM itself (src/lifecycle.ts): drain, finish requests, close, exit. +STOPSIGNAL SIGTERM +CMD ["node", "dist/main.js"] diff --git a/deploy/nginx.conf b/deploy/nginx.conf new file mode 100644 index 0000000..7a9eb76 --- /dev/null +++ b/deploy/nginx.conf @@ -0,0 +1,31 @@ +# The load balancer in docker-compose.yml: spreads requests over every `api` replica. +# Docker's DNS (127.0.0.11) answers "api" with one address per replica; resolving through a variable makes +# nginx re-resolve every 10 s, so `docker compose up --scale api=N` needs no restart here. +# TLS: terminate it here (listen 443 ssl + certificates) or in front of it (a CDN or a cloud load balancer); +# then count that proxy too in TRUST_PROXY. +events {} + +http { + resolver 127.0.0.11 valid=10s ipv6=off; + access_log off; + # the api's BODY_LIMIT_BYTES (2 MB) plus room + client_max_body_size 3m; + + server { + listen 80; + + location / { + set $api http://api:4000; + proxy_pass $api; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + # one trusted hop: TRUST_PROXY=1 on the api reads the right-most entry, the address nginx saw + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # a replica that is shutting down answers 503: try the next one (nginx never retries a POST that reached a replica) + proxy_next_upstream error timeout http_502 http_503; + proxy_read_timeout 120s; + } + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..eac92a0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,135 @@ +# The whole backend on one host: PostgreSQL ← PgBouncer ← N api replicas ← nginx, plus Redis and the worker. +# Guide: docs/17-scaling.md. +# +# cp .env.example .env # set JWT_SECRET, CORS_ORIGINS, POSTGRES_PASSWORD, … +# NODE_AUTH_TOKEN=… docker compose build # the kits come from GitHub Packages (read:packages) +# docker compose up -d # API_REPLICAS replicas (default 2) on http://localhost:8080 +# docker compose up -d --scale api=4 # more or fewer, at any time +# +# The api and the worker read .env too; the values below (database, Redis, proxy) win over it, because the +# services find each other by name inside this network. +name: mgpl + +# api and worker run the image that db-sync builds (one build, three commands) +x-api: &api + image: mgpl-api + pull_policy: never + restart: unless-stopped + env_file: + - path: .env + required: false + depends_on: + db-sync: { condition: service_completed_successfully } + pgbouncer: { condition: service_started } + redis: { condition: service_healthy } + # real signatures for reports and invoices: mount them read-only and set MGPL_SIGNATURE_*_FILE in .env + # volumes: ["/secure/mgpl-signatures:/secure:ro"] + +services: + postgres: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: mgpl + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 + # PgBouncer keeps the connections to PostgreSQL few; this only has to cover its pool plus db-sync and admin + command: postgres -c max_connections=${POSTGRES_MAX_CONNECTIONS:-100} + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d mgpl"] + interval: 5s + retries: 20 + + # Connection pooler: every api replica and worker connects here, not to PostgreSQL. Transaction pooling + # (the default) gives a server connection to a client only for the length of one transaction. + pgbouncer: + image: edoburu/pgbouncer:v1.25.2-p0 + restart: unless-stopped + environment: + DATABASE_URL: postgres://postgres:${POSTGRES_PASSWORD:-postgres}@postgres:5432/mgpl + AUTH_TYPE: scram-sha-256 + LISTEN_PORT: 6432 + POOL_MODE: ${PGBOUNCER_POOL_MODE:-transaction} + DEFAULT_POOL_SIZE: ${PGBOUNCER_DEFAULT_POOL_SIZE:-20} + MIN_POOL_SIZE: ${PGBOUNCER_MIN_POOL_SIZE:-0} + RESERVE_POOL_SIZE: ${PGBOUNCER_RESERVE_POOL_SIZE:-5} + MAX_CLIENT_CONN: ${PGBOUNCER_MAX_CLIENT_CONN:-500} + MAX_DB_CONNECTIONS: ${PGBOUNCER_MAX_DB_CONNECTIONS:-50} + ADMIN_USERS: postgres + depends_on: + postgres: { condition: service_healthy } + + redis: + image: redis:7-alpine + restart: unless-stopped + # AOF: queued emails and sessions survive a Redis restart + command: redis-server --appendonly yes + volumes: [redisdata:/data] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + retries: 20 + + # Creates missing tables once per `up`, straight on PostgreSQL, before any replica starts (they run with DB_SYNC=false). + db-sync: + image: mgpl-api + build: + context: . + secrets: [npm_token] + env_file: + - path: .env + required: false + environment: + DB_BACKEND: pg + DATABASE_URL: postgres://postgres:${POSTGRES_PASSWORD:-postgres}@postgres:5432/mgpl + command: ["node", "dist/sync.js"] + depends_on: + postgres: { condition: service_healthy } + restart: "no" + + api: + <<: *api + environment: + HOST: 0.0.0.0 + PORT: 4000 + DB_BACKEND: pg + DATABASE_URL: postgres://postgres:${POSTGRES_PASSWORD:-postgres}@pgbouncer:6432/mgpl + DB_SYNC: "false" + DB_POOL_MAX: ${DB_POOL_MAX:-10} + REDIS_URL: redis://redis:6379 + # nginx below is the one proxy in front of the replicas (add one per proxy in front of nginx) + TRUST_PROXY: ${TRUST_PROXY:-1} + deploy: + replicas: ${API_REPLICAS:-2} + # docker compose stop: SIGTERM, then SIGKILL after this; longer than SHUTDOWN_DRAIN_MS + SHUTDOWN_TIMEOUT_MS + stop_grace_period: 40s + + worker: + <<: *api + command: ["node", "dist/worker.js"] + environment: + DB_BACKEND: pg + DATABASE_URL: postgres://postgres:${POSTGRES_PASSWORD:-postgres}@pgbouncer:6432/mgpl + DB_SYNC: "false" + DB_POOL_MAX: ${DB_POOL_MAX:-10} + REDIS_URL: redis://redis:6379 + healthcheck: + disable: true + stop_grace_period: 40s + + nginx: + image: nginx:1.27-alpine + restart: unless-stopped + ports: ["${HTTP_PORT:-8080}:80"] + volumes: [./deploy/nginx.conf:/etc/nginx/nginx.conf:ro] + depends_on: [api] + +secrets: + npm_token: + environment: NODE_AUTH_TOKEN + +volumes: + pgdata: + redisdata: diff --git a/docs/09-configuration.md b/docs/09-configuration.md index 51d09b9..3a8125e 100644 --- a/docs/09-configuration.md +++ b/docs/09-configuration.md @@ -17,6 +17,11 @@ variables already set in the real environment win over the file. | `GRAPHQL_PATH` | `/graphql` | The one endpoint | | `DB_BACKEND` | `pg` | `pg` or `mongo` | | `DATABASE_URL` | local default in development; **required** in production | Postgres URL, or a MongoDB URI with `replicaSet=` | +| `DB_POOL_MAX` | `10` | Connections this process keeps open (Sequelize pool / MongoDB driver pool), per API server and worker. Behind PgBouncer: (replicas + workers) × this ≤ PgBouncer's `max_client_conn` ([17-scaling.md](17-scaling.md#sizing-the-pools)) | +| `DB_POOL_MIN` | `0` | Connections kept open when idle | +| `DB_POOL_IDLE_MS` | `10000` | Close a pooled connection after this long idle | +| `DB_POOL_ACQUIRE_MS` | `30000` | A query that waited this long for a free connection fails | +| `DB_SYNC` | `true` | Create missing tables / indexes on start. `false` on every replica when several run, with `npm run db:sync` once per release | | `REDIS_URL` | empty | Sessions, refresh tokens, login limits, db-kit's read cache and the job queues (BullMQ). Required for more than one server and for the worker | | `CORS_ORIGINS` | `*` in development, empty in production | Comma-separated origins. `*` is refused in production | | `GRAPHIQL` | on in development | GraphiQL at the endpoint (GET) | @@ -40,13 +45,20 @@ variables already set in the real environment win over the file. | `MGPL_SIGNATURE_HEADER_FILE` | — | PNG for the header variant (outside the repo) | | `MGPL_SIGNATURE_NONHEADER_FILE` | — | PNG for the non-header variant | | `MGPL_SIGNATURE_FILE` | — | One file for both | +| `SHUTDOWN_DRAIN_MS` | `5000` in production, `0` otherwise | On SIGTERM, `GET /health` answers 503 this long while requests are still served ([17-scaling.md](17-scaling.md#graceful-shutdown)) | +| `SHUTDOWN_TIMEOUT_MS` | `30000` | Exit anyway this long after the signal (API and worker) | + +`docker-compose.yml` reads a few more from `.env` (not the API): `API_REPLICAS`, `HTTP_PORT`, +`POSTGRES_PASSWORD`, `POSTGRES_MAX_CONNECTIONS` and the `PGBOUNCER_*` pool settings +([17-scaling.md](17-scaling.md#docker-compose)). Seed-only variables (`npm run seed`): `SEED_ORG_NAME`, `SEED_ORG_CODE`, `SEED_ORG_GSTIN` (optional), `SEED_BRANCH_CODE`, `SEED_BRANCH_NAME`, `SEED_BRANCH_STATE`, `SEED_OWNER_NAME`, `SEED_OWNER_EMAIL`, `SEED_OWNER_PASSWORD`. Test-only: `TEST_BACKENDS=pg,mongo` (default both), `DEBUG=1` prints resolver errors, -`MONGOMS_SYSTEM_BINARY` points mongodb-memory-server at a local `mongod`. +`MONGOMS_SYSTEM_BINARY` points mongodb-memory-server at a local `mongod`, `PGBOUNCER_BIN` the `pgbouncer` +binary for the PgBouncer and replica checks. ## Finance configuration diff --git a/docs/11-testing.md b/docs/11-testing.md index 334cdbd..a3d312a 100644 --- a/docs/11-testing.md +++ b/docs/11-testing.md @@ -18,6 +18,8 @@ flowchart LR | Area | What the suite proves | |---|---| | Server | `/health` and `{ health }`, introspection off when `INTROSPECTION=0`, depth limit 12 | +| PgBouncer | The whole PostgreSQL suite again through PgBouncer in transaction mode (`pgbouncer` on the PATH or `PGBOUNCER_BIN`; skipped with a note otherwise) | +| Replicas | Two API servers sharing PgBouncer and Redis: a session, a refresh and a logout cross replicas; 16 concurrent invoices with unique numbers over a 3-connection server pool; shared login limits; draining and shutdown | | Auth | no token → `UNAUTHENTICATED`, wrong password → `INVALID_CREDENTIALS`, login, viewer (null without a token), weak password, reset link through the mailer (unknown emails not revealed), reset ends old sessions, disable signs out at once, `mySessions`, logout | | Permissions | front desk cannot read the books; registers in branch A, not B; lists show only the viewer's branches; discounts need `invoice:discount`; nested vouchers hidden without `accounts:read` | | Tenancy | another organisation cannot read the invoice, the patient, or print it; cannot post into its ledgers | diff --git a/docs/12-deployment.md b/docs/12-deployment.md index 3b2a9c1..f85dbe9 100644 --- a/docs/12-deployment.md +++ b/docs/12-deployment.md @@ -21,7 +21,8 @@ flowchart LR - **Two or more servers need Redis**: sessions, refresh tokens and login limits must be shared. - **Workers** (`node dist/worker.js`) need Redis and the same environment as the API (database, `JWT_SECRET`, signature files). Run one or more; see [16-worker.md](16-worker.md). -- The process is stateless otherwise; scale horizontally. +- The process is stateless otherwise; scale horizontally. [17-scaling.md](17-scaling.md) covers N replicas, + PgBouncer, readiness and graceful shutdown; `docker-compose.yml` runs all of it on one host. ## Build and run @@ -32,8 +33,9 @@ NODE_ENV=production node dist/main.js # API NODE_ENV=production node dist/worker.js # worker (with REDIS_URL) ``` -A container image: Node 22 slim, copy `dist/`, `node_modules/` and `package.json`, run -`node dist/main.js`, expose `PORT`. The PDF engine needs no system fonts (fonts are bundled), but +A container image: the [`Dockerfile`](../Dockerfile) (Node 22 slim, `dist/`, production `node_modules/`; +`node dist/main.js`, `node dist/worker.js` or `node dist/sync.js`; port 4000). It needs `NODE_AUTH_TOKEN` as a +build secret for the kits. The PDF engine needs no system fonts (fonts are bundled), but report-kit's resvg is a native module: build the image on the same platform it runs on. ## Production checklist @@ -52,7 +54,9 @@ report-kit's resvg is a native module: build the image on the same platform it r | ☐ | First organisation seeded (`npm run seed`) with the branch `stateCode` and the org GSTIN | | ☐ | Tax rules seeded **and confirmed by the CA** (`verifyWithCA`) | | ☐ | Database backups (point-in-time for Postgres), encrypted | -| ☐ | Health check on `GET /health` (200 `{ ok: true }`, 503 when the database is down) | +| ☐ | Readiness check on `GET /health` (200 `{ ok: true }`; 503 when the database or Redis is down, or while the replica shuts down); liveness on `GET /health?live=1` | +| ☐ | More than one replica: `REDIS_URL` and the same `JWT_SECRET` on all, `DB_SYNC=false` with `node dist/sync.js` once per release, pools sized ([17-scaling.md](17-scaling.md)) | +| ☐ | The orchestrator waits longer than `SHUTDOWN_DRAIN_MS` + `SHUTDOWN_TIMEOUT_MS` between SIGTERM and SIGKILL | | ☐ | Logs shipped; request bodies are not logged (they hold patient data) | ## Sending password-reset emails diff --git a/docs/17-scaling.md b/docs/17-scaling.md new file mode 100644 index 0000000..6842e50 --- /dev/null +++ b/docs/17-scaling.md @@ -0,0 +1,152 @@ +# 17 · Scaling: replicas and PgBouncer + +The API runs as any number of identical replicas behind a reverse proxy. Every replica connects to +PostgreSQL through **PgBouncer** and keeps nothing per-user in its own memory: sessions, refresh tokens, +login limits and jobs live in **Redis**. + +```mermaid +flowchart LR + U["Browsers / apps"] -- HTTPS --> P["Reverse proxy
(nginx, a load balancer)"] + P -- "/graphql, /health" --> A1["api #1"] & A2["api #2"] & A3["api #N"] + A1 & A2 & A3 -- "DB_POOL_MAX each" --> B["PgBouncer
transaction pooling"] + W["worker(s)"] --> B + B -- "DEFAULT_POOL_SIZE" --> PG[("PostgreSQL")] + A1 & A2 & A3 & W --> R[("Redis
sessions · limits · queues · cache")] + S["db-sync
(once per release)"] -. DDL .-> PG +``` + +## What each replica needs + +| | Why | +|---|---| +| `REDIS_URL`, the same on every replica | Without it sessions, refresh tokens, login limits and jobs live in one process: a token issued by one replica fails on the next, and every replica allows its own 10 login attempts | +| `JWT_SECRET`, the same on every replica | Access tokens are verified locally; a per-process secret (the development default) fails on every other replica | +| `TRUST_PROXY` = the number of proxies in front | Login limits and sessions use the client IP from `X-Forwarded-For`, counted from the right through that many proxies | +| `DB_SYNC=false`, and `node dist/sync.js` once per release | Replicas starting together would race to create the same tables | +| A readiness check on `GET /health`, liveness on `GET /health?live=1` | See below | +| A stop timeout longer than `SHUTDOWN_DRAIN_MS` + `SHUTDOWN_TIMEOUT_MS` | So in-flight requests finish before SIGKILL | + +Where the state lives once `REDIS_URL` is set (checked in `src/container.ts` and `src/jobs/`): + +| State | Where | Without `REDIS_URL` | +|---|---|---| +| Sessions, refresh tokens, reset tokens (hashed) | Redis (auth-kit `redisSessionStore`, `redisKeyValueStore`) | process memory | +| Login rate limits | Redis (sliding window) | process memory | +| Background jobs and their results | Redis (BullMQ); any worker runs them, any replica reads `job(id)` | process memory | +| db-kit read cache | Redis (L1 per process is off) | none | +| Everything else | PostgreSQL / MongoDB | | +| DataLoader caches | one request only | | + +`npm test` proves it: the `[replicas]` checks run two API servers on one PgBouncer and one Redis. A session +started on one works on the other, a refresh token spent on one is refused on the other, a logout on one ends +the session on the other at once, and the login limit is reached after 10 attempts in all, not 10 per replica. + +## PgBouncer + +Point `DATABASE_URL` at PgBouncer instead of PostgreSQL, e.g. `postgres://USER:PASSWORD@pgbouncer:6432/mgpl`. +Nothing else changes in the API. + +**Pool mode: transaction** (recommended). A server connection belongs to a client only for one transaction, +so a few PostgreSQL connections serve many replicas. The API works under it because db-kit (Sequelize on +node-postgres) uses none of the session-level features that break there: + +| Feature that breaks under transaction pooling | In db-kit / the API | +|---|---| +| Named prepared statements | Not used: node-postgres sends unnamed statements (extended protocol), which PgBouncer passes through | +| `SET` without `LOCAL` | Only Sequelize's connection setup (`SET client_min_messages`, `SET TIME ZONE '+00:00'`). PgBouncer tracks `TimeZone` and re-applies it per client; `client_min_messages` only changes log noise. Timestamps are `timestamptz`, so values are exact either way | +| Advisory locks, `LISTEN`/`NOTIFY`, temporary tables, `WITH HOLD` cursors | Not used | +| Locks and isolation | `SELECT … FOR UPDATE` and the isolation level live inside `repos.transaction()`, which is one PgBouncer transaction | +| Counters (invoice and voucher numbers) | One `INSERT … ON CONFLICT … RETURNING` statement | + +`npm test` runs the whole PostgreSQL suite a second time through PgBouncer in transaction mode (`[pgbouncer]`), +then fires 16 invoices with payments at once from two replicas (2 × `DB_POOL_MAX=4` client connections) +over a 3-connection server pool: all are posted, every number is unique and the trial balance balances. + +Session mode also works, but then each client connection holds a server connection, so it saves little. +Statement mode does not work (transactions span several statements). + +`DB_SYNC` and `node dist/sync.js` run DDL; they work through PgBouncer, but `docker-compose.yml` points +db-sync straight at PostgreSQL, as migrations usually are. + +### Sizing the pools + +Two pools in series: + +``` +API replicas × DB_POOL_MAX + workers × DB_POOL_MAX ≤ PgBouncer max_client_conn +PgBouncer default_pool_size + reserve_pool_size ≤ max_db_connections < PostgreSQL max_connections +``` + +- `DB_POOL_MAX` (default 10) is per process. Behind PgBouncer these are cheap client connections; it + caps how many queries one replica runs at once. +- PgBouncer's `default_pool_size` (compose default 20) is what PostgreSQL actually sees. Start near + 2 × the database server's CPU cores and raise it only if clients wait (`SHOW POOLS` → `cl_waiting`). +- Leave PostgreSQL a few connections above PgBouncer's cap for db-sync, backups and an admin `psql`. + +Example: 4 replicas + 2 workers × `DB_POOL_MAX=10` = 60 client connections (`max_client_conn` 500 is +plenty); `default_pool_size=20`, `reserve_pool_size=5`, `max_db_connections=50`; PostgreSQL +`max_connections=100`. + +**MongoDB** needs no PgBouncer: the driver pools by itself, and `DB_POOL_MAX` / `DB_POOL_MIN` / +`DB_POOL_IDLE_MS` / `DB_POOL_ACQUIRE_MS` set `maxPoolSize`, `minPoolSize`, `maxIdleTimeMS` and +`waitQueueTimeoutMS`. Keep replicas × `DB_POOL_MAX` within the cluster's connection limit. + +## Health and readiness + +| Request | 200 when | 503 when | Use it for | +|---|---|---|---| +| `GET /health` | the database answers, Redis answers (when configured), and the replica is not shutting down | any of those fails | readiness: the load balancer sends traffic only to replicas that answer 200 | +| `GET /health?live=1` | the process is running | never | liveness: restart a replica only when the process hangs, not when the database is down (that would restart every replica at once) | + +The body says which part failed: `{ "ok": false, "database": "ok", "redis": "unreachable" }` or +`{ "ok": false, "draining": true }`. The `{ health }` GraphQL field still reports the database. + +## Graceful shutdown + +On SIGTERM (or SIGINT) the API (`src/lifecycle.ts`): + +1. answers `GET /health` with 503 at once, while it keeps serving every other request; +2. waits `SHUTDOWN_DRAIN_MS` (5 s in production) so the load balancer sees the 503 and stops routing here; +3. stops accepting connections, closes idle keep-alive connections, answers 503 to new requests on open + ones and waits for the requests in flight; +4. closes the job queue, the database pool and Redis, and exits 0. + +A second signal, or `SHUTDOWN_TIMEOUT_MS` (30 s) after the first, exits at once with code 1. The worker +finishes its jobs in hand and stops; after `SHUTDOWN_TIMEOUT_MS` it exits and BullMQ retries the job that +was cut short. Give the orchestrator a longer stop timeout than drain + timeout: `stop_grace_period: 40s` +in compose, `terminationGracePeriodSeconds: 40` in Kubernetes. + +Rolling updates then drop nothing: start the new replicas, wait for them to be ready, stop the old ones. +Stopping one replica of three under constant traffic through nginx gave 40 of 40 requests answered 200. + +## Docker Compose + +[`docker-compose.yml`](../docker-compose.yml) runs the whole backend on one host: PostgreSQL, PgBouncer +(transaction mode), Redis (AOF on), a one-shot `db-sync`, `API_REPLICAS` API containers, a worker and +nginx ([`deploy/nginx.conf`](../deploy/nginx.conf)) in front of the replicas on `HTTP_PORT`. + +```bash +cp .env.example .env # NODE_ENV=production, JWT_SECRET, CORS_ORIGINS, POSTGRES_PASSWORD, SMTP_URL, … +NODE_AUTH_TOKEN=… docker compose build # the kits come from GitHub Packages (read:packages) +docker compose up -d # http://localhost:8080/graphql +docker compose up -d --scale api=4 # change the number of replicas at any time +docker compose exec -e PGPASSWORD=… postgres psql -h pgbouncer -p 6432 -U postgres pgbouncer -c 'SHOW POOLS' +``` + +- The API containers get `DATABASE_URL` (PgBouncer), `REDIS_URL`, `DB_SYNC=false` and `TRUST_PROXY=1` + from the compose file; everything else comes from `.env`. +- nginx re-resolves `api` every 10 s, so scaling needs no restart, and retries a request on another + replica when one answers 503 or refuses the connection. +- TLS: terminate it in front (a CDN or cloud load balancer, then `TRUST_PROXY=2`) or add a `listen 443 ssl` + server to `deploy/nginx.conf`. +- Signatures: mount the files read-only into `api` and `worker` and set `MGPL_SIGNATURE_*_FILE`. +- The compose file is PostgreSQL only. For MongoDB, drop `postgres`, `pgbouncer` and `db-sync`, and set + `DB_BACKEND=mongo`, `DATABASE_URL` (a replica set) and `DB_SYNC` on one replica. + +## Kubernetes or another orchestrator + +The same pieces map one to one: a Deployment of the image with `replicas: N`, a readiness probe on +`/health`, a liveness probe on `/health?live=1`, `terminationGracePeriodSeconds: 40`; a Job (or an init +step of the release) running `node dist/sync.js`; PgBouncer as its own Deployment and Service (or a managed +pooler such as the provider's PgBouncer); Redis managed or as a StatefulSet; the worker as a second +Deployment with `command: ["node", "dist/worker.js"]`. diff --git a/docs/README.md b/docs/README.md index e33dd32..ab11de9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ flowchart TD B --> M["13 Adding a feature"] D --> N["14 Client"] B --> O["16 Worker"] + L --> P["17 Scaling"] ``` | # | Page | What it answers | @@ -38,5 +39,6 @@ flowchart TD | 14 | [Client](14-client.md) | Using `@mgpl-diagnostics/client` against this API | | 15 | [Glossary](15-glossary.md) | Lab, billing and accounting terms | | 16 | [Worker](16-worker.md) | Background jobs on Redis + BullMQ: emails, documents, running workers | +| 17 | [Scaling](17-scaling.md) | N replicas behind a proxy, PgBouncer, readiness, graceful shutdown, docker-compose | | — | [examples/](examples/) | Runnable `.graphql` operations | | — | [LOCAL-AGENT-BRIEF.md](LOCAL-AGENT-BRIEF.md) | What a local agent (or a person) must do next, outside this repo | diff --git a/llms-full.txt b/llms-full.txt index 5199fcb..5591afe 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1,6 +1,6 @@ # @mgpl-diagnostics/api — full documentation and source -> Generated by scripts/llms-full.mjs. Do not edit by hand. 86 files. +> Generated by scripts/llms-full.mjs. Do not edit by hand. 89 files. ## Contents @@ -22,6 +22,7 @@ - docs/14-client.md - docs/15-glossary.md - docs/16-worker.md +- docs/17-scaling.md - docs/LOCAL-AGENT-BRIEF.md - docs/README.md - kits.json @@ -59,6 +60,7 @@ - src/jobs/mailer.ts - src/jobs/processors.ts - src/jobs/types.ts +- src/lifecycle.ts - src/main.ts - src/schema/auth.graphql.ts - src/schema/base.graphql.ts @@ -88,6 +90,7 @@ - src/services/report.service.ts - src/services/settlement.service.ts - src/services/util.ts +- src/sync.ts - src/worker.ts - test/run.ts @@ -215,6 +218,8 @@ the kits, used as internal libraries. Read [docs/README.md](docs/README.md) for ``` src/main.ts start (config → container → server) src/worker.ts start the worker (config → container → BullMQ workers; needs REDIS_URL) +src/sync.ts create missing tables / indexes once, then exit (`npm run db:sync`; replicas run DB_SYNC=false) +src/lifecycle.ts readiness while draining, graceful shutdown on SIGTERM src/server.ts Fastify + Yoga + GET /health (no other routes); security-kit headers and client IP src/container.ts createDb (posting: financePosting) + createAuth (JWT) + job queue + services, once per process src/jobs/ queues (BullMQ or in-process), job payloads, processors, webhook mailer @@ -228,6 +233,7 @@ test/run.ts end-to-end over HTTP on PostgreSQL + MongoDB, plus docs/e docs/ A-to-Z docs (Mermaid), examples/*.graphql, LOCAL-AGENT-BRIEF.md schema.graphql printed schema (generated) kits.json kit refs for `npm run kits` (building unreleased kit branches from source) +Dockerfile, docker-compose.yml, deploy/ image; PostgreSQL ← PgBouncer ← N api replicas ← nginx, Redis, worker (docs/17-scaling.md) ``` ## Rules @@ -258,7 +264,12 @@ kits.json kit refs for `npm run kits` (building unreleased kit bran signals are collected by security-kit in the client; `src/graphql/device.ts` maps DeviceInput to auth-kit. Security headers and the client IP come from security-kit's `fastifySecurity` (never Fastify's own `req.ip` with a trust-all proxy setting: the left X-Forwarded-For entry is client-controlled). -14. Never commit patient data, signatures, secrets or `.env` files. Test fixtures use fictional names. +14. **Replicas.** Nothing per-user lives in process memory when `REDIS_URL` is set: sessions, refresh tokens, + login limits and jobs are in Redis, data behind PgBouncer. Keep it that way (no module-level caches of + users or sessions), and keep the db layer free of session-level PostgreSQL features (named prepared + statements, `SET` without `LOCAL`, advisory locks held across transactions, `LISTEN`, temp tables): + PgBouncer runs in transaction mode and `npm test` runs the suite through it. +15. Never commit patient data, signatures, secrets or `.env` files. Test fixtures use fictional names. ## Commands @@ -1347,6 +1358,11 @@ variables already set in the real environment win over the file. | `GRAPHQL_PATH` | `/graphql` | The one endpoint | | `DB_BACKEND` | `pg` | `pg` or `mongo` | | `DATABASE_URL` | local default in development; **required** in production | Postgres URL, or a MongoDB URI with `replicaSet=` | +| `DB_POOL_MAX` | `10` | Connections this process keeps open (Sequelize pool / MongoDB driver pool), per API server and worker. Behind PgBouncer: (replicas + workers) × this ≤ PgBouncer's `max_client_conn` ([17-scaling.md](17-scaling.md#sizing-the-pools)) | +| `DB_POOL_MIN` | `0` | Connections kept open when idle | +| `DB_POOL_IDLE_MS` | `10000` | Close a pooled connection after this long idle | +| `DB_POOL_ACQUIRE_MS` | `30000` | A query that waited this long for a free connection fails | +| `DB_SYNC` | `true` | Create missing tables / indexes on start. `false` on every replica when several run, with `npm run db:sync` once per release | | `REDIS_URL` | empty | Sessions, refresh tokens, login limits, db-kit's read cache and the job queues (BullMQ). Required for more than one server and for the worker | | `CORS_ORIGINS` | `*` in development, empty in production | Comma-separated origins. `*` is refused in production | | `GRAPHIQL` | on in development | GraphiQL at the endpoint (GET) | @@ -1370,13 +1386,20 @@ variables already set in the real environment win over the file. | `MGPL_SIGNATURE_HEADER_FILE` | — | PNG for the header variant (outside the repo) | | `MGPL_SIGNATURE_NONHEADER_FILE` | — | PNG for the non-header variant | | `MGPL_SIGNATURE_FILE` | — | One file for both | +| `SHUTDOWN_DRAIN_MS` | `5000` in production, `0` otherwise | On SIGTERM, `GET /health` answers 503 this long while requests are still served ([17-scaling.md](17-scaling.md#graceful-shutdown)) | +| `SHUTDOWN_TIMEOUT_MS` | `30000` | Exit anyway this long after the signal (API and worker) | + +`docker-compose.yml` reads a few more from `.env` (not the API): `API_REPLICAS`, `HTTP_PORT`, +`POSTGRES_PASSWORD`, `POSTGRES_MAX_CONNECTIONS` and the `PGBOUNCER_*` pool settings +([17-scaling.md](17-scaling.md#docker-compose)). Seed-only variables (`npm run seed`): `SEED_ORG_NAME`, `SEED_ORG_CODE`, `SEED_ORG_GSTIN` (optional), `SEED_BRANCH_CODE`, `SEED_BRANCH_NAME`, `SEED_BRANCH_STATE`, `SEED_OWNER_NAME`, `SEED_OWNER_EMAIL`, `SEED_OWNER_PASSWORD`. Test-only: `TEST_BACKENDS=pg,mongo` (default both), `DEBUG=1` prints resolver errors, -`MONGOMS_SYSTEM_BINARY` points mongodb-memory-server at a local `mongod`. +`MONGOMS_SYSTEM_BINARY` points mongodb-memory-server at a local `mongod`, `PGBOUNCER_BIN` the `pgbouncer` +binary for the PgBouncer and replica checks. ## Finance configuration @@ -1492,6 +1515,8 @@ flowchart LR | Area | What the suite proves | |---|---| | Server | `/health` and `{ health }`, introspection off when `INTROSPECTION=0`, depth limit 12 | +| PgBouncer | The whole PostgreSQL suite again through PgBouncer in transaction mode (`pgbouncer` on the PATH or `PGBOUNCER_BIN`; skipped with a note otherwise) | +| Replicas | Two API servers sharing PgBouncer and Redis: a session, a refresh and a logout cross replicas; 16 concurrent invoices with unique numbers over a 3-connection server pool; shared login limits; draining and shutdown | | Auth | no token → `UNAUTHENTICATED`, wrong password → `INVALID_CREDENTIALS`, login, viewer (null without a token), weak password, reset link through the mailer (unknown emails not revealed), reset ends old sessions, disable signs out at once, `mySessions`, logout | | Permissions | front desk cannot read the books; registers in branch A, not B; lists show only the viewer's branches; discounts need `invoice:discount`; nested vouchers hidden without `accounts:read` | | Tenancy | another organisation cannot read the invoice, the patient, or print it; cannot post into its ledgers | @@ -1566,7 +1591,8 @@ flowchart LR - **Two or more servers need Redis**: sessions, refresh tokens and login limits must be shared. - **Workers** (`node dist/worker.js`) need Redis and the same environment as the API (database, `JWT_SECRET`, signature files). Run one or more; see [16-worker.md](16-worker.md). -- The process is stateless otherwise; scale horizontally. +- The process is stateless otherwise; scale horizontally. [17-scaling.md](17-scaling.md) covers N replicas, + PgBouncer, readiness and graceful shutdown; `docker-compose.yml` runs all of it on one host. ## Build and run @@ -1577,8 +1603,9 @@ NODE_ENV=production node dist/main.js # API NODE_ENV=production node dist/worker.js # worker (with REDIS_URL) ``` -A container image: Node 22 slim, copy `dist/`, `node_modules/` and `package.json`, run -`node dist/main.js`, expose `PORT`. The PDF engine needs no system fonts (fonts are bundled), but +A container image: the [`Dockerfile`](../Dockerfile) (Node 22 slim, `dist/`, production `node_modules/`; +`node dist/main.js`, `node dist/worker.js` or `node dist/sync.js`; port 4000). It needs `NODE_AUTH_TOKEN` as a +build secret for the kits. The PDF engine needs no system fonts (fonts are bundled), but report-kit's resvg is a native module: build the image on the same platform it runs on. ## Production checklist @@ -1597,7 +1624,9 @@ report-kit's resvg is a native module: build the image on the same platform it r | ☐ | First organisation seeded (`npm run seed`) with the branch `stateCode` and the org GSTIN | | ☐ | Tax rules seeded **and confirmed by the CA** (`verifyWithCA`) | | ☐ | Database backups (point-in-time for Postgres), encrypted | -| ☐ | Health check on `GET /health` (200 `{ ok: true }`, 503 when the database is down) | +| ☐ | Readiness check on `GET /health` (200 `{ ok: true }`; 503 when the database or Redis is down, or while the replica shuts down); liveness on `GET /health?live=1` | +| ☐ | More than one replica: `REDIS_URL` and the same `JWT_SECRET` on all, `DB_SYNC=false` with `node dist/sync.js` once per release, pools sized ([17-scaling.md](17-scaling.md)) | +| ☐ | The orchestrator waits longer than `SHUTDOWN_DRAIN_MS` + `SHUTDOWN_TIMEOUT_MS` between SIGTERM and SIGKILL | | ☐ | Logs shipped; request bodies are not logged (they hold patient data) | ## Sending password-reset emails @@ -1960,6 +1989,164 @@ openBlob(documentBlob(done.document!)); Repeatable jobs (a nightly day-end reminder, say) use `queue.upsertJobScheduler(...)` in the worker entry. +--- + +## File: docs/17-scaling.md + +# 17 · Scaling: replicas and PgBouncer + +The API runs as any number of identical replicas behind a reverse proxy. Every replica connects to +PostgreSQL through **PgBouncer** and keeps nothing per-user in its own memory: sessions, refresh tokens, +login limits and jobs live in **Redis**. + +```mermaid +flowchart LR + U["Browsers / apps"] -- HTTPS --> P["Reverse proxy
(nginx, a load balancer)"] + P -- "/graphql, /health" --> A1["api #1"] & A2["api #2"] & A3["api #N"] + A1 & A2 & A3 -- "DB_POOL_MAX each" --> B["PgBouncer
transaction pooling"] + W["worker(s)"] --> B + B -- "DEFAULT_POOL_SIZE" --> PG[("PostgreSQL")] + A1 & A2 & A3 & W --> R[("Redis
sessions · limits · queues · cache")] + S["db-sync
(once per release)"] -. DDL .-> PG +``` + +## What each replica needs + +| | Why | +|---|---| +| `REDIS_URL`, the same on every replica | Without it sessions, refresh tokens, login limits and jobs live in one process: a token issued by one replica fails on the next, and every replica allows its own 10 login attempts | +| `JWT_SECRET`, the same on every replica | Access tokens are verified locally; a per-process secret (the development default) fails on every other replica | +| `TRUST_PROXY` = the number of proxies in front | Login limits and sessions use the client IP from `X-Forwarded-For`, counted from the right through that many proxies | +| `DB_SYNC=false`, and `node dist/sync.js` once per release | Replicas starting together would race to create the same tables | +| A readiness check on `GET /health`, liveness on `GET /health?live=1` | See below | +| A stop timeout longer than `SHUTDOWN_DRAIN_MS` + `SHUTDOWN_TIMEOUT_MS` | So in-flight requests finish before SIGKILL | + +Where the state lives once `REDIS_URL` is set (checked in `src/container.ts` and `src/jobs/`): + +| State | Where | Without `REDIS_URL` | +|---|---|---| +| Sessions, refresh tokens, reset tokens (hashed) | Redis (auth-kit `redisSessionStore`, `redisKeyValueStore`) | process memory | +| Login rate limits | Redis (sliding window) | process memory | +| Background jobs and their results | Redis (BullMQ); any worker runs them, any replica reads `job(id)` | process memory | +| db-kit read cache | Redis (L1 per process is off) | none | +| Everything else | PostgreSQL / MongoDB | | +| DataLoader caches | one request only | | + +`npm test` proves it: the `[replicas]` checks run two API servers on one PgBouncer and one Redis. A session +started on one works on the other, a refresh token spent on one is refused on the other, a logout on one ends +the session on the other at once, and the login limit is reached after 10 attempts in all, not 10 per replica. + +## PgBouncer + +Point `DATABASE_URL` at PgBouncer instead of PostgreSQL, e.g. `postgres://USER:PASSWORD@pgbouncer:6432/mgpl`. +Nothing else changes in the API. + +**Pool mode: transaction** (recommended). A server connection belongs to a client only for one transaction, +so a few PostgreSQL connections serve many replicas. The API works under it because db-kit (Sequelize on +node-postgres) uses none of the session-level features that break there: + +| Feature that breaks under transaction pooling | In db-kit / the API | +|---|---| +| Named prepared statements | Not used: node-postgres sends unnamed statements (extended protocol), which PgBouncer passes through | +| `SET` without `LOCAL` | Only Sequelize's connection setup (`SET client_min_messages`, `SET TIME ZONE '+00:00'`). PgBouncer tracks `TimeZone` and re-applies it per client; `client_min_messages` only changes log noise. Timestamps are `timestamptz`, so values are exact either way | +| Advisory locks, `LISTEN`/`NOTIFY`, temporary tables, `WITH HOLD` cursors | Not used | +| Locks and isolation | `SELECT … FOR UPDATE` and the isolation level live inside `repos.transaction()`, which is one PgBouncer transaction | +| Counters (invoice and voucher numbers) | One `INSERT … ON CONFLICT … RETURNING` statement | + +`npm test` runs the whole PostgreSQL suite a second time through PgBouncer in transaction mode (`[pgbouncer]`), +then fires 16 invoices with payments at once from two replicas (2 × `DB_POOL_MAX=4` client connections) +over a 3-connection server pool: all are posted, every number is unique and the trial balance balances. + +Session mode also works, but then each client connection holds a server connection, so it saves little. +Statement mode does not work (transactions span several statements). + +`DB_SYNC` and `node dist/sync.js` run DDL; they work through PgBouncer, but `docker-compose.yml` points +db-sync straight at PostgreSQL, as migrations usually are. + +### Sizing the pools + +Two pools in series: + +``` +API replicas × DB_POOL_MAX + workers × DB_POOL_MAX ≤ PgBouncer max_client_conn +PgBouncer default_pool_size + reserve_pool_size ≤ max_db_connections < PostgreSQL max_connections +``` + +- `DB_POOL_MAX` (default 10) is per process. Behind PgBouncer these are cheap client connections; it + caps how many queries one replica runs at once. +- PgBouncer's `default_pool_size` (compose default 20) is what PostgreSQL actually sees. Start near + 2 × the database server's CPU cores and raise it only if clients wait (`SHOW POOLS` → `cl_waiting`). +- Leave PostgreSQL a few connections above PgBouncer's cap for db-sync, backups and an admin `psql`. + +Example: 4 replicas + 2 workers × `DB_POOL_MAX=10` = 60 client connections (`max_client_conn` 500 is +plenty); `default_pool_size=20`, `reserve_pool_size=5`, `max_db_connections=50`; PostgreSQL +`max_connections=100`. + +**MongoDB** needs no PgBouncer: the driver pools by itself, and `DB_POOL_MAX` / `DB_POOL_MIN` / +`DB_POOL_IDLE_MS` / `DB_POOL_ACQUIRE_MS` set `maxPoolSize`, `minPoolSize`, `maxIdleTimeMS` and +`waitQueueTimeoutMS`. Keep replicas × `DB_POOL_MAX` within the cluster's connection limit. + +## Health and readiness + +| Request | 200 when | 503 when | Use it for | +|---|---|---|---| +| `GET /health` | the database answers, Redis answers (when configured), and the replica is not shutting down | any of those fails | readiness: the load balancer sends traffic only to replicas that answer 200 | +| `GET /health?live=1` | the process is running | never | liveness: restart a replica only when the process hangs, not when the database is down (that would restart every replica at once) | + +The body says which part failed: `{ "ok": false, "database": "ok", "redis": "unreachable" }` or +`{ "ok": false, "draining": true }`. The `{ health }` GraphQL field still reports the database. + +## Graceful shutdown + +On SIGTERM (or SIGINT) the API (`src/lifecycle.ts`): + +1. answers `GET /health` with 503 at once, while it keeps serving every other request; +2. waits `SHUTDOWN_DRAIN_MS` (5 s in production) so the load balancer sees the 503 and stops routing here; +3. stops accepting connections, closes idle keep-alive connections, answers 503 to new requests on open + ones and waits for the requests in flight; +4. closes the job queue, the database pool and Redis, and exits 0. + +A second signal, or `SHUTDOWN_TIMEOUT_MS` (30 s) after the first, exits at once with code 1. The worker +finishes its jobs in hand and stops; after `SHUTDOWN_TIMEOUT_MS` it exits and BullMQ retries the job that +was cut short. Give the orchestrator a longer stop timeout than drain + timeout: `stop_grace_period: 40s` +in compose, `terminationGracePeriodSeconds: 40` in Kubernetes. + +Rolling updates then drop nothing: start the new replicas, wait for them to be ready, stop the old ones. +Stopping one replica of three under constant traffic through nginx gave 40 of 40 requests answered 200. + +## Docker Compose + +[`docker-compose.yml`](../docker-compose.yml) runs the whole backend on one host: PostgreSQL, PgBouncer +(transaction mode), Redis (AOF on), a one-shot `db-sync`, `API_REPLICAS` API containers, a worker and +nginx ([`deploy/nginx.conf`](../deploy/nginx.conf)) in front of the replicas on `HTTP_PORT`. + +```bash +cp .env.example .env # NODE_ENV=production, JWT_SECRET, CORS_ORIGINS, POSTGRES_PASSWORD, SMTP_URL, … +NODE_AUTH_TOKEN=… docker compose build # the kits come from GitHub Packages (read:packages) +docker compose up -d # http://localhost:8080/graphql +docker compose up -d --scale api=4 # change the number of replicas at any time +docker compose exec -e PGPASSWORD=… postgres psql -h pgbouncer -p 6432 -U postgres pgbouncer -c 'SHOW POOLS' +``` + +- The API containers get `DATABASE_URL` (PgBouncer), `REDIS_URL`, `DB_SYNC=false` and `TRUST_PROXY=1` + from the compose file; everything else comes from `.env`. +- nginx re-resolves `api` every 10 s, so scaling needs no restart, and retries a request on another + replica when one answers 503 or refuses the connection. +- TLS: terminate it in front (a CDN or cloud load balancer, then `TRUST_PROXY=2`) or add a `listen 443 ssl` + server to `deploy/nginx.conf`. +- Signatures: mount the files read-only into `api` and `worker` and set `MGPL_SIGNATURE_*_FILE`. +- The compose file is PostgreSQL only. For MongoDB, drop `postgres`, `pgbouncer` and `db-sync`, and set + `DB_BACKEND=mongo`, `DATABASE_URL` (a replica set) and `DB_SYNC` on one replica. + +## Kubernetes or another orchestrator + +The same pieces map one to one: a Deployment of the image with `replicas: N`, a readiness probe on +`/health`, a liveness probe on `/health?live=1`, `terminationGracePeriodSeconds: 40`; a Job (or an init +step of the release) running `node dist/sync.js`; PgBouncer as its own Deployment and Service (or a managed +pooler such as the provider's PgBouncer); Redis managed or as a StatefulSet; the worker as a second +Deployment with `command: ["node", "dist/worker.js"]`. + + --- ## File: docs/LOCAL-AGENT-BRIEF.md @@ -2196,6 +2383,7 @@ flowchart TD B --> M["13 Adding a feature"] D --> N["14 Client"] B --> O["16 Worker"] + L --> P["17 Scaling"] ``` | # | Page | What it answers | @@ -2216,6 +2404,7 @@ flowchart TD | 14 | [Client](14-client.md) | Using `@mgpl-diagnostics/client` against this API | | 15 | [Glossary](15-glossary.md) | Lab, billing and accounting terms | | 16 | [Worker](16-worker.md) | Background jobs on Redis + BullMQ: emails, documents, running workers | +| 17 | [Scaling](17-scaling.md) | N replicas behind a proxy, PgBouncer, readiness, graceful shutdown, docker-compose | | — | [examples/](examples/) | Runnable `.graphql` operations | | — | [LOCAL-AGENT-BRIEF.md](LOCAL-AGENT-BRIEF.md) | What a local agent (or a person) must do next, outside this repo | @@ -2334,6 +2523,7 @@ Key facts for agents: "worker:dev": "tsx watch src/worker.ts", "start": "node dist/main.js", "worker": "node dist/worker.js", + "db:sync": "node dist/sync.js", "build": "tsup", "typecheck": "tsc --noEmit", "test": "tsx test/run.ts", @@ -4569,7 +4759,12 @@ export interface Config { port: number; /** URL path of the one GraphQL endpoint. */ graphqlPath: string; - db: { backend: DbBackend; url: string }; + /** + * The database. `url` may point at PgBouncer instead of PostgreSQL (docs/17-scaling.md); `pool` sizes this + * process's own connection pool (Sequelize for pg, the MongoDB driver for mongo); `sync` creates missing + * tables / indexes on start (turn it off on all but one replica, or run it once per release). + */ + db: { backend: DbBackend; url: string; sync: boolean; pool: { max: number; min: number; idleMs: number; acquireMs: number } }; /** Sessions, reset tokens, rate limits and the read cache. Without it they live in process memory (one server only). */ redisUrl: string | null; /** Origins allowed to call the API from a browser; "*" only in development. */ @@ -4605,6 +4800,12 @@ export interface Config { smtp: { url: string; from: string } | null; /** Optional HTTP relay for emails sent by the worker: POST { to, subject, text } as JSON. */ mailWebhookUrl: string | null; + /** + * Graceful shutdown on SIGTERM / SIGINT: GET /health answers 503 at once, the server keeps serving for + * `drainMs` so the load balancer stops sending traffic, then stops accepting connections and waits for the + * requests in flight; after `timeoutMs` in all it exits anyway. + */ + shutdown: { drainMs: number; timeoutMs: number }; } /** @@ -4637,6 +4838,15 @@ function hops(v: string | undefined): number { return int(v, 0); } +/** DB_POOL_*: connections this process keeps open (to PgBouncer or the database). Replicas × max must fit the server side. */ +function dbPool(env: Record): Config["db"]["pool"] { + const max = int(env.DB_POOL_MAX, 10); + const min = int(env.DB_POOL_MIN, 0); + if (max < 1) throw new Error("DB_POOL_MAX must be at least 1"); + if (min > max) throw new Error(`DB_POOL_MIN (${min}) cannot be larger than DB_POOL_MAX (${max})`); + return { max, min, idleMs: int(env.DB_POOL_IDLE_MS, 10_000), acquireMs: int(env.DB_POOL_ACQUIRE_MS, 30_000) }; +} + export function loadConfig(env: Record = process.env): Config { const mode = env.NODE_ENV === "production" ? "production" : env.NODE_ENV === "test" ? "test" : "development"; const dev = mode !== "production"; @@ -4659,7 +4869,12 @@ export function loadConfig(env: Record = process.env host: env.HOST ?? "0.0.0.0", port: int(env.PORT, 4000), graphqlPath: env.GRAPHQL_PATH ?? "/graphql", - db: { backend, url }, + db: { + backend, + url, + sync: bool(env.DB_SYNC, true), + pool: dbPool(env), + }, redisUrl: env.REDIS_URL || null, corsOrigins, graphiql: bool(env.GRAPHIQL, dev), @@ -4679,6 +4894,7 @@ export function loadConfig(env: Record = process.env jobs: { prefix: env.QUEUE_PREFIX || "mgpl", concurrency: Math.max(1, int(env.WORKER_CONCURRENCY, 4)), keepResultsMs: int(env.JOB_RESULT_TTL_MS, 3600_000) }, smtp: smtpUrl ? { url: smtpUrl, from: env.MAIL_FROM || "MGPL Diagnostics " } : null, mailWebhookUrl: env.MAIL_WEBHOOK_URL || null, + shutdown: { drainMs: int(env.SHUTDOWN_DRAIN_MS, mode === "production" ? 5_000 : 0), timeoutMs: int(env.SHUTDOWN_TIMEOUT_MS, 30_000) }, }; } ``` @@ -4724,6 +4940,11 @@ export interface Container { close(): Promise; } +// DB_POOL_*: the same knobs for both backends. With PgBouncer in transaction mode the pool holds client +// connections to PgBouncer, which multiplexes them over its own (smaller) pool of server connections. +const pgPool = ({ db: { pool } }: Config) => ({ max: pool.max, min: pool.min, idle: pool.idleMs, acquire: pool.acquireMs }); +const mongoPool = ({ db: { pool } }: Config) => ({ maxPoolSize: pool.max, minPoolSize: pool.min, maxIdleTimeMS: pool.idleMs, waitQueueTimeoutMS: pool.acquireMs }); + export async function createContainer(o: ContainerOptions): Promise { const { config } = o; const ownsDb = !o.db; @@ -4731,7 +4952,8 @@ export async function createContainer(o: ContainerOptions): Promise { o.db ?? (await createDb({ backend: config.db.backend, - ...(config.db.backend === "pg" ? { pg: { url: config.db.url } } : { mongo: { uri: config.db.url } }), + ...(config.db.backend === "pg" ? { pg: { url: config.db.url, options: { pool: pgPool(config) } } } : { mongo: { uri: config.db.url, options: mongoPool(config) } }), + sync: config.db.sync, redis: config.redisUrl ?? undefined, // invoices and payments post to the books through finance-kit → ledger-kit posting: financePosting, @@ -5928,6 +6150,7 @@ export function buildSchema(modules: GraphQLModule[] = MODULES): GraphQLSchema { export { loadConfig, type Config } from "./config/env"; export { createContainer, type Container, type ContainerOptions } from "./container"; export { buildServer } from "./server"; +export { isDraining, shutdown, shutdownOnSignals, type ShutdownOptions } from "./lifecycle"; export { buildSchema, MODULES, type GraphQLModule } from "./graphql/schema"; export { ERROR_CODES, type ErrorCode } from "./graphql/errors"; export type { RequestContext, Actor } from "./graphql/context"; @@ -6277,6 +6500,71 @@ export class JobFailure extends Error { ``` +--- + +## File: src/lifecycle.ts + +```ts +// Process lifecycle for running several API replicas behind a load balancer: readiness while draining, +// and a graceful shutdown on SIGTERM (what Docker, Kubernetes and systemd send before killing a process). +import type { FastifyInstance } from "fastify"; + +const draining = new WeakSet(); + +/** True once shutdown has begun: GET /health answers 503 so the load balancer stops routing here. */ +export const isDraining = (app: FastifyInstance): boolean => draining.has(app); + +export interface ShutdownOptions { + /** Keep serving this long after the signal, with /health at 503, so the load balancer notices first. */ + drainMs: number; + /** Exit with code 1 if shutting down takes longer than this in all. */ + timeoutMs: number; + log?: (msg: string) => void; +} + +/** + * Stops `app` gracefully: mark it draining → wait `drainMs` → `app.close()` (stops accepting connections, + * answers 503 to new requests on open ones, waits for the requests in flight, then runs onClose hooks, which + * close the container) → exit 0. A second signal, or `timeoutMs`, exits at once with 1. + */ +export async function shutdown(app: FastifyInstance, o: ShutdownOptions): Promise { + if (draining.has(app)) return; + draining.add(app); + const timer = setTimeout(() => { + o.log?.(`[api] shutdown took longer than ${o.timeoutMs} ms: exiting`); + process.exit(1); + }, o.timeoutMs); + timer.unref(); + if (o.drainMs > 0) { + o.log?.(`[api] draining for ${o.drainMs} ms (GET /health → 503)`); + await new Promise((r) => setTimeout(r, o.drainMs)); + } + await app.close(); + clearTimeout(timer); +} + +/** Wires SIGTERM and SIGINT to `shutdown`, then exits. Used by `main.ts`. */ +export function shutdownOnSignals(app: FastifyInstance, o: ShutdownOptions): void { + for (const signal of ["SIGTERM", "SIGINT"] as const) { + process.on(signal, () => { + if (draining.has(app)) { + o.log?.(`[api] ${signal} again: exiting now`); + process.exit(1); + } + o.log?.(`[api] ${signal}: shutting down`); + shutdown(app, o).then( + () => process.exit(0), + (e) => { + o.log?.(`[api] shutdown failed: ${(e as Error).message}`); + process.exit(1); + }, + ); + }); + } +} +``` + + --- ## File: src/main.ts @@ -6285,12 +6573,13 @@ export class JobFailure extends Error { // Starts the API: `npm run dev` (watch) or `npm run build && npm start`. import { loadConfig, loadEnvFile } from "./config/env"; import { createContainer } from "./container"; +import { shutdownOnSignals } from "./lifecycle"; import { buildServer } from "./server"; loadEnvFile(); const config = loadConfig(); const container = await createContainer({ config }); -if (config.env === "production" && !config.redisUrl) console.warn("[api] REDIS_URL is not set: sessions and jobs live in this process only (one server, lost on restart)"); +if (config.env === "production" && !config.redisUrl) console.warn("[api] REDIS_URL is not set: sessions, login limits and jobs live in this process only (run ONE replica; lost on restart)"); if (config.jwt.generated) console.warn("[api] JWT_SECRET is not set: using a random secret, so tokens die when the process restarts"); // With REDIS_URL, emails and background documents are sent and rendered by `npm run worker`. // Without it they run here; reset links go out through mail-kit (SMTP_URL), MAIL_WEBHOOK_URL, or createContainer({ config, mailer }). @@ -6299,11 +6588,9 @@ const app = await buildServer(container); await app.listen({ host: config.host, port: config.port }); console.log(`[api] GraphQL on http://${config.host}:${config.port}${config.graphqlPath} (${config.db.backend})`); -for (const signal of ["SIGINT", "SIGTERM"] as const) { - process.once(signal, () => { - app.close().then(() => process.exit(0), () => process.exit(1)); - }); -} +// SIGTERM / SIGINT: /health → 503, drain for SHUTDOWN_DRAIN_MS, finish the requests in flight, close the +// database, Redis and the job queue, exit (docs/17-scaling.md). +shutdownOnSignals(app, { ...config.shutdown, log: (m) => console.log(m) }); ``` @@ -7323,6 +7610,7 @@ import type { Container } from "./container"; import { createContext, type RequestContext } from "./graphql/context"; import { depthLimit } from "./graphql/limits"; import { buildSchema } from "./graphql/schema"; +import { isDraining } from "./lifecycle"; type ServerContext = { req: FastifyRequest; reply: FastifyReply }; @@ -7382,13 +7670,21 @@ export async function buildServer(container: Container): Promise { - try { - await container.db.rawRepos.orgs.count(); - return { ok: true }; - } catch { - return reply.status(503).send({ ok: false }); - } + // GET /health: readiness for load balancers. 200 when this replica can serve (database, and Redis when + // configured, answer; not shutting down), else 503. GET /health?live=1: liveness, 200 while the process + // runs (restart on failure only; a database outage should not restart every replica). + app.get<{ Querystring: { live?: string } }>("/health", async (req, reply) => { + if (req.query.live !== undefined) return { ok: true, live: true }; + if (isDraining(app)) return reply.status(503).send({ ok: false, draining: true }); + // a down Redis queues commands instead of failing them: give each check 2 s + const check = (p: Promise) => + Promise.race([p, new Promise((_, reject) => setTimeout(reject, 2_000).unref())]).then(() => "ok" as const, () => "unreachable" as const); + const [database, redis] = await Promise.all([ + check(container.db.rawRepos.orgs.count()), + container.db.redis ? check(container.db.redis.ping()) : Promise.resolve(undefined), + ]); + const ok = database === "ok" && redis !== "unreachable"; + return reply.status(ok ? 200 : 503).send({ ok, database, ...(redis ? { redis } : {}) }); }); app.addHook("onClose", async () => container.close()); @@ -9009,6 +9305,32 @@ export async function scopedFind( ``` +--- + +## File: src/sync.ts + +```ts +// Creates missing tables (PostgreSQL) or indexes (MongoDB), then exits: `npm run db:sync` or `node dist/sync.js`. +// With several API replicas, run this once per release and start the replicas with DB_SYNC=false, so they do +// not all race to create the same tables at once (docs/17-scaling.md). Point DATABASE_URL straight at the +// database for it (DDL does not need PgBouncer); it never alters or drops an existing table. +import { createDb } from "@mgpl-diagnostics/db-kit"; +import { financePosting } from "@mgpl-diagnostics/finance-kit"; +import { loadConfig, loadEnvFile } from "./config/env"; + +loadEnvFile(); +const config = loadConfig(); +const db = await createDb({ + backend: config.db.backend, + ...(config.db.backend === "pg" ? { pg: { url: config.db.url, options: { pool: { max: 1 } } } } : { mongo: { uri: config.db.url } }), + posting: financePosting, + sync: true, +} as Parameters[0]); +await db.close(); +console.log(`[sync] ${config.db.backend}: tables and indexes are in place`); +``` + + --- ## File: src/worker.ts @@ -9033,7 +9355,8 @@ console.log(`[worker] queues ${workers.workers.map((w) => w.name).join(", ")} on for (const signal of ["SIGINT", "SIGTERM"] as const) { process.once(signal, () => { - // finish the jobs in hand, then stop + // finish the jobs in hand, then stop; give up after SHUTDOWN_TIMEOUT_MS (BullMQ retries a job cut short) + setTimeout(() => process.exit(1), config.shutdown.timeoutMs).unref(); workers.close().then(() => container.close()).then(() => process.exit(0), () => process.exit(1)); }); } @@ -9050,7 +9373,7 @@ for (const signal of ["SIGINT", "SIGTERM"] as const) { // npm test both backends (first run downloads the database binaries) // TEST_BACKENDS=pg npm test PostgreSQL only (when MongoDB binaries cannot be downloaded) import { spawn, type ChildProcess } from "node:child_process"; -import { readdirSync, readFileSync, rmSync } from "node:fs"; +import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { hashPassword, SYSTEM_ROLES } from "@mgpl-diagnostics/auth-kit"; @@ -9059,10 +9382,12 @@ import { financePosting, istDay } from "@mgpl-diagnostics/finance-kit"; import { createMgplGraphQLClient, OPERATIONS } from "@mgpl-diagnostics/client"; import { parse, validate } from "graphql"; import { Redis } from "ioredis"; +import { QueryTypes, Sequelize } from "sequelize"; import { loadConfig } from "../src/config/env"; import { buildSchema } from "../src/graphql/schema"; import { createContainer } from "../src/container"; import { buildServer } from "../src/server"; +import { shutdown } from "../src/lifecycle"; import { startWorkers } from "../src/jobs"; import type { Mailer } from "../src/services/auth.service"; import { createMailer, memoryTransport } from "@mgpl-diagnostics/mail-kit"; @@ -9424,6 +9749,147 @@ async function workerSuite(pgUrl: string) { } } +/** + * PgBouncer in transaction mode in front of PostgreSQL: the whole pg suite again through it, then two API + * replicas sharing PgBouncer and Redis (docs/17-scaling.md). Needs `pgbouncer` (PGBOUNCER_BIN) and Redis + * (REDIS_URL, or `redis-server`); skipped with a note otherwise. CI installs both. + */ +async function pgbouncerSuite(pgPort: number, database: string) { + console.log("\n[pgbouncer]"); + const bin = process.env.PGBOUNCER_BIN || "pgbouncer"; + const dir = join(tmpdir(), `mgpl-api-pgbouncer-${process.pid}`); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true, mode: 0o755 }); + const port = 64331; + const poolSize = 3; + // the same settings as deploy/docker-compose.yml: transaction pooling, a small server pool + writeFileSync(join(dir, "userlist.txt"), '"postgres" "postgres"\n', { mode: 0o644 }); + writeFileSync( + join(dir, "pgbouncer.ini"), + [ + "[databases]", + `${database} = host=127.0.0.1 port=${pgPort} dbname=${database}`, + "[pgbouncer]", + "listen_addr = 127.0.0.1", + `listen_port = ${port}`, + "unix_socket_dir =", + "auth_type = scram-sha-256", + `auth_file = ${join(dir, "userlist.txt")}`, + "pool_mode = transaction", + `default_pool_size = ${poolSize}`, + "max_client_conn = 200", + "ignore_startup_parameters = extra_float_digits", + "", + ].join("\n"), + { mode: 0o644 }, + ); + // pgbouncer refuses to run as root + const args = [...(process.getuid?.() === 0 ? ["-u", "nobody"] : []), join(dir, "pgbouncer.ini")]; + let bouncer: ChildProcess; + try { + bouncer = spawn(bin, args, { stdio: ["ignore", "ignore", "pipe"] }); + await new Promise((resolve, reject) => { + bouncer.once("error", reject); + bouncer.once("exit", (c) => reject(new Error(`pgbouncer exited with ${c}`))); + setTimeout(resolve, 500); + }); + } catch (e) { + console.log(` (skipped: ${(e as Error).message}; install pgbouncer or set PGBOUNCER_BIN)`); + rmSync(dir, { recursive: true, force: true }); + return; + } + bouncer.removeAllListeners("exit"); + const url = `postgres://postgres:postgres@127.0.0.1:${port}/${database}`; + let redisUrl = process.env.REDIS_URL; + let redisServer: ChildProcess | undefined; + try { + // 1. every pg check again, through PgBouncer in transaction mode + const db = await createDb({ backend: "pg", pg: { url }, posting: financePosting }); + try { + await suite(db, "pgbouncer"); + } finally { + await db.close(); + } + + // 2. two replicas behind one load balancer: shared PgBouncer, shared Redis, nothing in process memory + console.log("\n[replicas]"); + if (!redisUrl) { + try { + redisServer = spawn("redis-server", ["--port", "6392", "--save", "", "--appendonly", "no"], { stdio: "ignore" }); + await new Promise((resolve, reject) => { + redisServer!.once("error", reject); + setTimeout(resolve, 400); + }); + redisUrl = "redis://127.0.0.1:6392"; + } catch { + console.log(" (skipped: set REDIS_URL or install redis-server)"); + return; + } + } + // what every replica shares: the database (through PgBouncer), Redis and JWT_SECRET + const replica = async () => { + const config = loadConfig({ NODE_ENV: "test", DB_BACKEND: "pg", DATABASE_URL: url, REDIS_URL: redisUrl, JWT_SECRET: "replicas-share-one-jwt-secret-0123456789", DB_POOL_MAX: "4", DB_SYNC: "false", TRUST_PROXY: "1", QUEUE_PREFIX: `mgpl-replicas-${process.pid}` }); + return buildServer(await createContainer({ config })); + }; + const [a, b] = [await replica(), await replica()]; + const gql = async (app: typeof a, query: string, variables?: Record, token?: string, ip = "203.0.113.20"): Promise => + (await app.inject({ method: "POST", url: "/graphql", headers: { "content-type": "application/json", "x-forwarded-for": ip, ...(token ? { authorization: `Bearer ${token}` } : {}) }, payload: JSON.stringify({ query, variables }) })).json() as Gql; + try { + const health = await a.inject({ url: "/health" }); + ok(health.statusCode === 200 && health.json().database === "ok" && health.json().redis === "ok", "[replicas] GET /health checks the database (through PgBouncer) and Redis"); + + const login = (await gql(a, 'mutation { login(input: { email: "owner@apl.test", password: "Lab-Owner-2026!", device: { deviceId: "dev-replica-0001" } }) { accessToken refreshToken } }')).data?.login; + ok((await gql(b, "{ organization { code } }", {}, login?.accessToken)).data?.organization.code === "APL", "[replicas] a session started on replica A works on replica B (sessions in Redis)"); + const next = (await gql(b, 'mutation($r: String!) { refreshSession(refreshToken: $r, device: { deviceId: "dev-replica-0001" }) { accessToken refreshToken } }', { r: login?.refreshToken })).data?.refreshSession; + ok(!!next?.accessToken && code(await gql(a, 'mutation($r: String!) { refreshSession(refreshToken: $r, device: { deviceId: "dev-replica-0001" }) { accessToken } }', { r: login?.refreshToken })) === "INVALID_TOKEN", "[replicas] refresh on B; the spent refresh token is refused on A (single use across replicas)"); + + // many writes at once from both replicas: 2 × DB_POOL_MAX client connections over a 3-connection server pool + const token = next.accessToken; + const branchId = (await gql(a, "{ organization { branches { id code } } }", {}, token)).data?.organization.branches.find((x: any) => x.code === "A01")?.id; + const patientId = (await gql(a, '{ patients(search: "Asha") { id } }', {}, token)).data?.patients[0]?.id; + const issued = await Promise.all( + Array.from({ length: 16 }, (_, i) => + gql(i % 2 ? a : b, "mutation($i: IssueInvoiceInput!) { issueInvoice(input: $i) { invoiceNo } }", { i: { branchId, patientId, items: [{ code: "CBC" }], payments: [{ amountPaise: 10000, mode: "cash" }] } }, token), + ), + ); + const numbers = issued.map((r) => r.data?.issueInvoice.invoiceNo); + ok(numbers.every(Boolean) && new Set(numbers).size === 16, `[replicas] 16 concurrent invoices (with payments) on both replicas: all posted, all numbers unique${issued.find((r) => r.errors) ? ` (${JSON.stringify(issued.find((r) => r.errors)?.errors)})` : ""}`); + const direct = new Sequelize(`postgres://postgres:postgres@127.0.0.1:${pgPort}/${database}`, { dialect: "postgres", logging: false }); + const [row] = await direct.query<{ n: string }>("SELECT count(*) AS n FROM pg_stat_activity WHERE datname = :database AND pid <> pg_backend_pid()", { replacements: { database }, type: QueryTypes.SELECT }); + const servers = Number(row?.n); + await direct.close(); + ok(servers <= poolSize, `[replicas] PostgreSQL saw ${servers} connection(s) from PgBouncer for 2 replicas × DB_POOL_MAX=4 (pool ${poolSize})`); + const tb = (await gql(b, "{ trialBalance { totalDebitPaise totalCreditPaise } }", {}, token)).data?.trialBalance; + ok(!!tb && tb.totalDebitPaise === tb.totalCreditPaise, "[replicas] the books still balance after the concurrent burst"); + + ok((await gql(b, "mutation { logout }", {}, token)).data?.logout === true && code(await gql(a, "{ organization { id } }", {}, token)) === "UNAUTHENTICATED", "[replicas] logout on B ends the session on A at once"); + + // login limits are shared: 10 attempts per window in all, not 10 per replica + let limitedAt = 0; + for (let i = 1; i <= 12 && !limitedAt; i++) { + const r = await gql(i % 2 ? a : b, 'mutation { login(input: { email: "nobody@replicas.test", password: "wrong-password" }) { token } }', {}, undefined, "198.51.100.7"); + if (code(r) === "RATE_LIMITED") limitedAt = i; + } + ok(limitedAt > 0 && limitedAt <= 11, `[replicas] the login rate limit is shared across replicas (limited at attempt ${limitedAt || "never"})`); + + // graceful shutdown: A drains (/health 503, still serving), then closes; B carries on + const stopping = shutdown(a, { drainMs: 400, timeoutMs: 10_000 }); + const drainingHealth = await a.inject({ url: "/health" }); + const servedWhileDraining = (await gql(a, "{ health { ok } }")).data?.health.ok; + ok(drainingHealth.statusCode === 503 && drainingHealth.json().draining === true && servedWhileDraining === true && (await a.inject({ url: "/health?live=1" })).statusCode === 200, "[replicas] on shutdown /health → 503 (draining) while requests are still served; ?live=1 stays 200"); + await stopping; + ok((await b.inject({ url: "/health" })).statusCode === 200, "[replicas] replica A shut down cleanly; B is still ready"); + } finally { + await a.close(); + await b.close(); + } + } finally { + redisServer?.kill(); + bouncer.kill(); + rmSync(dir, { recursive: true, force: true }); + } +} + /** Static checks: the documented example operations stay valid against the schema. */ function staticChecks() { console.log("\n[static]"); @@ -9483,6 +9949,8 @@ async function main() { await suite(db, "pg"); await fromConfig(url); await workerSuite(url); + await pg.createDatabase("mgpl_api_pgb"); + await pgbouncerSuite(54331, "mgpl_api_pgb"); } finally { await db.close(); await pg.stop(); diff --git a/package.json b/package.json index 4330b41..3de5601 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "worker:dev": "tsx watch src/worker.ts", "start": "node dist/main.js", "worker": "node dist/worker.js", + "db:sync": "node dist/sync.js", "build": "tsup", "typecheck": "tsc --noEmit", "test": "tsx test/run.ts", diff --git a/src/config/env.ts b/src/config/env.ts index 205070a..62af011 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -11,7 +11,12 @@ export interface Config { port: number; /** URL path of the one GraphQL endpoint. */ graphqlPath: string; - db: { backend: DbBackend; url: string }; + /** + * The database. `url` may point at PgBouncer instead of PostgreSQL (docs/17-scaling.md); `pool` sizes this + * process's own connection pool (Sequelize for pg, the MongoDB driver for mongo); `sync` creates missing + * tables / indexes on start (turn it off on all but one replica, or run it once per release). + */ + db: { backend: DbBackend; url: string; sync: boolean; pool: { max: number; min: number; idleMs: number; acquireMs: number } }; /** Sessions, reset tokens, rate limits and the read cache. Without it they live in process memory (one server only). */ redisUrl: string | null; /** Origins allowed to call the API from a browser; "*" only in development. */ @@ -47,6 +52,12 @@ export interface Config { smtp: { url: string; from: string } | null; /** Optional HTTP relay for emails sent by the worker: POST { to, subject, text } as JSON. */ mailWebhookUrl: string | null; + /** + * Graceful shutdown on SIGTERM / SIGINT: GET /health answers 503 at once, the server keeps serving for + * `drainMs` so the load balancer stops sending traffic, then stops accepting connections and waits for the + * requests in flight; after `timeoutMs` in all it exits anyway. + */ + shutdown: { drainMs: number; timeoutMs: number }; } /** @@ -79,6 +90,15 @@ function hops(v: string | undefined): number { return int(v, 0); } +/** DB_POOL_*: connections this process keeps open (to PgBouncer or the database). Replicas × max must fit the server side. */ +function dbPool(env: Record): Config["db"]["pool"] { + const max = int(env.DB_POOL_MAX, 10); + const min = int(env.DB_POOL_MIN, 0); + if (max < 1) throw new Error("DB_POOL_MAX must be at least 1"); + if (min > max) throw new Error(`DB_POOL_MIN (${min}) cannot be larger than DB_POOL_MAX (${max})`); + return { max, min, idleMs: int(env.DB_POOL_IDLE_MS, 10_000), acquireMs: int(env.DB_POOL_ACQUIRE_MS, 30_000) }; +} + export function loadConfig(env: Record = process.env): Config { const mode = env.NODE_ENV === "production" ? "production" : env.NODE_ENV === "test" ? "test" : "development"; const dev = mode !== "production"; @@ -101,7 +121,12 @@ export function loadConfig(env: Record = process.env host: env.HOST ?? "0.0.0.0", port: int(env.PORT, 4000), graphqlPath: env.GRAPHQL_PATH ?? "/graphql", - db: { backend, url }, + db: { + backend, + url, + sync: bool(env.DB_SYNC, true), + pool: dbPool(env), + }, redisUrl: env.REDIS_URL || null, corsOrigins, graphiql: bool(env.GRAPHIQL, dev), @@ -121,5 +146,6 @@ export function loadConfig(env: Record = process.env jobs: { prefix: env.QUEUE_PREFIX || "mgpl", concurrency: Math.max(1, int(env.WORKER_CONCURRENCY, 4)), keepResultsMs: int(env.JOB_RESULT_TTL_MS, 3600_000) }, smtp: smtpUrl ? { url: smtpUrl, from: env.MAIL_FROM || "MGPL Diagnostics " } : null, mailWebhookUrl: env.MAIL_WEBHOOK_URL || null, + shutdown: { drainMs: int(env.SHUTDOWN_DRAIN_MS, mode === "production" ? 5_000 : 0), timeoutMs: int(env.SHUTDOWN_TIMEOUT_MS, 30_000) }, }; } diff --git a/src/container.ts b/src/container.ts index cd311eb..bf13e26 100644 --- a/src/container.ts +++ b/src/container.ts @@ -33,6 +33,11 @@ export interface Container { close(): Promise; } +// DB_POOL_*: the same knobs for both backends. With PgBouncer in transaction mode the pool holds client +// connections to PgBouncer, which multiplexes them over its own (smaller) pool of server connections. +const pgPool = ({ db: { pool } }: Config) => ({ max: pool.max, min: pool.min, idle: pool.idleMs, acquire: pool.acquireMs }); +const mongoPool = ({ db: { pool } }: Config) => ({ maxPoolSize: pool.max, minPoolSize: pool.min, maxIdleTimeMS: pool.idleMs, waitQueueTimeoutMS: pool.acquireMs }); + export async function createContainer(o: ContainerOptions): Promise { const { config } = o; const ownsDb = !o.db; @@ -40,7 +45,8 @@ export async function createContainer(o: ContainerOptions): Promise { o.db ?? (await createDb({ backend: config.db.backend, - ...(config.db.backend === "pg" ? { pg: { url: config.db.url } } : { mongo: { uri: config.db.url } }), + ...(config.db.backend === "pg" ? { pg: { url: config.db.url, options: { pool: pgPool(config) } } } : { mongo: { uri: config.db.url, options: mongoPool(config) } }), + sync: config.db.sync, redis: config.redisUrl ?? undefined, // invoices and payments post to the books through finance-kit → ledger-kit posting: financePosting, diff --git a/src/index.ts b/src/index.ts index 5b3be91..8212a7d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ export { loadConfig, type Config } from "./config/env"; export { createContainer, type Container, type ContainerOptions } from "./container"; export { buildServer } from "./server"; +export { isDraining, shutdown, shutdownOnSignals, type ShutdownOptions } from "./lifecycle"; export { buildSchema, MODULES, type GraphQLModule } from "./graphql/schema"; export { ERROR_CODES, type ErrorCode } from "./graphql/errors"; export type { RequestContext, Actor } from "./graphql/context"; diff --git a/src/lifecycle.ts b/src/lifecycle.ts new file mode 100644 index 0000000..f348265 --- /dev/null +++ b/src/lifecycle.ts @@ -0,0 +1,57 @@ +// Process lifecycle for running several API replicas behind a load balancer: readiness while draining, +// and a graceful shutdown on SIGTERM (what Docker, Kubernetes and systemd send before killing a process). +import type { FastifyInstance } from "fastify"; + +const draining = new WeakSet(); + +/** True once shutdown has begun: GET /health answers 503 so the load balancer stops routing here. */ +export const isDraining = (app: FastifyInstance): boolean => draining.has(app); + +export interface ShutdownOptions { + /** Keep serving this long after the signal, with /health at 503, so the load balancer notices first. */ + drainMs: number; + /** Exit with code 1 if shutting down takes longer than this in all. */ + timeoutMs: number; + log?: (msg: string) => void; +} + +/** + * Stops `app` gracefully: mark it draining → wait `drainMs` → `app.close()` (stops accepting connections, + * answers 503 to new requests on open ones, waits for the requests in flight, then runs onClose hooks, which + * close the container) → exit 0. A second signal, or `timeoutMs`, exits at once with 1. + */ +export async function shutdown(app: FastifyInstance, o: ShutdownOptions): Promise { + if (draining.has(app)) return; + draining.add(app); + const timer = setTimeout(() => { + o.log?.(`[api] shutdown took longer than ${o.timeoutMs} ms: exiting`); + process.exit(1); + }, o.timeoutMs); + timer.unref(); + if (o.drainMs > 0) { + o.log?.(`[api] draining for ${o.drainMs} ms (GET /health → 503)`); + await new Promise((r) => setTimeout(r, o.drainMs)); + } + await app.close(); + clearTimeout(timer); +} + +/** Wires SIGTERM and SIGINT to `shutdown`, then exits. Used by `main.ts`. */ +export function shutdownOnSignals(app: FastifyInstance, o: ShutdownOptions): void { + for (const signal of ["SIGTERM", "SIGINT"] as const) { + process.on(signal, () => { + if (draining.has(app)) { + o.log?.(`[api] ${signal} again: exiting now`); + process.exit(1); + } + o.log?.(`[api] ${signal}: shutting down`); + shutdown(app, o).then( + () => process.exit(0), + (e) => { + o.log?.(`[api] shutdown failed: ${(e as Error).message}`); + process.exit(1); + }, + ); + }); + } +} diff --git a/src/main.ts b/src/main.ts index c3cabde..768feb0 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,12 +1,13 @@ // Starts the API: `npm run dev` (watch) or `npm run build && npm start`. import { loadConfig, loadEnvFile } from "./config/env"; import { createContainer } from "./container"; +import { shutdownOnSignals } from "./lifecycle"; import { buildServer } from "./server"; loadEnvFile(); const config = loadConfig(); const container = await createContainer({ config }); -if (config.env === "production" && !config.redisUrl) console.warn("[api] REDIS_URL is not set: sessions and jobs live in this process only (one server, lost on restart)"); +if (config.env === "production" && !config.redisUrl) console.warn("[api] REDIS_URL is not set: sessions, login limits and jobs live in this process only (run ONE replica; lost on restart)"); if (config.jwt.generated) console.warn("[api] JWT_SECRET is not set: using a random secret, so tokens die when the process restarts"); // With REDIS_URL, emails and background documents are sent and rendered by `npm run worker`. // Without it they run here; reset links go out through mail-kit (SMTP_URL), MAIL_WEBHOOK_URL, or createContainer({ config, mailer }). @@ -15,8 +16,6 @@ const app = await buildServer(container); await app.listen({ host: config.host, port: config.port }); console.log(`[api] GraphQL on http://${config.host}:${config.port}${config.graphqlPath} (${config.db.backend})`); -for (const signal of ["SIGINT", "SIGTERM"] as const) { - process.once(signal, () => { - app.close().then(() => process.exit(0), () => process.exit(1)); - }); -} +// SIGTERM / SIGINT: /health → 503, drain for SHUTDOWN_DRAIN_MS, finish the requests in flight, close the +// database, Redis and the job queue, exit (docs/17-scaling.md). +shutdownOnSignals(app, { ...config.shutdown, log: (m) => console.log(m) }); diff --git a/src/server.ts b/src/server.ts index a5970d7..270d0e0 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,6 +6,7 @@ import type { Container } from "./container"; import { createContext, type RequestContext } from "./graphql/context"; import { depthLimit } from "./graphql/limits"; import { buildSchema } from "./graphql/schema"; +import { isDraining } from "./lifecycle"; type ServerContext = { req: FastifyRequest; reply: FastifyReply }; @@ -65,13 +66,21 @@ export async function buildServer(container: Container): Promise { - try { - await container.db.rawRepos.orgs.count(); - return { ok: true }; - } catch { - return reply.status(503).send({ ok: false }); - } + // GET /health: readiness for load balancers. 200 when this replica can serve (database, and Redis when + // configured, answer; not shutting down), else 503. GET /health?live=1: liveness, 200 while the process + // runs (restart on failure only; a database outage should not restart every replica). + app.get<{ Querystring: { live?: string } }>("/health", async (req, reply) => { + if (req.query.live !== undefined) return { ok: true, live: true }; + if (isDraining(app)) return reply.status(503).send({ ok: false, draining: true }); + // a down Redis queues commands instead of failing them: give each check 2 s + const check = (p: Promise) => + Promise.race([p, new Promise((_, reject) => setTimeout(reject, 2_000).unref())]).then(() => "ok" as const, () => "unreachable" as const); + const [database, redis] = await Promise.all([ + check(container.db.rawRepos.orgs.count()), + container.db.redis ? check(container.db.redis.ping()) : Promise.resolve(undefined), + ]); + const ok = database === "ok" && redis !== "unreachable"; + return reply.status(ok ? 200 : 503).send({ ok, database, ...(redis ? { redis } : {}) }); }); app.addHook("onClose", async () => container.close()); diff --git a/src/sync.ts b/src/sync.ts new file mode 100644 index 0000000..50653db --- /dev/null +++ b/src/sync.ts @@ -0,0 +1,18 @@ +// Creates missing tables (PostgreSQL) or indexes (MongoDB), then exits: `npm run db:sync` or `node dist/sync.js`. +// With several API replicas, run this once per release and start the replicas with DB_SYNC=false, so they do +// not all race to create the same tables at once (docs/17-scaling.md). Point DATABASE_URL straight at the +// database for it (DDL does not need PgBouncer); it never alters or drops an existing table. +import { createDb } from "@mgpl-diagnostics/db-kit"; +import { financePosting } from "@mgpl-diagnostics/finance-kit"; +import { loadConfig, loadEnvFile } from "./config/env"; + +loadEnvFile(); +const config = loadConfig(); +const db = await createDb({ + backend: config.db.backend, + ...(config.db.backend === "pg" ? { pg: { url: config.db.url, options: { pool: { max: 1 } } } } : { mongo: { uri: config.db.url } }), + posting: financePosting, + sync: true, +} as Parameters[0]); +await db.close(); +console.log(`[sync] ${config.db.backend}: tables and indexes are in place`); diff --git a/src/worker.ts b/src/worker.ts index d8c06d3..c98e3e0 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -17,7 +17,8 @@ console.log(`[worker] queues ${workers.workers.map((w) => w.name).join(", ")} on for (const signal of ["SIGINT", "SIGTERM"] as const) { process.once(signal, () => { - // finish the jobs in hand, then stop + // finish the jobs in hand, then stop; give up after SHUTDOWN_TIMEOUT_MS (BullMQ retries a job cut short) + setTimeout(() => process.exit(1), config.shutdown.timeoutMs).unref(); workers.close().then(() => container.close()).then(() => process.exit(0), () => process.exit(1)); }); } diff --git a/test/run.ts b/test/run.ts index 85929fb..883b086 100644 --- a/test/run.ts +++ b/test/run.ts @@ -3,7 +3,7 @@ // npm test both backends (first run downloads the database binaries) // TEST_BACKENDS=pg npm test PostgreSQL only (when MongoDB binaries cannot be downloaded) import { spawn, type ChildProcess } from "node:child_process"; -import { readdirSync, readFileSync, rmSync } from "node:fs"; +import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { hashPassword, SYSTEM_ROLES } from "@mgpl-diagnostics/auth-kit"; @@ -12,10 +12,12 @@ import { financePosting, istDay } from "@mgpl-diagnostics/finance-kit"; import { createMgplGraphQLClient, OPERATIONS } from "@mgpl-diagnostics/client"; import { parse, validate } from "graphql"; import { Redis } from "ioredis"; +import { QueryTypes, Sequelize } from "sequelize"; import { loadConfig } from "../src/config/env"; import { buildSchema } from "../src/graphql/schema"; import { createContainer } from "../src/container"; import { buildServer } from "../src/server"; +import { shutdown } from "../src/lifecycle"; import { startWorkers } from "../src/jobs"; import type { Mailer } from "../src/services/auth.service"; import { createMailer, memoryTransport } from "@mgpl-diagnostics/mail-kit"; @@ -377,6 +379,147 @@ async function workerSuite(pgUrl: string) { } } +/** + * PgBouncer in transaction mode in front of PostgreSQL: the whole pg suite again through it, then two API + * replicas sharing PgBouncer and Redis (docs/17-scaling.md). Needs `pgbouncer` (PGBOUNCER_BIN) and Redis + * (REDIS_URL, or `redis-server`); skipped with a note otherwise. CI installs both. + */ +async function pgbouncerSuite(pgPort: number, database: string) { + console.log("\n[pgbouncer]"); + const bin = process.env.PGBOUNCER_BIN || "pgbouncer"; + const dir = join(tmpdir(), `mgpl-api-pgbouncer-${process.pid}`); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true, mode: 0o755 }); + const port = 64331; + const poolSize = 3; + // the same settings as deploy/docker-compose.yml: transaction pooling, a small server pool + writeFileSync(join(dir, "userlist.txt"), '"postgres" "postgres"\n', { mode: 0o644 }); + writeFileSync( + join(dir, "pgbouncer.ini"), + [ + "[databases]", + `${database} = host=127.0.0.1 port=${pgPort} dbname=${database}`, + "[pgbouncer]", + "listen_addr = 127.0.0.1", + `listen_port = ${port}`, + "unix_socket_dir =", + "auth_type = scram-sha-256", + `auth_file = ${join(dir, "userlist.txt")}`, + "pool_mode = transaction", + `default_pool_size = ${poolSize}`, + "max_client_conn = 200", + "ignore_startup_parameters = extra_float_digits", + "", + ].join("\n"), + { mode: 0o644 }, + ); + // pgbouncer refuses to run as root + const args = [...(process.getuid?.() === 0 ? ["-u", "nobody"] : []), join(dir, "pgbouncer.ini")]; + let bouncer: ChildProcess; + try { + bouncer = spawn(bin, args, { stdio: ["ignore", "ignore", "pipe"] }); + await new Promise((resolve, reject) => { + bouncer.once("error", reject); + bouncer.once("exit", (c) => reject(new Error(`pgbouncer exited with ${c}`))); + setTimeout(resolve, 500); + }); + } catch (e) { + console.log(` (skipped: ${(e as Error).message}; install pgbouncer or set PGBOUNCER_BIN)`); + rmSync(dir, { recursive: true, force: true }); + return; + } + bouncer.removeAllListeners("exit"); + const url = `postgres://postgres:postgres@127.0.0.1:${port}/${database}`; + let redisUrl = process.env.REDIS_URL; + let redisServer: ChildProcess | undefined; + try { + // 1. every pg check again, through PgBouncer in transaction mode + const db = await createDb({ backend: "pg", pg: { url }, posting: financePosting }); + try { + await suite(db, "pgbouncer"); + } finally { + await db.close(); + } + + // 2. two replicas behind one load balancer: shared PgBouncer, shared Redis, nothing in process memory + console.log("\n[replicas]"); + if (!redisUrl) { + try { + redisServer = spawn("redis-server", ["--port", "6392", "--save", "", "--appendonly", "no"], { stdio: "ignore" }); + await new Promise((resolve, reject) => { + redisServer!.once("error", reject); + setTimeout(resolve, 400); + }); + redisUrl = "redis://127.0.0.1:6392"; + } catch { + console.log(" (skipped: set REDIS_URL or install redis-server)"); + return; + } + } + // what every replica shares: the database (through PgBouncer), Redis and JWT_SECRET + const replica = async () => { + const config = loadConfig({ NODE_ENV: "test", DB_BACKEND: "pg", DATABASE_URL: url, REDIS_URL: redisUrl, JWT_SECRET: "replicas-share-one-jwt-secret-0123456789", DB_POOL_MAX: "4", DB_SYNC: "false", TRUST_PROXY: "1", QUEUE_PREFIX: `mgpl-replicas-${process.pid}` }); + return buildServer(await createContainer({ config })); + }; + const [a, b] = [await replica(), await replica()]; + const gql = async (app: typeof a, query: string, variables?: Record, token?: string, ip = "203.0.113.20"): Promise => + (await app.inject({ method: "POST", url: "/graphql", headers: { "content-type": "application/json", "x-forwarded-for": ip, ...(token ? { authorization: `Bearer ${token}` } : {}) }, payload: JSON.stringify({ query, variables }) })).json() as Gql; + try { + const health = await a.inject({ url: "/health" }); + ok(health.statusCode === 200 && health.json().database === "ok" && health.json().redis === "ok", "[replicas] GET /health checks the database (through PgBouncer) and Redis"); + + const login = (await gql(a, 'mutation { login(input: { email: "owner@apl.test", password: "Lab-Owner-2026!", device: { deviceId: "dev-replica-0001" } }) { accessToken refreshToken } }')).data?.login; + ok((await gql(b, "{ organization { code } }", {}, login?.accessToken)).data?.organization.code === "APL", "[replicas] a session started on replica A works on replica B (sessions in Redis)"); + const next = (await gql(b, 'mutation($r: String!) { refreshSession(refreshToken: $r, device: { deviceId: "dev-replica-0001" }) { accessToken refreshToken } }', { r: login?.refreshToken })).data?.refreshSession; + ok(!!next?.accessToken && code(await gql(a, 'mutation($r: String!) { refreshSession(refreshToken: $r, device: { deviceId: "dev-replica-0001" }) { accessToken } }', { r: login?.refreshToken })) === "INVALID_TOKEN", "[replicas] refresh on B; the spent refresh token is refused on A (single use across replicas)"); + + // many writes at once from both replicas: 2 × DB_POOL_MAX client connections over a 3-connection server pool + const token = next.accessToken; + const branchId = (await gql(a, "{ organization { branches { id code } } }", {}, token)).data?.organization.branches.find((x: any) => x.code === "A01")?.id; + const patientId = (await gql(a, '{ patients(search: "Asha") { id } }', {}, token)).data?.patients[0]?.id; + const issued = await Promise.all( + Array.from({ length: 16 }, (_, i) => + gql(i % 2 ? a : b, "mutation($i: IssueInvoiceInput!) { issueInvoice(input: $i) { invoiceNo } }", { i: { branchId, patientId, items: [{ code: "CBC" }], payments: [{ amountPaise: 10000, mode: "cash" }] } }, token), + ), + ); + const numbers = issued.map((r) => r.data?.issueInvoice.invoiceNo); + ok(numbers.every(Boolean) && new Set(numbers).size === 16, `[replicas] 16 concurrent invoices (with payments) on both replicas: all posted, all numbers unique${issued.find((r) => r.errors) ? ` (${JSON.stringify(issued.find((r) => r.errors)?.errors)})` : ""}`); + const direct = new Sequelize(`postgres://postgres:postgres@127.0.0.1:${pgPort}/${database}`, { dialect: "postgres", logging: false }); + const [row] = await direct.query<{ n: string }>("SELECT count(*) AS n FROM pg_stat_activity WHERE datname = :database AND pid <> pg_backend_pid()", { replacements: { database }, type: QueryTypes.SELECT }); + const servers = Number(row?.n); + await direct.close(); + ok(servers <= poolSize, `[replicas] PostgreSQL saw ${servers} connection(s) from PgBouncer for 2 replicas × DB_POOL_MAX=4 (pool ${poolSize})`); + const tb = (await gql(b, "{ trialBalance { totalDebitPaise totalCreditPaise } }", {}, token)).data?.trialBalance; + ok(!!tb && tb.totalDebitPaise === tb.totalCreditPaise, "[replicas] the books still balance after the concurrent burst"); + + ok((await gql(b, "mutation { logout }", {}, token)).data?.logout === true && code(await gql(a, "{ organization { id } }", {}, token)) === "UNAUTHENTICATED", "[replicas] logout on B ends the session on A at once"); + + // login limits are shared: 10 attempts per window in all, not 10 per replica + let limitedAt = 0; + for (let i = 1; i <= 12 && !limitedAt; i++) { + const r = await gql(i % 2 ? a : b, 'mutation { login(input: { email: "nobody@replicas.test", password: "wrong-password" }) { token } }', {}, undefined, "198.51.100.7"); + if (code(r) === "RATE_LIMITED") limitedAt = i; + } + ok(limitedAt > 0 && limitedAt <= 11, `[replicas] the login rate limit is shared across replicas (limited at attempt ${limitedAt || "never"})`); + + // graceful shutdown: A drains (/health 503, still serving), then closes; B carries on + const stopping = shutdown(a, { drainMs: 400, timeoutMs: 10_000 }); + const drainingHealth = await a.inject({ url: "/health" }); + const servedWhileDraining = (await gql(a, "{ health { ok } }")).data?.health.ok; + ok(drainingHealth.statusCode === 503 && drainingHealth.json().draining === true && servedWhileDraining === true && (await a.inject({ url: "/health?live=1" })).statusCode === 200, "[replicas] on shutdown /health → 503 (draining) while requests are still served; ?live=1 stays 200"); + await stopping; + ok((await b.inject({ url: "/health" })).statusCode === 200, "[replicas] replica A shut down cleanly; B is still ready"); + } finally { + await a.close(); + await b.close(); + } + } finally { + redisServer?.kill(); + bouncer.kill(); + rmSync(dir, { recursive: true, force: true }); + } +} + /** Static checks: the documented example operations stay valid against the schema. */ function staticChecks() { console.log("\n[static]"); @@ -436,6 +579,8 @@ async function main() { await suite(db, "pg"); await fromConfig(url); await workerSuite(url); + await pg.createDatabase("mgpl_api_pgb"); + await pgbouncerSuite(54331, "mgpl_api_pgb"); } finally { await db.close(); await pg.stop(); diff --git a/tsup.config.ts b/tsup.config.ts index 86f3513..233bdb7 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -2,7 +2,7 @@ import { defineConfig } from "tsup"; // The server bundle: our code only; the kits and drivers stay in node_modules. export default defineConfig({ - entry: { main: "src/main.ts", worker: "src/worker.ts", index: "src/index.ts" }, + entry: { main: "src/main.ts", worker: "src/worker.ts", sync: "src/sync.ts", index: "src/index.ts" }, format: ["esm"], dts: { entry: { index: "src/index.ts" } }, target: "node20", From 14f41fbf8ff41878a268b72c1b427619902e07e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 02:12:55 +0000 Subject: [PATCH 2/3] docs: PgBouncer and replicas in the README and llms.txt Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dcqyj37vsGrmgdnp7DLbkn --- README.md | 19 ++++++++++++++----- llms-full.txt | 24 +++++++++++++++++------- llms.txt | 5 +++-- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 28825b1..a19a51f 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,15 @@ **The MGPL Diagnostics backend: one GraphQL endpoint for the whole lab.** Sign-in, patients, bookings, samples, results, reports, billing, books, GST, receivables, settlements, -cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL or MongoDB. +cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL (through PgBouncer) or MongoDB, +Redis. Runs as any number of replicas behind a load balancer. [![Endpoint](https://img.shields.io/badge/endpoint-POST%20%2Fgraphql-14264A)](#) [![Databases](https://img.shields.io/badge/db-PostgreSQL%20%7C%20MongoDB-14264A)](#) +[![Pooling](https://img.shields.io/badge/pool-PgBouncer-14264A)](docs/17-scaling.md) [![License](https://img.shields.io/badge/license-All%20rights%20reserved-E9A23B)](LICENSE) -[Docs](docs/README.md) · [Architecture](docs/02-architecture.md) · [Getting started](docs/03-getting-started.md) · [API reference](docs/04-api-reference.md) · [Client](docs/14-client.md) +[Docs](docs/README.md) · [Architecture](docs/02-architecture.md) · [Getting started](docs/03-getting-started.md) · [API reference](docs/04-api-reference.md) · [Client](docs/14-client.md) · [Scaling](docs/17-scaling.md) @@ -21,9 +23,12 @@ cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL or Mon ```mermaid flowchart LR - APP["Apps
@mgpl-diagnostics/client"] -- "POST /graphql" --> API["api
controllers → services"] + APP["Apps
@mgpl-diagnostics/client"] -- "POST /graphql" --> LB["load balancer
(nginx)"] + LB --> API["api × N replicas
controllers → services"] API --> K["auth-kit · db-kit · catalog-kit
report-kit · invoice-kit · ledger-kit · finance-kit"] - K --> DB[("PostgreSQL / MongoDB")] + K --> PGB["PgBouncer
transaction pooling"] + PGB --> PG[("PostgreSQL")] + K --> MG[("or MongoDB")] K -.-> R[("Redis")] R -. "BullMQ jobs" .-> W["worker
emails · documents"] W --> K @@ -40,6 +45,10 @@ flowchart LR device; logout and disabling still apply at once. See [docs/06](docs/06-auth-and-permissions.md). - **A worker on Redis + BullMQ** for reset emails and background documents (`npm run worker`); without Redis the same jobs run in the API process. See [docs/16-worker.md](docs/16-worker.md). +- **Horizontal scaling through PgBouncer.** Run N identical replicas behind a load balancer: sessions, + login limits and jobs live in Redis, and PostgreSQL is reached through PgBouncer in transaction mode + (the whole test suite runs through it). `GET /health` is the readiness check, SIGTERM drains gracefully, + and `docker compose up --scale api=N` runs the whole stack. See [docs/17-scaling.md](docs/17-scaling.md). - **Layers.** `schema/` (SDL) → `controllers/` (permissions, mapping) → `services/` (kit calls) → kits. - **Safe by construction.** The organisation and user come from the session; every record is checked against them and against its branch; unknown errors are masked; depth and list limits. @@ -73,7 +82,7 @@ const hits = await mgpl.catalog.search("cbc"); Everything from A to Z is in [docs/](docs/README.md), with Mermaid diagrams: overview, architecture, getting started, the generated API reference, errors, auth and permissions, domain flows, documents, -configuration, data and money, testing, deployment, adding a feature, client, glossary, and runnable +configuration, data and money, testing, deployment, adding a feature, client, glossary, worker, scaling, and runnable [examples](docs/examples/). What has to happen outside this repo (releases, org settings, deployment) is in [docs/LOCAL-AGENT-BRIEF.md](docs/LOCAL-AGENT-BRIEF.md). diff --git a/llms-full.txt b/llms-full.txt index 6f19d12..361e24b 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -108,13 +108,15 @@ **The MGPL Diagnostics backend: one GraphQL endpoint for the whole lab.** Sign-in, patients, bookings, samples, results, reports, billing, books, GST, receivables, settlements, -cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL or MongoDB. +cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL (through PgBouncer) or MongoDB, +Redis. Runs as any number of replicas behind a load balancer. [![Endpoint](https://img.shields.io/badge/endpoint-POST%20%2Fgraphql-14264A)](#) [![Databases](https://img.shields.io/badge/db-PostgreSQL%20%7C%20MongoDB-14264A)](#) +[![Pooling](https://img.shields.io/badge/pool-PgBouncer-14264A)](docs/17-scaling.md) [![License](https://img.shields.io/badge/license-All%20rights%20reserved-E9A23B)](LICENSE) -[Docs](docs/README.md) · [Architecture](docs/02-architecture.md) · [Getting started](docs/03-getting-started.md) · [API reference](docs/04-api-reference.md) · [Client](docs/14-client.md) +[Docs](docs/README.md) · [Architecture](docs/02-architecture.md) · [Getting started](docs/03-getting-started.md) · [API reference](docs/04-api-reference.md) · [Client](docs/14-client.md) · [Scaling](docs/17-scaling.md) @@ -122,9 +124,12 @@ cash and bank, documents. TypeScript, GraphQL Yoga on Fastify, PostgreSQL or Mon ```mermaid flowchart LR - APP["Apps
@mgpl-diagnostics/client"] -- "POST /graphql" --> API["api
controllers → services"] + APP["Apps
@mgpl-diagnostics/client"] -- "POST /graphql" --> LB["load balancer
(nginx)"] + LB --> API["api × N replicas
controllers → services"] API --> K["auth-kit · db-kit · catalog-kit
report-kit · invoice-kit · ledger-kit · finance-kit"] - K --> DB[("PostgreSQL / MongoDB")] + K --> PGB["PgBouncer
transaction pooling"] + PGB --> PG[("PostgreSQL")] + K --> MG[("or MongoDB")] K -.-> R[("Redis")] R -. "BullMQ jobs" .-> W["worker
emails · documents"] W --> K @@ -141,6 +146,10 @@ flowchart LR device; logout and disabling still apply at once. See [docs/06](docs/06-auth-and-permissions.md). - **A worker on Redis + BullMQ** for reset emails and background documents (`npm run worker`); without Redis the same jobs run in the API process. See [docs/16-worker.md](docs/16-worker.md). +- **Horizontal scaling through PgBouncer.** Run N identical replicas behind a load balancer: sessions, + login limits and jobs live in Redis, and PostgreSQL is reached through PgBouncer in transaction mode + (the whole test suite runs through it). `GET /health` is the readiness check, SIGTERM drains gracefully, + and `docker compose up --scale api=N` runs the whole stack. See [docs/17-scaling.md](docs/17-scaling.md). - **Layers.** `schema/` (SDL) → `controllers/` (permissions, mapping) → `services/` (kit calls) → kits. - **Safe by construction.** The organisation and user come from the session; every record is checked against them and against its branch; unknown errors are masked; depth and list limits. @@ -174,7 +183,7 @@ const hits = await mgpl.catalog.search("cbc"); Everything from A to Z is in [docs/](docs/README.md), with Mermaid diagrams: overview, architecture, getting started, the generated API reference, errors, auth and permissions, domain flows, documents, -configuration, data and money, testing, deployment, adding a feature, client, glossary, and runnable +configuration, data and money, testing, deployment, adding a feature, client, glossary, worker, scaling, and runnable [examples](docs/examples/). What has to happen outside this repo (releases, org settings, deployment) is in [docs/LOCAL-AGENT-BRIEF.md](docs/LOCAL-AGENT-BRIEF.md). @@ -2468,12 +2477,13 @@ flowchart TD # @mgpl-diagnostics/api -> The MGPL Diagnostics backend (Node.js ≥ 20, TypeScript): one GraphQL endpoint, `POST /graphql` (GraphQL Yoga on Fastify), for sign-in, users and roles, the test catalog, patients, bookings, samples, results and reports, billing, double-entry books, GST returns, B2B receivables, partner settlements, cash and bank, lock dates, CSV exports, and rendered report/invoice documents (base64 PDF or HTML). The kits (auth, db, catalog, report, invoice, tally, finance) are internal libraries behind a controller → service layer. PostgreSQL or MongoDB; Redis optional. +> The MGPL Diagnostics backend (Node.js ≥ 20, TypeScript): one GraphQL endpoint, `POST /graphql` (GraphQL Yoga on Fastify), for sign-in, users and roles, the test catalog, patients, bookings, samples, results and reports, billing, double-entry books, GST returns, B2B receivables, partner settlements, cash and bank, lock dates, CSV exports, and rendered report/invoice documents (base64 PDF or HTML). The kits (auth, db, catalog, report, invoice, tally, finance) are internal libraries behind a controller → service layer. PostgreSQL (directly or through PgBouncer in transaction mode) or MongoDB; Redis optional for one server, required for more than one replica. Key facts for agents: - Layers: `src/schema` (SDL) → `src/controllers` (authorize, map) → `src/services` (kit calls) → kits. No REST routes except `GET /health`. - Auth: `login(input: { email, password, device })` returns an access JWT (15 min, Bearer) and a single-use refresh token; `refreshSession(refreshToken, device)` rotates them; a refresh from another deviceId ends the session (DEVICE_BINDING). Every request still checks the session and re-reads the user. `JWT_SECRET` required in production. - Worker: `src/worker.ts` (`npm run worker`) runs BullMQ jobs from Redis (`src/jobs/`): password-reset emails (MAIL_WEBHOOK_URL) and documents (`startDocumentJob`, `job(id)`). Without REDIS_URL jobs run in the API process. Permissions are `resource:action`, checked per branch with `authorize(ctx, perm, branchId)`. +- Scaling (docs/17-scaling.md): N replicas behind a load balancer share Redis (sessions, login limits, jobs), JWT_SECRET and PgBouncer (DATABASE_URL → pgbouncer:6432; `npm test` runs the suite through it). DB_POOL_* size each process's pool; DB_SYNC=false on replicas with `node dist/sync.js` once per release; GET /health is readiness (?live=1 liveness); SIGTERM drains (SHUTDOWN_DRAIN_MS, SHUTDOWN_TIMEOUT_MS). docker-compose.yml runs PostgreSQL ← PgBouncer ← api × N ← nginx, Redis, worker. - Tenant isolation: orgId and actorId come from the session, never from arguments; every id is checked against the organisation. - Errors carry `extensions.code`: UNAUTHENTICATED, FORBIDDEN, NOT_FOUND, BAD_USER_INPUT, CONFLICT, PERIOD_LOCKED, CREDIT_LIMIT_EXCEEDED, plus auth-kit codes. Unknown errors are masked. - Money is integer paise; days are IST "YYYY-MM-DD"; the financial year is April–March. Billing needs the branch's GST stateCode; GSTR-1/3B need the organisation's GSTIN. @@ -2483,7 +2493,7 @@ Key facts for agents: ## Docs - [README](README.md): what it is, quick start -- [docs/README.md](docs/README.md): index of the A-to-Z docs (architecture, getting started, API reference, errors, auth, flows, documents, configuration, data and money, testing, deployment, adding a feature, client, glossary) +- [docs/README.md](docs/README.md): index of the A-to-Z docs (architecture, getting started, API reference, errors, auth, flows, documents, configuration, data and money, testing, deployment, adding a feature, client, glossary, worker, scaling) - [docs/04-api-reference.md](docs/04-api-reference.md): every operation with its permission - [docs/LOCAL-AGENT-BRIEF.md](docs/LOCAL-AGENT-BRIEF.md): releases, org settings and deployment to do outside this repo - [AGENTS.md](AGENTS.md): map and rules diff --git a/llms.txt b/llms.txt index db86d90..009c48e 100644 --- a/llms.txt +++ b/llms.txt @@ -1,11 +1,12 @@ # @mgpl-diagnostics/api -> The MGPL Diagnostics backend (Node.js ≥ 20, TypeScript): one GraphQL endpoint, `POST /graphql` (GraphQL Yoga on Fastify), for sign-in, users and roles, the test catalog, patients, bookings, samples, results and reports, billing, double-entry books, GST returns, B2B receivables, partner settlements, cash and bank, lock dates, CSV exports, and rendered report/invoice documents (base64 PDF or HTML). The kits (auth, db, catalog, report, invoice, tally, finance) are internal libraries behind a controller → service layer. PostgreSQL or MongoDB; Redis optional. +> The MGPL Diagnostics backend (Node.js ≥ 20, TypeScript): one GraphQL endpoint, `POST /graphql` (GraphQL Yoga on Fastify), for sign-in, users and roles, the test catalog, patients, bookings, samples, results and reports, billing, double-entry books, GST returns, B2B receivables, partner settlements, cash and bank, lock dates, CSV exports, and rendered report/invoice documents (base64 PDF or HTML). The kits (auth, db, catalog, report, invoice, tally, finance) are internal libraries behind a controller → service layer. PostgreSQL (directly or through PgBouncer in transaction mode) or MongoDB; Redis optional for one server, required for more than one replica. Key facts for agents: - Layers: `src/schema` (SDL) → `src/controllers` (authorize, map) → `src/services` (kit calls) → kits. No REST routes except `GET /health`. - Auth: `login(input: { email, password, device })` returns an access JWT (15 min, Bearer) and a single-use refresh token; `refreshSession(refreshToken, device)` rotates them; a refresh from another deviceId ends the session (DEVICE_BINDING). Every request still checks the session and re-reads the user. `JWT_SECRET` required in production. - Worker: `src/worker.ts` (`npm run worker`) runs BullMQ jobs from Redis (`src/jobs/`): password-reset emails (MAIL_WEBHOOK_URL) and documents (`startDocumentJob`, `job(id)`). Without REDIS_URL jobs run in the API process. Permissions are `resource:action`, checked per branch with `authorize(ctx, perm, branchId)`. +- Scaling (docs/17-scaling.md): N replicas behind a load balancer share Redis (sessions, login limits, jobs), JWT_SECRET and PgBouncer (DATABASE_URL → pgbouncer:6432; `npm test` runs the suite through it). DB_POOL_* size each process's pool; DB_SYNC=false on replicas with `node dist/sync.js` once per release; GET /health is readiness (?live=1 liveness); SIGTERM drains (SHUTDOWN_DRAIN_MS, SHUTDOWN_TIMEOUT_MS). docker-compose.yml runs PostgreSQL ← PgBouncer ← api × N ← nginx, Redis, worker. - Tenant isolation: orgId and actorId come from the session, never from arguments; every id is checked against the organisation. - Errors carry `extensions.code`: UNAUTHENTICATED, FORBIDDEN, NOT_FOUND, BAD_USER_INPUT, CONFLICT, PERIOD_LOCKED, CREDIT_LIMIT_EXCEEDED, plus auth-kit codes. Unknown errors are masked. - Money is integer paise; days are IST "YYYY-MM-DD"; the financial year is April–March. Billing needs the branch's GST stateCode; GSTR-1/3B need the organisation's GSTIN. @@ -15,7 +16,7 @@ Key facts for agents: ## Docs - [README](README.md): what it is, quick start -- [docs/README.md](docs/README.md): index of the A-to-Z docs (architecture, getting started, API reference, errors, auth, flows, documents, configuration, data and money, testing, deployment, adding a feature, client, glossary) +- [docs/README.md](docs/README.md): index of the A-to-Z docs (architecture, getting started, API reference, errors, auth, flows, documents, configuration, data and money, testing, deployment, adding a feature, client, glossary, worker, scaling) - [docs/04-api-reference.md](docs/04-api-reference.md): every operation with its permission - [docs/LOCAL-AGENT-BRIEF.md](docs/LOCAL-AGENT-BRIEF.md): releases, org settings and deployment to do outside this repo - [AGENTS.md](AGENTS.md): map and rules From d262a6fc1133cd1abcf27e60af6d0bab646e3f1d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 02:14:16 +0000 Subject: [PATCH 3/3] docs(readme): security-kit, mail-kit, worker-kit and pgbouncer-kit in the repo table Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Dcqyj37vsGrmgdnp7DLbkn --- README.md | 2 ++ llms-full.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index a19a51f..3f2b04b 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,8 @@ npm run llms # llms-full.txt | [auth-kit](https://github.com/mgpl-diagnostics/auth-kit) · [db-kit](https://github.com/mgpl-diagnostics/db-kit) · [catalog-kit](https://github.com/mgpl-diagnostics/catalog-kit) | auth, data, test catalog | | [report-kit](https://github.com/mgpl-diagnostics/report-kit) · [invoice-kit](https://github.com/mgpl-diagnostics/invoice-kit) | report and invoice rendering | | [ledger-kit](https://github.com/mgpl-diagnostics/ledger-kit) · [finance-kit](https://github.com/mgpl-diagnostics/finance-kit) | books, GST and finance | +| [security-kit](https://github.com/mgpl-diagnostics/security-kit) · [mail-kit](https://github.com/mgpl-diagnostics/mail-kit) · [worker-kit](https://github.com/mgpl-diagnostics/worker-kit) | security headers, client IP and device signals; mail; background jobs | +| [pgbouncer-kit](https://github.com/mgpl-diagnostics/pgbouncer-kit) | PgBouncer setup (config, userlist, compose, TLS) for the connection pooler in front of PostgreSQL | | [report-studio](https://github.com/mgpl-diagnostics/report-studio) | Next.js reference app | © MGPL Diagnostics (Maa Gayatri Path Lab). All rights reserved; see [LICENSE](LICENSE). ledger-kit is our own software, not affiliated with Tally Solutions. diff --git a/llms-full.txt b/llms-full.txt index 361e24b..4c202b4 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -207,6 +207,8 @@ npm run llms # llms-full.txt | [auth-kit](https://github.com/mgpl-diagnostics/auth-kit) · [db-kit](https://github.com/mgpl-diagnostics/db-kit) · [catalog-kit](https://github.com/mgpl-diagnostics/catalog-kit) | auth, data, test catalog | | [report-kit](https://github.com/mgpl-diagnostics/report-kit) · [invoice-kit](https://github.com/mgpl-diagnostics/invoice-kit) | report and invoice rendering | | [ledger-kit](https://github.com/mgpl-diagnostics/ledger-kit) · [finance-kit](https://github.com/mgpl-diagnostics/finance-kit) | books, GST and finance | +| [security-kit](https://github.com/mgpl-diagnostics/security-kit) · [mail-kit](https://github.com/mgpl-diagnostics/mail-kit) · [worker-kit](https://github.com/mgpl-diagnostics/worker-kit) | security headers, client IP and device signals; mail; background jobs | +| [pgbouncer-kit](https://github.com/mgpl-diagnostics/pgbouncer-kit) | PgBouncer setup (config, userlist, compose, TLS) for the connection pooler in front of PostgreSQL | | [report-studio](https://github.com/mgpl-diagnostics/report-studio) | Next.js reference app | © MGPL Diagnostics (Maa Gayatri Path Lab). All rights reserved; see [LICENSE](LICENSE). ledger-kit is our own software, not affiliated with Tally Solutions.