From 954b3952cbee55690ca4355e8c1e2e29111007a7 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 13:43:49 -0700 Subject: [PATCH 1/8] ci(e2e): run checks for settings integration PRs Allow focused child PRs to receive normal CI before the completed suite returns to staging. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca2901c7871..9331d14d119 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main, staging, dev] pull_request: - branches: [main, staging, dev] + branches: [main, staging, dev, e2e/settings-playwright] # Docs content and markdown don't affect the app build or images; push # runs stay unfiltered because they feed the deploy pipeline. paths-ignore: From bdd0a488f2cb2d294ab3bba97e9256e6768a95fa Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos <157128530+BillLeoutsakosvl346@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:44:45 -0700 Subject: [PATCH 2/8] Add reusable Playwright E2E foundation (#5792) * test(e2e): add reusable Playwright foundation Establish a hermetic full-stack browser harness so settings and future surfaces can exercise real authentication, APIs, and isolated database state safely. * test(e2e): harden runner lifecycle and artifacts Fail closed on stale processes, preserve cleanup during interruption, and keep credentials and personal fixture data out of uploaded diagnostics. * test(e2e): tighten isolation and shutdown guards Make destructive database operations and service bindings strictly local while ensuring interrupted runs fully stop managed children before cleanup. * test(e2e): finalize harness safety boundaries Close the remaining lifecycle, credential-isolation, and CLI escape hatches so future persona and settings suites can build on a fail-closed foundation. * test(e2e): harden interrupted-run cleanup Use a detached cleanup supervisor so repeated signals cannot interrupt process-group shutdown or forced removal of the guarded run database. * test(e2e): guard signal cleanup edge cases Prevent empty process-group targets and verify forced-drop retries against final database state so interrupted runs cannot self-signal or report false cleanup failures. * test(e2e): require exact host and race-safe signals Reject dual-stack host mappings that cannot reach IPv4-only services and tolerate child exit races during process-group signal fallback. * test(e2e): force IPv4 for hosted test origin Accept safe dual-stack loopback resolution while pinning Chromium and Node traffic to 127.0.0.1 so CI reaches IPv4-only services reliably. * test(e2e): settle readiness exit race Mark successful readiness before the managed process completion branch can reject, preventing later normal shutdown from surfacing an abandoned promise failure. * test(e2e): propagate IPv4 preference to probes Keep Node-based Playwright requests on the same IPv4 path as Chromium and use direct loopback for orchestrator probes under dual-stack CI DNS. --------- Co-authored-by: Bill Leoutsakos --- .github/workflows/test-build.yml | 95 +++- .gitignore | 5 + apps/realtime/src/env.ts | 1 + apps/realtime/src/index.ts | 7 +- apps/realtime/src/routes/http.ts | 1 + apps/sim/app/api/health/route.test.ts | 15 +- apps/sim/app/api/health/route.ts | 3 + apps/sim/e2e/README.md | 100 ++++ apps/sim/e2e/fakes/stripe/server.ts | 530 ++++++++++++++++++ .../e2e/foundation/provider-isolation.spec.ts | 7 + apps/sim/e2e/foundation/runtime.spec.ts | 8 + apps/sim/e2e/foundation/safety.spec.ts | 178 ++++++ apps/sim/e2e/foundation/stripe-fake.spec.ts | 55 ++ apps/sim/e2e/scripts/options.ts | 64 +++ apps/sim/e2e/scripts/run.ts | 355 ++++++++++++ apps/sim/e2e/scripts/signal-cleanup.ts | 71 +++ .../e2e/settings/smoke/authenticated.spec.ts | 74 +++ .../settings/smoke/unauthenticated.spec.ts | 8 + apps/sim/e2e/support/database.ts | 120 ++++ apps/sim/e2e/support/deployment-profile.ts | 134 +++++ apps/sim/e2e/support/diagnostics.ts | 52 ++ apps/sim/e2e/support/env.ts | 102 ++++ apps/sim/e2e/support/hosts.ts | 41 ++ apps/sim/e2e/support/paths.ts | 11 + apps/sim/e2e/support/probes.ts | 47 ++ apps/sim/e2e/support/process.ts | 222 ++++++++ apps/sim/e2e/support/readiness.ts | 55 ++ apps/sim/e2e/support/signal-cleanup.ts | 8 + apps/sim/e2e/support/stack.ts | 136 +++++ apps/sim/lib/auth/auth.ts | 5 +- .../lib/billing/stripe-client-config.test.ts | 123 ++++ apps/sim/lib/billing/stripe-client-config.ts | 114 ++++ apps/sim/lib/billing/stripe-client.ts | 5 +- apps/sim/lib/core/config/env.ts | 2 + apps/sim/package.json | 3 + apps/sim/playwright.config.ts | 58 ++ apps/sim/vitest.config.ts | 2 +- biome.json | 3 + bun.lock | 9 + 39 files changed, 2817 insertions(+), 12 deletions(-) create mode 100644 apps/sim/e2e/README.md create mode 100644 apps/sim/e2e/fakes/stripe/server.ts create mode 100644 apps/sim/e2e/foundation/provider-isolation.spec.ts create mode 100644 apps/sim/e2e/foundation/runtime.spec.ts create mode 100644 apps/sim/e2e/foundation/safety.spec.ts create mode 100644 apps/sim/e2e/foundation/stripe-fake.spec.ts create mode 100644 apps/sim/e2e/scripts/options.ts create mode 100644 apps/sim/e2e/scripts/run.ts create mode 100644 apps/sim/e2e/scripts/signal-cleanup.ts create mode 100644 apps/sim/e2e/settings/smoke/authenticated.spec.ts create mode 100644 apps/sim/e2e/settings/smoke/unauthenticated.spec.ts create mode 100644 apps/sim/e2e/support/database.ts create mode 100644 apps/sim/e2e/support/deployment-profile.ts create mode 100644 apps/sim/e2e/support/diagnostics.ts create mode 100644 apps/sim/e2e/support/env.ts create mode 100644 apps/sim/e2e/support/hosts.ts create mode 100644 apps/sim/e2e/support/paths.ts create mode 100644 apps/sim/e2e/support/probes.ts create mode 100644 apps/sim/e2e/support/process.ts create mode 100644 apps/sim/e2e/support/readiness.ts create mode 100644 apps/sim/e2e/support/signal-cleanup.ts create mode 100644 apps/sim/e2e/support/stack.ts create mode 100644 apps/sim/lib/billing/stripe-client-config.test.ts create mode 100644 apps/sim/lib/billing/stripe-client-config.ts create mode 100644 apps/sim/playwright.config.ts diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index f20cae5f7d6..4d3fa2dbde4 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -245,4 +245,97 @@ jobs: AWS_REGION: 'us-west-2' ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only TURBO_CACHE_DIR: .turbo - run: bunx turbo run build --filter=sim \ No newline at end of file + run: bunx turbo run build --filter=sim + + settings-e2e: + name: Settings E2E (informational) + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + continue-on-error: true + + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 22 + + - name: Mount Bun cache (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.bun/install/cache + + - name: Mount node_modules (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ./node_modules + + - name: Mount Playwright browsers (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.cache/ms-playwright + + - name: Restore E2E Next.js build cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ./apps/sim/.next/cache + key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Configure E2E hostname + run: echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts + + - name: Install Chromium + working-directory: apps/sim + run: bun run test:e2e:install-browsers -- --with-deps + + - name: Run settings E2E foundation + timeout-minutes: 40 + working-directory: apps/sim + env: + CI: 'true' + E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' + run: bun run test:e2e + + - name: Upload E2E diagnostics + if: failure() || cancelled() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: settings-e2e-${{ github.run_id }} + path: | + apps/sim/playwright-report/ + apps/sim/test-results/ + apps/sim/e2e/.runs/ + !apps/sim/e2e/.runs/**/auth/** + if-no-files-found: ignore + include-hidden-files: true + retention-days: 7 \ No newline at end of file diff --git a/.gitignore b/.gitignore index a8a0d8e2fde..dc42d1c8e3f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,10 @@ package-lock.json # testing /coverage /apps/**/coverage +/apps/**/playwright-report/ +/apps/**/test-results/ +/apps/**/e2e/.runs/ +/apps/**/e2e/.auth/ # next.js /.next/ @@ -31,6 +35,7 @@ package-lock.json **/dist/ **/standalone/ sim-standalone.tar.gz +/.artifacts/ # redis dump.rdb diff --git a/apps/realtime/src/env.ts b/apps/realtime/src/env.ts index 5afdf3a20f3..2f75c4a595f 100644 --- a/apps/realtime/src/env.ts +++ b/apps/realtime/src/env.ts @@ -14,6 +14,7 @@ const EnvSchema = z.object({ INTERNAL_API_SECRET: z.string().min(32), NEXT_PUBLIC_APP_URL: z.string().url(), ALLOWED_ORIGINS: z.string().optional(), + REALTIME_HOST: z.string().min(1).default('0.0.0.0'), PORT: z.coerce.number().int().positive().default(3002), SIM_DB_ROLE: z.enum(['web', 'trigger', 'realtime']).optional(), DISABLE_AUTH: z diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index e43184c1f02..0c84867156b 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -31,6 +31,7 @@ async function createRoomManager(io: SocketIOServer): Promise { async function main() { const httpServer = createServer() const PORT = env.PORT + const HOST = env.REALTIME_HOST logger.info('Starting Socket.IO server...', { port: PORT, @@ -96,9 +97,9 @@ async function main() { await assertSchemaCompatibility() - httpServer.listen(PORT, '0.0.0.0', () => { - logger.info(`Socket.IO server running on port ${PORT}`) - logger.info(`Health check available at: http://localhost:${PORT}/health`) + httpServer.listen(PORT, HOST, () => { + logger.info(`Socket.IO server running on ${HOST}:${PORT}`) + logger.info(`Health check available at: http://${HOST}:${PORT}/health`) }) const shutdown = async () => { diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 0f8ed73cc52..0bfef282fcf 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -69,6 +69,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { status: 'ok', timestamp: new Date().toISOString(), connections, + ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}), }) ) } catch (error) { diff --git a/apps/sim/app/api/health/route.test.ts b/apps/sim/app/api/health/route.test.ts index 6abca827579..21d4d9d6a91 100644 --- a/apps/sim/app/api/health/route.test.ts +++ b/apps/sim/app/api/health/route.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { GET } from '@/app/api/health/route' describe('GET /api/health', () => { @@ -14,4 +14,17 @@ describe('GET /api/health', () => { timestamp: expect.any(String), }) }) + + it('returns the E2E run identity when configured', async () => { + vi.stubEnv('E2E_RUN_ID', 'run-health-check') + try { + const response = await GET() + await expect(response.json()).resolves.toMatchObject({ + status: 'ok', + runId: 'run-health-check', + }) + } finally { + vi.unstubAllEnvs() + } + }) }) diff --git a/apps/sim/app/api/health/route.ts b/apps/sim/app/api/health/route.ts index 5486272998c..517f9251724 100644 --- a/apps/sim/app/api/health/route.ts +++ b/apps/sim/app/api/health/route.ts @@ -1,11 +1,14 @@ /** * Health check endpoint for deployment platforms and container probes. */ +export const dynamic = 'force-dynamic' + export async function GET(): Promise { return Response.json( { status: 'ok', timestamp: new Date().toISOString(), + ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}), }, { status: 200 } ) diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md new file mode 100644 index 00000000000..49f108e8dd7 --- /dev/null +++ b/apps/sim/e2e/README.md @@ -0,0 +1,100 @@ +# Sim browser E2E + +This directory contains the reusable full-stack Playwright harness. It runs the +production Next.js app, realtime, deterministic external fakes, and a migrated +per-run pgvector database. + +## One-time setup + +0. Install Node 22 and Bun. Playwright workers require Node 22; set + `E2E_NODE_BINARY` to an alternate Node 22 executable when `node` on `PATH` + points elsewhere. + +1. Map the hosted E2E origin to loopback: + + ```bash + echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts + ``` + + The runner refuses to start unless every resolved address is loopback and an + IPv4 `127.0.0.1` result is present. Chromium and Node are configured to prefer + that IPv4 mapping, so CI environments that also synthesize `::1` remain safe. + +2. Start a local pgvector/Postgres admin instance: + + ```bash + docker run --rm --name sim-e2e-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=postgres \ + -p 5432:5432 \ + pgvector/pgvector:pg17 + ``` + + Cleanup uses `DROP DATABASE ... WITH (FORCE)`, which requires PostgreSQL 13 + or newer and is supported by the pinned pgvector/PostgreSQL 17 image. + +3. Install Chromium from `apps/sim`: + + ```bash + bun run test:e2e:install-browsers + ``` + +## Run the foundation + +From `apps/sim`: + +```bash +E2E_PG_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ + bun run test:e2e +``` + +The runner creates a unique `sim_e2e_` database, migrates it, starts the +Stripe fake and realtime, builds and starts Next.js, runs Playwright on Node 22, +then stops services and drops only that guarded database. + +On interruption, the runner launches a detached cleanup supervisor before +exiting. It terminates managed process groups, force-drops the guarded database, +and removes temporary auth/cloud-config directories even if another Ctrl-C +terminates the foreground package runner. + +Pass Playwright arguments after `--`: + +```bash +bun run test:e2e -- --project=hosted-billing-chromium-navigation +bun run test:e2e -- --grep "unauthenticated" +``` + +Do not invoke `playwright test` directly. Raw Playwright bypasses environment, +database, process, sharding, and teardown guards; the config rejects runs that +were not launched by the orchestrator. Report and trace viewer commands remain +safe because they do not execute tests. + +Sharding is supported only for the navigation project. The runner rejects +`--shard` for `hosted-billing-chromium-workflows`. + +## Diagnostics + +- HTML report: `playwright-report/` +- Traces and screenshots: `test-results/` +- App, realtime, migration, and fake logs: `e2e/.runs//logs/` + +Open the report: + +```bash +node ../../node_modules/@playwright/test/cli.js show-report playwright-report +``` + +Open a trace: + +```bash +node ../../node_modules/@playwright/test/cli.js show-trace test-results//trace.zip +``` + +The runner starts every child process from a fresh environment. It allowlists +only deterministic E2E values and shadows keys found in local `.env*` files, so +developer credentials are not used as test state or written to reports. + +Provider log scans are diagnostic tripwires, not proof of zero egress. The +primary boundaries are the default-deny child environment, provider disabling, +loopback-only service bindings, and guarded Stripe transport. diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts new file mode 100644 index 00000000000..8f89c48102e --- /dev/null +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -0,0 +1,530 @@ +import { createHash, timingSafeEqual } from 'node:crypto' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import type { AddressInfo } from 'node:net' + +export const STRIPE_FAKE_ENDPOINTS = { + health: '/health', + requestLog: '/__control/requests', + reset: '/__control/reset', +} as const + +const DEFAULT_MAX_BODY_BYTES = 64 * 1024 +const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded' + +export interface StripeFakeRequestRecord { + sequence: number + method: string + path: string + unexpected: boolean +} + +export interface StripeFakeServerOptions { + apiKey: string + hostname?: '127.0.0.1' + maxBodyBytes?: number + port?: number +} + +export interface StripeFakeServer { + readonly baseUrl: string | null + readonly requestLog: readonly StripeFakeRequestRecord[] + start(): Promise + stop(): Promise + reset(): void +} + +interface FakeCustomer { + id: string + object: 'customer' + created: number + email: string | null + livemode: false + metadata: Record + name: string | null +} + +interface StripeError { + type: 'api_error' | 'invalid_request_error' + code: string + message: string +} + +class RequestBodyError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } +} + +function isExpectedStripeRequest(method: string, path: string): boolean { + return ( + ((method === 'GET' || method === 'POST') && path === '/v1/customers/search') || + (method === 'GET' && path === '/v1/customers') || + (method === 'POST' && path === '/v1/customers') || + (method === 'GET' && /^\/v1\/customers\/cus_e2e_[a-f0-9]+$/.test(path)) + ) +} + +function secureEqual(actual: string | undefined, expected: string): boolean { + if (actual === undefined) return false + + const actualBuffer = Buffer.from(actual) + const expectedBuffer = Buffer.from(expected) + return ( + actualBuffer.length === expectedBuffer.length && timingSafeEqual(actualBuffer, expectedBuffer) + ) +} + +function cloneRequestLog(records: StripeFakeRequestRecord[]): StripeFakeRequestRecord[] { + return structuredClone(records) +} + +function sendJson( + response: ServerResponse, + status: number, + body: unknown, + requestId?: string +): void { + const serialized = JSON.stringify(body) + response.writeHead(status, { + 'content-length': Buffer.byteLength(serialized), + 'content-type': 'application/json; charset=utf-8', + ...(requestId ? { 'request-id': requestId } : {}), + }) + response.end(serialized) +} + +function sendStripeError( + response: ServerResponse, + status: number, + error: StripeError, + requestId: string +): void { + sendJson(response, status, { error }, requestId) +} + +async function readRequestBody(request: IncomingMessage, maxBodyBytes: number): Promise { + const chunks: Buffer[] = [] + let totalBytes = 0 + + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + totalBytes += buffer.length + if (totalBytes > maxBodyBytes) { + throw new RequestBodyError('Request body exceeds the Stripe fake limit', 413) + } + chunks.push(buffer) + } + + return Buffer.concat(chunks).toString('utf8') +} + +function parseFormBody(request: IncomingMessage, rawBody: string): URLSearchParams { + const contentType = request.headers['content-type']?.split(';', 1)[0]?.trim().toLowerCase() + if (contentType !== FORM_CONTENT_TYPE) { + throw new RequestBodyError(`Stripe fake requires ${FORM_CONTENT_TYPE}`, 400) + } + return new URLSearchParams(rawBody) +} + +function metadataFrom(parameters: URLSearchParams): Record { + const metadata: Record = {} + for (const [key, value] of parameters) { + const match = /^metadata\[([^\]]+)\]$/.exec(key) + if (match) metadata[match[1]] = value + } + return metadata +} + +function stableCustomerIdentity(parameters: URLSearchParams): string { + const metadata = metadataFrom(parameters) + const preferredIdentity = + metadata.userId || metadata.organizationId || parameters.get('email') || undefined + if (preferredIdentity) return preferredIdentity + + return [...parameters.entries()] + .sort(([leftKey, leftValue], [rightKey, rightValue]) => + `${leftKey}\u0000${leftValue}`.localeCompare(`${rightKey}\u0000${rightValue}`) + ) + .map(([key, value]) => `${key}=${value}`) + .join('&') +} + +function createDeterministicCustomer(parameters: URLSearchParams): FakeCustomer { + const digest = createHash('sha256').update(stableCustomerIdentity(parameters)).digest('hex') + return { + id: `cus_e2e_${digest.slice(0, 24)}`, + object: 'customer', + created: 1_700_000_000 + (Number.parseInt(digest.slice(0, 6), 16) % 1_000_000), + email: parameters.get('email'), + livemode: false, + metadata: metadataFrom(parameters), + name: parameters.get('name'), + } +} + +function unescapeSearchValue(value: string): string { + return value.replace(/\\(["\\])/g, '$1') +} + +function parseSupportedCustomerSearchEmail(query: string): string { + const match = /^email:"((?:\\.|[^"])*)" AND -metadata\["customerType"\]:"organization"$/.exec( + query + ) + if (!match) { + throw new RequestBodyError( + `Stripe fake does not implement customer search query: ${query}`, + 501 + ) + } + return unescapeSearchValue(match[1]) +} + +function parseLimit(parameters: URLSearchParams): number { + const rawLimit = parameters.get('limit') + if (rawLimit === null) return 10 + + const limit = Number(rawLimit) + if (!Number.isInteger(limit) || limit < 1 || limit > 100) { + throw new RequestBodyError('Stripe fake requires limit to be an integer from 1 to 100', 400) + } + return limit +} + +function validateTestApiKey(apiKey: string): void { + if (!apiKey.startsWith('sk_test_') || apiKey.length === 'sk_test_'.length) { + throw new Error('Stripe fake apiKey must be a non-empty sk_test_ secret key') + } +} + +function validateServerOptions(options: StripeFakeServerOptions): void { + validateTestApiKey(options.apiKey) + if ( + options.port !== undefined && + (!Number.isInteger(options.port) || options.port < 0 || options.port > 65535) + ) { + throw new Error('Stripe fake port must be an integer from 0 to 65535') + } + if ( + options.maxBodyBytes !== undefined && + (!Number.isInteger(options.maxBodyBytes) || options.maxBodyBytes < 1) + ) { + throw new Error('Stripe fake maxBodyBytes must be a positive integer') + } +} + +/** + * Creates a loopback-only Stripe fake. The returned server is inert until start() + * is called, allowing an orchestrator to own lifecycle and failure handling. + */ +export function createStripeFakeServer(options: StripeFakeServerOptions): StripeFakeServer { + validateServerOptions(options) + + const hostname = options.hostname ?? '127.0.0.1' + const port = options.port ?? 0 + const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES + const expectedAuthorization = `Bearer ${options.apiKey}` + const records: StripeFakeRequestRecord[] = [] + const customers = new Map() + let sequence = 0 + let baseUrl: string | null = null + + const reset = (): void => { + records.length = 0 + customers.clear() + sequence = 0 + } + + const handleControlRequest = ( + request: IncomingMessage, + response: ServerResponse, + method: string, + path: string + ): boolean => { + if (method === 'GET' && path === STRIPE_FAKE_ENDPOINTS.health) { + sendJson(response, 200, { status: 'ok', service: 'stripe-fake' }) + return true + } + + const isRequestLog = method === 'GET' && path === STRIPE_FAKE_ENDPOINTS.requestLog + const isReset = method === 'POST' && path === STRIPE_FAKE_ENDPOINTS.reset + if (!isRequestLog && !isReset) return false + + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { + sendStripeError( + response, + 401, + { + type: 'invalid_request_error', + code: 'api_key_invalid', + message: 'Invalid test Bearer authorization for Stripe fake control endpoint.', + }, + 'req_e2e_control' + ) + return true + } + + if (isRequestLog) { + sendJson(response, 200, { requests: cloneRequestLog(records) }) + } else { + reset() + sendJson(response, 200, { reset: true }) + } + return true + } + + const handleStripeRequest = async ( + request: IncomingMessage, + response: ServerResponse, + method: string, + url: URL + ): Promise => { + sequence += 1 + const requestId = `req_e2e_${String(sequence).padStart(6, '0')}` + const expected = isExpectedStripeRequest(method, url.pathname) + let formBody: URLSearchParams | null = null + + try { + if (method !== 'GET' && method !== 'HEAD') { + const rawBody = await readRequestBody(request, maxBodyBytes) + if (rawBody) { + if (request.headers['content-type']?.startsWith(FORM_CONTENT_TYPE)) { + formBody = new URLSearchParams(rawBody) + } + } + } + } catch (error) { + records.push({ + sequence, + method, + path: url.pathname, + unexpected: !expected, + }) + const bodyError = + error instanceof RequestBodyError + ? error + : new RequestBodyError('Unable to read Stripe fake request body', 400) + sendStripeError( + response, + bodyError.status, + { + type: 'invalid_request_error', + code: 'invalid_request_body', + message: bodyError.message, + }, + requestId + ) + return + } + + records.push({ + sequence, + method, + path: url.pathname, + unexpected: !expected, + }) + + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { + sendStripeError( + response, + 401, + { + type: 'invalid_request_error', + code: 'api_key_invalid', + message: 'Invalid test Bearer authorization for Stripe fake.', + }, + requestId + ) + return + } + + if (!expected) { + sendStripeError( + response, + 501, + { + type: 'api_error', + code: 'stripe_fake_unimplemented', + message: `Stripe fake does not implement ${method} ${url.pathname}.`, + }, + requestId + ) + return + } + + try { + if ((method === 'GET' || method === 'POST') && url.pathname === '/v1/customers/search') { + const parameters = + method === 'GET' ? url.searchParams : (formBody ?? parseFormBody(request, '')) + const query = parameters.get('query') + if (!query) throw new RequestBodyError('Stripe fake customer search requires query', 400) + const email = parseSupportedCustomerSearchEmail(query) + + const data = [...customers.values()] + .filter( + (customer) => + customer.email === email && customer.metadata.customerType !== 'organization' + ) + .slice(0, parseLimit(parameters)) + sendJson( + response, + 200, + { + object: 'search_result', + data, + has_more: false, + next_page: null, + url: '/v1/customers/search', + }, + requestId + ) + return + } + + if (method === 'GET' && url.pathname === '/v1/customers') { + const email = url.searchParams.get('email') + const data = [...customers.values()] + .filter((customer) => email === null || customer.email === email) + .slice(0, parseLimit(url.searchParams)) + sendJson( + response, + 200, + { + object: 'list', + data, + has_more: false, + url: '/v1/customers', + }, + requestId + ) + return + } + + if (method === 'GET' && url.pathname.startsWith('/v1/customers/')) { + const customerId = decodeURIComponent(url.pathname.slice('/v1/customers/'.length)) + const customer = customers.get(customerId) + if (!customer) { + sendStripeError( + response, + 404, + { + type: 'invalid_request_error', + code: 'resource_missing', + message: `No such customer: ${customerId}`, + }, + requestId + ) + return + } + sendJson(response, 200, customer, requestId) + return + } + + const parameters = formBody ?? parseFormBody(request, '') + const candidate = createDeterministicCustomer(parameters) + const customer = customers.get(candidate.id) ?? candidate + customers.set(customer.id, customer) + sendJson(response, 200, customer, requestId) + } catch (error) { + const bodyError = + error instanceof RequestBodyError + ? error + : new RequestBodyError('Stripe fake rejected malformed request parameters', 400) + if (bodyError.status === 501) { + const recorded = records.at(-1) + if (recorded) recorded.unexpected = true + } + sendStripeError( + response, + bodyError.status, + { + type: 'invalid_request_error', + code: 'invalid_request', + message: bodyError.message, + }, + requestId + ) + } + } + + const server: Server = createServer((request, response) => { + const method = request.method?.toUpperCase() ?? 'UNKNOWN' + const url = new URL(request.url ?? '/', 'http://stripe-fake.invalid') + if (handleControlRequest(request, response, method, url.pathname)) return + + void handleStripeRequest(request, response, method, url).catch(() => { + if (!response.headersSent) { + sendStripeError( + response, + 500, + { + type: 'api_error', + code: 'stripe_fake_internal_error', + message: 'Stripe fake encountered an internal error.', + }, + 'req_e2e_internal' + ) + } else { + response.destroy() + } + }) + }) + + return { + get baseUrl() { + return baseUrl + }, + get requestLog() { + return cloneRequestLog(records) + }, + async start() { + if (baseUrl) return baseUrl + + await new Promise((resolve, reject) => { + const handleError = (error: Error) => { + server.off('listening', handleListening) + reject(error) + } + const handleListening = () => { + server.off('error', handleError) + resolve() + } + server.once('error', handleError) + server.once('listening', handleListening) + server.listen(port, hostname) + }) + + const address = server.address() as AddressInfo + const urlHostname = address.family === 'IPv6' ? `[${address.address}]` : address.address + baseUrl = `http://${urlHostname}:${address.port}` + return baseUrl + }, + async stop() { + if (!server.listening) { + baseUrl = null + return + } + + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error) + else resolve() + }) + server.closeIdleConnections() + }) + baseUrl = null + }, + reset, + } +} + +/** Starts a Stripe fake in one call for orchestration code. */ +export async function startStripeFakeServer( + options: StripeFakeServerOptions +): Promise { + const server = createStripeFakeServer(options) + await server.start() + return server +} diff --git a/apps/sim/e2e/foundation/provider-isolation.spec.ts b/apps/sim/e2e/foundation/provider-isolation.spec.ts new file mode 100644 index 00000000000..f31c7a62644 --- /dev/null +++ b/apps/sim/e2e/foundation/provider-isolation.spec.ts @@ -0,0 +1,7 @@ +import { expect, test } from '@playwright/test' + +test('blacklisted local providers skip outbound model discovery', async ({ request }) => { + const response = await request.get('/api/providers/ollama/models') + expect(response.status()).toBe(200) + expect(await response.json()).toEqual({ models: [] }) +}) diff --git a/apps/sim/e2e/foundation/runtime.spec.ts b/apps/sim/e2e/foundation/runtime.spec.ts new file mode 100644 index 00000000000..0e49d712bb7 --- /dev/null +++ b/apps/sim/e2e/foundation/runtime.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from '@playwright/test' + +test('Playwright workers run on Node 22', () => { + expect(process.release.name).toBe('node') + expect(Number(process.versions.node.split('.')[0])).toBe(22) + expect(process.execPath.toLowerCase()).not.toContain('bun') + expect(process.env.NODE_OPTIONS).toContain('--dns-result-order=ipv4first') +}) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts new file mode 100644 index 00000000000..261180d16ed --- /dev/null +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -0,0 +1,178 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:net' +import os from 'node:os' +import path from 'node:path' +import { loadEnvConfig } from '@next/env' +import { expect, test } from '@playwright/test' +import { parseRunOptions } from '../scripts/options' +import { + assertLoopbackPostgresUrl, + assertSafeDatabaseName, + buildRunDatabaseUrl, +} from '../support/database' +import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' +import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' +import { + assertPortAvailable, + spawnManagedProcess, + waitForManagedProcessReady, +} from '../support/process' +import { parseProcessGroupIds } from '../support/signal-cleanup' + +test.describe('foundation safety guards', () => { + test('discovers env keys without leaking values and shadows unknown keys', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-env-')) + try { + writeFileSync( + path.join(directory, '.env'), + 'OPENAI_API_KEY=do-not-leak-this\nSAFE_VALUE=from-file\n' + ) + expect(discoverEnvFileKeys(directory)).toEqual(['OPENAI_API_KEY', 'SAFE_VALUE']) + + const result = buildChildEnvironment({ + values: { REQUIRED_VALUE: 'configured' }, + required: ['REQUIRED_VALUE'], + allowedSensitiveKeys: new Set(), + envDirectory: directory, + }) + expect(result.env.OPENAI_API_KEY).toBe('') + expect(result.env.SAFE_VALUE).toBe('') + expect(JSON.stringify(result)).not.toContain('do-not-leak-this') + expect(JSON.stringify(result)).not.toContain('from-file') + } finally { + rmSync(directory, { recursive: true, force: true }) + } + }) + + test('@next/env preserves non-empty and empty shadow values over an env file', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-next-env-')) + const key = `SIM_E2E_CANARY_${Date.now()}` + const emptyKey = `${key}_EMPTY` + try { + writeFileSync(path.join(directory, '.env'), `${key}=file-value\n${emptyKey}=must-not-load\n`) + process.env[key] = 'shadowed-value' + process.env[emptyKey] = '' + loadEnvConfig(directory, false, console, true) + expect(process.env[key]).toBe('shadowed-value') + expect(process.env[emptyKey]).toBe('') + } finally { + delete process.env[key] + delete process.env[emptyKey] + rmSync(directory, { recursive: true, force: true }) + } + }) + + test('database guards reject shared or remote targets', () => { + expect(() => assertSafeDatabaseName('simstudio')).toThrow() + expect(() => assertSafeDatabaseName('sim_e2e_valid_run')).not.toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@example.com/postgres') + ).toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@localhost/postgres') + ).toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@127.0.0.1/postgres?sslmode=disable') + ).toThrow() + expect( + buildRunDatabaseUrl( + 'postgresql://postgres:postgres@127.0.0.1:5432/postgres', + 'sim_e2e_valid_run' + ) + ).toContain('/sim_e2e_valid_run') + }) + + test('loopback detection rejects public addresses', () => { + expect(isLoopbackAddress('127.0.0.1')).toBe(true) + expect(isLoopbackAddress('127.10.20.30')).toBe(true) + expect(isLoopbackAddress('::1')).toBe(true) + expect(isLoopbackAddress('8.8.8.8')).toBe(false) + expect(areValidE2eHostAddresses(['127.0.0.1'])).toBe(true) + expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(true) + expect(areValidE2eHostAddresses(['::1'])).toBe(false) + expect(areValidE2eHostAddresses(['127.0.0.2'])).toBe(false) + }) + + test('sharding is limited to the navigation project', () => { + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-navigation', '--shard=1/2']) + ).not.toThrow() + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-workflows', '--shard=1/2']) + ).toThrow(/coupled workflows must remain unsharded/) + }) + + test('Playwright CLI arguments cannot override orchestration invariants', () => { + expect(() => parseRunOptions(['--workers=8'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--config=other.config.ts'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--project', 'hosted-billing-chromium-navigation'])).toThrow( + /canonical/ + ) + expect(() => parseRunOptions(['--pass-with-no-tests'])).toThrow(/cannot override/) + }) + + test('empty signal cleanup groups never become PID zero', () => { + expect(parseProcessGroupIds(undefined)).toEqual([]) + expect(parseProcessGroupIds('')).toEqual([]) + expect(parseProcessGroupIds(' 123, 0, -1, nope, 456 ')).toEqual([123, 456]) + }) + + test('port preflight rejects an existing listener', async () => { + const server = createServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + try { + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Expected TCP listener') + await expect(assertPortAvailable(address.port)).rejects.toThrow(/to be free/) + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ) + } + }) + + test('spawn failures finalize without hanging cleanup', async () => { + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-spawn-')) + try { + const managed = spawnManagedProcess({ + name: 'missing-command', + command: path.join(logsDirectory, 'does-not-exist'), + args: [], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + const completion = await managed.completion + expect(completion.error).toBeTruthy() + await expect(managed.stop()).resolves.toBeUndefined() + } finally { + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) + + test('process exit after readiness does not create an unhandled rejection', async () => { + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-ready-')) + let unhandled: unknown + const onUnhandled = (error: unknown) => { + unhandled = error + } + process.once('unhandledRejection', onUnhandled) + try { + const managed = spawnManagedProcess({ + name: 'ready-process', + command: 'sleep', + args: ['10'], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + await waitForManagedProcessReady(managed, async () => {}) + await managed.stop() + await new Promise((resolve) => setImmediate(resolve)) + expect(unhandled).toBeUndefined() + } finally { + process.off('unhandledRejection', onUnhandled) + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/sim/e2e/foundation/stripe-fake.spec.ts b/apps/sim/e2e/foundation/stripe-fake.spec.ts new file mode 100644 index 00000000000..8eec71d6870 --- /dev/null +++ b/apps/sim/e2e/foundation/stripe-fake.spec.ts @@ -0,0 +1,55 @@ +import { expect, test } from '@playwright/test' +import { startStripeFakeServer } from '../fakes/stripe/server' + +test('Stripe fake records allowlisted calls and rejects unknown routes', async () => { + const apiKey = 'sk_test_foundation_fake_spec' + const fake = await startStripeFakeServer({ apiKey }) + try { + const baseUrl = fake.baseUrl + expect(baseUrl).toBeTruthy() + + const health = await fetch(`${baseUrl}/health`) + expect(health.status).toBe(200) + + const customer = await fetch(`${baseUrl}/v1/customers`, { + method: 'POST', + headers: { + authorization: `Bearer ${apiKey}`, + 'content-type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + email: 'foundation@example.com', + 'metadata[userId]': 'user-foundation', + }), + }) + expect(customer.status).toBe(200) + expect((await customer.json()) as { id: string }).toMatchObject({ + id: expect.stringMatching(/^cus_e2e_/), + }) + + const search = await fetch( + `${baseUrl}/v1/customers/search?${new URLSearchParams({ + query: 'email:"foundation@example.com" AND -metadata["customerType"]:"organization"', + limit: '1', + })}`, + { headers: { authorization: `Bearer ${apiKey}` } } + ) + expect(search.status).toBe(200) + + const unsupportedSearch = await fetch( + `${baseUrl}/v1/customers/search?${new URLSearchParams({ + query: 'name:"Foundation"', + })}`, + { headers: { authorization: `Bearer ${apiKey}` } } + ) + expect(unsupportedSearch.status).toBe(501) + + const unknown = await fetch(`${baseUrl}/v1/invoices`, { + headers: { authorization: `Bearer ${apiKey}` }, + }) + expect(unknown.status).toBe(501) + expect(fake.requestLog.some(({ unexpected }) => unexpected)).toBe(true) + } finally { + await fake.stop() + } +}) diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts new file mode 100644 index 00000000000..fb8bdde3e0f --- /dev/null +++ b/apps/sim/e2e/scripts/options.ts @@ -0,0 +1,64 @@ +const NAVIGATION_PROJECT = 'hosted-billing-chromium-navigation' +const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' +const FORBIDDEN_OPTIONS = [ + '--config', + '-c', + '--workers', + '-j', + '--retries', + '--fully-parallel', + '--pass-with-no-tests', + '--list', +] as const + +export interface E2eRunOptions { + playwrightArgs: string[] +} + +export function parseRunOptions(argv: string[]): E2eRunOptions { + const normalizedArgs = argv[0] === '--' ? argv.slice(1) : [...argv] + if (normalizedArgs.includes('--skip-build')) { + throw new Error( + '--skip-build remains disabled until the planned profile/source-keyed build reuse experiment proves it safe' + ) + } + for (const option of FORBIDDEN_OPTIONS) { + if (hasOption(normalizedArgs, option)) { + throw new Error(`${option} cannot override E2E orchestration invariants`) + } + } + if (normalizedArgs.includes('--project')) { + throw new Error('Use canonical --project= syntax') + } + if (normalizedArgs.includes('--shard')) { + throw new Error('Use canonical --shard= syntax') + } + const playwrightArgs = [...normalizedArgs] + const projects = getEqualsOptionValues(playwrightArgs, '--project') + const unknownProject = projects.find( + (project) => project !== NAVIGATION_PROJECT && project !== WORKFLOWS_PROJECT + ) + if (unknownProject) throw new Error(`Unknown E2E Playwright project: ${unknownProject}`) + const hasShard = hasOption(playwrightArgs, '--shard') + + if ( + hasShard && + (projects.length === 0 || + projects.includes(WORKFLOWS_PROJECT) || + projects.some((project) => project !== NAVIGATION_PROJECT)) + ) { + throw new Error( + `--shard is supported only with --project=${NAVIGATION_PROJECT}; coupled workflows must remain unsharded` + ) + } + + return { playwrightArgs } +} + +function hasOption(args: string[], name: string): boolean { + return args.some((arg) => arg === name || arg.startsWith(`${name}=`)) +} + +function getEqualsOptionValues(args: string[], name: string): string[] { + return args.filter((arg) => arg.startsWith(`${name}=`)).map((arg) => arg.slice(name.length + 1)) +} diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts new file mode 100644 index 00000000000..9a1f102e921 --- /dev/null +++ b/apps/sim/e2e/scripts/run.ts @@ -0,0 +1,355 @@ +import { spawn, spawnSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' +import path from 'node:path' +import { type StripeFakeServer, startStripeFakeServer } from '../fakes/stripe/server' +import { + buildRunDatabaseUrl, + createRunDatabase, + createRunDatabaseName, + dropRunDatabase, + type RunDatabase, +} from '../support/database' +import { createHostedBillingProfile, E2E_ORIGIN, E2E_PROFILE } from '../support/deployment-profile' +import { + assertNoForbiddenProviderInitialization, + assertNoForbiddenProviderTraffic, +} from '../support/diagnostics' +import { E2E_OS_PASSTHROUGH_KEYS, formatRedactedEnvironmentSummary } from '../support/env' +import { assertE2eHostResolvesToLoopback } from '../support/hosts' +import { getRunDirectory, SIM_APP_DIR } from '../support/paths' +import { + assertAdminApiBoundary, + type FoundationProvisioningResult, + inspectFoundationUsers, +} from '../support/probes' +import { + assertPortAvailable, + getActiveManagedProcessGroupIds, + type ManagedProcess, + stopAllManagedProcesses, +} from '../support/process' +import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' +import { parseRunOptions } from './options' + +const DEFAULT_ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' +const STRIPE_TEST_KEY = 'sk_test_sim_e2e_foundation' + +async function main(): Promise { + const options = parseRunOptions(process.argv.slice(2)) + const runId = createRunId() + const runDirectory = getRunDirectory(runId) + const logsDirectory = path.join(runDirectory, 'logs') + const storageStateDirectory = path.join(runDirectory, 'auth') + const markerDirectory = path.join(runDirectory, 'markers') + const homeDirectory = path.join(runDirectory, 'home') + mkdirSync(logsDirectory, { recursive: true }) + mkdirSync(storageStateDirectory, { recursive: true }) + mkdirSync(markerDirectory, { recursive: true }) + mkdirSync(homeDirectory, { recursive: true }) + + const nodeExecutable = resolveNode22() + const bunExecutable = resolveBunExecutable() + const adminDatabaseUrl = process.env.E2E_PG_ADMIN_URL ?? DEFAULT_ADMIN_DATABASE_URL + + let runDatabase: RunDatabase | null = null + let databaseCreationComplete = false + let stripeFake: StripeFakeServer | null = null + let realtime: ManagedProcess | null = null + let app: ManagedProcess | null = null + let cleanupPromise: Promise | null = null + let failed = false + + const cleanup = (): Promise => { + if (cleanupPromise) return cleanupPromise + cleanupPromise = (async () => { + const failures: unknown[] = [] + try { + await stopAllManagedProcesses() + } catch (error) { + failures.push(error) + } + + if (stripeFake) { + try { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } catch (error) { + failures.push(error) + } + try { + await stripeFake.stop() + } catch (error) { + failures.push(error) + } + } + + for (const sensitiveDirectory of [storageStateDirectory, homeDirectory]) { + try { + rmSync(sensitiveDirectory, { recursive: true, force: true }) + } catch (error) { + failures.push(error) + } + } + + try { + if (runDatabase) await dropRunDatabase(adminDatabaseUrl, runDatabase.name) + } catch (error) { + failures.push(error) + } + + if (failures.length > 0) { + throw new AggregateError(failures, 'One or more E2E cleanup stages failed') + } + })() + return cleanupPromise + } + + const handleSignal = (signal: NodeJS.Signals): void => { + failed = true + const exitCode = signal === 'SIGINT' ? 130 : 143 + process.exitCode = exitCode + console.error(`Received ${signal}; cleaning up the E2E run`) + if (stripeFake) { + try { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } catch {} + } + if (runDatabase) { + const cleanupLogFd = openSync(path.join(logsDirectory, 'signal-cleanup.log'), 'a') + const cleanupProcess = spawn( + bunExecutable, + ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/signal-cleanup.ts')], + { + cwd: SIM_APP_DIR, + detached: true, + env: { + NODE_ENV: 'test', + PATH: process.env.PATH ?? '', + HOME: homeDirectory, + E2E_PG_ADMIN_URL: adminDatabaseUrl, + E2E_DATABASE_NAME: runDatabase.name, + E2E_DATABASE_CREATION_COMPLETE: String(databaseCreationComplete), + E2E_CLEANUP_PROCESS_GROUPS: getActiveManagedProcessGroupIds().join(','), + E2E_CLEANUP_DIRECTORIES: JSON.stringify([storageStateDirectory, homeDirectory]), + }, + stdio: ['ignore', cleanupLogFd, cleanupLogFd], + } + ) + cleanupProcess.unref() + closeSync(cleanupLogFd) + console.error(`Detached E2E cleanup supervisor started as PID ${cleanupProcess.pid}`) + } + process.exit(exitCode) + } + // `once` is intentional: a second signal uses the OS default force termination. + process.once('SIGINT', handleSignal) + process.once('SIGTERM', handleSignal) + + try { + const hostAddresses = await assertE2eHostResolvesToLoopback() + console.info(`E2E host resolved to loopback: ${hostAddresses.join(', ')}`) + await Promise.all([assertPortAvailable(3000), assertPortAvailable(3002)]) + + const runDatabaseName = createRunDatabaseName(runId) + runDatabase = { + name: runDatabaseName, + url: buildRunDatabaseUrl(adminDatabaseUrl, runDatabaseName), + } + await createRunDatabase(adminDatabaseUrl, runDatabaseName) + databaseCreationComplete = true + stripeFake = await startStripeFakeServer({ + apiKey: STRIPE_TEST_KEY, + hostname: '127.0.0.1', + port: 0, + }) + if (!stripeFake.baseUrl) throw new Error('Stripe fake did not expose a base URL') + + const profile = createHostedBillingProfile({ + runId, + databaseUrl: runDatabase.url, + stripeApiBaseUrl: stripeFake.baseUrl, + homeDirectory, + playwrightBrowsersPath: resolvePlaywrightBrowsersPath(), + ci: process.env.CI === 'true', + }) + console.info(formatRedactedEnvironmentSummary(profile.id, profile.childEnvironment)) + + const stackOptions = { + bunExecutable, + nodeExecutable, + env: profile.childEnvironment.env, + logsDirectory, + } + + await runMigrations(stackOptions) + await buildApp(stackOptions) + realtime = await startRealtime(stackOptions) + app = await startApp(stackOptions) + await assertAdminApiBoundary( + 'http://127.0.0.1:3000', + profile.childEnvironment.env.ADMIN_API_KEY + ) + assertNoForbiddenProviderInitialization([app.logPath, realtime.logPath]) + + const playwrightEnvironment = createPlaywrightEnvironment( + profile.childEnvironment.env, + runId, + storageStateDirectory, + markerDirectory + ) + await runPlaywright( + { + nodeExecutable, + env: playwrightEnvironment, + logsDirectory, + }, + options.playwrightArgs + ) + const provisioning = await inspectFoundationUsers(runDatabase.url, runId) + assertAuthenticatedSmokeEffectsIfPresent( + stripeFake, + provisioning, + hasFoundationCompletionMarker(markerDirectory) + ) + } catch (error) { + failed = true + console.error(error) + process.exitCode = 1 + } finally { + try { + assertNoForbiddenProviderTraffic( + [app?.logPath, realtime?.logPath].filter((value): value is string => Boolean(value)) + ) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + try { + await cleanup() + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + process.off('SIGINT', handleSignal) + process.off('SIGTERM', handleSignal) + console.info(`E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}`) + } +} + +function createRunId(): string { + const timestamp = new Date().toISOString().replace(/\D/g, '').slice(0, 14) + const random = randomUUID().replace(/-/g, '').slice(0, 10) + return `${timestamp}_${random}` +} + +function resolveBunExecutable(): string { + if (!process.versions.bun) { + throw new Error('The E2E orchestrator must be started with Bun') + } + return process.execPath +} + +function resolveNode22(): string { + const executable = process.env.E2E_NODE_BINARY ?? 'node' + const result = spawnSync(executable, ['--version'], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (result.status !== 0) { + throw new Error(`Unable to execute Node for Playwright: ${result.stderr}`) + } + const version = result.stdout.trim() + if (!/^v22\./.test(version)) { + throw new Error(`Playwright E2E requires Node 22, received ${version}`) + } + return executable +} + +function resolvePlaywrightBrowsersPath(): string { + if (process.env.PLAYWRIGHT_BROWSERS_PATH) return process.env.PLAYWRIGHT_BROWSERS_PATH + const parentHome = process.env.HOME + if (!parentHome) throw new Error('HOME is required to locate installed Playwright browsers') + return process.platform === 'darwin' + ? path.join(parentHome, 'Library/Caches/ms-playwright') + : path.join(parentHome, '.cache/ms-playwright') +} + +function createPlaywrightEnvironment( + stackEnvironment: Record, + runId: string, + storageStateDirectory: string, + markerDirectory: string +): Record { + const keys = [...E2E_OS_PASSTHROUGH_KEYS, 'HOME', 'NODE_OPTIONS', 'PLAYWRIGHT_BROWSERS_PATH'] + const env: Record = {} + for (const key of keys) { + const value = stackEnvironment[key] + if (value !== undefined) env[key] = value + } + return { + ...env, + E2E_PROFILE, + E2E_ORCHESTRATED: '1', + E2E_RUN_ID: runId, + E2E_BASE_URL: E2E_ORIGIN, + E2E_STORAGE_STATE_DIR: storageStateDirectory, + E2E_MARKER_DIR: markerDirectory, + } +} + +function assertAuthenticatedSmokeEffectsIfPresent( + stripeFake: StripeFakeServer, + provisioning: FoundationProvisioningResult, + completed: boolean +): void { + const created = stripeFake.requestLog.some( + ({ method, path }) => method === 'POST' && path === '/v1/customers' + ) + const unexpected = stripeFake.requestLog.filter((request) => request.unexpected) + if (unexpected.length > 0) { + throw new Error( + `Stripe fake received unsupported requests: ${unexpected + .map(({ method, path }) => `${method} ${path}`) + .join(', ')}` + ) + } + if (!completed) { + console.info('Authenticated foundation smoke was filtered out; skipping its post-run probes') + return + } + if (!created) throw new Error('Billing-enabled signup did not create a fake Stripe customer') + if (provisioning.count === 0) { + throw new Error('Authenticated smoke did not create a foundation user') + } + if (!provisioning.allHaveStripeCustomers) { + throw new Error('A foundation user was not persisted with its fake Stripe customer') + } + if (!provisioning.allHaveStats) { + throw new Error('A foundation user was not initialized with user_stats') + } +} + +function hasFoundationCompletionMarker(markerDirectory: string): boolean { + return ( + existsSync(markerDirectory) && + readdirSync(markerDirectory).some((name) => name.startsWith('foundation-authenticated-')) + ) +} + +await main() diff --git a/apps/sim/e2e/scripts/signal-cleanup.ts b/apps/sim/e2e/scripts/signal-cleanup.ts new file mode 100644 index 00000000000..5d24f82df5a --- /dev/null +++ b/apps/sim/e2e/scripts/signal-cleanup.ts @@ -0,0 +1,71 @@ +import { rmSync } from 'node:fs' +import { sleep } from '@sim/utils/helpers' +import { dropRunDatabase, dropRunDatabaseWithRetries } from '../support/database' +import { parseProcessGroupIds } from '../support/signal-cleanup' + +const adminUrl = process.env.E2E_PG_ADMIN_URL +const databaseName = process.env.E2E_DATABASE_NAME +const processGroupIds = parseProcessGroupIds(process.env.E2E_CLEANUP_PROCESS_GROUPS) +const sensitiveDirectories = JSON.parse(process.env.E2E_CLEANUP_DIRECTORIES ?? '[]') as string[] +const databaseCreationComplete = process.env.E2E_DATABASE_CREATION_COMPLETE === 'true' + +if (!adminUrl || !databaseName) { + console.error('Signal cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') + process.exit(1) +} + +const failures: unknown[] = [] +signalProcessGroups(processGroupIds, 'SIGTERM', failures) +await sleep(500) + +try { + if (databaseCreationComplete) { + await dropRunDatabase(adminUrl, databaseName) + } else { + await dropRunDatabaseWithRetries(adminUrl, databaseName) + } +} catch (error) { + failures.push(error) +} + +signalProcessGroups(processGroupIds, 'SIGKILL', failures) +await sleep(250) + +for (const directory of sensitiveDirectories) { + try { + rmSync(directory, { recursive: true, force: true }) + } catch (error) { + failures.push(error) + } +} + +for (const error of failures) console.error(error) +process.exit(failures.length > 0 ? 1 : 0) + +function signalProcessGroups( + groupIds: number[], + signal: NodeJS.Signals, + failures: unknown[] +): void { + for (const groupId of groupIds) { + if (!Number.isInteger(groupId) || groupId <= 0) continue + try { + if (process.platform !== 'win32') process.kill(-groupId, signal) + else process.kill(groupId, signal) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ESRCH') continue + if (code === 'EPERM' && process.platform !== 'win32') { + try { + process.kill(groupId, signal) + } catch (fallbackError) { + if ((fallbackError as NodeJS.ErrnoException).code !== 'ESRCH') { + failures.push(fallbackError) + } + } + } else { + failures.push(error) + } + } + } +} diff --git a/apps/sim/e2e/settings/smoke/authenticated.spec.ts b/apps/sim/e2e/settings/smoke/authenticated.spec.ts new file mode 100644 index 00000000000..a64c7e9a1db --- /dev/null +++ b/apps/sim/e2e/settings/smoke/authenticated.spec.ts @@ -0,0 +1,74 @@ +import { createHash } from 'node:crypto' +import { writeFileSync } from 'node:fs' +import path from 'node:path' +import { expect, test } from '@playwright/test' + +test('billing-enabled signup, login, and settings use real Sim boundaries', async ({ + browser, + page, +}, testInfo) => { + test.slow() + + const runId = process.env.E2E_RUN_ID ?? `${Date.now()}` + const testIdentity = createHash('sha256') + .update(`${runId}:${testInfo.project.name}:${testInfo.workerIndex}:${testInfo.repeatEachIndex}`) + .digest('hex') + .slice(0, 16) + const email = `e2e-foundation-${runId}-${testIdentity}@example.com` + const password = 'E2eFoundation1!' + const storageStatePath = path.join(requiredEnv('E2E_STORAGE_STATE_DIR'), `${testIdentity}.json`) + + await page.goto('/signup') + await expect(page.getByRole('heading', { name: 'Create an account' })).toBeVisible() + await page.getByLabel('Full name').fill('Playwright Foundation') + await page.getByLabel('Email').fill(email) + await page.getByRole('textbox', { name: 'Password' }).fill(password) + await page.getByRole('button', { name: 'Create account' }).click() + + await expect(page).toHaveURL(/\/workspace\/[^/]+\/(?:home|w(?:\/|$))/, { + timeout: 60_000, + }) + const workspaceId = new URL(page.url()).pathname.split('/')[2] + expect(workspaceId).toBeTruthy() + + const settingsPath = `/workspace/${workspaceId}/settings/general` + await page.goto(settingsPath) + await assertGeneralSettings(page) + + await page.getByRole('button', { name: 'Sign out' }).click() + await expect(page).toHaveURL(/\/login\?fromLogout=true$/) + + await page.getByLabel('Email').fill(email) + await page.getByRole('textbox', { name: 'Password' }).fill(password) + await page.getByRole('button', { name: 'Sign in' }).click() + await expect(page).toHaveURL(/\/workspace(?:\/|$)/) + + await page.context().storageState({ path: storageStatePath }) + const restoredContext = await browser.newContext({ storageState: storageStatePath }) + try { + const restoredPage = await restoredContext.newPage() + await restoredPage.goto(`${requiredEnv('E2E_BASE_URL')}${settingsPath}`) + await assertGeneralSettings(restoredPage) + } finally { + await restoredContext.close() + } + writeFileSync( + path.join(requiredEnv('E2E_MARKER_DIR'), `foundation-authenticated-${testIdentity}.json`), + JSON.stringify({ runId, testIdentity }) + ) +}) + +async function assertGeneralSettings(page: import('@playwright/test').Page): Promise { + await expect(page).toHaveURL(/\/settings\/general$/) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + await expect( + page.getByText('Manage your profile, appearance, and preferences.', { exact: true }) + ).toBeVisible() + await expect(page.getByText('Profile', { exact: true })).toBeVisible() +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing required Playwright environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts b/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts new file mode 100644 index 00000000000..ab3aaca910f --- /dev/null +++ b/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from '@playwright/test' + +test('protected settings redirect unauthenticated users to login', async ({ page }) => { + await page.goto('/account/settings/general') + + await expect(page).toHaveURL(/\/login(?:\?|$)/) + await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible() +}) diff --git a/apps/sim/e2e/support/database.ts b/apps/sim/e2e/support/database.ts new file mode 100644 index 00000000000..8284e707791 --- /dev/null +++ b/apps/sim/e2e/support/database.ts @@ -0,0 +1,120 @@ +import { sleep } from '@sim/utils/helpers' +import postgres from 'postgres' +import { isLoopbackAddress } from './hosts' + +const DATABASE_NAME_PATTERN = /^sim_e2e_[a-z0-9_]+$/ +const PROTECTED_DATABASES = new Set(['postgres', 'simstudio', 'template0', 'template1']) + +export interface RunDatabase { + name: string + url: string +} + +export function createRunDatabaseName(runId: string): string { + const suffix = runId + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_+|_+$/g, '') + const name = `sim_e2e_${suffix}` + assertSafeDatabaseName(name) + return name +} + +export function assertSafeDatabaseName(name: string): void { + if (!DATABASE_NAME_PATTERN.test(name) || PROTECTED_DATABASES.has(name)) { + throw new Error(`Refusing unsafe E2E database name: ${name}`) + } +} + +export function assertLoopbackPostgresUrl(rawUrl: string): URL { + const url = new URL(rawUrl) + const hostname = url.hostname.replace(/^\[|\]$/g, '') + if (!['postgres:', 'postgresql:'].includes(url.protocol)) { + throw new Error(`E2E PostgreSQL admin URL requires postgres protocol, received ${url.protocol}`) + } + if (url.search || url.hash) { + throw new Error('E2E PostgreSQL admin URL must not contain query parameters or a fragment') + } + if (!isLoopbackAddress(hostname)) { + throw new Error(`E2E PostgreSQL admin URL must be loopback, received ${url.hostname}`) + } + return url +} + +export function buildRunDatabaseUrl(adminUrl: string, databaseName: string): string { + assertSafeDatabaseName(databaseName) + const url = assertLoopbackPostgresUrl(adminUrl) + url.pathname = `/${databaseName}` + return url.toString() +} + +export async function createRunDatabase( + adminUrl: string, + databaseName: string +): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { max: 1, connect_timeout: 10 }) + try { + await sql.unsafe(`CREATE DATABASE "${databaseName}"`) + } finally { + await sql.end() + } + return { name: databaseName, url: buildRunDatabaseUrl(adminUrl, databaseName) } +} + +export async function dropRunDatabase(adminUrl: string, databaseName: string): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { + max: 1, + connect_timeout: 10, + onnotice: () => {}, + }) + try { + await sql.unsafe(`DROP DATABASE IF EXISTS "${databaseName}" WITH (FORCE)`) + } finally { + await sql.end() + } +} + +export async function dropRunDatabaseWithRetries( + adminUrl: string, + databaseName: string, + deadlineMs = 5_000, + intervalMs = 100 +): Promise { + const deadline = Date.now() + deadlineMs + let lastError: unknown + do { + try { + await dropRunDatabase(adminUrl, databaseName) + lastError = undefined + } catch (error) { + lastError = error + } + await sleep(intervalMs) + } while (Date.now() < deadline) + if (await runDatabaseExists(adminUrl, databaseName)) { + throw ( + lastError ?? + new Error(`Guarded E2E database still exists after forced-drop retry window: ${databaseName}`) + ) + } +} + +async function runDatabaseExists(adminUrl: string, databaseName: string): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) + try { + const rows = await sql>` + SELECT EXISTS( + SELECT 1 FROM pg_database WHERE datname = ${databaseName} + ) AS "exists" + ` + return rows[0]?.exists ?? false + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts new file mode 100644 index 00000000000..badcd7bb054 --- /dev/null +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -0,0 +1,134 @@ +import { buildChildEnvironment, type ChildEnvironment } from './env' + +export const E2E_PROFILE = 'hosted-billing-chromium' +export const E2E_HOST = 'e2e.sim.ai' +export const E2E_ORIGIN = `http://${E2E_HOST}:3000` +export const E2E_SOCKET_ORIGIN = `http://${E2E_HOST}:3002` + +const REQUIRED_KEYS = [ + 'NODE_ENV', + 'NEXT_PUBLIC_APP_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'DATABASE_URL', + 'MIGRATION_DATABASE_URL', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'BILLING_ENABLED', + 'NEXT_PUBLIC_BILLING_ENABLED', + 'STRIPE_SECRET_KEY', + 'STRIPE_API_BASE_URL', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', +] as const + +const ALLOWED_SENSITIVE_KEYS = new Set([ + 'BETTER_AUTH_SECRET', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'EMAIL_PASSWORD_SIGNUP_ENABLED', + 'NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED', + 'STRIPE_SECRET_KEY', + 'STRIPE_WEBHOOK_SECRET', +]) + +export interface HostedBillingProfileOptions { + runId: string + databaseUrl: string + stripeApiBaseUrl: string + homeDirectory: string + playwrightBrowsersPath: string + ci: boolean +} + +export interface HostedBillingProfile { + id: typeof E2E_PROFILE + origin: typeof E2E_ORIGIN + childEnvironment: ChildEnvironment +} + +export function createHostedBillingProfile({ + runId, + databaseUrl, + stripeApiBaseUrl, + homeDirectory, + playwrightBrowsersPath, + ci, +}: HostedBillingProfileOptions): HostedBillingProfile { + const values: Record = { + NODE_ENV: 'production', + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192 --dns-result-order=ipv4first', + NEXT_TELEMETRY_DISABLED: '1', + HOME: homeDirectory, + XDG_CONFIG_HOME: `${homeDirectory}/xdg`, + AWS_EC2_METADATA_DISABLED: 'true', + AWS_SHARED_CREDENTIALS_FILE: '/dev/null', + AWS_CONFIG_FILE: '/dev/null', + CLOUDSDK_CONFIG: `${homeDirectory}/gcloud`, + AZURE_CONFIG_DIR: `${homeDirectory}/azure`, + PLAYWRIGHT_BROWSERS_PATH: playwrightBrowsersPath, + E2E_PROFILE, + E2E_RUN_ID: runId, + E2E_BASE_URL: E2E_ORIGIN, + NEXT_PUBLIC_APP_URL: E2E_ORIGIN, + BETTER_AUTH_URL: E2E_ORIGIN, + BETTER_AUTH_SECRET: 'e2e-better-auth-secret-at-least-32-characters-long', + DATABASE_URL: databaseUrl, + MIGRATION_DATABASE_URL: databaseUrl, + ENCRYPTION_KEY: '11'.repeat(32), + API_ENCRYPTION_KEY: '22'.repeat(32), + INTERNAL_API_SECRET: 'e2e-internal-api-secret-at-least-32-characters', + ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', + BILLING_ENABLED: 'true', + NEXT_PUBLIC_BILLING_ENABLED: 'true', + EMAIL_VERIFICATION_ENABLED: 'false', + EMAIL_PASSWORD_SIGNUP_ENABLED: 'true', + NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED: 'true', + DISABLE_REGISTRATION: 'false', + DISABLE_EMAIL_SIGNUP: 'false', + SIGNUP_MX_VALIDATION_ENABLED: 'false', + NEXT_PUBLIC_POSTHOG_ENABLED: 'false', + BLACKLISTED_PROVIDERS: 'ollama,ollama-cloud,vllm,litellm,openrouter,together,fireworks,baseten', + STRIPE_SECRET_KEY: 'sk_test_sim_e2e_foundation', + STRIPE_WEBHOOK_SECRET: 'whsec_sim_e2e_foundation', + STRIPE_FREE_PRICE_ID: 'price_e2e_free', + STRIPE_API_BASE_URL: stripeApiBaseUrl, + SOCKET_SERVER_URL: 'http://127.0.0.1:3002', + NEXT_PUBLIC_SOCKET_URL: E2E_SOCKET_ORIGIN, + CI: ci ? 'true' : 'false', + } + + validateProfileValues(values) + + return { + id: E2E_PROFILE, + origin: E2E_ORIGIN, + childEnvironment: buildChildEnvironment({ + values, + required: REQUIRED_KEYS, + allowedSensitiveKeys: ALLOWED_SENSITIVE_KEYS, + }), + } +} + +function validateProfileValues(values: Record): void { + if (values.NEXT_PUBLIC_APP_URL !== E2E_ORIGIN || values.BETTER_AUTH_URL !== E2E_ORIGIN) { + throw new Error('E2E app and Better Auth origins must exactly match') + } + if (values.BILLING_ENABLED !== values.NEXT_PUBLIC_BILLING_ENABLED) { + throw new Error('BILLING_ENABLED and NEXT_PUBLIC_BILLING_ENABLED must match') + } + if (!values.DATABASE_URL.includes('/sim_e2e_')) { + throw new Error('E2E profile requires a sim_e2e_ database') + } + const stripeUrl = new URL(values.STRIPE_API_BASE_URL) + if (stripeUrl.hostname !== '127.0.0.1') { + throw new Error('E2E Stripe API must use numeric IPv4 loopback 127.0.0.1') + } +} diff --git a/apps/sim/e2e/support/diagnostics.ts b/apps/sim/e2e/support/diagnostics.ts new file mode 100644 index 00000000000..370b3212153 --- /dev/null +++ b/apps/sim/e2e/support/diagnostics.ts @@ -0,0 +1,52 @@ +import { existsSync, readFileSync } from 'node:fs' + +const FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS = [ + /api\.stripe\.com/i, + /api\.agentmail\.to/i, + /telemetry\.simstudio\.ai/i, + /Failed to fetch Ollama models/i, +] + +const FORBIDDEN_PROVIDER_STARTUP_PATTERNS = [ + ...FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS, + /Failed to initialize .*provider/i, + /Failed to initialize .*client/i, +] + +/** + * Scan only service startup logs. Later settings tests intentionally exercise + * URL-validation denials whose error copy must not be treated as provider boot. + */ +export function assertNoForbiddenProviderInitialization(logPaths: string[]): void { + assertLogsDoNotMatch( + logPaths, + FORBIDDEN_PROVIDER_STARTUP_PATTERNS, + 'Shadowed or forbidden provider initialization was detected' + ) +} + +/** + * Re-scan after browser activity using only concrete outbound signatures. + * Intentional URL-validation errors in later workflow suites remain allowed. + */ +export function assertNoForbiddenProviderTraffic(logPaths: string[]): void { + assertLogsDoNotMatch( + logPaths, + FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS, + 'Forbidden provider traffic was detected' + ) +} + +function assertLogsDoNotMatch(logPaths: string[], patterns: RegExp[], message: string): void { + const violations: string[] = [] + for (const logPath of logPaths) { + if (!existsSync(logPath)) continue + const contents = readFileSync(logPath, 'utf8') + for (const pattern of patterns) { + if (pattern.test(contents)) violations.push(`${logPath}: ${pattern}`) + } + } + if (violations.length > 0) { + throw new Error(`${message}:\n${violations.join('\n')}`) + } +} diff --git a/apps/sim/e2e/support/env.ts b/apps/sim/e2e/support/env.ts new file mode 100644 index 00000000000..7527596b41e --- /dev/null +++ b/apps/sim/e2e/support/env.ts @@ -0,0 +1,102 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { SIM_APP_DIR } from './paths' + +const ENV_KEY_PATTERN = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/ +const SENSITIVE_KEY_PATTERN = + /(?:^|_)(?:API_?KEY|SECRET|TOKEN|PASSWORD|PRIVATE_?KEY|CLIENT_?SECRET|WEBHOOK)(?:_|$)/i + +const OS_PASSTHROUGH_KEYS = [ + 'PATH', + 'USER', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SYSTEMROOT', + 'CI', + 'GITHUB_ACTIONS', +] as const + +export interface ChildEnvironment { + env: Record + discoveredKeys: string[] + shadowedKeys: string[] +} + +export interface BuildChildEnvironmentOptions { + values: Record + required: readonly string[] + allowedSensitiveKeys: ReadonlySet + envDirectory?: string +} + +export function discoverEnvFileKeys(directory = SIM_APP_DIR): string[] { + if (!existsSync(directory)) return [] + + const keys = new Set() + const files = readdirSync(directory) + .filter((name) => name === '.env' || name.startsWith('.env.')) + .sort() + + for (const file of files) { + const contents = readFileSync(path.join(directory, file), 'utf8') + for (const line of contents.split(/\r?\n/)) { + const match = ENV_KEY_PATTERN.exec(line) + if (match) keys.add(match[1]) + } + } + + return [...keys].sort() +} + +export function buildChildEnvironment({ + values, + required, + allowedSensitiveKeys, + envDirectory = SIM_APP_DIR, +}: BuildChildEnvironmentOptions): ChildEnvironment { + const missing = required.filter((key) => !values[key]?.trim()) + if (missing.length > 0) { + throw new Error(`Missing required E2E environment values: ${missing.join(', ')}`) + } + + const env: Record = {} + for (const key of OS_PASSTHROUGH_KEYS) { + const value = process.env[key] + if (value !== undefined) env[key] = value + } + + for (const [key, value] of Object.entries(values)) { + if (SENSITIVE_KEY_PATTERN.test(key) && !allowedSensitiveKeys.has(key)) { + throw new Error(`Sensitive E2E environment key is not explicitly allowed: ${key}`) + } + env[key] = value + } + + const discoveredKeys = discoverEnvFileKeys(envDirectory) + const shadowedKeys: string[] = [] + for (const key of discoveredKeys) { + if (key in env) continue + env[key] = '' + shadowedKeys.push(key) + } + + return { env, discoveredKeys, shadowedKeys } +} + +export function formatRedactedEnvironmentSummary( + profile: string, + childEnvironment: ChildEnvironment +): string { + return [ + `E2E profile: ${profile}`, + `Allowed keys: ${Object.keys(childEnvironment.env) + .filter((key) => !childEnvironment.shadowedKeys.includes(key)) + .sort() + .join(', ')}`, + `Shadowed local keys: ${childEnvironment.shadowedKeys.join(', ') || '(none)'}`, + ].join('\n') +} + +export const E2E_OS_PASSTHROUGH_KEYS = OS_PASSTHROUGH_KEYS diff --git a/apps/sim/e2e/support/hosts.ts b/apps/sim/e2e/support/hosts.ts new file mode 100644 index 00000000000..81e6127755c --- /dev/null +++ b/apps/sim/e2e/support/hosts.ts @@ -0,0 +1,41 @@ +import { promises as dns } from 'node:dns' +import { isIP } from 'node:net' +import { E2E_HOST } from './deployment-profile' + +export function isLoopbackAddress(address: string): boolean { + if (address === '::1' || address === '0:0:0:0:0:0:0:1') return true + if (isIP(address) === 4) { + const first = Number(address.split('.')[0]) + return first === 127 + } + return false +} + +export function areValidE2eHostAddresses(addresses: string[]): boolean { + return ( + addresses.length > 0 && addresses.every(isLoopbackAddress) && addresses.includes('127.0.0.1') + ) +} + +export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Promise { + let records: Array<{ address: string }> + try { + records = await dns.lookup(hostname, { all: true, verbatim: true }) + } catch { + throw new Error(getHostMappingError(hostname, [])) + } + + const addresses = [...new Set(records.map(({ address }) => address))] + if (!areValidE2eHostAddresses(addresses)) { + throw new Error(getHostMappingError(hostname, addresses)) + } + return addresses +} + +function getHostMappingError(hostname: string, addresses: string[]): string { + const observed = addresses.length > 0 ? addresses.join(', ') : 'no addresses' + return [ + `${hostname} must resolve only to loopback and include IPv4 127.0.0.1; observed ${observed}.`, + `Add it once with: echo "127.0.0.1 ${hostname}" | sudo tee -a /etc/hosts`, + ].join(' ') +} diff --git a/apps/sim/e2e/support/paths.ts b/apps/sim/e2e/support/paths.ts new file mode 100644 index 00000000000..44ec033e0e4 --- /dev/null +++ b/apps/sim/e2e/support/paths.ts @@ -0,0 +1,11 @@ +import path from 'node:path' + +export const SIM_APP_DIR = path.resolve(process.cwd()) +export const REPO_ROOT = path.resolve(SIM_APP_DIR, '../..') +export const DB_PACKAGE_DIR = path.join(REPO_ROOT, 'packages/db') +export const REALTIME_APP_DIR = path.join(REPO_ROOT, 'apps/realtime') +export const PLAYWRIGHT_CLI = path.join(REPO_ROOT, 'node_modules/@playwright/test/cli.js') + +export function getRunDirectory(runId: string): string { + return path.join(SIM_APP_DIR, 'e2e/.runs', runId) +} diff --git a/apps/sim/e2e/support/probes.ts b/apps/sim/e2e/support/probes.ts new file mode 100644 index 00000000000..40f4e42bae1 --- /dev/null +++ b/apps/sim/e2e/support/probes.ts @@ -0,0 +1,47 @@ +import postgres from 'postgres' + +export async function assertAdminApiBoundary(origin: string, adminKey: string): Promise { + const endpoint = `${origin}/api/v1/admin/users?limit=1&offset=0` + const unauthorized = await fetch(endpoint) + if (unauthorized.status !== 401) { + throw new Error(`Admin API missing-key probe returned ${unauthorized.status}, expected 401`) + } + + const authorized = await fetch(endpoint, { + headers: { 'x-admin-key': adminKey }, + }) + if (!authorized.ok) { + throw new Error(`Configured Admin API probe returned ${authorized.status}`) + } +} + +export interface FoundationProvisioningResult { + count: number + allHaveStripeCustomers: boolean + allHaveStats: boolean +} + +export async function inspectFoundationUsers( + databaseUrl: string, + runId: string +): Promise { + const sql = postgres(databaseUrl, { max: 1, connect_timeout: 10 }) + try { + const prefix = `e2e-foundation-${runId}-` + const rows = await sql>` + SELECT + u.stripe_customer_id AS "stripeCustomerId", + (s.user_id IS NOT NULL) AS "hasStats" + FROM "user" u + LEFT JOIN user_stats s ON s.user_id = u.id + WHERE starts_with(u.email, ${prefix}) + ` + return { + count: rows.length, + allHaveStripeCustomers: rows.every((row) => row.stripeCustomerId?.startsWith('cus_e2e_')), + allHaveStats: rows.every((row) => row.hasStats), + } + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts new file mode 100644 index 00000000000..406502275af --- /dev/null +++ b/apps/sim/e2e/support/process.ts @@ -0,0 +1,222 @@ +import { type ChildProcess, spawn, spawnSync } from 'node:child_process' +import { closeSync, mkdirSync, openSync } from 'node:fs' +import { createServer } from 'node:net' +import path from 'node:path' + +export interface CommandOptions { + name: string + command: string + args: string[] + cwd: string + env: Record + logsDirectory: string +} + +export interface ManagedProcess { + name: string + child: ChildProcess + completion: Promise + logPath: string + stop(): Promise +} + +interface ProcessCompletion { + code: number | null + signal: NodeJS.Signals | null + error?: Error +} + +const activeProcesses = new Set() + +export function spawnManagedProcess(options: CommandOptions): ManagedProcess { + mkdirSync(options.logsDirectory, { recursive: true }) + const logPath = path.join(options.logsDirectory, `${options.name}.log`) + const logFd = openSync(logPath, 'a') + const child: ChildProcess = spawn(options.command, options.args, { + cwd: options.cwd, + detached: process.platform !== 'win32', + env: options.env as NodeJS.ProcessEnv, + stdio: ['ignore', logFd, logFd], + }) + let resolveCompletion!: (completion: ProcessCompletion) => void + const completion = new Promise((resolve) => { + resolveCompletion = resolve + }) + let finalized = false + const managed: ManagedProcess = { + name: options.name, + child, + completion, + logPath, + stop: () => stopProcess(managed), + } + const finalize = (result: ProcessCompletion): void => { + if (finalized) return + finalized = true + activeProcesses.delete(managed) + closeSync(logFd) + resolveCompletion(result) + } + activeProcesses.add(managed) + child.once('error', (error) => finalize({ code: null, signal: null, error })) + child.once('exit', (code, signal) => finalize({ code, signal })) + return managed +} + +export async function runCommand(options: CommandOptions): Promise { + const managed = spawnManagedProcess(options) + const result = await managed.completion + if (result.error) { + throw new Error(`${options.name} failed to spawn. See ${managed.logPath}`, { + cause: result.error, + }) + } + if (result.code !== 0) { + throw new Error( + `${options.name} exited with ${result.code ?? result.signal ?? 'unknown status'}. See ${managed.logPath}` + ) + } +} + +export async function stopProcesses(processes: ManagedProcess[]): Promise { + const results = await Promise.allSettled( + [...processes].reverse().map((process) => process.stop()) + ) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + if (failures.length > 0) { + throw new AggregateError(failures, 'Failed to stop one or more managed E2E processes') + } +} + +export async function stopAllManagedProcesses(): Promise { + await stopProcesses([...activeProcesses]) +} + +export function getActiveManagedProcessGroupIds(): number[] { + return [...new Set([...activeProcesses].flatMap(({ child }) => (child.pid ? [child.pid] : [])))] +} + +export async function waitForManagedProcessReady( + process: ManagedProcess, + readiness: (signal: AbortSignal) => Promise +): Promise { + if (process.child.exitCode !== null || process.child.signalCode !== null) { + throw new Error(`${process.name} exited before readiness. See ${process.logPath}`) + } + + const controller = new AbortController() + let ready = false + const exited = process.completion.then((result) => { + if (ready) return + throw new Error( + `${process.name} exited before readiness with ${result.error?.message ?? result.code ?? result.signal ?? 'unknown status'}. See ${process.logPath}` + ) + }) + const becameReady = readiness(controller.signal).then(() => { + ready = true + }) + + try { + await Promise.race([becameReady, exited]) + } finally { + controller.abort(new Error(`${process.name} readiness polling cancelled`)) + } +} + +export async function assertPortAvailable(port: number, host = '127.0.0.1'): Promise { + await new Promise((resolve, reject) => { + const server = createServer() + server.once('error', (error) => { + reject(new Error(`E2E requires ${host}:${port} to be free: ${String(error)}`)) + }) + server.listen({ host, port, exclusive: true }, () => { + server.close((error) => { + if (error) reject(error) + else resolve() + }) + }) + }) +} + +async function stopProcess(managed: ManagedProcess): Promise { + const { child } = managed + if (!child.pid) { + await managed.completion + return + } + if (child.exitCode !== null || child.signalCode !== null) return + const processIds = [child.pid] + sendSignalToPids(processIds, 'SIGTERM') + + const stopped = await Promise.race([ + managed.completion.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), + ]) + if (stopped) return + + sendSignalToPids(processIds.filter(isPidRunning), 'SIGKILL') + const killed = await Promise.race([ + managed.completion.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 2_000)), + ]) + if (!killed) { + throw new Error(`Managed process ${managed.name} did not exit after SIGKILL`) + } +} + +function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { + for (const processId of processIds) { + try { + if (process.platform !== 'win32') process.kill(-processId, signal) + else process.kill(processId, signal) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ESRCH') continue + if (code === 'EPERM' && process.platform !== 'win32') { + try { + process.kill(processId, signal) + } catch (fallbackError) { + if ((fallbackError as NodeJS.ErrnoException).code !== 'ESRCH') { + throw fallbackError + } + } + continue + } + throw error + } + } +} + +function isPidRunning(processId: number): boolean { + return getRunningPids([processId]).length > 0 +} + +function getRunningPids(processIds: number[]): number[] { + if (processIds.length === 0) return [] + if (process.platform === 'win32') { + return processIds.filter((processId) => { + try { + process.kill(processId, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false + throw error + } + }) + } + + const status = spawnSync('ps', ['-o', 'pid=,stat=', '-p', processIds.join(',')], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (status.status !== 0 && status.status !== 1) { + throw new Error(`Unable to inspect managed E2E processes: ${status.stderr}`) + } + return status.stdout.split('\n').flatMap((line) => { + const [pidText, processStatus] = line.trim().split(/\s+/) + const pid = Number(pidText) + return Number.isInteger(pid) && !processStatus?.startsWith('Z') ? [pid] : [] + }) +} diff --git a/apps/sim/e2e/support/readiness.ts b/apps/sim/e2e/support/readiness.ts new file mode 100644 index 00000000000..628976bfa14 --- /dev/null +++ b/apps/sim/e2e/support/readiness.ts @@ -0,0 +1,55 @@ +export interface ReadinessOptions { + name: string + url: string + timeoutMs?: number + intervalMs?: number + signal?: AbortSignal + validate?: (response: Response) => Promise | boolean +} + +export async function waitForHttpReady({ + name, + url, + timeoutMs = 120_000, + intervalMs = 500, + signal, + validate = (response) => response.ok, +}: ReadinessOptions): Promise { + const deadline = Date.now() + timeoutMs + let lastError: unknown + + while (Date.now() < deadline) { + if (signal?.aborted) throw signal.reason + try { + const timeoutSignal = AbortSignal.timeout(5_000) + const response = await fetch(url, { + redirect: 'manual', + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) + if (await validate(response)) return + lastError = new Error(`${name} returned ${response.status}`) + } catch (error) { + if (signal?.aborted) throw signal.reason + lastError = error + } + await waitForInterval(intervalMs, signal) + } + + throw new Error(`${name} did not become ready at ${url}: ${String(lastError)}`) +} + +async function waitForInterval(milliseconds: number, signal?: AbortSignal): Promise { + await new Promise((resolve, reject) => { + const complete = () => { + signal?.removeEventListener('abort', abort) + resolve() + } + const timeout = setTimeout(complete, milliseconds) + const abort = () => { + clearTimeout(timeout) + reject(signal?.reason) + } + if (signal?.aborted) abort() + else signal?.addEventListener('abort', abort, { once: true }) + }) +} diff --git a/apps/sim/e2e/support/signal-cleanup.ts b/apps/sim/e2e/support/signal-cleanup.ts new file mode 100644 index 00000000000..18fcfe1e0d8 --- /dev/null +++ b/apps/sim/e2e/support/signal-cleanup.ts @@ -0,0 +1,8 @@ +export function parseProcessGroupIds(rawValue: string | undefined): number[] { + return (rawValue ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean) + .map(Number) + .filter((value) => Number.isInteger(value) && value > 0) +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts new file mode 100644 index 00000000000..d1ce203d6b6 --- /dev/null +++ b/apps/sim/e2e/support/stack.ts @@ -0,0 +1,136 @@ +import path from 'node:path' +import { DB_PACKAGE_DIR, PLAYWRIGHT_CLI, REALTIME_APP_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' +import { + type ManagedProcess, + runCommand, + spawnManagedProcess, + waitForManagedProcessReady, +} from './process' +import { waitForHttpReady } from './readiness' + +export interface StackCommandOptions { + bunExecutable: string + nodeExecutable: string + env: Record + logsDirectory: string +} + +export async function runMigrations(options: StackCommandOptions): Promise { + await runCommand({ + name: 'migrate', + command: options.bunExecutable, + args: ['--no-env-file', path.join(DB_PACKAGE_DIR, 'scripts/migrate.ts')], + cwd: DB_PACKAGE_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function buildApp(options: StackCommandOptions): Promise { + await runCommand({ + name: 'sandbox-bundles', + command: options.bunExecutable, + args: ['--no-env-file', 'run', 'build:sandbox-bundles'], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) + await runCommand({ + name: 'next-build', + command: options.nodeExecutable, + args: [path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), 'build'], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function startRealtime(options: StackCommandOptions): Promise { + const realtime = spawnManagedProcess({ + name: 'realtime', + command: options.bunExecutable, + args: ['--no-env-file', 'src/index.ts'], + cwd: REALTIME_APP_DIR, + env: { + ...options.env, + SIM_DB_ROLE: 'realtime', + DB_APP_NAME: 'sim-realtime', + REALTIME_HOST: '127.0.0.1', + PORT: '3002', + }, + logsDirectory: options.logsDirectory, + }) + try { + await waitForManagedProcessReady(realtime, (signal) => + waitForHttpReady({ + name: 'Realtime', + url: 'http://127.0.0.1:3002/health', + signal, + validate: async (response) => { + if (!response.ok) return false + const body = (await response.json()) as { status?: string; runId?: string } + return body.status === 'ok' && body.runId === options.env.E2E_RUN_ID + }, + }) + ) + return realtime + } catch (error) { + await realtime.stop() + throw error + } +} + +export async function startApp(options: StackCommandOptions): Promise { + const app = spawnManagedProcess({ + name: 'app', + command: options.nodeExecutable, + args: [ + path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), + 'start', + '-p', + '3000', + '-H', + '127.0.0.1', + ], + cwd: SIM_APP_DIR, + env: { + ...options.env, + SIM_DB_ROLE: 'web', + DB_APP_NAME: 'sim-app', + PORT: '3000', + }, + logsDirectory: options.logsDirectory, + }) + try { + await waitForManagedProcessReady(app, (signal) => + waitForHttpReady({ + name: 'Next.js', + url: 'http://127.0.0.1:3000/api/health', + signal, + validate: async (response) => { + if (!response.ok) return false + const body = (await response.json()) as { status?: string; runId?: string } + return body.status === 'ok' && body.runId === options.env.E2E_RUN_ID + }, + }) + ) + return app + } catch (error) { + await app.stop() + throw error + } +} + +export async function runPlaywright( + options: Omit, + args: string[] +): Promise { + await runCommand({ + name: 'playwright', + command: options.nodeExecutable, + args: [PLAYWRIGHT_CLI, 'test', '--config=playwright.config.ts', ...args], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index ea2d1f0b227..b0dba3441aa 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -54,6 +54,7 @@ import { import { pauseProSubscriptionForOrgCoverage } from '@/lib/billing/organizations/membership' import { isPro, isTeam } from '@/lib/billing/plan-helpers' import { getPlans, resolvePlanFromStripeSubscription } from '@/lib/billing/plans' +import { buildStripeClientConfig } from '@/lib/billing/stripe-client-config' import { syncSeatsFromStripeQuantity } from '@/lib/billing/validation/seat-management' import { handleAbandonedCheckout } from '@/lib/billing/webhooks/checkout' import { handleChargeDispute, handleDisputeClosed } from '@/lib/billing/webhooks/disputes' @@ -188,9 +189,7 @@ const validStripeKey = env.STRIPE_SECRET_KEY let stripeClient = null if (validStripeKey) { - stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', { - apiVersion: '2025-08-27.basil', - }) + stripeClient = new Stripe(validStripeKey, buildStripeClientConfig(env)) } export const auth = betterAuth({ diff --git a/apps/sim/lib/billing/stripe-client-config.test.ts b/apps/sim/lib/billing/stripe-client-config.test.ts new file mode 100644 index 00000000000..e15d41f203b --- /dev/null +++ b/apps/sim/lib/billing/stripe-client-config.test.ts @@ -0,0 +1,123 @@ +/** @vitest-environment node */ + +import { describe, expect, it } from 'vitest' +import { + buildStripeClientConfig, + STRIPE_API_VERSION, + STRIPE_E2E_PROFILE, + type StripeClientConfigEnvironment, +} from '@/lib/billing/stripe-client-config' + +const guardedEnvironment: StripeClientConfigEnvironment = { + DATABASE_URL: 'postgresql://sim:sim@127.0.0.1:5432/sim_e2e_config_test', + E2E_PROFILE: STRIPE_E2E_PROFILE, + STRIPE_API_BASE_URL: 'http://127.0.0.1:12111', + STRIPE_SECRET_KEY: 'sk_test_e2e_config', +} + +describe('buildStripeClientConfig', () => { + it('preserves the normal Stripe configuration without an override', () => { + expect( + buildStripeClientConfig({ + DATABASE_URL: 'postgresql://db.example.com/sim', + STRIPE_SECRET_KEY: 'sk_live_normal_deployment', + }) + ).toEqual({ + apiVersion: STRIPE_API_VERSION, + }) + }) + + it('builds an HTTP transport for a guarded loopback fake', () => { + expect(buildStripeClientConfig(guardedEnvironment)).toEqual({ + apiVersion: STRIPE_API_VERSION, + host: '127.0.0.1', + port: '12111', + protocol: 'http', + }) + }) + + it('uses the HTTP default port when the override omits one', () => { + expect( + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: 'http://localhost', + }) + ).toMatchObject({ + host: 'localhost', + port: 80, + protocol: 'http', + }) + }) + + it.each(['sk_test_e2e_config', 'sk_live_e2e_config'])( + 'fails closed when the E2E profile configures %s without an override', + (stripeSecretKey) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: undefined, + STRIPE_SECRET_KEY: stripeSecretKey, + }) + ).toThrow('STRIPE_API_BASE_URL is required') + } + ) + + it.each([ + { + name: 'a missing E2E profile', + environment: { E2E_PROFILE: undefined }, + message: 'requires E2E_PROFILE', + }, + { + name: 'a different E2E profile', + environment: { E2E_PROFILE: 'self-hosted-chromium' }, + message: 'requires E2E_PROFILE', + }, + { + name: 'a live Stripe key', + environment: { STRIPE_SECRET_KEY: 'sk_live_not_redirectable' }, + message: 'requires a Stripe test secret key', + }, + { + name: 'a public database', + environment: { DATABASE_URL: 'postgresql://db.example.com/sim' }, + message: 'requires a guarded sim_e2e_* database', + }, + { + name: 'an empty E2E database suffix', + environment: { DATABASE_URL: 'postgresql://127.0.0.1/sim_e2e_' }, + message: 'requires a guarded sim_e2e_* database', + }, + { + name: 'a malformed database URL', + environment: { DATABASE_URL: 'not-a-database-url' }, + message: 'requires a guarded sim_e2e_* database', + }, + ])('rejects an override with $name', ({ environment, message }) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + ...environment, + }) + ).toThrow(message) + }) + + it.each([ + 'https://127.0.0.1:12111', + 'http://stripe.example.com:12111', + 'http://0.0.0.0:12111', + 'http://user:password@127.0.0.1:12111', + 'http://127.0.0.1:12111/v1', + 'http://127.0.0.1:12111?mode=test', + 'http://127.0.0.1:12111#fragment', + ' http://127.0.0.1:12111', + 'not-a-url', + ])('rejects an unsafe or malformed override: %s', (stripeApiBaseUrl) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: stripeApiBaseUrl, + }) + ).toThrow() + }) +}) diff --git a/apps/sim/lib/billing/stripe-client-config.ts b/apps/sim/lib/billing/stripe-client-config.ts new file mode 100644 index 00000000000..320cee74b77 --- /dev/null +++ b/apps/sim/lib/billing/stripe-client-config.ts @@ -0,0 +1,114 @@ +import type Stripe from 'stripe' + +export const STRIPE_API_VERSION = '2025-08-27.basil' as const +export const STRIPE_E2E_PROFILE = 'hosted-billing-chromium' as const + +export interface StripeClientConfigEnvironment { + DATABASE_URL?: string + E2E_PROFILE?: string + STRIPE_API_BASE_URL?: string + STRIPE_SECRET_KEY?: string +} + +const E2E_DATABASE_NAME_PATTERN = /^sim_e2e_[A-Za-z0-9_-]+$/ + +function getDatabaseName(databaseUrl: string | undefined): string | null { + if (!databaseUrl) return null + + try { + const parsed = new URL(databaseUrl) + if (parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:') return null + + const pathSegments = parsed.pathname.split('/').filter(Boolean) + if (pathSegments.length !== 1) return null + + return decodeURIComponent(pathSegments[0]) + } catch { + return null + } +} + +function isLoopbackHostname(hostname: string): boolean { + const normalized = hostname.toLowerCase() + if (normalized === 'localhost') return true + + const octets = normalized.split('.') + return ( + octets.length === 4 && + octets.every((octet) => /^\d+$/.test(octet) && Number(octet) <= 255) && + Number(octets[0]) === 127 + ) +} + +function parseStripeApiBaseUrl(rawUrl: string): URL { + if (rawUrl.trim() !== rawUrl) { + throw new Error('STRIPE_API_BASE_URL must not contain surrounding whitespace') + } + + let parsed: URL + try { + parsed = new URL(rawUrl) + } catch { + throw new Error('STRIPE_API_BASE_URL must be a valid absolute URL') + } + + if (parsed.protocol !== 'http:') { + throw new Error('STRIPE_API_BASE_URL must use HTTP for the loopback E2E fake') + } + if (!isLoopbackHostname(parsed.hostname)) { + throw new Error('STRIPE_API_BASE_URL must use a loopback hostname') + } + if (parsed.username || parsed.password) { + throw new Error('STRIPE_API_BASE_URL must not contain credentials') + } + if (parsed.search || parsed.hash) { + throw new Error('STRIPE_API_BASE_URL must not contain a query or hash') + } + if (parsed.pathname !== '/') { + throw new Error('STRIPE_API_BASE_URL must not contain a non-root path') + } + + return parsed +} + +/** + * Builds the Stripe SDK configuration while keeping the E2E transport override + * unavailable to normal deployments and production credentials. + */ +export function buildStripeClientConfig( + environment: StripeClientConfigEnvironment +): Stripe.StripeConfig { + const baseConfig: Stripe.StripeConfig = { + apiVersion: STRIPE_API_VERSION, + } + const override = environment.STRIPE_API_BASE_URL + + if (override === undefined || override === '') { + if (environment.E2E_PROFILE === STRIPE_E2E_PROFILE && environment.STRIPE_SECRET_KEY) { + throw new Error( + 'STRIPE_API_BASE_URL is required when the hosted billing E2E profile configures Stripe' + ) + } + return baseConfig + } + + if (environment.E2E_PROFILE !== STRIPE_E2E_PROFILE) { + throw new Error(`STRIPE_API_BASE_URL requires E2E_PROFILE=${STRIPE_E2E_PROFILE}`) + } + if (!environment.STRIPE_SECRET_KEY?.startsWith('sk_test_')) { + throw new Error('STRIPE_API_BASE_URL requires a Stripe test secret key') + } + + const databaseName = getDatabaseName(environment.DATABASE_URL) + if (!databaseName || !E2E_DATABASE_NAME_PATTERN.test(databaseName)) { + throw new Error('STRIPE_API_BASE_URL requires a guarded sim_e2e_* database') + } + + const endpoint = parseStripeApiBaseUrl(override) + return { + ...baseConfig, + host: endpoint.hostname, + port: endpoint.port || 80, + protocol: 'http', + } +} diff --git a/apps/sim/lib/billing/stripe-client.ts b/apps/sim/lib/billing/stripe-client.ts index 13bb089845b..9f1d6edc808 100644 --- a/apps/sim/lib/billing/stripe-client.ts +++ b/apps/sim/lib/billing/stripe-client.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import Stripe from 'stripe' +import { buildStripeClientConfig } from '@/lib/billing/stripe-client-config' import { env } from '@/lib/core/config/env' const logger = createLogger('StripeClient') @@ -37,9 +38,7 @@ const createStripeClientSingleton = () => { try { isInitializing = true - stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', { - apiVersion: '2025-08-27.basil', - }) + stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', buildStripeClientConfig(env)) logger.info('Stripe client initialized successfully') return stripeClient diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index c86eb9e7ff4..0972bd75634 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -68,7 +68,9 @@ export const env = createEnv({ REDIS_TLS_SERVERNAME: z.string().min(1).optional(), // TLS SNI override; required when REDIS_URL targets an IP over rediss:// (e.g. trigger.dev PrivateLink VPCE IP) so cert hostname verification matches the ElastiCache cert's CN // Payment & Billing + E2E_PROFILE: z.string().min(1).optional(), // Server-only E2E deployment profile selector STRIPE_SECRET_KEY: z.string().min(1).optional(), // Stripe secret key for payment processing + STRIPE_API_BASE_URL: z.string().url().optional(), // Guarded server-only Stripe API override for E2E fakes STRIPE_WEBHOOK_SECRET: z.string().min(1).optional(), // General Stripe webhook secret STRIPE_FREE_PRICE_ID: z.string().min(1).optional(), // Stripe price ID for free tier FREE_TIER_COST_LIMIT: z.number().optional(), // Cost limit for free tier users diff --git a/apps/sim/package.json b/apps/sim/package.json index a7800a9f904..288bf38b3c7 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -24,6 +24,8 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", + "test:e2e": "exec bun --no-env-file e2e/scripts/run.ts", + "test:e2e:install-browsers": "node ../../node_modules/@playwright/test/cli.js install chromium", "email:dev": "email dev --dir components/emails", "type-check": "NODE_OPTIONS='--max-old-space-size=8192' tsc --noEmit", "lint": "biome check --write --unsafe .", @@ -220,6 +222,7 @@ "zustand": "^5.0.13" }, "devDependencies": { + "@playwright/test": "1.61.1", "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@tailwindcss/typography": "0.5.19", diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts new file mode 100644 index 00000000000..70cb2641644 --- /dev/null +++ b/apps/sim/playwright.config.ts @@ -0,0 +1,58 @@ +import { defineConfig, devices } from '@playwright/test' + +if (process.env.E2E_ORCHESTRATED !== '1') { + throw new Error( + 'Playwright tests must run through `bun run test:e2e` so database, environment, and teardown guards are active' + ) +} + +const isCI = process.env.CI === 'true' +const baseURL = process.env.E2E_BASE_URL ?? 'http://e2e.sim.ai:3000' + +export default defineConfig({ + testDir: './e2e', + testMatch: '**/*.spec.ts', + fullyParallel: true, + forbidOnly: isCI, + retries: 0, + workers: 1, + timeout: 60_000, + expect: { + timeout: 10_000, + }, + reporter: [ + ['list'], + ['html', { open: 'never', outputFolder: 'playwright-report' }], + ...(isCI ? ([['github']] as const) : []), + ], + outputDir: 'test-results', + use: { + ...devices['Desktop Chrome'], + baseURL, + launchOptions: { + args: ['--host-resolver-rules=MAP e2e.sim.ai 127.0.0.1'], + }, + actionTimeout: 15_000, + navigationTimeout: 30_000, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'off', + }, + projects: [ + { + name: 'hosted-billing-chromium-navigation', + testMatch: [ + '**/foundation/**/*.spec.ts', + '**/settings/smoke/unauthenticated.spec.ts', + '**/settings/navigation/**/*.spec.ts', + ], + workers: 1, + }, + { + name: 'hosted-billing-chromium-workflows', + testMatch: ['**/settings/smoke/authenticated.spec.ts', '**/settings/workflows/**/*.spec.ts'], + fullyParallel: false, + workers: 1, + }, + ], +}) diff --git a/apps/sim/vitest.config.ts b/apps/sim/vitest.config.ts index a966ad5233e..02425c6f712 100644 --- a/apps/sim/vitest.config.ts +++ b/apps/sim/vitest.config.ts @@ -16,7 +16,7 @@ export default defineConfig({ globals: true, environment: 'node', include: ['**/*.test.{ts,tsx}'], - exclude: [...configDefaults.exclude, '**/node_modules/**', '**/dist/**'], + exclude: [...configDefaults.exclude, '**/node_modules/**', '**/dist/**', '**/e2e/**'], setupFiles: ['./vitest.setup.ts'], pool: 'threads', isolate: true, diff --git a/biome.json b/biome.json index 271e701c8b4..3271ed0dde1 100644 --- a/biome.json +++ b/biome.json @@ -22,6 +22,9 @@ "!**/.env", "!**/.vercel", "!**/coverage", + "!**/playwright-report", + "!**/test-results", + "!**/e2e/.runs", "!**/public/sw.js", "!**/public/workbox-*.js", "!**/public/worker-*.js", diff --git a/bun.lock b/bun.lock index f3ad75a78dd..51ef90840cb 100644 --- a/bun.lock +++ b/bun.lock @@ -288,6 +288,7 @@ "zustand": "^5.0.13", }, "devDependencies": { + "@playwright/test": "1.61.1", "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@tailwindcss/typography": "0.5.19", @@ -1331,6 +1332,8 @@ "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], + "@playwright/test": ["@playwright/test@1.61.1", "", { "dependencies": { "playwright": "1.61.1" }, "bin": { "playwright": "cli.js" } }, "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig=="], + "@posthog/core": ["@posthog/core@1.24.4", "", { "dependencies": { "cross-spawn": "^7.0.6" } }, "sha512-S+TolwBHSSJz7WWtgaELQWQqXviSm3uf1e+qorWUts0bZcgPwWzhnmhCUZAhvn0NVpTQHDJ3epv+hHbPLl5dHg=="], "@posthog/types": ["@posthog/types@1.364.4", "", {}, "sha512-U7NpIy9XWrzz1q/66xyDu8Wm12a7avNRKRn5ISPT5kuCJQRaeAaHuf+dpgrFnuqjCCgxg+oIY/ReJdlZ+8/z4Q=="], @@ -3467,6 +3470,10 @@ "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + "playwright": ["playwright@1.61.1", "", { "dependencies": { "playwright-core": "1.61.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ=="], + + "playwright-core": ["playwright-core@1.61.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg=="], + "points-on-curve": ["points-on-curve@0.2.0", "", {}, "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A=="], "points-on-path": ["points-on-path@0.2.1", "", { "dependencies": { "path-data-parser": "0.1.0", "points-on-curve": "0.2.0" } }, "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g=="], @@ -4781,6 +4788,8 @@ "pino-pretty/pino-abstract-transport": ["pino-abstract-transport@3.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg=="], + "playwright/fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + "postcss-nested/postcss-selector-parser": ["postcss-selector-parser@6.1.4", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ=="], "posthog-js/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.208.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg=="], From b0b0470951e039381db8ae1b69c5bc05673c25be Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos <157128530+BillLeoutsakosvl346@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:06:12 -0700 Subject: [PATCH 3/8] Add deterministic E2E settings personas (#5813) * Add safe E2E build reuse and environment isolation Separate build-time inputs from per-run services so verified artifacts can accelerate local iteration without weakening fresh database or provider boundaries. * Add validated E2E persona world model Model settings personas as deterministic, serializable resource graphs and provide production-first factories with exact-ID cleanup so impossible or cross-world fixture states fail before browser tests run. Co-authored-by: Cursor * Seed and verify E2E settings personas Provision the validated world through real product boundaries, capture isolated sessions through the login UI, and assert persona contracts plus two-worker cross-world isolation without exposing credentials to Playwright. Co-authored-by: Cursor * Harden E2E persona orchestration and remove deferred cleanup Generate runtime secrets per run, fail closed when diagnostics cannot be scanned, and harden cache and process cleanup while removing the unused exact-ID cleanup module until retained-stack support has a real consumer. * Fix CI option policy assertions Make local-only parser scenarios explicit so the safety suite tests the intended policy instead of inheriting the GitHub runner environment. * Fix organization invitation fixture semantics Match production organization invitations even when workspace grants are attached, and remove an unused cookie-reset method from the E2E client. --------- Co-authored-by: Bill Leoutsakos Co-authored-by: Cursor --- .github/workflows/test-build.yml | 12 +- .gitignore | 1 + apps/sim/e2e/README.md | 99 ++- apps/sim/e2e/fakes/stripe/server.ts | 8 + apps/sim/e2e/fixtures/e2e-world.ts | 269 ++++++ apps/sim/e2e/fixtures/factories/billing.ts | 165 ++++ .../sim/e2e/fixtures/factories/invitations.ts | 49 ++ .../e2e/fixtures/factories/organizations.ts | 52 ++ .../fixtures/factories/permission-groups.ts | 62 ++ apps/sim/e2e/fixtures/factories/platform.ts | 16 + apps/sim/e2e/fixtures/factories/users.ts | 49 ++ apps/sim/e2e/fixtures/factories/workspaces.ts | 49 ++ apps/sim/e2e/fixtures/http-client.ts | 148 ++++ apps/sim/e2e/fixtures/namespace.ts | 86 ++ apps/sim/e2e/fixtures/persona-test.ts | 59 ++ apps/sim/e2e/fixtures/scenario.ts | 184 ++++ apps/sim/e2e/fixtures/validate-scenario.ts | 746 ++++++++++++++++ .../sim/e2e/foundation/build-manifest.spec.ts | 174 ++++ apps/sim/e2e/foundation/http-client.spec.ts | 90 ++ apps/sim/e2e/foundation/leak-canary.spec.ts | 84 ++ apps/sim/e2e/foundation/safety.spec.ts | 173 +++- .../foundation/scenario-validation.spec.ts | 329 ++++++++ apps/sim/e2e/foundation/stripe-fake.spec.ts | 12 + apps/sim/e2e/scripts/capture-auth-states.ts | 167 ++++ apps/sim/e2e/scripts/options.ts | 79 +- apps/sim/e2e/scripts/run.ts | 228 ++++- apps/sim/e2e/scripts/seed-world.ts | 795 ++++++++++++++++++ apps/sim/e2e/scripts/signal-cleanup.ts | 45 +- .../e2e/settings/persona-contracts.spec.ts | 169 ++++ .../e2e/settings/persona-isolation.spec.ts | 75 ++ apps/sim/e2e/settings/personas.ts | 462 ++++++++++ .../e2e/settings/smoke/authenticated.spec.ts | 10 +- apps/sim/e2e/support/build-manifest.ts | 386 +++++++++ apps/sim/e2e/support/deployment-profile.ts | 257 +++++- apps/sim/e2e/support/env.ts | 14 +- apps/sim/e2e/support/leak-canary.ts | 132 +++ apps/sim/e2e/support/paths.ts | 4 +- apps/sim/e2e/support/probes.ts | 34 + apps/sim/e2e/support/process.ts | 76 +- apps/sim/e2e/support/run-lock.ts | 168 ++++ apps/sim/e2e/support/runtime-secrets.ts | 29 + apps/sim/e2e/support/sandbox-bundles.ts | 67 ++ apps/sim/e2e/support/seed-safety.ts | 33 + apps/sim/e2e/support/signal-cleanup.ts | 11 + apps/sim/e2e/support/stack.ts | 82 +- .../lib/execution/sandbox/bundles/build.ts | 101 ++- .../execution/sandbox/bundles/integrity.json | 20 + apps/sim/package.json | 2 + apps/sim/playwright.config.ts | 17 +- bun.lock | 72 +- 50 files changed, 6268 insertions(+), 183 deletions(-) create mode 100644 apps/sim/e2e/fixtures/e2e-world.ts create mode 100644 apps/sim/e2e/fixtures/factories/billing.ts create mode 100644 apps/sim/e2e/fixtures/factories/invitations.ts create mode 100644 apps/sim/e2e/fixtures/factories/organizations.ts create mode 100644 apps/sim/e2e/fixtures/factories/permission-groups.ts create mode 100644 apps/sim/e2e/fixtures/factories/platform.ts create mode 100644 apps/sim/e2e/fixtures/factories/users.ts create mode 100644 apps/sim/e2e/fixtures/factories/workspaces.ts create mode 100644 apps/sim/e2e/fixtures/http-client.ts create mode 100644 apps/sim/e2e/fixtures/namespace.ts create mode 100644 apps/sim/e2e/fixtures/persona-test.ts create mode 100644 apps/sim/e2e/fixtures/scenario.ts create mode 100644 apps/sim/e2e/fixtures/validate-scenario.ts create mode 100644 apps/sim/e2e/foundation/build-manifest.spec.ts create mode 100644 apps/sim/e2e/foundation/http-client.spec.ts create mode 100644 apps/sim/e2e/foundation/leak-canary.spec.ts create mode 100644 apps/sim/e2e/foundation/scenario-validation.spec.ts create mode 100644 apps/sim/e2e/scripts/capture-auth-states.ts create mode 100644 apps/sim/e2e/scripts/seed-world.ts create mode 100644 apps/sim/e2e/settings/persona-contracts.spec.ts create mode 100644 apps/sim/e2e/settings/persona-isolation.spec.ts create mode 100644 apps/sim/e2e/settings/personas.ts create mode 100644 apps/sim/e2e/support/build-manifest.ts create mode 100644 apps/sim/e2e/support/leak-canary.ts create mode 100644 apps/sim/e2e/support/run-lock.ts create mode 100644 apps/sim/e2e/support/runtime-secrets.ts create mode 100644 apps/sim/e2e/support/sandbox-bundles.ts create mode 100644 apps/sim/e2e/support/seed-safety.ts create mode 100644 apps/sim/lib/execution/sandbox/bundles/integrity.json diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4d3fa2dbde4..579c4fc0d48 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -300,14 +300,6 @@ jobs: key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ~/.cache/ms-playwright - - name: Restore E2E Next.js build cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: ./apps/sim/.next/cache - key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}- - - name: Install dependencies run: bun install --frozen-lockfile @@ -327,7 +319,7 @@ jobs: run: bun run test:e2e - name: Upload E2E diagnostics - if: failure() || cancelled() + if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: settings-e2e-${{ github.run_id }} @@ -336,6 +328,8 @@ jobs: apps/sim/test-results/ apps/sim/e2e/.runs/ !apps/sim/e2e/.runs/**/auth/** + !apps/sim/e2e/.runs/**/private/** + !apps/sim/e2e/.runs/**/homes/** if-no-files-found: ignore include-hidden-files: true retention-days: 7 \ No newline at end of file diff --git a/.gitignore b/.gitignore index dc42d1c8e3f..109495320d0 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ package-lock.json /apps/**/test-results/ /apps/**/e2e/.runs/ /apps/**/e2e/.auth/ +/apps/**/e2e/.cache/ # next.js /.next/ diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index 49f108e8dd7..de632d015f9 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -50,13 +50,22 @@ E2E_PG_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ ``` The runner creates a unique `sim_e2e_` database, migrates it, starts the -Stripe fake and realtime, builds and starts Next.js, runs Playwright on Node 22, -then stops services and drops only that guarded database. +Stripe fake and realtime, builds and starts Next.js, seeds the validated persona +world through production APIs plus narrow trusted arrangements, captures each +persona through the real login UI, runs Playwright on Node 22, then stops +services and drops only that guarded database. +Server telemetry currently shares the strict loopback Stripe-fake process; the +generic modular fake-service refactor remains scoped to the roadmap's +`e2e/06b-enterprise-integrations` phase. +An exclusive checkout-level orchestrator lock prevents concurrent runs from +racing on `.next`, the shared build cache, or the fixed app/realtime ports. On interruption, the runner launches a detached cleanup supervisor before exiting. It terminates managed process groups, force-drops the guarded database, and removes temporary auth/cloud-config directories even if another Ctrl-C terminates the foreground package runner. +Cleanup failures retain the lock and require the reported resources to be +inspected and cleaned before manually removing `e2e/.cache/orchestrator.lock`. Pass Playwright arguments after `--`: @@ -65,19 +74,61 @@ bun run test:e2e -- --project=hosted-billing-chromium-navigation bun run test:e2e -- --grep "unauthenticated" ``` +Projects form a dependency chain to keep shared boundaries serialized. Selecting +the personas project therefore also runs navigation and workflows by default, +and an upstream failure skips its dependents. For focused local iteration, +`--no-deps` is explicitly supported: + +```bash +bun run test:e2e -- --project=hosted-billing-chromium-personas --no-deps +``` + +`--no-deps` requires exactly one explicit canonical project. It skips only +Playwright project dependencies; the guarded one-shot stack still performs full +seed and auth setup. Do not use it for full verification; the runner rejects it in CI. + +For a local follow-up run, reuse only a verified build while still creating a +new database, Stripe fake, app, realtime process, and browser run: + +```bash +bun run test:e2e -- --reuse-build --project=hosted-billing-chromium-navigation +``` + +The cache lives under ignored `e2e/.cache/builds/`. A hit requires matching +source contents (including uncommitted/untracked files), build/public profile, +Node/Bun/Next versions, platform, `BUILD_ID`, and the cached artifact checksum. +Any mismatch performs and caches a fresh local build; only the most recent cache +entry is retained because this application's production artifact is several +gigabytes. CI rejects `--reuse-build` and does not copy or hash a disposable +cache artifact. Plain local runs also skip cache identity computation, +multi-gigabyte copying, and cache population; only an explicit `--reuse-build` +request pays those costs. Cache restore also removes abandoned temporary stores +from interrupted writes before accepting a hit. `--skip-build` remains unsupported. + +Keep-stack/rerun supervision is intentionally unavailable. The initial safety +experiment requires descriptor ownership, mutation observation, state snapshots, +and teardown to ship as one unit; until all of those are proven, each invocation +uses the normal one-shot lifecycle. + Do not invoke `playwright test` directly. Raw Playwright bypasses environment, database, process, sharding, and teardown guards; the config rejects runs that were not launched by the orchestrator. Report and trace viewer commands remain safe because they do not execute tests. Sharding is supported only for the navigation project. The runner rejects -`--shard` for `hosted-billing-chromium-workflows`. +`--shard` for workflows, persona contracts, and the dedicated two-worker +cross-world isolation project. Project dependencies serialize navigation, +workflows, and persona contracts before the isolation project opens its +two-worker pool. ## Diagnostics - HTML report: `playwright-report/` - Traces and screenshots: `test-results/` -- App, realtime, migration, and fake logs: `e2e/.runs//logs/` +- App, realtime, migration, seed, auth-capture, and fake logs: + `e2e/.runs//logs/` +- Non-secret persona manifest and auth-capture failure screenshots: + `e2e/.runs//` Open the report: @@ -91,9 +142,43 @@ Open a trace: node ../../node_modules/@playwright/test/cli.js show-trace test-results//trace.zip ``` -The runner starts every child process from a fresh environment. It allowlists -only deterministic E2E values and shadows keys found in local `.env*` files, so -developer credentials are not used as test state or written to reports. +The runner starts every child process from a fresh, purpose-specific +environment. Next build receives deterministic sentinels instead of the run +database/fake endpoint; app, realtime, migrations, seeding, auth capture, and +Playwright each receive only their required values. Playwright receives the +non-secret manifest and storage-state directory, never passwords, the admin API +key, or the database URL. Next build/start shadow +keys found in local `.env*` files, while children that cannot load those files +omit denied keys entirely. Developer credentials are not used as test state or +written to reports. Named persona credentials and a separate all-synthetic-user +canary list live outside every child `HOME` in a private run directory. Auth +capture receives only the persona file, which is deleted immediately after +capture; storage states are mode `0600` and excluded from CI artifacts. Captured +passwords are loaded into orchestrator memory, then both secret files are +deleted before Playwright starts. After managed processes stop and logs flush, +the in-memory canary scans the manifest, logs, report files, and trace archives +and fails if a synthetic password, invitation token, or runtime secret escaped +the excluded private directories. Cancelled CI runs do not upload unscanned +diagnostics, and an unreadable canary or incomplete archive scan causes all +potentially unscanned diagnostic roots to be scrubbed. Storage-state session cookies are intentionally not canaried +because authenticated Playwright traces contain them by design; they are +synthetic and invalid once the run database is dropped. +Fresh-session recapture is deliberately deferred. Future membership-mutation +coverage must explicitly restore a private credential handoff and re-review its +access boundary rather than assuming credentials persist through Playwright. + +E2E builds verify the pinned Bun executable plus reviewed sandbox-bundle +source, direct dependency, and output fingerprints and never regenerate +committed `.cjs` files. +`bun run build:sandbox-bundles:integrity` is the explicit maintenance command +that regenerates bundles and their reviewed integrity manifest together. +Unrelated monorepo lockfile changes do not invalidate the bundle fingerprint; +the committed output hashes still detect any transitive change that alters a +bundle. + +Reset/reseed cleanup remains deferred with keep-stack supervision. Ordinary +runs own a unique guarded database and remove it wholesale rather than carrying +untested row-level deletion code. Provider log scans are diagnostic tripwires, not proof of zero egress. The primary boundaries are the default-deny child environment, provider disabling, diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts index 8f89c48102e..9fe5d07826c 100644 --- a/apps/sim/e2e/fakes/stripe/server.ts +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -6,6 +6,8 @@ export const STRIPE_FAKE_ENDPOINTS = { health: '/health', requestLog: '/__control/requests', reset: '/__control/reset', + // Co-locate the only other server-side external boundary so the E2E stack needs one loopback fake. + telemetry: '/v1/traces', } as const const DEFAULT_MAX_BODY_BYTES = 64 * 1024 @@ -63,6 +65,7 @@ function isExpectedStripeRequest(method: string, path: string): boolean { ((method === 'GET' || method === 'POST') && path === '/v1/customers/search') || (method === 'GET' && path === '/v1/customers') || (method === 'POST' && path === '/v1/customers') || + (method === 'POST' && path === STRIPE_FAKE_ENDPOINTS.telemetry) || (method === 'GET' && /^\/v1\/customers\/cus_e2e_[a-f0-9]+$/.test(path)) ) } @@ -326,6 +329,11 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe unexpected: !expected, }) + if (method === 'POST' && url.pathname === STRIPE_FAKE_ENDPOINTS.telemetry) { + sendJson(response, 200, { partialSuccess: {} }, requestId) + return + } + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { sendStripeError( response, diff --git a/apps/sim/e2e/fixtures/e2e-world.ts b/apps/sim/e2e/fixtures/e2e-world.ts new file mode 100644 index 00000000000..fe7014f611b --- /dev/null +++ b/apps/sim/e2e/fixtures/e2e-world.ts @@ -0,0 +1,269 @@ +import { randomUUID } from 'node:crypto' +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { z } from 'zod' +import type { ResolvedScenario, ScenarioPersona } from './scenario' + +export const PERSONA_MANIFEST_VERSION = 1 as const +const SAFE_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/ +const STORAGE_STATE_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\.json$/ +const safeIdentifierSchema = z.string().regex(SAFE_IDENTIFIER_PATTERN) +const storageStateFilenameSchema = z + .string() + .regex(STORAGE_STATE_FILENAME_PATTERN) + .refine((value) => !value.includes('..'), 'storage-state filename must not contain ".."') + +const workspaceExpectationSchema = z.object({ + workspaceId: z.string(), + workspaceKey: z.string(), + access: z.enum(['none', 'read', 'write', 'admin']), + roleSource: z.enum(['none', 'owner', 'explicit', 'org-admin']), + hostContext: z.object({ + isOwner: z.boolean(), + hostMembership: z.enum(['owner', 'member', 'external']), + payerScope: z.enum(['user', 'organization']), + plan: z.enum(['free', 'pro_6000', 'pro_25000', 'team_6000', 'team_25000', 'enterprise']), + hosted: z.boolean(), + billingEnabled: z.boolean(), + }), +}) + +export const personaManifestEntrySchema = z.object({ + key: safeIdentifierSchema, + world: safeIdentifierSchema, + userId: z.string(), + email: z.string().email(), + name: z.string(), + expectedActiveOrganizationId: z.string().nullable(), + storageStatePath: storageStateFilenameSchema, + canonicalRoute: z.string().startsWith('/'), + workspaces: z.array(workspaceExpectationSchema).min(1), + permissionGroupIds: z.array(z.string()), + expectedPlatformRole: z.enum(['user', 'admin']), + expectedSuperUserMode: z.boolean(), +}) + +const worldManifestSchema = z.object({ + namespace: z.object({ + run: z.string(), + world: z.string(), + prefix: z.string(), + }), + userIds: z.record(z.string(), z.string()), + userIdentities: z.record( + z.string(), + z.object({ id: z.string(), email: z.string().email(), name: z.string() }) + ), + organizationIds: z.record(z.string(), z.string()), + organizationIdentities: z.record( + z.string(), + z.object({ id: z.string(), name: z.string(), slug: z.string() }) + ), + organizationMemberIds: z.record(z.string(), z.string()), + workspaceIds: z.record(z.string(), z.string()), + workspaceIdentities: z.record(z.string(), z.object({ id: z.string(), name: z.string() })), + subscriptionIds: z.record(z.string(), z.string()), + permissionIds: z.record(z.string(), z.string()), + permissionGroupIds: z.record(z.string(), z.string()), + permissionGroupMemberIds: z.record(z.string(), z.string()), + invitationIds: z.record(z.string(), z.string()), + invitationGrantIds: z.record(z.string(), z.string()), +}) + +export const scenarioManifestSchema = z.object({ + schemaVersion: z.literal(PERSONA_MANIFEST_VERSION), + runId: z.string(), + createdAt: z.string(), + authCaptureComplete: z.boolean(), + worlds: z.record(safeIdentifierSchema, worldManifestSchema), + personas: z.record(safeIdentifierSchema, personaManifestEntrySchema), +}) + +export const personaCredentialsSchema = z.object({ + schemaVersion: z.literal(PERSONA_MANIFEST_VERSION), + runId: z.string(), + personas: z.record( + safeIdentifierSchema, + z.object({ + email: z.string().email(), + password: z.string().min(12), + }) + ), +}) + +export type ScenarioManifest = z.infer +export type PersonaManifestEntry = z.infer +export type PersonaCredentials = z.infer + +export interface CreatedWorldRecords { + users: Map + organizations: Map + organizationMembers: Map + workspaces: Map + subscriptions: Map + permissions: Map + permissionGroups: Map + permissionGroupMembers: Map + invitations: Map + invitationGrants: Map +} + +export interface E2EWorld { + scenario: ResolvedScenario + records: CreatedWorldRecords +} + +export function createWorldRecords(): CreatedWorldRecords { + return { + users: new Map(), + organizations: new Map(), + organizationMembers: new Map(), + workspaces: new Map(), + subscriptions: new Map(), + permissions: new Map(), + permissionGroups: new Map(), + permissionGroupMembers: new Map(), + invitations: new Map(), + invitationGrants: new Map(), + } +} + +export function buildScenarioManifest(runId: string, worlds: E2EWorld[]): ScenarioManifest { + const manifest: ScenarioManifest = { + schemaVersion: PERSONA_MANIFEST_VERSION, + runId, + createdAt: new Date().toISOString(), + authCaptureComplete: false, + worlds: {}, + personas: {}, + } + + for (const world of worlds) { + const worldKey = world.scenario.definition.namespace.world + if (manifest.worlds[worldKey]) { + throw new Error(`Duplicate world namespace: ${worldKey}`) + } + manifest.worlds[worldKey] = { + namespace: world.scenario.definition.namespace, + userIds: mapValues(world.records.users, ({ id }) => id), + userIdentities: Object.fromEntries(world.records.users), + organizationIds: mapValues(world.records.organizations, ({ id }) => id), + organizationIdentities: Object.fromEntries( + [...world.records.organizations].map(([key, { id, name, slug }]) => [ + key, + { id, name, slug }, + ]) + ), + organizationMemberIds: Object.fromEntries(world.records.organizationMembers), + workspaceIds: mapValues(world.records.workspaces, ({ id }) => id), + workspaceIdentities: Object.fromEntries(world.records.workspaces), + subscriptionIds: Object.fromEntries(world.records.subscriptions), + permissionIds: Object.fromEntries(world.records.permissions), + permissionGroupIds: Object.fromEntries(world.records.permissionGroups), + permissionGroupMemberIds: Object.fromEntries(world.records.permissionGroupMembers), + invitationIds: Object.fromEntries(world.records.invitations), + invitationGrantIds: Object.fromEntries(world.records.invitationGrants), + } + for (const persona of world.scenario.definition.personas) { + if (manifest.personas[persona.key]) { + throw new Error(`Duplicate cross-world persona key: ${persona.key}`) + } + manifest.personas[persona.key] = buildPersonaManifest(world, persona) + } + } + assertManifestContainsNoSecrets(manifest) + return scenarioManifestSchema.parse(manifest) +} + +export function readScenarioManifest(filePath: string): ScenarioManifest { + return scenarioManifestSchema.parse(JSON.parse(readFileSync(filePath, 'utf8'))) +} + +export function readPersonaCredentials(filePath: string): PersonaCredentials { + return personaCredentialsSchema.parse(JSON.parse(readFileSync(filePath, 'utf8'))) +} + +export function resolveStorageStatePath(directory: string, filename: string): string { + const safeFilename = storageStateFilenameSchema.parse(filename) + const root = path.resolve(directory) + const resolved = path.resolve(root, safeFilename) + if (path.dirname(resolved) !== root) { + throw new Error(`Storage-state path escapes its run directory: ${filename}`) + } + return resolved +} + +export function writeJsonAtomic(filePath: string, value: unknown, mode = 0o600): void { + mkdirSync(path.dirname(filePath), { recursive: true }) + const temporaryPath = `${filePath}.tmp-${process.pid}-${randomUUID()}` + writeFileSync(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { mode }) + renameSync(temporaryPath, filePath) +} + +export function assertManifestContainsNoSecrets(value: unknown): void { + const serialized = JSON.stringify(value) + for (const forbidden of [ + 'password', + 'cookie', + 'adminApiKey', + 'databaseUrl', + 'invitationToken', + 'stripeSecret', + ]) { + if (serialized.toLowerCase().includes(forbidden.toLowerCase())) { + throw new Error(`Non-secret scenario manifest contains forbidden field: ${forbidden}`) + } + } +} + +function buildPersonaManifest(world: E2EWorld, persona: ScenarioPersona): PersonaManifestEntry { + const user = required(world.records.users, persona.userKey, 'user') + const membership = world.scenario.definition.organizationMemberships.find( + ({ userKey }) => userKey === persona.userKey + ) + const organizationId = membership + ? required(world.records.organizations, membership.organizationKey, 'organization').id + : null + const canonicalWorkspace = required( + world.records.workspaces, + persona.canonicalRoute.workspaceKey, + 'canonical workspace' + ) + return { + key: persona.key, + world: world.scenario.definition.namespace.world, + userId: user.id, + email: user.email, + name: user.name, + expectedActiveOrganizationId: organizationId, + storageStatePath: persona.storageStateFilename, + canonicalRoute: `/workspace/${encodeURIComponent(canonicalWorkspace.id)}/settings/${ + persona.canonicalRoute.settingsSection + }`, + workspaces: persona.workspaces.map((expectation) => ({ + workspaceId: required(world.records.workspaces, expectation.workspaceKey, 'workspace').id, + workspaceKey: expectation.workspaceKey, + access: expectation.access, + roleSource: expectation.roleSource, + hostContext: expectation.hostContext, + })), + permissionGroupIds: persona.permissionGroupKeys.map((key) => + required(world.records.permissionGroups, key, 'permission group') + ), + expectedPlatformRole: persona.expectedPlatformRole, + expectedSuperUserMode: persona.expectedSuperUserMode, + } +} + +function mapValues( + values: ReadonlyMap, + select: (value: T) => string +): Record { + return Object.fromEntries([...values].map(([key, value]) => [key, select(value)])) +} + +function required(values: ReadonlyMap, key: string, label: string): T { + const value = values.get(key) + if (!value) throw new Error(`Missing created ${label}: ${key}`) + return value +} diff --git a/apps/sim/e2e/fixtures/factories/billing.ts b/apps/sim/e2e/fixtures/factories/billing.ts new file mode 100644 index 00000000000..be6507c0c87 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/billing.ts @@ -0,0 +1,165 @@ +import { db } from '@sim/db' +import { member, organization, subscription, user, userStats } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' + +const CREDITS_PER_DOLLAR = 200 +const FREE_USAGE_LIMIT_DOLLARS = '5' + +export interface SubscriptionArrangement { + referenceId: string + plan: 'pro_6000' | 'pro_25000' | 'team_6000' | 'team_25000' | 'enterprise' + status?: 'active' | 'past_due' | 'canceled' + seats?: number + memberUserIds?: string[] + enterprise?: { + monthlyPrice: number + } +} + +export async function arrangeSubscription(input: SubscriptionArrangement): Promise<{ id: string }> { + const entitled = await db + .select({ id: subscription.id }) + .from(subscription) + .where( + and( + eq(subscription.referenceId, input.referenceId), + inArray(subscription.status, ['active', 'past_due']) + ) + ) + .limit(1) + if (entitled.length > 0 && (input.status ?? 'active') !== 'canceled') { + throw new Error(`Billing reference already has an entitled subscription: ${input.referenceId}`) + } + + const now = new Date() + const periodEnd = new Date(now.getTime() + 30 * 24 * 60 * 60 * 1_000) + const id = generateId() + const status = input.status ?? 'active' + const stripeCustomerId = await resolveCanonicalStripeCustomerId(input.referenceId) + const metadata = + input.plan === 'enterprise' + ? { + plan: 'enterprise', + referenceId: input.referenceId, + monthlyPrice: input.enterprise?.monthlyPrice ?? 500, + seats: input.seats ?? 1, + } + : null + + await db.transaction(async (tx) => { + await tx.insert(subscription).values({ + id, + plan: input.plan, + referenceId: input.referenceId, + stripeCustomerId, + // Stripe lifecycle calls are outside Step 2; do not invent a remote subscription identity. + stripeSubscriptionId: null, + status, + periodStart: now, + periodEnd, + cancelAtPeriodEnd: false, + canceledAt: status === 'canceled' ? now : null, + endedAt: status === 'canceled' ? now : null, + seats: input.seats, + billingInterval: 'month', + metadata, + }) + + if (input.plan.startsWith('pro_')) { + await tx + .update(userStats) + .set({ + currentUsageLimit: String(planCredits(input.plan) / CREDITS_PER_DOLLAR), + usageLimitUpdatedAt: now, + billingBlocked: false, + billingBlockedReason: null, + }) + .where(eq(userStats.userId, input.referenceId)) + return + } + + const orgUsageLimit = + input.plan === 'enterprise' + ? String(metadata?.monthlyPrice ?? 500) + : String((planCredits(input.plan) / CREDITS_PER_DOLLAR) * (input.seats ?? 1)) + await tx + .update(organization) + .set({ orgUsageLimit, updatedAt: now }) + .where(eq(organization.id, input.referenceId)) + if (input.memberUserIds && input.memberUserIds.length > 0) { + await tx + .update(userStats) + .set({ + currentUsageLimit: null, + usageLimitUpdatedAt: now, + billingBlocked: false, + billingBlockedReason: null, + }) + .where(inArray(userStats.userId, input.memberUserIds)) + } + }) + + return { id } +} + +async function resolveCanonicalStripeCustomerId(referenceId: string): Promise { + const direct = await db + .select({ stripeCustomerId: user.stripeCustomerId }) + .from(user) + .where(eq(user.id, referenceId)) + .limit(1) + const directCustomerId = direct[0]?.stripeCustomerId + if (directCustomerId) return directCustomerId + + const owner = await db + .select({ stripeCustomerId: user.stripeCustomerId }) + .from(member) + .innerJoin(user, eq(user.id, member.userId)) + .where(and(eq(member.organizationId, referenceId), eq(member.role, 'owner'))) + .limit(1) + const ownerCustomerId = owner[0]?.stripeCustomerId + if (ownerCustomerId) return ownerCustomerId + throw new Error(`Billing reference has no canonical Stripe customer: ${referenceId}`) +} + +export async function lapseOrganizationSubscription(input: { + subscriptionId: string + organizationId: string + memberUserIds: string[] +}): Promise { + const now = new Date() + await db.transaction(async (tx) => { + await tx + .update(subscription) + .set({ + status: 'canceled', + cancelAtPeriodEnd: false, + cancelAt: now, + canceledAt: now, + endedAt: now, + periodEnd: now, + }) + .where(eq(subscription.id, input.subscriptionId)) + await tx + .update(organization) + .set({ orgUsageLimit: null, updatedAt: now }) + .where(eq(organization.id, input.organizationId)) + if (input.memberUserIds.length > 0) { + await tx + .update(userStats) + .set({ + currentUsageLimit: FREE_USAGE_LIMIT_DOLLARS, + usageLimitUpdatedAt: now, + billingBlocked: false, + billingBlockedReason: null, + }) + .where(inArray(userStats.userId, input.memberUserIds)) + } + }) +} + +function planCredits(plan: SubscriptionArrangement['plan']): number { + const match = plan.match(/_(\d+)$/) + return match ? Number(match[1]) : 0 +} diff --git a/apps/sim/e2e/fixtures/factories/invitations.ts b/apps/sim/e2e/fixtures/factories/invitations.ts new file mode 100644 index 00000000000..76904948baa --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/invitations.ts @@ -0,0 +1,49 @@ +import { db } from '@sim/db' +import { invitation, invitationWorkspaceGrant } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' + +export async function arrangePendingInvitation(input: { + email: string + token: string + inviterId: string + organizationId: string + role: 'admin' | 'member' + expiresAt: Date + workspaceGrants: Array<{ + workspaceId: string + permission: 'admin' | 'write' | 'read' + }> +}): Promise<{ invitationId: string; grantIds: string[] }> { + const invitationId = generateId() + const grantIds = input.workspaceGrants.map(() => generateId()) + const now = new Date() + await db.transaction(async (tx) => { + await tx.insert(invitation).values({ + id: invitationId, + kind: 'organization', + email: input.email.trim().toLowerCase(), + inviterId: input.inviterId, + organizationId: input.organizationId, + membershipIntent: 'internal', + role: input.role, + status: 'pending', + token: input.token, + expiresAt: input.expiresAt, + createdAt: now, + updatedAt: now, + }) + if (input.workspaceGrants.length > 0) { + await tx.insert(invitationWorkspaceGrant).values( + input.workspaceGrants.map((grant, index) => ({ + id: grantIds[index], + invitationId, + workspaceId: grant.workspaceId, + permission: grant.permission, + createdAt: now, + updatedAt: now, + })) + ) + } + }) + return { invitationId, grantIds } +} diff --git a/apps/sim/e2e/fixtures/factories/organizations.ts b/apps/sim/e2e/fixtures/factories/organizations.ts new file mode 100644 index 00000000000..d374b7fda49 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/organizations.ts @@ -0,0 +1,52 @@ +import { z } from 'zod' +import { E2eHttpClient } from '../http-client' + +const organizationSchema = z.object({ + id: z.string(), + name: z.string(), + slug: z.string(), + memberId: z.string(), +}) +const organizationMemberSchema = z.object({ + id: z.string(), + userId: z.string(), + organizationId: z.string(), + role: z.string(), + action: z.enum(['created', 'updated', 'already_member']), +}) + +export function createAdminClient(baseUrl: string, adminApiKey: string): E2eHttpClient { + return new E2eHttpClient({ + baseUrl, + defaultHeaders: { 'x-admin-key': adminApiKey }, + }) +} + +export async function createOrganization( + adminClient: E2eHttpClient, + input: { name: string; slug: string; ownerId: string } +): Promise> { + const response = await adminClient.request({ + method: 'POST', + path: '/api/v1/admin/organizations', + body: input, + schema: z.object({ data: organizationSchema }), + expectedStatus: 200, + }) + return response.data +} + +export async function addOrganizationMember( + adminClient: E2eHttpClient, + organizationId: string, + input: { userId: string; role: 'admin' | 'member' } +): Promise> { + const response = await adminClient.request({ + method: 'POST', + path: `/api/v1/admin/organizations/${organizationId}/members`, + body: input, + schema: z.object({ data: organizationMemberSchema }), + expectedStatus: 200, + }) + return response.data +} diff --git a/apps/sim/e2e/fixtures/factories/permission-groups.ts b/apps/sim/e2e/fixtures/factories/permission-groups.ts new file mode 100644 index 00000000000..25615fc1f97 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/permission-groups.ts @@ -0,0 +1,62 @@ +import { z } from 'zod' +import type { E2eHttpClient } from '../http-client' + +const permissionGroupSchema = z.object({ + id: z.string(), + organizationId: z.string(), + name: z.string(), + createdBy: z.string(), + isDefault: z.boolean(), + workspaceIds: z.array(z.string()), + config: z.record(z.string(), z.unknown()), +}) + +export interface RestrictedPermissionConfig { + hideSecretsTab: boolean + hideApiKeysTab: boolean + hideInboxTab: boolean + disableMcpTools: boolean + disableCustomTools: boolean +} + +export async function createPermissionGroup( + ownerClient: E2eHttpClient, + organizationId: string, + input: { + name: string + description?: string + workspaceIds: string[] + config: RestrictedPermissionConfig + isDefault: boolean + } +): Promise> { + const response = await ownerClient.request({ + method: 'POST', + path: `/api/organizations/${organizationId}/permission-groups`, + body: input, + schema: z.object({ permissionGroup: permissionGroupSchema }), + expectedStatus: 201, + }) + return response.permissionGroup +} + +export async function addPermissionGroupMember( + ownerClient: E2eHttpClient, + organizationId: string, + permissionGroupId: string, + userId: string +): Promise { + const response = await ownerClient.request({ + method: 'POST', + path: `/api/organizations/${organizationId}/permission-groups/${permissionGroupId}/members`, + body: { userId }, + schema: z.object({ + member: z.object({ + id: z.string(), + userId: z.string(), + }), + }), + expectedStatus: 201, + }) + return response.member.id +} diff --git a/apps/sim/e2e/fixtures/factories/platform.ts b/apps/sim/e2e/fixtures/factories/platform.ts new file mode 100644 index 00000000000..ddb4e3f67a7 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/platform.ts @@ -0,0 +1,16 @@ +import { db } from '@sim/db' +import { settings, user } from '@sim/db/schema' +import { eq } from 'drizzle-orm' + +export async function arrangeEffectivePlatformAdmin(userId: string): Promise { + await db.transaction(async (tx) => { + await tx.update(user).set({ role: 'admin', updatedAt: new Date() }).where(eq(user.id, userId)) + await tx + .insert(settings) + .values({ id: userId, userId, superUserModeEnabled: true }) + .onConflictDoUpdate({ + target: settings.userId, + set: { superUserModeEnabled: true }, + }) + }) +} diff --git a/apps/sim/e2e/fixtures/factories/users.ts b/apps/sim/e2e/fixtures/factories/users.ts new file mode 100644 index 00000000000..e1635e49e9a --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/users.ts @@ -0,0 +1,49 @@ +import { z } from 'zod' +import { E2eHttpClient } from '../http-client' + +const authUserSchema = z.object({ + id: z.string().min(1), + email: z.string().email(), + name: z.string(), +}) +const authResponseSchema = z + .object({ + user: authUserSchema, + }) + .passthrough() + +export interface SyntheticLogin { + name: string + email: string + password: string +} + +export async function createSyntheticUser( + client: E2eHttpClient, + login: SyntheticLogin +): Promise> { + const response = await client.request({ + method: 'POST', + path: '/api/auth/sign-up/email', + body: login, + schema: authResponseSchema, + expectedStatus: 200, + }) + return response.user +} + +export async function createAuthenticatedClient( + baseUrl: string, + login: SyntheticLogin, + onAttempt?: ConstructorParameters[0]['onAttempt'] +): Promise { + const client = new E2eHttpClient({ baseUrl, onAttempt }) + await client.request({ + method: 'POST', + path: '/api/auth/sign-in/email', + body: { email: login.email, password: login.password }, + schema: authResponseSchema, + expectedStatus: 200, + }) + return client +} diff --git a/apps/sim/e2e/fixtures/factories/workspaces.ts b/apps/sim/e2e/fixtures/factories/workspaces.ts new file mode 100644 index 00000000000..c078fae2bbb --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/workspaces.ts @@ -0,0 +1,49 @@ +import { z } from 'zod' +import type { E2eHttpClient } from '../http-client' + +const workspaceSchema = z + .object({ + id: z.string(), + name: z.string(), + ownerId: z.string(), + organizationId: z.string().nullable(), + billedAccountUserId: z.string(), + workspaceMode: z.enum(['personal', 'organization', 'grandfathered_shared']), + }) + .passthrough() +const permissionMutationSchema = z.object({ + id: z.string(), + workspaceId: z.string(), + userId: z.string(), + permissions: z.enum(['admin', 'write', 'read']), + action: z.enum(['created', 'updated', 'already_member']), +}) + +export async function createWorkspace( + ownerClient: E2eHttpClient, + input: { name: string; color?: string } +): Promise> { + const response = await ownerClient.request({ + method: 'POST', + path: '/api/workspaces', + body: { ...input, skipDefaultWorkflow: true }, + schema: z.object({ workspace: workspaceSchema }), + expectedStatus: 200, + }) + return response.workspace +} + +export async function grantWorkspacePermission( + adminClient: E2eHttpClient, + workspaceId: string, + input: { userId: string; permissions: 'admin' | 'write' | 'read' } +): Promise> { + const response = await adminClient.request({ + method: 'POST', + path: `/api/v1/admin/workspaces/${workspaceId}/members`, + body: input, + schema: z.object({ data: permissionMutationSchema }), + expectedStatus: 200, + }) + return response.data +} diff --git a/apps/sim/e2e/fixtures/http-client.ts b/apps/sim/e2e/fixtures/http-client.ts new file mode 100644 index 00000000000..42cd8034659 --- /dev/null +++ b/apps/sim/e2e/fixtures/http-client.ts @@ -0,0 +1,148 @@ +import { sleep } from '@sim/utils/helpers' +import type { z } from 'zod' + +// Better Auth does not emit Retry-After for every limiter. The bounded fallback spans its +// one-minute signup window while still preferring an explicit server header when available. +const DEFAULT_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000] as const +const MAX_RETRY_DELAY_MS = 30_000 +const MAX_TOTAL_RETRY_DELAY_MS = 90_000 + +export interface E2eHttpClientOptions { + baseUrl: string + defaultHeaders?: Record + fetchImplementation?: typeof fetch + sleepImplementation?: (milliseconds: number) => Promise + retryDelaysMs?: readonly number[] + onAttempt?: (attempt: { method: string; path: string; number: number; status?: number }) => void +} + +export class E2eHttpClient { + private readonly baseUrl: string + private readonly defaultHeaders: Record + private readonly fetchImplementation: typeof fetch + private readonly sleepImplementation: (milliseconds: number) => Promise + private readonly retryDelaysMs: readonly number[] + private readonly onAttempt?: E2eHttpClientOptions['onAttempt'] + private readonly cookies = new Map() + + constructor(options: E2eHttpClientOptions) { + this.baseUrl = options.baseUrl.replace(/\/$/, '') + this.defaultHeaders = options.defaultHeaders ?? {} + this.fetchImplementation = options.fetchImplementation ?? fetch + this.sleepImplementation = options.sleepImplementation ?? sleep + this.retryDelaysMs = options.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS + this.onAttempt = options.onAttempt + } + + async request(options: { + method?: string + path: string + body?: unknown + schema: TSchema + expectedStatus?: number | readonly number[] + }): Promise> { + const method = options.method ?? 'GET' + const expectedStatuses = Array.isArray(options.expectedStatus) + ? options.expectedStatus + : [options.expectedStatus ?? 200] + let totalRetryDelay = 0 + + for (let attempt = 1; ; attempt += 1) { + const response = await this.fetchImplementation(`${this.baseUrl}${options.path}`, { + method, + headers: { + accept: 'application/json', + ...(options.body === undefined ? {} : { 'content-type': 'application/json' }), + ...this.defaultHeaders, + ...(this.cookies.size > 0 ? { cookie: this.serializeCookies() } : {}), + }, + body: options.body === undefined ? undefined : JSON.stringify(options.body), + redirect: 'manual', + }) + this.captureCookies(response.headers) + this.onAttempt?.({ method, path: options.path, number: attempt, status: response.status }) + + if (response.status === 429 && attempt <= this.retryDelaysMs.length) { + const requestedDelay = + parseRetryAfter(response.headers.get('retry-after')) ?? this.retryDelaysMs[attempt - 1] + const remainingDelayBudget = MAX_TOTAL_RETRY_DELAY_MS - totalRetryDelay + if (remainingDelayBudget > 0) { + const delay = Math.min(requestedDelay, MAX_RETRY_DELAY_MS, remainingDelayBudget) + totalRetryDelay += delay + await this.sleepImplementation(delay) + continue + } + } + + const payload = await readJsonResponse(response, method, options.path) + if (!expectedStatuses.includes(response.status)) { + throw new Error( + `${method} ${options.path} returned ${response.status}; expected ${expectedStatuses.join( + '/' + )}; response body redacted` + ) + } + const parsed = options.schema.safeParse(payload) + if (!parsed.success) { + throw new Error( + `${method} ${options.path} returned an invalid response: ${parsed.error.issues + .map((issue) => `${issue.path.join('.') || ''}: ${issue.message}`) + .join('; ')}` + ) + } + return parsed.data + } + } + + getCookieHeader(): string { + return this.serializeCookies() + } + + private captureCookies(headers: Headers): void { + const values = + 'getSetCookie' in headers && typeof headers.getSetCookie === 'function' + ? headers.getSetCookie() + : splitCombinedSetCookie(headers.get('set-cookie')) + for (const value of values) { + const pair = value.split(';', 1)[0] + const separator = pair.indexOf('=') + if (separator <= 0) continue + const name = pair.slice(0, separator).trim() + const cookieValue = pair.slice(separator + 1).trim() + if (cookieValue) this.cookies.set(name, cookieValue) + else this.cookies.delete(name) + } + } + + private serializeCookies(): string { + return [...this.cookies.entries()].map(([name, value]) => `${name}=${value}`).join('; ') + } +} + +function parseRetryAfter(value: string | null): number | null { + if (!value) return null + const seconds = Number(value) + if (Number.isFinite(seconds) && seconds >= 0) return Math.ceil(seconds * 1_000) + const date = Date.parse(value) + if (!Number.isNaN(date)) return Math.max(0, date - Date.now()) + return null +} + +function splitCombinedSetCookie(value: string | null): string[] { + if (!value) return [] + return value.split(/,(?=\s*[^;,=\s]+=[^;,]*)/) +} + +async function readJsonResponse( + response: Response, + method: string, + path: string +): Promise { + const text = await response.text() + if (!text) return null + try { + return JSON.parse(text) + } catch { + throw new Error(`${method} ${path} returned non-JSON content with status ${response.status}`) + } +} diff --git a/apps/sim/e2e/fixtures/namespace.ts b/apps/sim/e2e/fixtures/namespace.ts new file mode 100644 index 00000000000..4d47df866fa --- /dev/null +++ b/apps/sim/e2e/fixtures/namespace.ts @@ -0,0 +1,86 @@ +import { createHash } from 'node:crypto' +import type { ScenarioNamespaceDescriptor } from './scenario' + +const EMAIL_DOMAIN = 'example.com' + +export interface ScenarioNamespace extends ScenarioNamespaceDescriptor { + email(label: string): string + slug(label: string): string + name(label: string): string + invitationToken(label: string): string + storageStateFilename(personaKey: string): string +} + +/** + * Namespaces only values the fixture controls. Production IDs are deliberately absent from this API. + */ +export function createScenarioNamespace(run: string, world: string): ScenarioNamespace { + const normalizedRun = normalizePart(run, 'run') + const normalizedWorld = normalizePart(world, 'world') + const digest = hash(`${run}\0${world}`).slice(0, 8) + const prefix = `e2e-${normalizedRun.slice(0, 10)}-${normalizedWorld.slice(0, 10)}-${digest}` + + return Object.freeze({ + run, + world, + prefix, + email(label: string): string { + return `${prefix}-${shortLabel(label, 11)}-${hash(label).slice(0, 8)}@${EMAIL_DOMAIN}` + }, + slug(label: string): string { + return `${prefix}-${shortLabel(label, 18)}-${hash(label).slice(0, 8)}` + }, + name(label: string): string { + return `E Two E ${humanize(world)} ${humanize(label)} ${alphabeticHash( + `${prefix}\0${label}` + )}` + }, + invitationToken(label: string): string { + const normalizedLabel = normalizeLabel(label).slice(0, 24) + return `${prefix}.invite.${normalizedLabel}.${hash(`${prefix}\0${label}`).slice(0, 12)}` + }, + storageStateFilename(personaKey: string): string { + return `${prefix}-${shortLabel(personaKey, 20)}-${hash(personaKey).slice(0, 8)}.json` + }, + }) +} + +function normalizePart(value: string, label: string): string { + const normalized = normalizeLabel(value) + if (!normalized) throw new Error(`Scenario namespace ${label} must contain a letter or number`) + return normalized +} + +function normalizeLabel(value: string): string { + const normalized = value + .normalize('NFKD') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + if (!normalized) throw new Error('Namespaced value label must contain a letter or number') + return normalized +} + +function humanize(value: string): string { + const normalized = normalizeLabel(value) + return normalized + .split('-') + .map((part) => `${part[0].toUpperCase()}${part.slice(1)}`) + .join(' ') +} + +function shortLabel(value: string, length: number): string { + return normalizeLabel(value).slice(0, length).replace(/-+$/, '') +} + +function hash(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +function alphabeticHash(value: string): string { + return hash(value) + .slice(0, 8) + .replace(/[0-9a-f]/g, (character) => + String.fromCharCode('a'.charCodeAt(0) + Number.parseInt(character, 16)) + ) +} diff --git a/apps/sim/e2e/fixtures/persona-test.ts b/apps/sim/e2e/fixtures/persona-test.ts new file mode 100644 index 00000000000..1bfe4d84e3f --- /dev/null +++ b/apps/sim/e2e/fixtures/persona-test.ts @@ -0,0 +1,59 @@ +import { test as base } from '@playwright/test' +import { + type PersonaManifestEntry, + readScenarioManifest, + resolveStorageStatePath, + type ScenarioManifest, +} from './e2e-world' + +interface PersonaFixtures { + personaManifest: ScenarioManifest + contextForPersona: (personaKey: string) => Promise +} + +export const test = base.extend({ + personaManifest: [ + async ({ browserName: _browserName }, use) => { + const manifest = readScenarioManifest(requiredEnv('E2E_MANIFEST_PATH')) + if (!manifest.authCaptureComplete) { + throw new Error('Persona storage states were not captured successfully') + } + await use(manifest) + }, + { scope: 'test' }, + ], + contextForPersona: async ({ browser, contextOptions, personaManifest }, use) => { + const contexts = new Set() + await use(async (personaKey) => { + const persona = requirePersona(personaManifest, personaKey) + const context = await browser.newContext({ + ...contextOptions, + baseURL: requiredEnv('E2E_BASE_URL'), + storageState: resolveStorageStatePath( + requiredEnv('E2E_STORAGE_STATE_DIR'), + persona.storageStatePath + ), + }) + contexts.add(context) + return context + }) + await Promise.all([...contexts].map((context) => context.close())) + }, +}) + +export { expect } from '@playwright/test' + +export function requirePersona( + manifest: ScenarioManifest, + personaKey: string +): PersonaManifestEntry { + const persona = manifest.personas[personaKey] + if (!persona) throw new Error(`Unknown persona: ${personaKey}`) + return persona +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing persona fixture environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/fixtures/scenario.ts b/apps/sim/e2e/fixtures/scenario.ts new file mode 100644 index 00000000000..9367c70979b --- /dev/null +++ b/apps/sim/e2e/fixtures/scenario.ts @@ -0,0 +1,184 @@ +export const SCENARIO_VERSION = 1 as const + +export type ScenarioVersion = typeof SCENARIO_VERSION +export type ResourceKey = string +export type WorkspaceAccess = 'read' | 'write' | 'admin' +export type ExpectedWorkspaceAccess = WorkspaceAccess | 'none' +export type RoleSource = 'owner' | 'explicit' | 'org-admin' | 'none' +export type OrganizationRole = 'owner' | 'admin' | 'member' +export type SubscriptionPlan = 'pro_6000' | 'pro_25000' | 'team_6000' | 'team_25000' | 'enterprise' +export type SubscriptionStatus = 'active' | 'past_due' | 'lapsed' +export type SettingsSection = 'general' | 'secrets' | 'api-keys' | 'inbox' | 'mcp' | 'custom-tools' + +export interface ScenarioNamespaceDescriptor { + run: string + world: string + prefix: string +} + +export interface ScenarioDeployment { + hosted: boolean + billingEnabled: boolean +} + +export interface ScenarioUser { + key: ResourceKey + email: string + name: string + hosted: boolean + billingEnabled: boolean + platformRole?: 'user' | 'admin' + superUserModeEnabled?: boolean +} + +export interface ScenarioOrganization { + key: ResourceKey + name: string + slug: string + ownerUserKey: ResourceKey + hosted: boolean + billingEnabled: boolean +} + +export interface ScenarioOrganizationMembership { + organizationKey: ResourceKey + userKey: ResourceKey + role: OrganizationRole +} + +export interface EnterpriseSubscriptionMetadata { + plan: 'enterprise' + monthlyPrice: number + seats: number +} + +export interface ScenarioSubscription { + key: ResourceKey + plan: SubscriptionPlan + status: SubscriptionStatus + billingReference: + | { kind: 'user'; userKey: ResourceKey } + | { kind: 'organization'; organizationKey: ResourceKey } + hosted: boolean + billingEnabled: boolean + seats?: number + enterprise?: EnterpriseSubscriptionMetadata +} + +export interface ScenarioWorkspace { + key: ResourceKey + name: string + ownerUserKey: ResourceKey + organizationKey?: ResourceKey + payer: + | { kind: 'user'; userKey: ResourceKey } + | { kind: 'organization'; organizationKey: ResourceKey } + subscriptionKey?: ResourceKey + hosted: boolean + billingEnabled: boolean +} + +export interface ScenarioWorkspaceGrant { + workspaceKey: ResourceKey + userKey: ResourceKey + access: WorkspaceAccess +} + +export interface PermissionGroupRestrictions { + hiddenSettings: readonly SettingsSection[] + disabledFeatures: readonly ('mcp' | 'custom-tools')[] +} + +export interface ScenarioPermissionGroup { + key: ResourceKey + name: string + organizationKey: ResourceKey + workspaceKeys: readonly ResourceKey[] + memberUserKeys: readonly ResourceKey[] + restrictions: PermissionGroupRestrictions +} + +export interface ScenarioInvitation { + key: ResourceKey + organizationKey: ResourceKey + email: string + token: string + role: Exclude + expiresAt: string + workspaceGrants: readonly { + workspaceKey: ResourceKey + access: WorkspaceAccess + }[] +} + +export interface PersonaWorkspaceExpectation { + workspaceKey: ResourceKey + access: ExpectedWorkspaceAccess + roleSource: RoleSource + hostContext: { + isOwner: boolean + hostMembership: 'owner' | 'member' | 'external' + payerScope: 'user' | 'organization' + plan: 'free' | SubscriptionPlan + hosted: boolean + billingEnabled: boolean + } +} + +export interface ScenarioPersona { + key: ResourceKey + userKey: ResourceKey + storageStateFilename: string + workspaces: readonly PersonaWorkspaceExpectation[] + permissionGroupKeys: readonly ResourceKey[] + canonicalRoute: { + workspaceKey: ResourceKey + settingsSection: SettingsSection + } + expectedPlatformRole: 'user' | 'admin' + expectedSuperUserMode: boolean +} + +/** + * Pure declaration consumed by later factories. Keys are graph-local references, never production IDs. + * Factories must record API-generated IDs separately and keep them opaque. + */ +export interface ScenarioDefinition { + version: ScenarioVersion + namespace: ScenarioNamespaceDescriptor + deployment: ScenarioDeployment + users: readonly ScenarioUser[] + organizations: readonly ScenarioOrganization[] + organizationMemberships: readonly ScenarioOrganizationMembership[] + subscriptions: readonly ScenarioSubscription[] + workspaces: readonly ScenarioWorkspace[] + workspaceGrants: readonly ScenarioWorkspaceGrant[] + permissionGroups: readonly ScenarioPermissionGroup[] + invitations: readonly ScenarioInvitation[] + personas: readonly ScenarioPersona[] +} + +export interface ResolvedPersona { + definition: ScenarioPersona + user: ScenarioUser + workspaces: readonly { + expectation: PersonaWorkspaceExpectation + workspace: ScenarioWorkspace + }[] + permissionGroups: readonly ScenarioPermissionGroup[] +} + +export interface ResolvedScenario { + definition: ScenarioDefinition + usersByKey: ReadonlyMap + organizationsByKey: ReadonlyMap + subscriptionsByKey: ReadonlyMap + workspacesByKey: ReadonlyMap + permissionGroupsByKey: ReadonlyMap + invitationsByKey: ReadonlyMap + personasByKey: ReadonlyMap +} + +export function canonicalSettingsRoute(workspaceId: string, section: SettingsSection): string { + return `/workspace/${encodeURIComponent(workspaceId)}/settings/${section}` +} diff --git a/apps/sim/e2e/fixtures/validate-scenario.ts b/apps/sim/e2e/fixtures/validate-scenario.ts new file mode 100644 index 00000000000..0d2ac3e7698 --- /dev/null +++ b/apps/sim/e2e/fixtures/validate-scenario.ts @@ -0,0 +1,746 @@ +import type { + ExpectedWorkspaceAccess, + OrganizationRole, + ResolvedPersona, + ResolvedScenario, + ResourceKey, + ScenarioDefinition, + ScenarioOrganization, + ScenarioPermissionGroup, + ScenarioSubscription, + ScenarioUser, + ScenarioWorkspace, + ScenarioWorkspaceGrant, +} from './scenario' +import { SCENARIO_VERSION } from './scenario' + +export class ScenarioValidationError extends Error { + readonly issues: readonly string[] + + constructor(issues: readonly string[]) { + super(`Invalid E2E scenario:\n- ${issues.join('\n- ')}`) + this.name = 'ScenarioValidationError' + this.issues = issues + } +} + +const RESOURCE_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/ +const STORAGE_STATE_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\.json$/ + +export function validateScenario(definition: ScenarioDefinition): ResolvedScenario { + const issues: string[] = [] + if (definition.version !== SCENARIO_VERSION) { + issues.push(`version must be ${SCENARIO_VERSION}`) + } + if (!definition.namespace.run || !definition.namespace.world || !definition.namespace.prefix) { + issues.push('namespace run, world, and prefix must be non-empty') + } + + const usersByKey = indexByKey('user', definition.users, issues) + const organizationsByKey = indexByKey('organization', definition.organizations, issues) + const subscriptionsByKey = indexByKey('subscription', definition.subscriptions, issues) + const workspacesByKey = indexByKey('workspace', definition.workspaces, issues) + const permissionGroupsByKey = indexByKey('permission group', definition.permissionGroups, issues) + const invitationsByKey = indexByKey('invitation', definition.invitations, issues) + indexByKey('persona', definition.personas, issues) + + validateControllableIdentities(definition, issues) + validateDeploymentFlags(definition, issues) + validateMemberships(definition, usersByKey, organizationsByKey, issues) + validateOrganizations(definition, usersByKey, issues) + validateSubscriptions(definition, usersByKey, organizationsByKey, issues) + validateWorkspaces(definition, usersByKey, organizationsByKey, subscriptionsByKey, issues) + validateWorkspaceGrants(definition.workspaceGrants, usersByKey, workspacesByKey, issues) + validatePermissionGroups(definition, usersByKey, organizationsByKey, workspacesByKey, issues) + validateInvitations(definition, organizationsByKey, workspacesByKey, issues) + + const resolvedPersonas = new Map() + for (const persona of definition.personas) { + const user = usersByKey.get(persona.userKey) + if (!user) { + issues.push(`persona "${persona.key}" references missing user "${persona.userKey}"`) + continue + } + if (persona.workspaces.filter(({ access }) => access !== 'none').length === 0) { + issues.push(`persona "${persona.key}" has zero accessible seeded workspaces`) + } + validateUniqueValues( + `persona "${persona.key}" workspace expectation`, + persona.workspaces.map(({ workspaceKey }) => workspaceKey), + issues + ) + validateUniqueValues( + `persona "${persona.key}" permission group`, + persona.permissionGroupKeys, + issues + ) + + const resolvedWorkspaces: ResolvedPersona['workspaces'][number][] = [] + for (const expectation of persona.workspaces) { + const workspace = workspacesByKey.get(expectation.workspaceKey) + if (!workspace) { + issues.push( + `persona "${persona.key}" references missing workspace "${expectation.workspaceKey}"` + ) + continue + } + validatePersonaWorkspaceExpectation( + definition, + persona.key, + user, + workspace, + expectation, + subscriptionsByKey, + issues + ) + resolvedWorkspaces.push({ expectation, workspace }) + } + + const routeExpectation = persona.workspaces.find( + ({ workspaceKey }) => workspaceKey === persona.canonicalRoute.workspaceKey + ) + if (!routeExpectation || routeExpectation.access === 'none') { + issues.push( + `persona "${persona.key}" canonical route must target an accessible declared workspace` + ) + } + + const expectedGroups = groupsForUser(definition, user.key) + if ( + !sameSet( + persona.permissionGroupKeys, + expectedGroups.map(({ key }) => key) + ) + ) { + issues.push(`persona "${persona.key}" permission-group expectations do not match its grants`) + } + const resolvedGroups = persona.permissionGroupKeys + .map((key) => permissionGroupsByKey.get(key)) + .filter((group): group is ScenarioPermissionGroup => group !== undefined) + + const platformRole = user.platformRole ?? 'user' + if (persona.expectedPlatformRole !== platformRole) { + issues.push(`persona "${persona.key}" has an inconsistent expected platform role`) + } + if (persona.expectedSuperUserMode !== (user.superUserModeEnabled ?? false)) { + issues.push(`persona "${persona.key}" has an inconsistent expected superuser setting`) + } + if (platformRole === 'admin' && !user.superUserModeEnabled) { + issues.push(`platform admin user "${user.key}" must enable superuser mode`) + } + if (user.superUserModeEnabled && platformRole !== 'admin') { + issues.push(`superuser user "${user.key}" must also have the platform admin role`) + } + + resolvedPersonas.set(persona.key, { + definition: persona, + user, + workspaces: resolvedWorkspaces, + permissionGroups: resolvedGroups, + }) + } + + if (issues.length > 0) throw new ScenarioValidationError(issues) + + return { + definition, + usersByKey, + organizationsByKey, + subscriptionsByKey, + workspacesByKey, + permissionGroupsByKey, + invitationsByKey, + personasByKey: resolvedPersonas, + } +} + +export function validateScenarioSet(scenarios: readonly ResolvedScenario[]): void { + const issues: string[] = [] + validateUniqueValues( + 'world namespace', + scenarios.map(({ definition }) => definition.namespace.world), + issues + ) + validateUniqueValues( + 'world namespace prefix', + scenarios.map(({ definition }) => definition.namespace.prefix), + issues + ) + validateUniqueValues( + 'cross-world email', + scenarios.flatMap(({ definition }) => [ + ...definition.users.map(({ email }) => email.toLowerCase()), + ...definition.invitations.map(({ email }) => email.toLowerCase()), + ]), + issues + ) + validateUniqueValues( + 'cross-world organization slug', + scenarios.flatMap(({ definition }) => + definition.organizations.map(({ slug }) => slug.toLowerCase()) + ), + issues + ) + validateUniqueValues( + 'cross-world persona key', + scenarios.flatMap(({ definition }) => definition.personas.map(({ key }) => key)), + issues + ) + validateUniqueValues( + 'cross-world storage-state filename', + scenarios.flatMap(({ definition }) => + definition.personas.map(({ storageStateFilename }) => storageStateFilename) + ), + issues + ) + if (issues.length > 0) throw new ScenarioValidationError(issues) +} + +function indexByKey( + label: string, + values: readonly T[], + issues: string[] +): Map { + const result = new Map() + for (const value of values) { + if (!value.key.trim()) { + issues.push(`${label} key must be non-empty`) + } else if (!RESOURCE_KEY_PATTERN.test(value.key)) { + issues.push(`${label} key "${value.key}" must be a safe identifier`) + } else if (result.has(value.key)) { + issues.push(`duplicate ${label} key "${value.key}"`) + } else { + result.set(value.key, value) + } + } + return result +} + +function validateControllableIdentities(definition: ScenarioDefinition, issues: string[]): void { + validateUniqueValues( + 'email', + [ + ...definition.users.map(({ email }) => email.toLowerCase()), + ...definition.invitations.map(({ email }) => email.toLowerCase()), + ], + issues + ) + validateUniqueValues( + 'user name', + definition.users.map(({ name }) => name), + issues + ) + validateUniqueValues( + 'organization name', + definition.organizations.map(({ name }) => name), + issues + ) + validateUniqueValues( + 'organization slug', + definition.organizations.map(({ slug }) => slug.toLowerCase()), + issues + ) + validateUniqueValues( + 'workspace name', + definition.workspaces.map(({ name }) => name), + issues + ) + validateUniqueValues( + 'permission-group name', + definition.permissionGroups.map( + ({ organizationKey, name }) => `${organizationKey}\0${name.toLowerCase()}` + ), + issues + ) + validateUniqueValues( + 'invitation token', + definition.invitations.map(({ token }) => token), + issues + ) + validateUniqueValues( + 'storage-state filename', + definition.personas.map(({ storageStateFilename }) => storageStateFilename), + issues + ) + for (const { storageStateFilename } of definition.personas) { + if ( + !STORAGE_STATE_FILENAME_PATTERN.test(storageStateFilename) || + storageStateFilename.includes('..') + ) { + issues.push( + `storage-state filename "${storageStateFilename}" must be a separator-free JSON basename` + ) + } + } +} + +function validateUniqueValues(label: string, values: readonly string[], issues: string[]): void { + const seen = new Set() + for (const value of values) { + if (!value.trim()) { + issues.push(`${label} must be non-empty`) + } else if (seen.has(value)) { + issues.push(`duplicate ${label} "${value.replace('\0', '/')}"`) + } else { + seen.add(value) + } + } +} + +function validateDeploymentFlags(definition: ScenarioDefinition, issues: string[]): void { + const resources = [ + ...definition.users.map((resource) => ({ kind: 'user', resource })), + ...definition.organizations.map((resource) => ({ kind: 'organization', resource })), + ...definition.subscriptions.map((resource) => ({ kind: 'subscription', resource })), + ...definition.workspaces.map((resource) => ({ kind: 'workspace', resource })), + ] + for (const { kind, resource } of resources) { + if (resource.hosted !== definition.deployment.hosted) { + issues.push(`${kind} "${resource.key}" has an inconsistent hosted flag`) + } + if (resource.billingEnabled !== definition.deployment.billingEnabled) { + issues.push(`${kind} "${resource.key}" has an inconsistent billing flag`) + } + } + if (!definition.deployment.billingEnabled && definition.subscriptions.length > 0) { + issues.push('billing-disabled scenarios cannot contain subscriptions') + } +} + +function validateMemberships( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + issues: string[] +): void { + const pairs = new Set() + const organizationByUser = new Map() + for (const membership of definition.organizationMemberships) { + const pair = `${membership.organizationKey}\0${membership.userKey}` + if (pairs.has(pair)) { + issues.push( + `duplicate organization membership "${membership.organizationKey}/${membership.userKey}"` + ) + } + pairs.add(pair) + if (!organizationsByKey.has(membership.organizationKey)) { + issues.push(`membership references missing organization "${membership.organizationKey}"`) + } + if (!usersByKey.has(membership.userKey)) { + issues.push(`membership references missing user "${membership.userKey}"`) + } + const previousOrganization = organizationByUser.get(membership.userKey) + if (previousOrganization && previousOrganization !== membership.organizationKey) { + issues.push(`user "${membership.userKey}" cannot belong to more than one organization`) + } + organizationByUser.set(membership.userKey, membership.organizationKey) + } +} + +function validateOrganizations( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + issues: string[] +): void { + for (const organization of definition.organizations) { + if (!usersByKey.has(organization.ownerUserKey)) { + issues.push( + `organization "${organization.key}" references missing owner "${organization.ownerUserKey}"` + ) + } + const ownerMemberships = definition.organizationMemberships.filter( + (membership) => + membership.organizationKey === organization.key && + membership.userKey === organization.ownerUserKey && + membership.role === 'owner' + ) + if (ownerMemberships.length !== 1) { + issues.push(`organization "${organization.key}" must have exactly one owner membership`) + } + const otherOwners = definition.organizationMemberships.filter( + (membership) => + membership.organizationKey === organization.key && + membership.role === 'owner' && + membership.userKey !== organization.ownerUserKey + ) + if (otherOwners.length > 0) { + issues.push(`organization "${organization.key}" cannot have a second owner membership`) + } + } +} + +function validateSubscriptions( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + issues: string[] +): void { + const entitledBillingReferences = new Set() + for (const subscription of definition.subscriptions) { + const reference = billingReferenceKey(subscription) + if (subscription.billingReference.kind === 'user') { + if (!usersByKey.has(subscription.billingReference.userKey)) { + issues.push( + `subscription "${subscription.key}" references missing user "${subscription.billingReference.userKey}"` + ) + } + if (subscription.plan.startsWith('team_') || subscription.plan === 'enterprise') { + issues.push(`subscription "${subscription.key}" has an invalid plan for a user payer`) + } + if (subscription.status === 'lapsed') { + issues.push( + `subscription "${subscription.key}" uses unsupported lapsed status for a user payer` + ) + } + } else { + const organizationKey = subscription.billingReference.organizationKey + if (!organizationsByKey.has(organizationKey)) { + issues.push( + `subscription "${subscription.key}" references missing organization "${organizationKey}"` + ) + } + if (subscription.plan === 'pro_6000' || subscription.plan === 'pro_25000') { + issues.push( + `subscription "${subscription.key}" has an invalid plan for an organization payer` + ) + } + const memberCount = definition.organizationMemberships.filter( + (membership) => membership.organizationKey === organizationKey + ).length + if (!subscription.seats || subscription.seats < memberCount) { + issues.push(`subscription "${subscription.key}" seats do not cover organization members`) + } + } + + if (subscription.status === 'active' || subscription.status === 'past_due') { + if (entitledBillingReferences.has(reference)) { + issues.push(`billing reference "${reference}" has duplicate entitled subscriptions`) + } + entitledBillingReferences.add(reference) + } + + if (subscription.plan === 'enterprise') { + const metadata = subscription.enterprise + if ( + !metadata || + metadata.plan !== 'enterprise' || + !Number.isFinite(metadata.monthlyPrice) || + metadata.monthlyPrice <= 0 || + !Number.isInteger(metadata.seats) || + metadata.seats < 1 || + metadata.seats !== subscription.seats + ) { + issues.push(`subscription "${subscription.key}" has invalid Enterprise metadata`) + } + } else if (subscription.enterprise) { + issues.push(`non-Enterprise subscription "${subscription.key}" has Enterprise metadata`) + } + } +} + +function validateWorkspaces( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + subscriptionsByKey: ReadonlyMap, + issues: string[] +): void { + for (const workspace of definition.workspaces) { + if (!usersByKey.has(workspace.ownerUserKey)) { + issues.push( + `workspace "${workspace.key}" references missing owner "${workspace.ownerUserKey}"` + ) + } + const subscription = workspace.subscriptionKey + ? subscriptionsByKey.get(workspace.subscriptionKey) + : undefined + if (workspace.subscriptionKey && !subscription) { + issues.push( + `workspace "${workspace.key}" references missing subscription "${workspace.subscriptionKey}"` + ) + } + + if (workspace.organizationKey) { + if (!organizationsByKey.has(workspace.organizationKey)) { + issues.push( + `workspace "${workspace.key}" references missing organization "${workspace.organizationKey}"` + ) + } + if ( + workspace.payer.kind !== 'organization' || + workspace.payer.organizationKey !== workspace.organizationKey + ) { + issues.push(`organization workspace "${workspace.key}" has an incoherent payer`) + } + const ownerMembership = membershipFor( + definition, + workspace.organizationKey, + workspace.ownerUserKey + ) + const organization = organizationsByKey.get(workspace.organizationKey) + if ( + !ownerMembership || + ownerMembership.role !== 'owner' || + organization?.ownerUserKey !== workspace.ownerUserKey + ) { + issues.push( + `organization workspace "${workspace.key}" creator must be the organization owner` + ) + } + if (!subscription) { + issues.push(`organization workspace "${workspace.key}" must retain its subscription record`) + } else if ( + subscription.billingReference.kind !== 'organization' || + subscription.billingReference.organizationKey !== workspace.organizationKey + ) { + issues.push(`organization workspace "${workspace.key}" has an incoherent subscription`) + } + if (subscription?.status === 'past_due') { + issues.push( + `organization workspace "${workspace.key}" cannot be provisioned from a past-due subscription` + ) + } + } else { + if (workspace.payer.kind !== 'user' || workspace.payer.userKey !== workspace.ownerUserKey) { + issues.push(`personal workspace "${workspace.key}" has an incoherent payer/owner`) + } + if ( + subscription && + (subscription.billingReference.kind !== 'user' || + subscription.billingReference.userKey !== workspace.ownerUserKey) + ) { + issues.push(`personal workspace "${workspace.key}" has an incoherent subscription`) + } + } + } +} + +function validateWorkspaceGrants( + grants: readonly ScenarioWorkspaceGrant[], + usersByKey: ReadonlyMap, + workspacesByKey: ReadonlyMap, + issues: string[] +): void { + const pairs = new Set() + for (const grant of grants) { + const pair = `${grant.workspaceKey}\0${grant.userKey}` + if (pairs.has(pair)) { + issues.push(`duplicate workspace grant "${grant.workspaceKey}/${grant.userKey}"`) + } + pairs.add(pair) + if (!workspacesByKey.has(grant.workspaceKey)) { + issues.push(`workspace grant references missing workspace "${grant.workspaceKey}"`) + } + if (!usersByKey.has(grant.userKey)) { + issues.push(`workspace grant references missing user "${grant.userKey}"`) + } + } +} + +function validatePermissionGroups( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + workspacesByKey: ReadonlyMap, + issues: string[] +): void { + for (const group of definition.permissionGroups) { + if (!organizationsByKey.has(group.organizationKey)) { + issues.push( + `permission group "${group.key}" references missing organization "${group.organizationKey}"` + ) + } + if (group.workspaceKeys.length === 0) { + issues.push(`permission group "${group.key}" must have Enterprise workspace scope`) + } + if (group.memberUserKeys.length === 0) { + issues.push( + `permission group "${group.key}" must name explicit members; default and all-member groups are not modeled` + ) + } + validateUniqueValues( + `permission group "${group.key}" workspace scope`, + group.workspaceKeys, + issues + ) + validateUniqueValues(`permission group "${group.key}" member`, group.memberUserKeys, issues) + + const enterpriseSubscription = definition.subscriptions.find( + (subscription) => + subscription.billingReference.kind === 'organization' && + subscription.billingReference.organizationKey === group.organizationKey && + subscription.plan === 'enterprise' && + subscription.status === 'active' + ) + if (!enterpriseSubscription) { + issues.push(`permission group "${group.key}" requires an active Enterprise organization`) + } + for (const workspaceKey of group.workspaceKeys) { + const workspace = workspacesByKey.get(workspaceKey) + if (!workspace) { + issues.push( + `permission group "${group.key}" references missing workspace "${workspaceKey}"` + ) + } else if ( + workspace.organizationKey !== group.organizationKey || + workspace.subscriptionKey !== enterpriseSubscription?.key + ) { + issues.push( + `permission group "${group.key}" workspace "${workspaceKey}" is outside its Enterprise scope` + ) + } + } + for (const userKey of group.memberUserKeys) { + if (!usersByKey.has(userKey)) { + issues.push(`permission group "${group.key}" references missing user "${userKey}"`) + } + if (!membershipFor(definition, group.organizationKey, userKey)) { + issues.push(`permission group "${group.key}" member "${userKey}" lacks host membership`) + } + } + } +} + +function validateInvitations( + definition: ScenarioDefinition, + organizationsByKey: ReadonlyMap, + workspacesByKey: ReadonlyMap, + issues: string[] +): void { + for (const invitation of definition.invitations) { + if (!organizationsByKey.has(invitation.organizationKey)) { + issues.push( + `invitation "${invitation.key}" references missing organization "${invitation.organizationKey}"` + ) + } + if (Number.isNaN(Date.parse(invitation.expiresAt))) { + issues.push(`invitation "${invitation.key}" has an invalid expiry`) + } + validateUniqueValues( + `invitation "${invitation.key}" workspace grant`, + invitation.workspaceGrants.map(({ workspaceKey }) => workspaceKey), + issues + ) + for (const grant of invitation.workspaceGrants) { + const workspace = workspacesByKey.get(grant.workspaceKey) + if (!workspace) { + issues.push( + `invitation "${invitation.key}" references missing workspace "${grant.workspaceKey}"` + ) + } else if (workspace.organizationKey !== invitation.organizationKey) { + issues.push(`invitation "${invitation.key}" grants a workspace outside its organization`) + } + } + } +} + +function validatePersonaWorkspaceExpectation( + definition: ScenarioDefinition, + personaKey: ResourceKey, + user: ScenarioUser, + workspace: ScenarioWorkspace, + expected: { + access: ExpectedWorkspaceAccess + roleSource: 'owner' | 'explicit' | 'org-admin' | 'none' + hostContext: { + isOwner: boolean + hostMembership: 'owner' | 'member' | 'external' + payerScope: 'user' | 'organization' + plan: 'free' | ScenarioSubscription['plan'] + hosted: boolean + billingEnabled: boolean + } + }, + subscriptionsByKey: ReadonlyMap, + issues: string[] +): void { + const actual = deriveWorkspaceAccess(definition, user.key, workspace) + if (expected.access !== actual.access || expected.roleSource !== actual.roleSource) { + issues.push( + `persona "${personaKey}" has incoherent access/roleSource for workspace "${workspace.key}"` + ) + } + if ( + (actual.isOwner || + actual.organizationRole === 'owner' || + actual.organizationRole === 'admin') && + expected.access !== 'admin' + ) { + issues.push( + `persona "${personaKey}" owner/admin expectation for "${workspace.key}" is below admin` + ) + } + const subscription = workspace.subscriptionKey + ? subscriptionsByKey.get(workspace.subscriptionKey) + : undefined + const actualPlan = !subscription || subscription.status === 'lapsed' ? 'free' : subscription.plan + const actualMembership = actual.isOwner + ? 'owner' + : actual.organizationRole + ? 'member' + : 'external' + if ( + expected.hostContext.isOwner !== actual.isOwner || + expected.hostContext.hostMembership !== actualMembership || + expected.hostContext.payerScope !== workspace.payer.kind || + expected.hostContext.plan !== actualPlan || + expected.hostContext.hosted !== workspace.hosted || + expected.hostContext.billingEnabled !== workspace.billingEnabled + ) { + issues.push(`persona "${personaKey}" has an incoherent host context for "${workspace.key}"`) + } +} + +function deriveWorkspaceAccess( + definition: ScenarioDefinition, + userKey: ResourceKey, + workspace: ScenarioWorkspace +): { + access: ExpectedWorkspaceAccess + roleSource: 'owner' | 'explicit' | 'org-admin' | 'none' + isOwner: boolean + organizationRole?: OrganizationRole +} { + const isOwner = workspace.ownerUserKey === userKey + const organizationRole = workspace.organizationKey + ? membershipFor(definition, workspace.organizationKey, userKey)?.role + : undefined + if (isOwner) return { access: 'admin', roleSource: 'owner', isOwner, organizationRole } + if (organizationRole === 'owner' || organizationRole === 'admin') { + return { access: 'admin', roleSource: 'org-admin', isOwner, organizationRole } + } + const grant = definition.workspaceGrants.find( + (candidate) => candidate.workspaceKey === workspace.key && candidate.userKey === userKey + ) + if (grant) { + return { access: grant.access, roleSource: 'explicit', isOwner, organizationRole } + } + return { access: 'none', roleSource: 'none', isOwner, organizationRole } +} + +function groupsForUser( + definition: ScenarioDefinition, + userKey: ResourceKey +): ScenarioPermissionGroup[] { + return definition.permissionGroups.filter((group) => group.memberUserKeys.includes(userKey)) +} + +function membershipFor( + definition: ScenarioDefinition, + organizationKey: ResourceKey, + userKey: ResourceKey +): ScenarioDefinition['organizationMemberships'][number] | undefined { + return definition.organizationMemberships.find( + (membership) => membership.organizationKey === organizationKey && membership.userKey === userKey + ) +} + +function billingReferenceKey(subscription: ScenarioSubscription): string { + return subscription.billingReference.kind === 'user' + ? `user/${subscription.billingReference.userKey}` + : `organization/${subscription.billingReference.organizationKey}` +} + +function sameSet(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value) => right.includes(value)) +} diff --git a/apps/sim/e2e/foundation/build-manifest.spec.ts b/apps/sim/e2e/foundation/build-manifest.spec.ts new file mode 100644 index 00000000000..b993bbd56cd --- /dev/null +++ b/apps/sim/e2e/foundation/build-manifest.spec.ts @@ -0,0 +1,174 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync, +} from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { expect, test } from '@playwright/test' +import { + type BuildArtifactPaths, + type BuildIdentity, + clearActiveNextBuild, + pruneBuildCache, + restoreCachedBuild, + storeCompletedBuild, +} from '../support/build-manifest' +import { acquireE2eRunLock } from '../support/run-lock' + +const identity: BuildIdentity = { + schemaVersion: 1, + nextBuildHash: 'next-build-hash', + sourceHash: 'source-hash', + profileHash: 'profile-hash', + nodeVersion: 'v22.0.0', + bunVersion: '1.0.0', + nextVersion: '16.0.0', + platform: process.platform, + architecture: process.arch, +} + +test.describe('verified Next build cache', () => { + test('stores, restores, and rejects artifact corruption', () => { + withBuildPaths((paths) => { + writeBuild(paths.activeNextDirectory, 'build-one', 'original') + storeCompletedBuild(identity, paths) + clearActiveNextBuild(paths.activeNextDirectory) + const staleRestore = `${paths.activeNextDirectory}.e2e-restore-123` + const staleBackup = `${paths.activeNextDirectory}.e2e-backup-123` + const interruptedStore = path.join(paths.buildCacheDirectory, 'other-hash.tmp-123') + mkdirSync(staleRestore) + mkdirSync(staleBackup) + mkdirSync(interruptedStore) + + expect(restoreCachedBuild(identity, paths)).toMatchObject({ + reused: true, + reason: 'verified cache hit', + }) + expect(readFileSync(path.join(paths.activeNextDirectory, 'server.js'), 'utf8')).toBe( + 'original' + ) + expect(existsSync(staleRestore)).toBe(false) + expect(existsSync(staleBackup)).toBe(false) + expect(existsSync(interruptedStore)).toBe(false) + + writeFileSync( + path.join(paths.buildCacheDirectory, identity.nextBuildHash, '.next', 'server.js'), + 'tampered' + ) + expect(restoreCachedBuild(identity, paths)).toMatchObject({ + reused: false, + reason: 'cached artifact checksum does not match the manifest', + }) + }) + }) + + test('rejects manifest identity and BUILD_ID corruption', () => { + withBuildPaths((paths) => { + writeBuild(paths.activeNextDirectory, 'build-one', 'original') + storeCompletedBuild(identity, paths) + const cacheDirectory = path.join(paths.buildCacheDirectory, identity.nextBuildHash) + const manifestPath = path.join(cacheDirectory, 'manifest.json') + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as Record + writeFileSync(manifestPath, JSON.stringify({ ...manifest, sourceHash: 'wrong' })) + expect(restoreCachedBuild(identity, paths).reason).toBe( + 'cache manifest identity does not match' + ) + + writeFileSync(manifestPath, JSON.stringify(manifest)) + writeFileSync(path.join(cacheDirectory, '.next', 'BUILD_ID'), 'wrong') + expect(restoreCachedBuild(identity, paths).reason).toBe( + 'cached BUILD_ID does not match the manifest' + ) + }) + }) + + test('prunes old build entries while retaining the selected cache', () => { + withBuildPaths((paths) => { + const now = Date.now() / 1_000 + for (const [index, name] of ['oldest', 'newer', identity.nextBuildHash].entries()) { + const directory = path.join(paths.buildCacheDirectory, name) + mkdirSync(directory, { recursive: true }) + utimesSync(directory, now + index, now + index) + } + const abandonedStore = path.join(paths.buildCacheDirectory, 'abandoned.tmp-123') + mkdirSync(abandonedStore) + expect(pruneBuildCache(identity.nextBuildHash, 2, paths.buildCacheDirectory)).toEqual([ + 'oldest', + ]) + expect(existsSync(abandonedStore)).toBe(false) + expect(restoreCachedBuild(identity, paths).reason).toBe( + 'cache artifact or completed manifest is missing' + ) + }) + }) +}) + +test('orchestrator lock rejects live ownership and recovers stale descriptors', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-run-lock-')) + const lockPath = path.join(directory, 'orchestrator.lock') + try { + const lock = acquireE2eRunLock(lockPath) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/Another E2E orchestrator/) + lock.transfer(process.pid) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/Another E2E orchestrator/) + lock.retain('manual cleanup required') + lock.transfer(process.pid) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/manual cleanup required/) + lock.release() + + mkdirSync(lockPath) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/is acquiring/) + rmSync(lockPath, { recursive: true }) + + mkdirSync(lockPath) + writeFileSync( + path.join(lockPath, 'owner.json'), + JSON.stringify({ + pid: 2_147_483_647, + token: 'stale', + startedAt: '2000-01-01T00:00:00Z', + processStartIdentity: null, + }) + ) + const recovered = acquireE2eRunLock(lockPath) + recovered.release() + + mkdirSync(lockPath) + writeFileSync( + path.join(lockPath, 'owner.json'), + JSON.stringify({ + pid: process.pid, + token: 'reused-pid', + startedAt: '2000-01-01T00:00:00Z', + processStartIdentity: 'not-the-current-process-start', + }) + ) + const reusedPidRecovered = acquireE2eRunLock(lockPath) + reusedPidRecovered.release() + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function withBuildPaths(run: (paths: BuildArtifactPaths) => void): void { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-build-cache-')) + try { + run({ + activeNextDirectory: path.join(directory, '.next'), + buildCacheDirectory: path.join(directory, 'cache'), + }) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +} + +function writeBuild(directory: string, buildId: string, contents: string): void { + mkdirSync(directory, { recursive: true }) + writeFileSync(path.join(directory, 'BUILD_ID'), buildId) + writeFileSync(path.join(directory, 'server.js'), contents) +} diff --git a/apps/sim/e2e/foundation/http-client.spec.ts b/apps/sim/e2e/foundation/http-client.spec.ts new file mode 100644 index 00000000000..c9d2f04a704 --- /dev/null +++ b/apps/sim/e2e/foundation/http-client.spec.ts @@ -0,0 +1,90 @@ +import { expect, test } from '@playwright/test' +import { z } from 'zod' +import { E2eHttpClient } from '../fixtures/http-client' + +test.describe('E2E HTTP client', () => { + test('honors Retry-After only for 429 responses and records attempts', async () => { + const delays: number[] = [] + const attempts: number[] = [] + let requestCount = 0 + const client = new E2eHttpClient({ + baseUrl: 'http://127.0.0.1:1', + fetchImplementation: async () => { + requestCount += 1 + return requestCount === 1 + ? new Response(JSON.stringify({ error: 'limited' }), { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '0.01' }, + }) + : Response.json({ ok: true }) + }, + sleepImplementation: async (milliseconds) => { + delays.push(milliseconds) + }, + onAttempt: ({ number }) => attempts.push(number), + }) + + await expect( + client.request({ + path: '/retry', + schema: z.object({ ok: z.literal(true) }), + }) + ).resolves.toEqual({ ok: true }) + expect(delays).toEqual([10]) + expect(attempts).toEqual([1, 2]) + }) + + test('clamps oversized Retry-After values', async () => { + const delays: number[] = [] + let requestCount = 0 + const client = new E2eHttpClient({ + baseUrl: 'http://127.0.0.1:1', + retryDelaysMs: [1], + fetchImplementation: async () => { + requestCount += 1 + return requestCount === 1 + ? new Response(JSON.stringify({ error: 'limited' }), { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '3600' }, + }) + : Response.json({ ok: true }) + }, + sleepImplementation: async (milliseconds) => { + delays.push(milliseconds) + }, + }) + await client.request({ path: '/retry', schema: z.object({ ok: z.literal(true) }) }) + expect(delays).toEqual([30_000]) + }) + + test('keeps independent cookie jars and redacts failed response bodies', async () => { + const cookieHeaders: Array = [] + const createFetch = + (secret: string): typeof fetch => + async (_input, init) => { + cookieHeaders.push(new Headers(init?.headers).get('cookie')) + if (cookieHeaders.length <= 2) { + return new Response(JSON.stringify({ ok: true }), { + headers: { 'set-cookie': `session=${secret}; Path=/; HttpOnly` }, + }) + } + return new Response(JSON.stringify({ error: `do not print ${secret}` }), { status: 400 }) + } + const fetchImplementation = createFetch('synthetic-secret') + const first = new E2eHttpClient({ baseUrl: 'http://127.0.0.1:1', fetchImplementation }) + const second = new E2eHttpClient({ baseUrl: 'http://127.0.0.1:1', fetchImplementation }) + const schema = z.object({ ok: z.literal(true) }) + + await first.request({ path: '/login-a', schema }) + await second.request({ path: '/login-b', schema }) + expect(first.getCookieHeader()).toBe('session=synthetic-secret') + expect(second.getCookieHeader()).toBe('session=synthetic-secret') + + await expect(first.request({ path: '/failure', schema })).rejects.not.toThrow( + /synthetic-secret/ + ) + expect(cookieHeaders[0]).toBeNull() + expect(cookieHeaders[1]).toBeNull() + expect(cookieHeaders[2]).toBe('session=synthetic-secret') + }) +}) diff --git a/apps/sim/e2e/foundation/leak-canary.spec.ts b/apps/sim/e2e/foundation/leak-canary.spec.ts new file mode 100644 index 00000000000..0a1014437ac --- /dev/null +++ b/apps/sim/e2e/foundation/leak-canary.spec.ts @@ -0,0 +1,84 @@ +import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { expect, test } from '@playwright/test' +import JSZip from 'jszip' +import { writeJsonAtomic } from '../fixtures/e2e-world' +import { + assertNoSyntheticSecretLeaks, + loadSyntheticSecretCanaryForScan, + readSyntheticSecretCanarySecrets, + scrubUnscannableArtifacts, + writeSyntheticSecretCanary, +} from '../support/leak-canary' + +test('credential leak canary scans artifacts but excludes its private source', async () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-leak-canary-')) + const secretsPath = path.join(directory, 'private', 'synthetic-secrets.json') + const artifactsPath = path.join(directory, 'artifacts') + const password = 'known-synthetic-password' + try { + writeSyntheticSecretCanary(secretsPath, 'leak-canary', [password, 'setup-only-password']) + expect(loadSyntheticSecretCanaryForScan([], secretsPath)).toEqual([ + password, + 'setup-only-password', + ]) + expect(loadSyntheticSecretCanaryForScan(['already-loaded'], secretsPath)).toEqual([ + 'already-loaded', + password, + 'setup-only-password', + ]) + writeJsonAtomic(path.join(artifactsPath, 'clean.json'), { status: 'clean' }) + + await expect( + assertNoSyntheticSecretLeaks({ + secrets: readSyntheticSecretCanarySecrets(secretsPath), + roots: [directory], + excludedPaths: [path.dirname(secretsPath)], + }) + ).resolves.toBeUndefined() + + const tracePath = path.join(artifactsPath, 'trace.txt') + writeFileSync(tracePath, 'request body: setup-only-password') + await expect( + assertNoSyntheticSecretLeaks({ + secrets: readSyntheticSecretCanarySecrets(secretsPath), + roots: [directory], + excludedPaths: [path.dirname(secretsPath)], + }) + ).rejects.toThrow(/secret leaked/) + + writeFileSync(tracePath, `request body: ${password}`) + const zip = new JSZip() + zip.file('trace.txt', `request body: ${password}`) + writeFileSync( + path.join(artifactsPath, 'trace.zip'), + await zip.generateAsync({ type: 'nodebuffer' }) + ) + unlinkSync(tracePath) + await expect( + assertNoSyntheticSecretLeaks({ + secrets: readSyntheticSecretCanarySecrets(secretsPath), + roots: [directory], + excludedPaths: [path.dirname(secretsPath)], + }) + ).rejects.toThrow(/secret leaked/) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) + +test('malformed canaries fail closed by scrubbing unscannable diagnostics', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-malformed-canary-')) + const secretsPath = path.join(directory, 'synthetic-secrets.json') + const diagnosticsPath = path.join(directory, 'diagnostics') + try { + writeFileSync(secretsPath, '{"schemaVersion":1,"secrets":["truncated') + writeJsonAtomic(path.join(diagnosticsPath, 'report.json'), { status: 'unscanned' }) + expect(() => loadSyntheticSecretCanaryForScan(['runtime-secret'], secretsPath)).toThrow() + scrubUnscannableArtifacts([diagnosticsPath]) + expect(existsSync(diagnosticsPath)).toBe(false) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 261180d16ed..131ff2ed4cf 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -10,6 +10,7 @@ import { assertSafeDatabaseName, buildRunDatabaseUrl, } from '../support/database' +import { createHostedBillingProfile } from '../support/deployment-profile' import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' import { @@ -17,7 +18,10 @@ import { spawnManagedProcess, waitForManagedProcessReady, } from '../support/process' -import { parseProcessGroupIds } from '../support/signal-cleanup' +import { createE2eRuntimeSecrets } from '../support/runtime-secrets' +import { verifySandboxBundleIntegrity } from '../support/sandbox-bundles' +import { assertSafeSeedEnvironment } from '../support/seed-safety' +import { isProcessGroupAlive, parseProcessGroupIds } from '../support/signal-cleanup' test.describe('foundation safety guards', () => { test('discovers env keys without leaking values and shadows unknown keys', () => { @@ -62,26 +66,97 @@ test.describe('foundation safety guards', () => { } }) + test('deployment profile projects least-privilege build and runtime environments', () => { + const profile = createHostedBillingProfile({ + runId: 'projection_test', + databaseUrl: 'postgresql://127.0.0.1:5432/sim_e2e_projection_test', + stripeApiBaseUrl: 'http://127.0.0.1:40123', + runtimeHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-runtime'), + setupHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-setup'), + authCaptureHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-auth'), + playwrightHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-playwright'), + playwrightBrowsersPath: path.join(os.tmpdir(), 'sim-e2e-browsers'), + runtimeSecrets: createE2eRuntimeSecrets(), + ci: false, + }) + const { build, app, realtime, migration, seed, authCapture, playwright } = profile.environments + + expect(build.env.DATABASE_URL).toContain('/sim_e2e_build_sentinel') + expect(build.env.DATABASE_URL).not.toBe(app.env.DATABASE_URL) + expect(build.env.STRIPE_API_BASE_URL).toBe('http://127.0.0.1:1') + expect(build.env.TELEMETRY_ENDPOINT).toBe('http://127.0.0.1:1/v1/traces') + expect(app.env.TELEMETRY_ENDPOINT).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/v1\/traces$/) + expect(build.env.E2E_RUN_ID).toBe('build_sentinel') + for (const key of [ + 'BETTER_AUTH_SECRET', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'STRIPE_SECRET_KEY', + 'STRIPE_WEBHOOK_SECRET', + ]) { + expect(build.env[key], `${key} must use a build-only sentinel`).not.toBe(app.env[key]) + } + + for (const key of Object.keys(app.env).filter((key) => key.startsWith('NEXT_PUBLIC_'))) { + expect(build.env[key], `${key} must be identical at build and runtime`).toBe(app.env[key]) + } + + expect(seed.env.ADMIN_API_KEY).toBe(app.env.ADMIN_API_KEY) + expect(seed.env.DATABASE_URL).toBe(app.env.DATABASE_URL) + expect(authCapture.env.ADMIN_API_KEY).toBeUndefined() + expect(authCapture.env.DATABASE_URL).toBeUndefined() + expect(playwright.env.ADMIN_API_KEY).toBeUndefined() + expect(playwright.env.DATABASE_URL).toBeUndefined() + expect(realtime.env.ADMIN_API_KEY).toBeUndefined() + expect(realtime.env.STRIPE_SECRET_KEY).toBeUndefined() + expect(migration.env.ADMIN_API_KEY).toBeUndefined() + expect(migration.env.MIGRATION_DATABASE_URL).toBe(app.env.DATABASE_URL) + expect(app.env.HOME).not.toBe(seed.env.HOME) + expect(seed.env.HOME).not.toBe(authCapture.env.HOME) + expect(authCapture.env.HOME).not.toBe(playwright.env.HOME) + }) + test('database guards reject shared or remote targets', () => { expect(() => assertSafeDatabaseName('simstudio')).toThrow() expect(() => assertSafeDatabaseName('sim_e2e_valid_run')).not.toThrow() + expect(() => assertLoopbackPostgresUrl('postgresql://example.com/postgres')).toThrow() + expect(() => assertLoopbackPostgresUrl('postgresql://localhost/postgres')).toThrow() expect(() => - assertLoopbackPostgresUrl('postgresql://postgres:postgres@example.com/postgres') - ).toThrow() - expect(() => - assertLoopbackPostgresUrl('postgresql://postgres:postgres@localhost/postgres') - ).toThrow() - expect(() => - assertLoopbackPostgresUrl('postgresql://postgres:postgres@127.0.0.1/postgres?sslmode=disable') + assertLoopbackPostgresUrl('postgresql://127.0.0.1/postgres?sslmode=disable') ).toThrow() expect( - buildRunDatabaseUrl( - 'postgresql://postgres:postgres@127.0.0.1:5432/postgres', - 'sim_e2e_valid_run' - ) + buildRunDatabaseUrl('postgresql://127.0.0.1:5432/postgres', 'sim_e2e_valid_run') ).toContain('/sim_e2e_valid_run') }) + test('standalone seeding requires its exact guarded run target', () => { + const environment = { + E2E_ORCHESTRATED: '1', + E2E_PROFILE: 'hosted-billing-chromium', + E2E_RUN_ID: 'seed_guard', + E2E_BASE_URL: 'http://e2e.sim.ai:3000', + DATABASE_URL: 'postgresql://127.0.0.1:5432/sim_e2e_seed_guard', + } + expect(() => assertSafeSeedEnvironment(environment)).not.toThrow() + expect(() => assertSafeSeedEnvironment({ ...environment, E2E_ORCHESTRATED: '0' })).toThrow( + /guarded E2E orchestrator/ + ) + expect(() => + assertSafeSeedEnvironment({ + ...environment, + DATABASE_URL: 'postgresql://127.0.0.1:5432/simstudio', + }) + ).toThrow(/unsafe E2E database/) + expect(() => + assertSafeSeedEnvironment({ + ...environment, + DATABASE_URL: 'postgresql://192.0.2.1:5432/sim_e2e_seed_guard', + }) + ).toThrow(/must be loopback/) + }) + test('loopback detection rejects public addresses', () => { expect(isLoopbackAddress('127.0.0.1')).toBe(true) expect(isLoopbackAddress('127.10.20.30')).toBe(true) @@ -99,16 +174,62 @@ test.describe('foundation safety guards', () => { ).not.toThrow() expect(() => parseRunOptions(['--project=hosted-billing-chromium-workflows', '--shard=1/2']) - ).toThrow(/coupled workflows must remain unsharded/) + ).toThrow(/coupled E2E projects must remain unsharded/) + expect(() => parseRunOptions(['--project=hosted-billing-chromium-personas'])).not.toThrow() + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-personas', '--no-deps'], { ci: false }) + ).not.toThrow() + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-personas', '--no-deps'], { ci: true }) + ).toThrow(/--no-deps is local-only/) + expect(() => parseRunOptions(['--no-deps'], { ci: false })).toThrow( + /exactly one explicit canonical/ + ) + expect(() => + parseRunOptions( + [ + '--project=hosted-billing-chromium-navigation', + '--project=hosted-billing-chromium-personas', + '--no-deps', + ], + { ci: false } + ) + ).toThrow(/exactly one explicit canonical/) + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-persona-isolation', '--shard=1/2']) + ).toThrow(/coupled E2E projects must remain unsharded/) }) test('Playwright CLI arguments cannot override orchestration invariants', () => { expect(() => parseRunOptions(['--workers=8'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['-j8'])).toThrow(/cannot override/) expect(() => parseRunOptions(['--config=other.config.ts'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['-cother.config.ts'])).toThrow(/cannot override/) expect(() => parseRunOptions(['--project', 'hosted-billing-chromium-navigation'])).toThrow( /canonical/ ) expect(() => parseRunOptions(['--pass-with-no-tests'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--output=/tmp/elsewhere'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--reporter=line'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--update-snapshots'])).toThrow(/not a supported/) + expect(parseRunOptions(['--grep', 'persona contract', '--headed']).playwrightArgs).toEqual([ + '--grep', + 'persona contract', + '--headed', + ]) + expect(parseRunOptions(['--reuse-build'], { ci: false })).toEqual({ + playwrightArgs: [], + reuseBuild: true, + }) + expect(() => parseRunOptions(['--reuse-build'], { ci: true })).toThrow(/local-only/) + expect(() => parseRunOptions(['--keep-stack'], { ci: false })).toThrow(/deferred/) + }) + + test('committed sandbox bundles match the reviewed fingerprint', () => { + expect(() => verifySandboxBundleIntegrity({ runningBunVersion: '1.3.13' })).not.toThrow() + expect(() => verifySandboxBundleIntegrity({ runningBunVersion: '1.3.14' })).toThrow( + /require Bun 1\.3\.13/ + ) }) test('empty signal cleanup groups never become PID zero', () => { @@ -150,6 +271,32 @@ test.describe('foundation safety guards', () => { } }) + test('cleanup retains and terminates descendants after their group leader exits', async () => { + test.skip(process.platform === 'win32', 'Windows does not expose POSIX process groups') + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-descendant-')) + try { + const managed = spawnManagedProcess({ + name: 'exited-group-leader', + command: process.execPath, + args: [ + '-e', + "const { spawn } = require('node:child_process'); const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' }); child.unref();", + ], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + const groupId = managed.child.pid + expect(groupId).toBeTruthy() + await managed.completion + expect(isProcessGroupAlive(groupId ?? 0)).toBe(true) + await managed.stop() + expect(isProcessGroupAlive(groupId ?? 0)).toBe(false) + } finally { + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) + test('process exit after readiness does not create an unhandled rejection', async () => { const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-ready-')) let unhandled: unknown diff --git a/apps/sim/e2e/foundation/scenario-validation.spec.ts b/apps/sim/e2e/foundation/scenario-validation.spec.ts new file mode 100644 index 00000000000..019a4a7f059 --- /dev/null +++ b/apps/sim/e2e/foundation/scenario-validation.spec.ts @@ -0,0 +1,329 @@ +import { expect, test } from '@playwright/test' +import { resolveStorageStatePath } from '../fixtures/e2e-world' +import { createScenarioNamespace } from '../fixtures/namespace' +import { + canonicalSettingsRoute, + type ScenarioDefinition, + type ScenarioOrganizationMembership, + type ScenarioWorkspaceGrant, +} from '../fixtures/scenario' +import { + ScenarioValidationError, + validateScenario, + validateScenarioSet, +} from '../fixtures/validate-scenario' +import { + createPrimarySettingsScenario, + createSettingsPersonaScenarios, + SETTINGS_PERSONA_KEYS, +} from '../settings/personas' + +test.describe('pure scenario validation', () => { + test('resolves the literal settings cast and minimal isolation twin', () => { + const scenarios = createSettingsPersonaScenarios('run-42') + const primary = validateScenario(scenarios.primary) + const twin = validateScenario(scenarios.isolationTwin) + + expect([...primary.personasByKey.keys()]).toEqual(SETTINGS_PERSONA_KEYS) + expect(twin.personasByKey.size).toBe(1) + expect(twin.workspacesByKey.size).toBe(1) + expect(twin.organizationsByKey.size).toBe(0) + expect(() => validateScenarioSet([primary, twin])).not.toThrow() + + expect(workspaceExpectation(primary, 'personalPaidOwner').hostContext.plan).toBe('pro_6000') + expect(workspaceExpectation(primary, 'personalMaxOwner').hostContext.plan).toBe('pro_25000') + expect(workspaceExpectation(primary, 'workspaceReadMember')).toMatchObject({ + access: 'read', + roleSource: 'explicit', + hostContext: { hostMembership: 'member', plan: 'team_6000' }, + }) + expect(workspaceExpectation(primary, 'workspaceWriteMember').access).toBe('write') + expect(workspaceExpectation(primary, 'workspaceAdminMember')).toMatchObject({ + access: 'admin', + roleSource: 'explicit', + hostContext: { hostMembership: 'member' }, + }) + expect(workspaceExpectation(primary, 'externalWorkspaceAdmin')).toMatchObject({ + access: 'admin', + roleSource: 'explicit', + hostContext: { hostMembership: 'external' }, + }) + expect( + primary.definition.organizationMemberships.some( + ({ userKey }) => userKey === 'external-workspace-admin' + ) + ).toBe(false) + + expect(workspaceExpectation(primary, 'enterpriseOrganizationAdmin')).toMatchObject({ + access: 'admin', + roleSource: 'org-admin', + hostContext: { isOwner: false, plan: 'enterprise' }, + }) + expect( + primary.definition.workspaceGrants.some( + ({ userKey }) => userKey === 'enterprise-organization-admin' + ) + ).toBe(false) + expect(workspaceExpectation(primary, 'freeOrganizationOwner').hostContext.plan).toBe('free') + expect(primary.subscriptionsByKey.get('lapsed-team-subscription')?.status).toBe('lapsed') + + const restricted = primary.personasByKey.get('permissionGroupRestricted') + expect(restricted?.permissionGroups[0]?.restrictions).toEqual({ + hiddenSettings: ['secrets', 'api-keys', 'inbox'], + disabledFeatures: ['mcp', 'custom-tools'], + }) + expect(primary.personasByKey.get('platformAdmin')?.user).toMatchObject({ + platformRole: 'admin', + superUserModeEnabled: true, + }) + for (const persona of primary.personasByKey.values()) { + expect(persona.workspaces.some(({ expectation }) => expectation.access !== 'none')).toBe(true) + expect(persona.definition.canonicalRoute.settingsSection).toBe('general') + } + expect(workspaceExpectation(primary, 'personalFreeOwner', 'team-workspace')).toMatchObject({ + access: 'none', + roleSource: 'none', + }) + }) + + test('rejects unsafe storage paths and cross-world identity collisions', () => { + const unsafe = validScenario() + unsafe.personas = unsafe.personas.map((persona, index) => + index === 0 ? { ...persona, storageStateFilename: '../../escaped.json' } : persona + ) + expectInvalid(unsafe, /separator-free JSON basename/) + expect(() => resolveStorageStatePath('/tmp/e2e-auth', '../../escaped.json')).toThrow() + + const definitions = createSettingsPersonaScenarios('cross-world') + const primary = validateScenario(definitions.primary) + const twin = validateScenario({ + ...definitions.isolationTwin, + namespace: { + ...definitions.isolationTwin.namespace, + world: definitions.primary.namespace.world, + }, + }) + expect(() => validateScenarioSet([primary, twin])).toThrow(/duplicate world namespace/) + }) + + test('namespaces controllable values deterministically without manufacturing API IDs', () => { + const first = createScenarioNamespace('Run 42/Alpha', 'Primary World') + const again = createScenarioNamespace('Run 42/Alpha', 'Primary World') + const otherWorld = createScenarioNamespace('Run 42/Alpha', 'Twin World') + + expect(first).toMatchObject({ + run: 'Run 42/Alpha', + world: 'Primary World', + }) + expect(first.email('Owner')).toBe(again.email('Owner')) + expect(first.slug('Organization')).toBe(again.slug('Organization')) + expect(first.name('Workspace')).toBe(again.name('Workspace')) + expect(first.invitationToken('Invite')).toBe(again.invitationToken('Invite')) + expect(first.storageStateFilename('personalFreeOwner')).toBe( + again.storageStateFilename('personalFreeOwner') + ) + expect(first.email('Owner')).not.toBe(otherWorld.email('Owner')) + expect(first.storageStateFilename('personalFreeOwner')).toMatch(/\.json$/) + expect(Object.keys(first).some((key) => /(^|api)id$/i.test(key))).toBe(false) + expect(canonicalSettingsRoute('opaque/id:generated-by-api', 'api-keys')).toBe( + '/workspace/opaque%2Fid%3Agenerated-by-api/settings/api-keys' + ) + }) + + test('rejects duplicate organization memberships', () => { + const scenario = validScenario() + scenario.organizationMemberships = [ + ...scenario.organizationMemberships, + scenario.organizationMemberships[0], + ] as ScenarioOrganizationMembership[] + expectInvalid(scenario, /duplicate organization membership/) + }) + + test('rejects incoherent organization workspace payer and subscription identity', () => { + const scenario = validScenario() + scenario.workspaces = scenario.workspaces.map((workspace) => + workspace.key === 'team-workspace' + ? { + ...workspace, + payer: { kind: 'organization', organizationKey: 'enterprise-organization' }, + } + : workspace + ) + expectInvalid(scenario, /incoherent payer/) + }) + + test('rejects owner or organization-admin expectations below admin', () => { + const scenario = validScenario() + scenario.personas = scenario.personas.map((persona) => + persona.key === 'enterpriseOrganizationAdmin' + ? { + ...persona, + workspaces: persona.workspaces.map((expectation) => ({ + ...expectation, + access: 'read', + })), + } + : persona + ) + expectInvalid(scenario, /below admin/) + }) + + test('rejects invalid and misplaced Enterprise metadata', () => { + const invalidEnterprise = validScenario() + invalidEnterprise.subscriptions = invalidEnterprise.subscriptions.map((subscription) => + subscription.key === 'enterprise-subscription' + ? { ...subscription, enterprise: { ...subscription.enterprise!, monthlyPrice: 0 } } + : subscription + ) + expectInvalid(invalidEnterprise, /invalid Enterprise metadata/) + + const metadataOnTeam = validScenario() + metadataOnTeam.subscriptions = metadataOnTeam.subscriptions.map((subscription) => + subscription.key === 'team-subscription' + ? { + ...subscription, + enterprise: { + plan: 'enterprise', + monthlyPrice: 1, + seats: 4, + }, + } + : subscription + ) + expectInvalid(metadataOnTeam, /non-Enterprise subscription/) + }) + + test('rejects unsupported lapsed personal subscriptions before seeding', () => { + const scenario = validScenario() + scenario.subscriptions = scenario.subscriptions.map((subscription) => + subscription.key === 'personal-paid-subscription' + ? { ...subscription, status: 'lapsed' as const } + : subscription + ) + expectInvalid(scenario, /unsupported lapsed status for a user payer/) + }) + + test('rejects organization workspace provisioning from past-due subscriptions', () => { + const scenario = validScenario() + scenario.subscriptions = scenario.subscriptions.map((subscription) => + subscription.key === 'team-subscription' + ? { ...subscription, status: 'past_due' as const } + : subscription + ) + expectInvalid(scenario, /cannot be provisioned from a past-due subscription/) + }) + + test('rejects permission groups without active Enterprise organization/workspace scope', () => { + const scenario = validScenario() + scenario.permissionGroups = scenario.permissionGroups.map((group) => ({ + ...group, + organizationKey: 'team-organization', + workspaceKeys: ['team-workspace'], + })) + expectInvalid(scenario, /requires an active Enterprise organization/) + }) + + test('rejects duplicate explicit grants and unsupported all-member groups', () => { + const duplicateGrant = validScenario() + duplicateGrant.workspaceGrants = [ + ...duplicateGrant.workspaceGrants, + duplicateGrant.workspaceGrants[0], + ] as ScenarioWorkspaceGrant[] + expectInvalid(duplicateGrant, /duplicate workspace grant/) + + const allMemberGroup = validScenario() + allMemberGroup.permissionGroups = allMemberGroup.permissionGroups.map((group) => ({ + ...group, + memberUserKeys: [], + })) + expectInvalid(allMemberGroup, /default and all-member groups are not modeled/) + }) + + test('rejects inconsistent hosted and billing flags', () => { + const hosted = validScenario() + hosted.workspaces = hosted.workspaces.map((workspace, index) => + index === 0 ? { ...workspace, hosted: false } : workspace + ) + expectInvalid(hosted, /inconsistent hosted flag/) + + const billing = validScenario() + billing.users = billing.users.map((user, index) => + index === 0 ? { ...user, billingEnabled: false } : user + ) + expectInvalid(billing, /inconsistent billing flag/) + }) + + test('rejects zero-workspace personas and none-only foreign denial declarations', () => { + const empty = validScenario() + empty.personas = empty.personas.map((persona) => + persona.key === 'personalPaidOwner' ? { ...persona, workspaces: [] } : persona + ) + expectInvalid(empty, /zero accessible seeded workspaces/) + + const noneOnly = validScenario() + noneOnly.personas = noneOnly.personas.map((persona) => + persona.key === 'personalFreeOwner' + ? { + ...persona, + workspaces: persona.workspaces.filter(({ access }) => access === 'none'), + canonicalRoute: { workspaceKey: 'team-workspace', settingsSection: 'general' }, + } + : persona + ) + expectInvalid(noneOnly, /zero accessible seeded workspaces/) + }) + + test('rejects invalid references and duplicate controllable identities', () => { + const badReference = validScenario() + badReference.workspaceGrants = badReference.workspaceGrants.map((grant, index) => + index === 0 ? { ...grant, userKey: 'missing-user' } : grant + ) + expectInvalid(badReference, /references missing user/) + + const duplicateEmail = validScenario() + duplicateEmail.users = duplicateEmail.users.map((user, index, users) => + index === 1 ? { ...user, email: users[0].email.toUpperCase() } : user + ) + expectInvalid(duplicateEmail, /duplicate email/) + + const duplicateStorageState = validScenario() + duplicateStorageState.personas = duplicateStorageState.personas.map( + (persona, index, personas) => + index === 1 + ? { ...persona, storageStateFilename: personas[0].storageStateFilename } + : persona + ) + expectInvalid(duplicateStorageState, /duplicate storage-state filename/) + }) +}) + +function validScenario(): MutableScenario { + return structuredClone( + createPrimarySettingsScenario(createScenarioNamespace('validation-run', 'primary')) + ) as MutableScenario +} + +type MutableScenario = { + -readonly [Key in keyof ScenarioDefinition]: ScenarioDefinition[Key] extends readonly (infer Item)[] + ? Item[] + : ScenarioDefinition[Key] +} + +function expectInvalid(scenario: ScenarioDefinition, message: RegExp): void { + expect(() => validateScenario(scenario)).toThrow(ScenarioValidationError) + expect(() => validateScenario(scenario)).toThrow(message) +} + +function workspaceExpectation( + scenario: ReturnType, + personaKey: string, + workspaceKey?: string +) { + const persona = scenario.personasByKey.get(personaKey) + if (!persona) throw new Error(`Missing persona "${personaKey}"`) + const expectation = workspaceKey + ? persona.definition.workspaces.find((candidate) => candidate.workspaceKey === workspaceKey) + : persona.definition.workspaces.find((candidate) => candidate.access !== 'none') + if (!expectation) throw new Error(`Missing workspace expectation for "${personaKey}"`) + return expectation +} diff --git a/apps/sim/e2e/foundation/stripe-fake.spec.ts b/apps/sim/e2e/foundation/stripe-fake.spec.ts index 8eec71d6870..8a0b839164b 100644 --- a/apps/sim/e2e/foundation/stripe-fake.spec.ts +++ b/apps/sim/e2e/foundation/stripe-fake.spec.ts @@ -36,6 +36,18 @@ test('Stripe fake records allowlisted calls and rejects unknown routes', async ( ) expect(search.status).toBe(200) + const telemetry = await fetch(`${baseUrl}/v1/traces`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ resourceSpans: [] }), + }) + expect(telemetry.status).toBe(200) + expect( + fake.requestLog.some( + ({ method, path, unexpected }) => method === 'POST' && path === '/v1/traces' && !unexpected + ) + ).toBe(true) + const unsupportedSearch = await fetch( `${baseUrl}/v1/customers/search?${new URLSearchParams({ query: 'name:"Foundation"', diff --git a/apps/sim/e2e/scripts/capture-auth-states.ts b/apps/sim/e2e/scripts/capture-auth-states.ts new file mode 100644 index 00000000000..da1b773921a --- /dev/null +++ b/apps/sim/e2e/scripts/capture-auth-states.ts @@ -0,0 +1,167 @@ +import { chmodSync, mkdirSync, statSync } from 'node:fs' +import path from 'node:path' +import { chromium, expect } from '@playwright/test' +import { sleep } from '@sim/utils/helpers' +import { + readPersonaCredentials, + readScenarioManifest, + resolveStorageStatePath, + scenarioManifestSchema, + writeJsonAtomic, +} from '../fixtures/e2e-world' + +const baseUrl = requiredEnv('E2E_BASE_URL') +const manifestPath = requiredEnv('E2E_MANIFEST_PATH') +const credentialsPath = requiredEnv('E2E_CREDENTIALS_PATH') +const storageStateDirectory = requiredEnv('E2E_STORAGE_STATE_DIR') +const screenshotsDirectory = requiredEnv('E2E_AUTH_SCREENSHOT_DIR') +const UI_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000] as const + +async function main(): Promise { + const manifest = readScenarioManifest(manifestPath) + const credentials = readPersonaCredentials(credentialsPath) + if (manifest.runId !== credentials.runId || manifest.runId !== requiredEnv('E2E_RUN_ID')) { + throw new Error('Persona auth inputs belong to different E2E runs') + } + mkdirSync(storageStateDirectory, { recursive: true }) + mkdirSync(screenshotsDirectory, { recursive: true }) + + const browser = await chromium.launch({ + args: ['--host-resolver-rules=MAP e2e.sim.ai 127.0.0.1'], + }) + try { + for (const [personaKey, persona] of Object.entries(manifest.personas)) { + const login = credentials.personas[personaKey] + if (!login) throw new Error(`Missing private login for persona: ${personaKey}`) + const context = await browser.newContext({ baseURL: baseUrl }) + const page = await context.newPage() + try { + await page.goto('/login') + await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible() + await page.getByLabel('Email').fill(login.email) + await page.getByRole('textbox', { name: 'Password' }).fill(login.password) + await signInThroughUi(page, personaKey) + + const sessionResponse = await context.request.get('/api/auth/get-session') + if (!sessionResponse.ok()) { + throw new Error(`Session probe failed for ${personaKey}: ${sessionResponse.status()}`) + } + const session = (await sessionResponse.json()) as { + user?: { id?: string; email?: string } + session?: { activeOrganizationId?: string | null } + } + if (session.user?.id !== persona.userId || session.user.email !== persona.email) { + throw new Error(`Session identity mismatch for persona: ${personaKey}`) + } + if ( + (session.session?.activeOrganizationId ?? null) !== persona.expectedActiveOrganizationId + ) { + throw new Error(`Active organization mismatch for persona: ${personaKey}`) + } + // Deliberately duplicated by persona contracts: capture must reject lazy workspace creation + // before persisting auth, while the later contract independently verifies the saved session. + const workspaceResponse = await context.request.get('/api/workspaces?scope=all') + if (!workspaceResponse.ok()) { + throw new Error( + `Workspace inventory failed for ${personaKey}: ${workspaceResponse.status()}` + ) + } + const workspacePayload = (await workspaceResponse.json()) as { + workspaces?: Array<{ id?: string }> + } + const actualWorkspaceIds = (workspacePayload.workspaces ?? []) + .map(({ id }) => id) + .filter((id): id is string => Boolean(id)) + .sort() + const expectedWorkspaceIds = persona.workspaces + .filter(({ access }) => access !== 'none') + .map(({ workspaceId }) => workspaceId) + .sort() + if (JSON.stringify(actualWorkspaceIds) !== JSON.stringify(expectedWorkspaceIds)) { + throw new Error(`Workspace inventory mismatch for persona: ${personaKey}`) + } + + await page.goto(persona.canonicalRoute) + await expect(page).toHaveURL(new URL(persona.canonicalRoute, baseUrl).toString()) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + const storageStatePath = resolveStorageStatePath( + storageStateDirectory, + persona.storageStatePath + ) + await context.storageState({ path: storageStatePath }) + chmodSync(storageStatePath, 0o600) + if ((statSync(storageStatePath).mode & 0o777) !== 0o600) { + throw new Error(`Storage state permissions are not private for persona: ${personaKey}`) + } + } catch (error) { + await page + .getByRole('textbox', { name: 'Password' }) + .fill('') + .catch(() => {}) + await page + .screenshot({ + path: path.join(screenshotsDirectory, `${personaKey}.failure.png`), + fullPage: true, + }) + .catch(() => {}) + throw error + } finally { + await context.close() + } + } + + manifest.authCaptureComplete = true + writeJsonAtomic(manifestPath, scenarioManifestSchema.parse(manifest)) + } finally { + await browser.close() + } +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing auth capture environment value: ${key}`) + return value +} + +async function signInThroughUi( + page: import('@playwright/test').Page, + personaKey: string +): Promise { + for (let attempt = 0; attempt <= UI_RETRY_DELAYS_MS.length; attempt += 1) { + const responsePromise = page.waitForResponse((response) => { + const url = new URL(response.url()) + return url.pathname === '/api/auth/sign-in/email' + }) + await page.getByRole('button', { name: 'Sign in' }).click() + const response = await responsePromise + if (response.status() === 200) { + try { + await page.waitForURL(/\/workspace(?:\/|$)/, { timeout: 30_000 }) + } catch { + throw new Error(`UI sign-in did not redirect for persona: ${personaKey}`) + } + return + } + if (response.status() !== 429 || attempt === UI_RETRY_DELAYS_MS.length) { + throw new Error( + `UI sign-in failed for persona ${personaKey} with status ${response.status()}; response body redacted` + ) + } + await sleep(UI_RETRY_DELAYS_MS[attempt]) + } +} + +main().catch((error) => { + const raw = error instanceof Error ? (error.stack ?? error.message) : String(error) + let redacted = raw + try { + const credentials = readPersonaCredentials(credentialsPath) + for (const { password } of Object.values(credentials.personas)) { + redacted = redacted.replaceAll(password, '[REDACTED]') + } + } catch { + redacted = 'Auth capture failed; credentials unavailable for safe diagnostic formatting' + } + console.error(redacted) + process.exitCode = 1 +}) diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts index fb8bdde3e0f..ae4dc26150d 100644 --- a/apps/sim/e2e/scripts/options.ts +++ b/apps/sim/e2e/scripts/options.ts @@ -1,5 +1,13 @@ const NAVIGATION_PROJECT = 'hosted-billing-chromium-navigation' const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' +const PERSONAS_PROJECT = 'hosted-billing-chromium-personas' +const PERSONA_ISOLATION_PROJECT = 'hosted-billing-chromium-persona-isolation' +const E2E_PROJECTS = new Set([ + NAVIGATION_PROJECT, + WORKFLOWS_PROJECT, + PERSONAS_PROJECT, + PERSONA_ISOLATION_PROJECT, +]) const FORBIDDEN_OPTIONS = [ '--config', '-c', @@ -9,19 +17,50 @@ const FORBIDDEN_OPTIONS = [ '--fully-parallel', '--pass-with-no-tests', '--list', + '--output', + '--output-dir', + '--reporter', + '--trace', + '--timeout', + '--debug', + '--ui', ] as const +const SAFE_BOOLEAN_OPTIONS = new Set(['--no-deps', '--headed', '--quiet']) +const SAFE_VALUE_OPTIONS = new Set(['--grep', '--grep-invert', '-g']) export interface E2eRunOptions { playwrightArgs: string[] + reuseBuild: boolean } -export function parseRunOptions(argv: string[]): E2eRunOptions { +export function parseRunOptions( + argv: string[], + environment: { ci: boolean } = { ci: process.env.CI === 'true' } +): E2eRunOptions { const normalizedArgs = argv[0] === '--' ? argv.slice(1) : [...argv] if (normalizedArgs.includes('--skip-build')) { throw new Error( '--skip-build remains disabled until the planned profile/source-keyed build reuse experiment proves it safe' ) } + if (hasOption(normalizedArgs, '--keep-stack')) { + throw new Error( + '--keep-stack is unavailable: the all-or-nothing retained-stack safety experiment was deferred' + ) + } + if (normalizedArgs.some((arg) => arg.startsWith('--reuse-build='))) { + throw new Error('Use --reuse-build without a value') + } + const reuseBuild = normalizedArgs.includes('--reuse-build') + if (reuseBuild && environment.ci) { + throw new Error('--reuse-build is local-only; CI must run a fresh one-shot build') + } + if (hasOption(normalizedArgs, '--no-deps') && environment.ci) { + throw new Error('--no-deps is local-only; CI must run the complete project dependency chain') + } + if (normalizedArgs.some((arg) => arg.startsWith('--no-deps='))) { + throw new Error('Use --no-deps without a value') + } for (const option of FORBIDDEN_OPTIONS) { if (hasOption(normalizedArgs, option)) { throw new Error(`${option} cannot override E2E orchestration invariants`) @@ -33,12 +72,14 @@ export function parseRunOptions(argv: string[]): E2eRunOptions { if (normalizedArgs.includes('--shard')) { throw new Error('Use canonical --shard= syntax') } - const playwrightArgs = [...normalizedArgs] + const playwrightArgs = normalizedArgs.filter((arg) => arg !== '--reuse-build') + assertSupportedPlaywrightArgs(playwrightArgs) const projects = getEqualsOptionValues(playwrightArgs, '--project') - const unknownProject = projects.find( - (project) => project !== NAVIGATION_PROJECT && project !== WORKFLOWS_PROJECT - ) + const unknownProject = projects.find((project) => !E2E_PROJECTS.has(project)) if (unknownProject) throw new Error(`Unknown E2E Playwright project: ${unknownProject}`) + if (normalizedArgs.includes('--no-deps') && projects.length !== 1) { + throw new Error('--no-deps requires exactly one explicit canonical --project=') + } const hasShard = hasOption(playwrightArgs, '--shard') if ( @@ -48,14 +89,38 @@ export function parseRunOptions(argv: string[]): E2eRunOptions { projects.some((project) => project !== NAVIGATION_PROJECT)) ) { throw new Error( - `--shard is supported only with --project=${NAVIGATION_PROJECT}; coupled workflows must remain unsharded` + `--shard is supported only with --project=${NAVIGATION_PROJECT}; coupled E2E projects must remain unsharded` ) } - return { playwrightArgs } + return { playwrightArgs, reuseBuild } +} + +function assertSupportedPlaywrightArgs(args: string[]): void { + for (let index = 0; index < args.length; index += 1) { + const argument = args[index] + if (!argument.startsWith('-')) continue + if (SAFE_BOOLEAN_OPTIONS.has(argument)) continue + if (argument.startsWith('--project=') || argument.startsWith('--shard=')) continue + const equalsName = argument.includes('=') ? argument.slice(0, argument.indexOf('=')) : argument + if (SAFE_VALUE_OPTIONS.has(equalsName) && argument.includes('=')) continue + if (SAFE_VALUE_OPTIONS.has(argument)) { + if (args[index + 1] === undefined) { + throw new Error(`${argument} requires a value`) + } + index += 1 + continue + } + throw new Error( + `${argument} is not a supported E2E Playwright option; artifact and orchestration overrides are denied` + ) + } } function hasOption(args: string[], name: string): boolean { + if (name.startsWith('-') && !name.startsWith('--')) { + return args.some((arg) => arg === name || arg.startsWith(name)) + } return args.some((arg) => arg === name || arg.startsWith(`${name}=`)) } diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 9a1f102e921..c6b6924934c 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -23,11 +23,17 @@ import { assertNoForbiddenProviderInitialization, assertNoForbiddenProviderTraffic, } from '../support/diagnostics' -import { E2E_OS_PASSTHROUGH_KEYS, formatRedactedEnvironmentSummary } from '../support/env' +import { formatRedactedEnvironmentSummary } from '../support/env' import { assertE2eHostResolvesToLoopback } from '../support/hosts' +import { + assertNoSyntheticSecretLeaks, + loadSyntheticSecretCanaryForScan, + scrubUnscannableArtifacts, +} from '../support/leak-canary' import { getRunDirectory, SIM_APP_DIR } from '../support/paths' import { assertAdminApiBoundary, + assertManifestWorkspaceIdentities, type FoundationProvisioningResult, inspectFoundationUsers, } from '../support/probes' @@ -37,11 +43,24 @@ import { type ManagedProcess, stopAllManagedProcesses, } from '../support/process' -import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' +import { acquireE2eRunLock } from '../support/run-lock' +import { + createE2eRuntimeSecrets, + FOUNDATION_TEST_PASSWORD, + runtimeSecretValues, +} from '../support/runtime-secrets' +import { + buildApp, + capturePersonaAuthStates, + runMigrations, + runPlaywright, + seedE2eWorld, + startApp, + startRealtime, +} from '../support/stack' import { parseRunOptions } from './options' const DEFAULT_ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' -const STRIPE_TEST_KEY = 'sk_test_sim_e2e_foundation' async function main(): Promise { const options = parseRunOptions(process.argv.slice(2)) @@ -50,15 +69,39 @@ async function main(): Promise { const logsDirectory = path.join(runDirectory, 'logs') const storageStateDirectory = path.join(runDirectory, 'auth') const markerDirectory = path.join(runDirectory, 'markers') - const homeDirectory = path.join(runDirectory, 'home') + const privateDirectory = path.join(runDirectory, 'private') + const homesDirectory = path.join(runDirectory, 'homes') + const runtimeHomeDirectory = path.join(homesDirectory, 'runtime') + const setupHomeDirectory = path.join(homesDirectory, 'setup') + const authCaptureHomeDirectory = path.join(homesDirectory, 'auth-capture') + const playwrightHomeDirectory = path.join(homesDirectory, 'playwright') + const manifestPath = path.join(runDirectory, 'persona-manifest.json') + const credentialsPath = path.join(privateDirectory, 'persona-credentials.json') + const canarySecretsPath = path.join(privateDirectory, 'synthetic-secrets.json') + const authScreenshotsDirectory = path.join(runDirectory, 'auth-capture-screenshots') + const diagnosticRoots = [ + runDirectory, + path.join(SIM_APP_DIR, 'playwright-report'), + path.join(SIM_APP_DIR, 'test-results'), + ] mkdirSync(logsDirectory, { recursive: true }) - mkdirSync(storageStateDirectory, { recursive: true }) + mkdirSync(storageStateDirectory, { recursive: true, mode: 0o700 }) mkdirSync(markerDirectory, { recursive: true }) - mkdirSync(homeDirectory, { recursive: true }) + mkdirSync(privateDirectory, { recursive: true, mode: 0o700 }) + for (const directory of [ + runtimeHomeDirectory, + setupHomeDirectory, + authCaptureHomeDirectory, + playwrightHomeDirectory, + ]) { + mkdirSync(directory, { recursive: true, mode: 0o700 }) + } const nodeExecutable = resolveNode22() const bunExecutable = resolveBunExecutable() const adminDatabaseUrl = process.env.E2E_PG_ADMIN_URL ?? DEFAULT_ADMIN_DATABASE_URL + const runtimeSecrets = createE2eRuntimeSecrets() + const runLock = acquireE2eRunLock() let runDatabase: RunDatabase | null = null let databaseCreationComplete = false @@ -66,6 +109,12 @@ async function main(): Promise { let realtime: ManagedProcess | null = null let app: ManagedProcess | null = null let cleanupPromise: Promise | null = null + let leakCanarySecrets: string[] = [ + FOUNDATION_TEST_PASSWORD, + ...runtimeSecretValues(runtimeSecrets), + ] + let canaryCoverageComplete = true + let diagnosticsRetained = true let failed = false const cleanup = (): Promise => { @@ -94,9 +143,12 @@ async function main(): Promise { } } - for (const sensitiveDirectory of [storageStateDirectory, homeDirectory]) { + for (const sensitiveDirectory of [storageStateDirectory, privateDirectory, homesDirectory]) { try { rmSync(sensitiveDirectory, { recursive: true, force: true }) + if (existsSync(sensitiveDirectory)) { + throw new Error(`Sensitive directory still exists: ${sensitiveDirectory}`) + } } catch (error) { failures.push(error) } @@ -128,6 +180,7 @@ async function main(): Promise { ) } catch {} } + let lockTransferred = false if (runDatabase) { const cleanupLogFd = openSync(path.join(logsDirectory, 'signal-cleanup.log'), 'a') const cleanupProcess = spawn( @@ -139,20 +192,31 @@ async function main(): Promise { env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '', - HOME: homeDirectory, + HOME: setupHomeDirectory, E2E_PG_ADMIN_URL: adminDatabaseUrl, E2E_DATABASE_NAME: runDatabase.name, E2E_DATABASE_CREATION_COMPLETE: String(databaseCreationComplete), E2E_CLEANUP_PROCESS_GROUPS: getActiveManagedProcessGroupIds().join(','), - E2E_CLEANUP_DIRECTORIES: JSON.stringify([storageStateDirectory, homeDirectory]), + E2E_CLEANUP_DIRECTORIES: JSON.stringify([ + storageStateDirectory, + privateDirectory, + homesDirectory, + ]), + E2E_RUN_LOCK_PATH: runLock.path, + E2E_RUN_LOCK_TOKEN: runLock.token, }, stdio: ['ignore', cleanupLogFd, cleanupLogFd], } ) + if (cleanupProcess.pid) { + runLock.transfer(cleanupProcess.pid) + lockTransferred = true + } cleanupProcess.unref() closeSync(cleanupLogFd) console.error(`Detached E2E cleanup supervisor started as PID ${cleanupProcess.pid}`) } + if (!lockTransferred) runLock.release() process.exit(exitCode) } // `once` is intentional: a second signal uses the OS default force termination. @@ -172,7 +236,7 @@ async function main(): Promise { await createRunDatabase(adminDatabaseUrl, runDatabaseName) databaseCreationComplete = true stripeFake = await startStripeFakeServer({ - apiKey: STRIPE_TEST_KEY, + apiKey: runtimeSecrets.stripeSecretKey, hostname: '127.0.0.1', port: 0, }) @@ -182,34 +246,78 @@ async function main(): Promise { runId, databaseUrl: runDatabase.url, stripeApiBaseUrl: stripeFake.baseUrl, - homeDirectory, + runtimeHomeDirectory, + setupHomeDirectory, + authCaptureHomeDirectory, + playwrightHomeDirectory, playwrightBrowsersPath: resolvePlaywrightBrowsersPath(), + runtimeSecrets, ci: process.env.CI === 'true', }) - console.info(formatRedactedEnvironmentSummary(profile.id, profile.childEnvironment)) + for (const [name, environment] of Object.entries(profile.environments)) { + console.info(formatRedactedEnvironmentSummary(`${profile.id}/${name}`, environment)) + } - const stackOptions = { + const commandOptions = { bunExecutable, nodeExecutable, - env: profile.childEnvironment.env, logsDirectory, } - await runMigrations(stackOptions) - await buildApp(stackOptions) - realtime = await startRealtime(stackOptions) - app = await startApp(stackOptions) + await runMigrations({ + ...commandOptions, + env: profile.environments.migration.env, + }) + await buildApp({ + ...commandOptions, + buildEnvironment: profile.environments.build, + reuseBuild: options.reuseBuild, + ci: process.env.CI === 'true', + }) + realtime = await startRealtime({ + ...commandOptions, + env: profile.environments.realtime.env, + }) + app = await startApp({ + ...commandOptions, + env: profile.environments.app.env, + }) await assertAdminApiBoundary( 'http://127.0.0.1:3000', - profile.childEnvironment.env.ADMIN_API_KEY + profile.environments.app.env.ADMIN_API_KEY ) assertNoForbiddenProviderInitialization([app.logPath, realtime.logPath]) + await seedE2eWorld({ + ...commandOptions, + env: { + ...profile.environments.seed.env, + E2E_ORCHESTRATED: '1', + E2E_MANIFEST_PATH: manifestPath, + E2E_CREDENTIALS_PATH: credentialsPath, + E2E_CANARY_SECRETS_PATH: canarySecretsPath, + }, + }) + await capturePersonaAuthStates({ + ...commandOptions, + env: { + ...profile.environments.authCapture.env, + E2E_MANIFEST_PATH: manifestPath, + E2E_CREDENTIALS_PATH: credentialsPath, + E2E_STORAGE_STATE_DIR: storageStateDirectory, + E2E_AUTH_SCREENSHOT_DIR: authScreenshotsDirectory, + }, + }) + leakCanarySecrets = loadSyntheticSecretCanaryForScan(leakCanarySecrets, canarySecretsPath) + rmSync(credentialsPath, { force: true }) + rmSync(canarySecretsPath, { force: true }) + const playwrightEnvironment = createPlaywrightEnvironment( - profile.childEnvironment.env, + profile.environments.playwright.env, runId, storageStateDirectory, - markerDirectory + markerDirectory, + manifestPath ) await runPlaywright( { @@ -230,25 +338,81 @@ async function main(): Promise { console.error(error) process.exitCode = 1 } finally { + if (runDatabase && existsSync(manifestPath)) { + try { + await assertManifestWorkspaceIdentities(runDatabase.url, manifestPath) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + } try { - assertNoForbiddenProviderTraffic( - [app?.logPath, realtime?.logPath].filter((value): value is string => Boolean(value)) - ) + leakCanarySecrets = loadSyntheticSecretCanaryForScan(leakCanarySecrets, canarySecretsPath) } catch (error) { + canaryCoverageComplete = false failed = true process.exitCode = 1 console.error(error) + } finally { + rmSync(credentialsPath, { force: true }) + rmSync(canarySecretsPath, { force: true }) } + let cleanupSucceeded = false try { await cleanup() + cleanupSucceeded = true } catch (error) { failed = true process.exitCode = 1 console.error(error) } + try { + assertNoForbiddenProviderTraffic( + [app?.logPath, realtime?.logPath].filter((value): value is string => Boolean(value)) + ) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + if (canaryCoverageComplete && leakCanarySecrets.length > 0) { + try { + await assertNoSyntheticSecretLeaks({ + secrets: leakCanarySecrets, + roots: diagnosticRoots, + excludedPaths: [privateDirectory, storageStateDirectory], + }) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + diagnosticsRetained = scrubDiagnostics(diagnosticRoots) + if (diagnosticsRetained) cleanupSucceeded = false + } + } else if (!canaryCoverageComplete) { + diagnosticsRetained = scrubDiagnostics(diagnosticRoots) + if (diagnosticsRetained) cleanupSucceeded = false + } process.off('SIGINT', handleSignal) process.off('SIGTERM', handleSignal) - console.info(`E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}`) + if (cleanupSucceeded) runLock.release() + else runLock.retain('normal cleanup failed; inspect diagnostics and clean resources manually') + console.info( + diagnosticsRetained + ? `E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}` + : 'E2E failed; diagnostics were scrubbed because complete secret scanning was impossible' + ) + } +} + +function scrubDiagnostics(roots: string[]): boolean { + try { + scrubUnscannableArtifacts(roots) + return false + } catch (error) { + console.error(error) + return true } } @@ -291,25 +455,21 @@ function resolvePlaywrightBrowsersPath(): string { } function createPlaywrightEnvironment( - stackEnvironment: Record, + baseEnvironment: Record, runId: string, storageStateDirectory: string, - markerDirectory: string + markerDirectory: string, + manifestPath: string ): Record { - const keys = [...E2E_OS_PASSTHROUGH_KEYS, 'HOME', 'NODE_OPTIONS', 'PLAYWRIGHT_BROWSERS_PATH'] - const env: Record = {} - for (const key of keys) { - const value = stackEnvironment[key] - if (value !== undefined) env[key] = value - } return { - ...env, + ...baseEnvironment, E2E_PROFILE, E2E_ORCHESTRATED: '1', E2E_RUN_ID: runId, E2E_BASE_URL: E2E_ORIGIN, E2E_STORAGE_STATE_DIR: storageStateDirectory, E2E_MARKER_DIR: markerDirectory, + E2E_MANIFEST_PATH: manifestPath, } } diff --git a/apps/sim/e2e/scripts/seed-world.ts b/apps/sim/e2e/scripts/seed-world.ts new file mode 100644 index 00000000000..530658b991e --- /dev/null +++ b/apps/sim/e2e/scripts/seed-world.ts @@ -0,0 +1,795 @@ +import { randomBytes } from 'node:crypto' +import { db } from '@sim/db' +import { + invitation, + invitationWorkspaceGrant, + member, + permissions, + subscription, + user, + userStats, +} from '@sim/db/schema' +import { and, eq, inArray } from 'drizzle-orm' +import { z } from 'zod' +import { + buildScenarioManifest, + createWorldRecords, + type E2EWorld, + PERSONA_MANIFEST_VERSION, + type PersonaCredentials, + writeJsonAtomic, +} from '../fixtures/e2e-world' +import { arrangeSubscription, lapseOrganizationSubscription } from '../fixtures/factories/billing' +import { arrangePendingInvitation } from '../fixtures/factories/invitations' +import { + addOrganizationMember, + createAdminClient, + createOrganization, +} from '../fixtures/factories/organizations' +import { + addPermissionGroupMember, + createPermissionGroup, +} from '../fixtures/factories/permission-groups' +import { arrangeEffectivePlatformAdmin } from '../fixtures/factories/platform' +import { + createAuthenticatedClient, + createSyntheticUser, + type SyntheticLogin, +} from '../fixtures/factories/users' +import { createWorkspace, grantWorkspacePermission } from '../fixtures/factories/workspaces' +import { E2eHttpClient } from '../fixtures/http-client' +import type { ResolvedScenario, ScenarioSubscription } from '../fixtures/scenario' +import { validateScenario, validateScenarioSet } from '../fixtures/validate-scenario' +import { createSettingsPersonaScenarios } from '../settings/personas' +import { writeSyntheticSecretCanary } from '../support/leak-canary' +import { assertSafeSeedEnvironment } from '../support/seed-safety' + +const requiredEnvSchema = z.object({ + E2E_RUN_ID: z.string().min(1), + E2E_ORCHESTRATED: z.literal('1'), + E2E_PROFILE: z.string().min(1), + E2E_BASE_URL: z.string().url(), + ADMIN_API_KEY: z.string().min(1), + DATABASE_URL: z.string().url(), + E2E_MANIFEST_PATH: z.string().min(1), + E2E_CREDENTIALS_PATH: z.string().min(1), + E2E_CANARY_SECRETS_PATH: z.string().min(1), +}) + +async function main(): Promise { + const env = requiredEnvSchema.parse(process.env) + assertSafeSeedEnvironment(env) + const definitions = createSettingsPersonaScenarios(env.E2E_RUN_ID) + const scenarios = [ + validateScenario(definitions.primary), + validateScenario(definitions.isolationTwin), + ] + validateScenarioSet(scenarios) + const adminClient = createAdminClient(env.E2E_BASE_URL, env.ADMIN_API_KEY) + const attemptCounts = new Map() + const worlds: E2EWorld[] = [] + const loginsByWorldAndUser = buildSyntheticLogins(scenarios) + const ownerClients = new Map() + writeSyntheticSecretCanary(env.E2E_CANARY_SECRETS_PATH, env.E2E_RUN_ID, [ + ...[...loginsByWorldAndUser.values()].map(({ password }) => password), + ...scenarios.flatMap(({ definition }) => definition.invitations.map(({ token }) => token)), + ]) + writeJsonAtomic( + env.E2E_CREDENTIALS_PATH, + buildPersonaCredentials(env.E2E_RUN_ID, scenarios, loginsByWorldAndUser) + ) + + for (const scenario of scenarios) { + const world: E2EWorld = { scenario, records: createWorldRecords() } + await createUsers(world, env.E2E_BASE_URL, attemptCounts, loginsByWorldAndUser) + await createOrganizationsAndSubscriptions(world, adminClient) + await addMemberships(world, adminClient) + await createScenarioWorkspaces( + world, + env.E2E_BASE_URL, + loginsByWorldAndUser, + ownerClients, + attemptCounts + ) + await lapsePlannedSubscriptions(world) + await createGrantsAndPermissionGroups(world, adminClient, ownerClients) + await arrangePlatformAdminsAndInvitations(world) + await recordProductionPermissionIds(world) + await assertTrustedWorldInvariants(world) + worlds.push(world) + } + + await assertSecondOrganizationIsRejected(worlds, adminClient) + const manifest = buildScenarioManifest(env.E2E_RUN_ID, worlds) + writeJsonAtomic(env.E2E_MANIFEST_PATH, manifest) + + console.info( + `Seeded ${Object.keys(manifest.personas).length} personas across ${worlds.length} isolated worlds` + ) + console.info( + `Auth attempts: ${[...attemptCounts.entries()] + .map(([operation, count]) => `${operation}=${count}`) + .join(', ')}` + ) +} + +async function createUsers( + world: E2EWorld, + baseUrl: string, + attemptCounts: Map, + logins: Map +): Promise { + for (const user of world.scenario.definition.users) { + const login = required(logins, worldUserKey(world.scenario, user.key), 'synthetic login') + const created = await createSyntheticUser( + new E2eHttpClient({ + baseUrl, + onAttempt: ({ path }) => increment(attemptCounts, `signup:${path}`), + }), + login + ) + if (created.email !== user.email || created.name !== user.name) { + throw new Error(`Production signup returned an unexpected identity for ${user.key}`) + } + world.records.users.set(user.key, created) + } +} + +async function createOrganizationsAndSubscriptions( + world: E2EWorld, + adminClient: E2eHttpClient +): Promise { + for (const organization of world.scenario.definition.organizations) { + const owner = required(world.records.users, organization.ownerUserKey, 'organization owner') + const created = await createOrganization(adminClient, { + name: organization.name, + slug: organization.slug, + ownerId: owner.id, + }) + if (created.name !== organization.name || created.slug !== organization.slug) { + throw new Error(`Production organization identity mismatch for ${organization.key}`) + } + world.records.organizations.set(organization.key, { + id: created.id, + memberId: created.memberId, + name: created.name, + slug: created.slug, + }) + world.records.organizationMembers.set( + `${organization.key}:${organization.ownerUserKey}`, + created.memberId + ) + } + + for (const subscription of world.scenario.definition.subscriptions) { + const referenceId = resolveBillingReference(world, subscription) + const organizationKey = + subscription.billingReference.kind === 'organization' + ? subscription.billingReference.organizationKey + : undefined + const memberUserIds = organizationKey + ? world.scenario.definition.organizationMemberships + .filter((membership) => membership.organizationKey === organizationKey) + .map(({ userKey }) => required(world.records.users, userKey, 'subscription member').id) + : undefined + const created = await arrangeSubscription({ + referenceId, + plan: subscription.plan, + status: subscription.status === 'lapsed' ? 'active' : subscription.status, + seats: subscription.seats, + memberUserIds, + enterprise: subscription.enterprise + ? { monthlyPrice: subscription.enterprise.monthlyPrice } + : undefined, + }) + world.records.subscriptions.set(subscription.key, created.id) + } +} + +async function addMemberships(world: E2EWorld, adminClient: E2eHttpClient): Promise { + for (const membership of world.scenario.definition.organizationMemberships) { + if (membership.role === 'owner') continue + const organization = required( + world.records.organizations, + membership.organizationKey, + 'organization' + ) + const user = required(world.records.users, membership.userKey, 'organization member') + const created = await addOrganizationMember(adminClient, organization.id, { + userId: user.id, + role: membership.role, + }) + world.records.organizationMembers.set( + `${membership.organizationKey}:${membership.userKey}`, + created.id + ) + } +} + +async function createScenarioWorkspaces( + world: E2EWorld, + baseUrl: string, + logins: Map, + ownerClients: Map, + attemptCounts: Map +): Promise { + for (const workspace of world.scenario.definition.workspaces) { + const login = required( + logins, + worldUserKey(world.scenario, workspace.ownerUserKey), + 'owner login' + ) + const clientKey = worldUserKey(world.scenario, workspace.ownerUserKey) + let client = ownerClients.get(clientKey) + if (!client) { + client = await createAuthenticatedClient(baseUrl, login, ({ path }) => + increment(attemptCounts, `signin:${path}`) + ) + ownerClients.set(clientKey, client) + } + const created = await createWorkspace(client, { name: workspace.name }) + assertCreatedWorkspace(world, workspace, created) + world.records.workspaces.set(workspace.key, { id: created.id, name: created.name }) + } +} + +async function lapsePlannedSubscriptions(world: E2EWorld): Promise { + for (const subscription of world.scenario.definition.subscriptions) { + if (subscription.status !== 'lapsed' || subscription.billingReference.kind !== 'organization') { + continue + } + const organizationKey = subscription.billingReference.organizationKey + const organizationId = required( + world.records.organizations, + organizationKey, + 'lapsed organization' + ).id + const memberUserIds = world.scenario.definition.organizationMemberships + .filter((membership) => membership.organizationKey === organizationKey) + .map(({ userKey }) => required(world.records.users, userKey, 'lapsed member').id) + await lapseOrganizationSubscription({ + subscriptionId: required(world.records.subscriptions, subscription.key, 'subscription'), + organizationId, + memberUserIds, + }) + } +} + +async function createGrantsAndPermissionGroups( + world: E2EWorld, + adminClient: E2eHttpClient, + ownerClients: Map +): Promise { + for (const grant of world.scenario.definition.workspaceGrants) { + const created = await grantWorkspacePermission( + adminClient, + required(world.records.workspaces, grant.workspaceKey, 'workspace').id, + { + userId: required(world.records.users, grant.userKey, 'permission user').id, + permissions: grant.access, + } + ) + world.records.permissions.set(`${grant.workspaceKey}:${grant.userKey}`, created.id) + } + + for (const group of world.scenario.definition.permissionGroups) { + const organization = required( + world.records.organizations, + group.organizationKey, + 'permission group organization' + ) + const ownerKey = required( + world.scenario.organizationsByKey, + group.organizationKey, + 'permission group organization definition' + ).ownerUserKey + const ownerClient = required( + ownerClients, + worldUserKey(world.scenario, ownerKey), + 'permission group owner session' + ) + const created = await createPermissionGroup(ownerClient, organization.id, { + name: group.name, + workspaceIds: group.workspaceKeys.map( + (key) => required(world.records.workspaces, key, 'permission group workspace').id + ), + config: { + hideSecretsTab: group.restrictions.hiddenSettings.includes('secrets'), + hideApiKeysTab: group.restrictions.hiddenSettings.includes('api-keys'), + hideInboxTab: group.restrictions.hiddenSettings.includes('inbox'), + disableMcpTools: group.restrictions.disabledFeatures.includes('mcp'), + disableCustomTools: group.restrictions.disabledFeatures.includes('custom-tools'), + }, + isDefault: false, + }) + if (created.isDefault) { + throw new Error(`Permission-group default state does not match scenario: ${group.key}`) + } + world.records.permissionGroups.set(group.key, created.id) + for (const userKey of group.memberUserKeys) { + const memberId = await addPermissionGroupMember( + ownerClient, + organization.id, + created.id, + required(world.records.users, userKey, 'permission group member').id + ) + world.records.permissionGroupMembers.set(`${group.key}:${userKey}`, memberId) + } + } +} + +async function arrangePlatformAdminsAndInvitations(world: E2EWorld): Promise { + for (const user of world.scenario.definition.users) { + if (user.platformRole === 'admin' && user.superUserModeEnabled) { + await arrangeEffectivePlatformAdmin( + required(world.records.users, user.key, 'platform admin').id + ) + } + } + for (const invitation of world.scenario.definition.invitations) { + const organization = required( + world.records.organizations, + invitation.organizationKey, + 'invitation organization' + ) + const organizationDefinition = required( + world.scenario.organizationsByKey, + invitation.organizationKey, + 'invitation organization definition' + ) + const created = await arrangePendingInvitation({ + email: invitation.email, + token: invitation.token, + inviterId: required(world.records.users, organizationDefinition.ownerUserKey, 'inviter').id, + organizationId: organization.id, + role: invitation.role, + expiresAt: new Date(invitation.expiresAt), + workspaceGrants: invitation.workspaceGrants.map((grant) => ({ + workspaceId: required(world.records.workspaces, grant.workspaceKey, 'invited workspace').id, + permission: grant.access, + })), + }) + world.records.invitations.set(invitation.key, created.invitationId) + invitation.workspaceGrants.forEach((grant, index) => { + world.records.invitationGrants.set( + `${invitation.key}:${grant.workspaceKey}`, + created.grantIds[index] + ) + }) + } +} + +async function recordProductionPermissionIds(world: E2EWorld): Promise { + for (const [workspaceKey, workspaceRecord] of world.records.workspaces) { + const rows = await db + .select({ id: permissions.id, userId: permissions.userId }) + .from(permissions) + .where( + and(eq(permissions.entityType, 'workspace'), eq(permissions.entityId, workspaceRecord.id)) + ) + for (const row of rows) { + const userKey = [...world.records.users].find(([, user]) => user.id === row.userId)?.[0] + if (userKey) world.records.permissions.set(`${workspaceKey}:${userKey}`, row.id) + } + } +} + +async function assertTrustedWorldInvariants(world: E2EWorld): Promise { + for (const persona of world.scenario.definition.personas) { + const userId = required(world.records.users, persona.userKey, 'invariant user').id + for (const expectation of persona.workspaces) { + if (expectation.access === 'none') continue + const workspaceDefinition = required( + world.scenario.workspacesByKey, + expectation.workspaceKey, + 'invariant workspace definition' + ) + const workspaceId = required( + world.records.workspaces, + expectation.workspaceKey, + 'invariant workspace' + ).id + if (expectation.roleSource === 'org-admin') { + const explicitRows = await db + .select({ id: permissions.id }) + .from(permissions) + .where( + and( + eq(permissions.entityType, 'workspace'), + eq(permissions.entityId, workspaceId), + eq(permissions.userId, userId) + ) + ) + if (explicitRows.length !== 0) { + throw new Error( + `Organization-derived admin unexpectedly received an explicit grant: ${persona.key}/${expectation.workspaceKey}` + ) + } + } + if ( + expectation.hostContext.hostMembership === 'external' && + workspaceDefinition.organizationKey + ) { + const organizationId = required( + world.records.organizations, + workspaceDefinition.organizationKey, + 'external host organization' + ).id + const memberships = await db + .select({ id: member.id }) + .from(member) + .where(and(eq(member.userId, userId), eq(member.organizationId, organizationId))) + if (memberships.length !== 0) { + throw new Error( + `External workspace persona unexpectedly received host membership: ${persona.key}` + ) + } + } + } + } + + const userIds = [...world.records.users.values()].map(({ id }) => id) + const persistedUsers = await db + .select({ + id: user.id, + stripeCustomerId: user.stripeCustomerId, + currentUsageLimit: userStats.currentUsageLimit, + billingBlocked: userStats.billingBlocked, + billingBlockedReason: userStats.billingBlockedReason, + }) + .from(user) + .innerJoin(userStats, eq(userStats.userId, user.id)) + .where(inArray(user.id, userIds)) + if (persistedUsers.length !== userIds.length) { + throw new Error('Every synthetic user must retain exactly one user_stats row') + } + for (const definition of world.scenario.definition.users) { + const userId = required(world.records.users, definition.key, 'persisted user').id + const row = persistedUsers.find((candidate) => candidate.id === userId) + if ( + !row?.stripeCustomerId?.startsWith('cus_e2e_') || + row.currentUsageLimit !== expectedUsageLimit(world.scenario, definition.key) || + row.billingBlocked || + row.billingBlockedReason !== null + ) { + throw new Error(`Persisted user billing state does not match scenario: ${definition.key}`) + } + } + + const organizationIds = [...world.records.organizations.values()].map(({ id }) => id) + if (organizationIds.length > 0) { + const persistedMemberships = await db + .select({ + organizationId: member.organizationId, + userId: member.userId, + role: member.role, + }) + .from(member) + .where(inArray(member.organizationId, organizationIds)) + const expectedMemberships = world.scenario.definition.organizationMemberships + .map((definition) => ({ + organizationId: required( + world.records.organizations, + definition.organizationKey, + 'membership organization' + ).id, + userId: required(world.records.users, definition.userKey, 'membership user').id, + role: definition.role, + })) + .sort(byMembership) + if ( + JSON.stringify(persistedMemberships.sort(byMembership)) !== + JSON.stringify(expectedMemberships) + ) { + throw new Error('Persisted organization membership set does not match scenario') + } + } + + const subscriptionIds = [...world.records.subscriptions.values()] + if (subscriptionIds.length > 0) { + const rows = await db + .select({ + id: subscription.id, + plan: subscription.plan, + status: subscription.status, + referenceId: subscription.referenceId, + stripeCustomerId: subscription.stripeCustomerId, + stripeSubscriptionId: subscription.stripeSubscriptionId, + periodStart: subscription.periodStart, + periodEnd: subscription.periodEnd, + cancelAtPeriodEnd: subscription.cancelAtPeriodEnd, + canceledAt: subscription.canceledAt, + endedAt: subscription.endedAt, + seats: subscription.seats, + metadata: subscription.metadata, + }) + .from(subscription) + .where(inArray(subscription.id, subscriptionIds)) + if (rows.length !== subscriptionIds.length) { + throw new Error('Seeded subscription registry does not match persisted rows') + } + for (const definition of world.scenario.definition.subscriptions) { + const id = required(world.records.subscriptions, definition.key, 'subscription') + const row = rows.find((candidate) => candidate.id === id) + if ( + !row || + row.plan !== definition.plan || + row.referenceId !== resolveBillingReference(world, definition) || + row.status !== (definition.status === 'lapsed' ? 'canceled' : definition.status) || + !row.stripeCustomerId?.startsWith('cus_e2e_') || + row.stripeSubscriptionId !== null || + !row.periodStart || + !row.periodEnd || + row.cancelAtPeriodEnd || + row.seats !== (definition.seats ?? null) || + (definition.status === 'lapsed' + ? !row.canceledAt || !row.endedAt + : row.canceledAt !== null || row.endedAt !== null) || + !matchesEnterpriseMetadata(row.metadata, definition, row.referenceId) + ) { + throw new Error(`Persisted subscription does not match scenario: ${definition.key}`) + } + } + } + + for (const definition of world.scenario.definition.invitations) { + const invitationId = required(world.records.invitations, definition.key, 'invitation') + const [row] = await db + .select({ + kind: invitation.kind, + email: invitation.email, + inviterId: invitation.inviterId, + organizationId: invitation.organizationId, + membershipIntent: invitation.membershipIntent, + role: invitation.role, + status: invitation.status, + expiresAt: invitation.expiresAt, + token: invitation.token, + }) + .from(invitation) + .where(eq(invitation.id, invitationId)) + .limit(1) + if ( + !row || + row.kind !== 'organization' || + row.email !== definition.email || + row.inviterId !== + required( + world.records.users, + required( + world.scenario.organizationsByKey, + definition.organizationKey, + 'invitation organization definition' + ).ownerUserKey, + 'invitation inviter' + ).id || + row.organizationId !== + required(world.records.organizations, definition.organizationKey, 'invitation organization') + .id || + row.role !== definition.role || + row.status !== 'pending' || + row.membershipIntent !== 'internal' || + row.token !== definition.token || + row.expiresAt.toISOString() !== new Date(definition.expiresAt).toISOString() + ) { + throw new Error(`Persisted invitation does not match scenario: ${definition.key}`) + } + const grants = await db + .select({ + id: invitationWorkspaceGrant.id, + workspaceId: invitationWorkspaceGrant.workspaceId, + permission: invitationWorkspaceGrant.permission, + }) + .from(invitationWorkspaceGrant) + .where(eq(invitationWorkspaceGrant.invitationId, invitationId)) + const expectedGrants = definition.workspaceGrants.map((grant) => ({ + id: required( + world.records.invitationGrants, + `${definition.key}:${grant.workspaceKey}`, + 'invitation grant' + ), + workspaceId: required(world.records.workspaces, grant.workspaceKey, 'invited workspace').id, + permission: grant.access, + })) + if (JSON.stringify(grants.sort(byId)) !== JSON.stringify(expectedGrants.sort(byId))) { + throw new Error(`Persisted invitation grants do not match scenario: ${definition.key}`) + } + } +} + +async function assertSecondOrganizationIsRejected( + worlds: E2EWorld[], + adminClient: E2eHttpClient +): Promise { + for (const world of worlds) { + for (const membership of world.scenario.definition.organizationMemberships) { + const target = world.scenario.definition.organizations.find( + (organization) => + organization.key !== membership.organizationKey && + !world.scenario.definition.organizationMemberships.some( + (candidate) => + candidate.organizationKey === organization.key && + candidate.userKey === membership.userKey + ) + ) + if (!target) continue + const user = required(world.records.users, membership.userKey, 'constraint user') + const organization = required( + world.records.organizations, + target.key, + 'constraint organization' + ) + await adminClient.request({ + method: 'POST', + path: `/api/v1/admin/organizations/${organization.id}/members`, + body: { userId: user.id, role: 'member' }, + schema: z.object({ + error: z.object({ + code: z.literal('BAD_REQUEST'), + message: z.string().min(1), + }), + }), + expectedStatus: 400, + }) + const unexpectedMembership = await db + .select({ id: member.id }) + .from(member) + .where(and(eq(member.organizationId, organization.id), eq(member.userId, user.id))) + .limit(1) + if (unexpectedMembership.length > 0) { + throw new Error('Rejected second-organization request still created a membership') + } + return + } + } + throw new Error('Scenario does not contain a cross-organization membership constraint probe') +} + +function buildPersonaCredentials( + runId: string, + scenarios: ResolvedScenario[], + logins: Map +): PersonaCredentials { + const credentials: PersonaCredentials = { + schemaVersion: PERSONA_MANIFEST_VERSION, + runId, + personas: {}, + } + for (const scenario of scenarios) { + for (const persona of scenario.definition.personas) { + const login = required(logins, worldUserKey(scenario, persona.userKey), 'persona login') + credentials.personas[persona.key] = { email: login.email, password: login.password } + } + } + return credentials +} + +function buildSyntheticLogins(scenarios: ResolvedScenario[]): Map { + const logins = new Map() + for (const scenario of scenarios) { + for (const user of scenario.definition.users) { + logins.set(worldUserKey(scenario, user.key), { + name: user.name, + email: user.email, + password: randomBytes(24).toString('base64url'), + }) + } + } + return logins +} + +function resolveBillingReference(world: E2EWorld, subscription: ScenarioSubscription): string { + return subscription.billingReference.kind === 'user' + ? required(world.records.users, subscription.billingReference.userKey, 'subscription user').id + : required( + world.records.organizations, + subscription.billingReference.organizationKey, + 'subscription organization' + ).id +} + +function expectedUsageLimit(scenario: ResolvedScenario, userKey: string): string | null { + const membership = scenario.definition.organizationMemberships.find( + (candidate) => candidate.userKey === userKey + ) + if (membership) { + const organizationSubscription = scenario.definition.subscriptions.find( + (candidate) => + candidate.billingReference.kind === 'organization' && + candidate.billingReference.organizationKey === membership.organizationKey + ) + if ( + organizationSubscription?.status === 'active' || + organizationSubscription?.status === 'past_due' + ) { + return null + } + } + const personalSubscription = scenario.definition.subscriptions.find( + (candidate) => + candidate.billingReference.kind === 'user' && + candidate.billingReference.userKey === userKey && + (candidate.status === 'active' || candidate.status === 'past_due') + ) + if (personalSubscription?.plan.startsWith('pro_')) { + return String(Number(personalSubscription.plan.split('_')[1]) / 200) + } + return '5' +} + +function matchesEnterpriseMetadata( + metadata: unknown, + definition: ScenarioSubscription, + referenceId: string +): boolean { + if (definition.plan !== 'enterprise') return metadata === null + const persisted = metadata as { + plan?: unknown + referenceId?: unknown + monthlyPrice?: unknown + seats?: unknown + } | null + return ( + persisted?.plan === 'enterprise' && + persisted.referenceId === referenceId && + persisted.monthlyPrice === definition.enterprise?.monthlyPrice && + persisted.seats === definition.enterprise?.seats + ) +} + +function assertCreatedWorkspace( + world: E2EWorld, + expected: (typeof world.scenario.definition.workspaces)[number], + created: { + name: string + organizationId: string | null + ownerId: string + billedAccountUserId: string + workspaceMode: string + } +): void { + const ownerId = required(world.records.users, expected.ownerUserKey, 'workspace owner').id + const organizationId = expected.organizationKey + ? required(world.records.organizations, expected.organizationKey, 'workspace organization').id + : null + if ( + created.name !== expected.name || + created.ownerId !== ownerId || + created.billedAccountUserId !== ownerId || + created.organizationId !== organizationId || + created.workspaceMode !== (organizationId ? 'organization' : 'personal') + ) { + throw new Error(`Production workspace policy created an unexpected shape for ${expected.key}`) + } +} + +function worldUserKey(scenario: ResolvedScenario, userKey: string): string { + return `${scenario.definition.namespace.world}:${userKey}` +} + +function increment(values: Map, key: string): void { + values.set(key, (values.get(key) ?? 0) + 1) +} + +function byId(left: { id: string }, right: { id: string }): number { + return left.id.localeCompare(right.id) +} + +function byMembership( + left: { organizationId: string; userId: string; role: string }, + right: { organizationId: string; userId: string; role: string } +): number { + return ( + left.organizationId.localeCompare(right.organizationId) || + left.userId.localeCompare(right.userId) || + left.role.localeCompare(right.role) + ) +} + +function required(values: ReadonlyMap, key: K, label: string): V { + const value = values.get(key) + if (!value) throw new Error(`Missing ${label}: ${String(key)}`) + return value +} + +await main() diff --git a/apps/sim/e2e/scripts/signal-cleanup.ts b/apps/sim/e2e/scripts/signal-cleanup.ts index 5d24f82df5a..dfa9559ae68 100644 --- a/apps/sim/e2e/scripts/signal-cleanup.ts +++ b/apps/sim/e2e/scripts/signal-cleanup.ts @@ -1,22 +1,42 @@ -import { rmSync } from 'node:fs' +import { existsSync, rmSync } from 'node:fs' import { sleep } from '@sim/utils/helpers' import { dropRunDatabase, dropRunDatabaseWithRetries } from '../support/database' -import { parseProcessGroupIds } from '../support/signal-cleanup' +import { releaseE2eRunLock, retainE2eRunLock } from '../support/run-lock' +import { isProcessGroupAlive, parseProcessGroupIds } from '../support/signal-cleanup' const adminUrl = process.env.E2E_PG_ADMIN_URL const databaseName = process.env.E2E_DATABASE_NAME const processGroupIds = parseProcessGroupIds(process.env.E2E_CLEANUP_PROCESS_GROUPS) const sensitiveDirectories = JSON.parse(process.env.E2E_CLEANUP_DIRECTORIES ?? '[]') as string[] const databaseCreationComplete = process.env.E2E_DATABASE_CREATION_COMPLETE === 'true' +const runLockPath = process.env.E2E_RUN_LOCK_PATH +const runLockToken = process.env.E2E_RUN_LOCK_TOKEN if (!adminUrl || !databaseName) { console.error('Signal cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') + retainRunLock('signal cleanup was missing database coordinates') process.exit(1) } const failures: unknown[] = [] signalProcessGroups(processGroupIds, 'SIGTERM', failures) await sleep(500) +signalProcessGroups(processGroupIds, 'SIGKILL', failures) +await sleep(250) +for (const groupId of processGroupIds) { + if (isProcessGroupAlive(groupId)) { + failures.push(new Error(`Managed process group still exists after SIGKILL: ${groupId}`)) + } +} + +for (const directory of sensitiveDirectories) { + try { + rmSync(directory, { recursive: true, force: true }) + if (existsSync(directory)) throw new Error(`Sensitive directory still exists: ${directory}`) + } catch (error) { + failures.push(error) + } +} try { if (databaseCreationComplete) { @@ -28,19 +48,22 @@ try { failures.push(error) } -signalProcessGroups(processGroupIds, 'SIGKILL', failures) -await sleep(250) +for (const error of failures) console.error(error) +if (failures.length === 0) releaseRunLock() +else retainRunLock(`signal cleanup failed with ${failures.length} error(s)`) +process.exit(failures.length > 0 ? 1 : 0) -for (const directory of sensitiveDirectories) { - try { - rmSync(directory, { recursive: true, force: true }) - } catch (error) { - failures.push(error) +function releaseRunLock(): void { + if (runLockPath && runLockToken) { + releaseE2eRunLock(runLockPath, runLockToken) } } -for (const error of failures) console.error(error) -process.exit(failures.length > 0 ? 1 : 0) +function retainRunLock(reason: string): void { + if (runLockPath && runLockToken) { + retainE2eRunLock(runLockPath, runLockToken, reason) + } +} function signalProcessGroups( groupIds: number[], diff --git a/apps/sim/e2e/settings/persona-contracts.spec.ts b/apps/sim/e2e/settings/persona-contracts.spec.ts new file mode 100644 index 00000000000..c898ba8c140 --- /dev/null +++ b/apps/sim/e2e/settings/persona-contracts.spec.ts @@ -0,0 +1,169 @@ +import { existsSync } from 'node:fs' +import path from 'node:path' +import { expect, requirePersona, test } from '../fixtures/persona-test' +import { SETTINGS_PERSONA_KEYS } from './personas' + +test('persona workers receive no privileged setup values', () => { + expect(process.env.ADMIN_API_KEY).toBeUndefined() + expect(process.env.DATABASE_URL).toBeUndefined() + expect(process.env.E2E_CREDENTIALS_PATH).toBeUndefined() + for (const key of [ + 'BETTER_AUTH_SECRET', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'STRIPE_SECRET_KEY', + 'STRIPE_WEBHOOK_SECRET', + ]) { + expect(process.env[key]).toBeUndefined() + } + expect(process.env.HOME).toMatch(/[\\/]homes[\\/]playwright$/) + const privateDirectory = path.join(path.dirname(requiredEnv('E2E_MANIFEST_PATH')), 'private') + expect(existsSync(path.join(privateDirectory, 'persona-credentials.json'))).toBe(false) + expect(existsSync(path.join(privateDirectory, 'synthetic-secrets.json'))).toBe(false) +}) + +for (const personaKey of SETTINGS_PERSONA_KEYS) { + test(`${personaKey} matches its real API contract`, async ({ + contextForPersona, + personaManifest, + }) => { + const persona = requirePersona(personaManifest, personaKey) + const context = await contextForPersona(personaKey) + const page = await context.newPage() + const canonicalUrl = new URL(persona.canonicalRoute, requiredEnv('E2E_BASE_URL')).toString() + await page.goto(canonicalUrl) + await expect(page).toHaveURL(canonicalUrl) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + if (personaKey === 'permissionGroupRestricted') { + await expect(page.getByRole('button', { name: 'General', exact: true })).toBeVisible() + for (const label of ['Secrets', 'Sim API keys', 'Sim mailer', 'MCP tools', 'Custom tools']) { + await expect(page.getByRole('button', { name: label, exact: true })).toHaveCount(0) + } + const workspaceId = persona.workspaces.find(({ access }) => access !== 'none')?.workspaceId + expect(workspaceId).toBeTruthy() + for (const section of ['secrets', 'api-keys', 'inbox', 'mcp', 'custom-tools']) { + const deniedUrl = new URL( + `/workspace/${encodeURIComponent(workspaceId ?? '')}/settings/${section}`, + requiredEnv('E2E_BASE_URL') + ).toString() + const response = await page.goto(deniedUrl) + expect(response?.status(), `${section} must be denied by the server route gate`).toBe(404) + } + } + + const sessionResponse = await context.request.get('/api/auth/get-session') + expect(sessionResponse.status()).toBe(200) + const session = (await sessionResponse.json()) as { + user?: { id?: string; email?: string; role?: string } + session?: { activeOrganizationId?: string | null } + } + expect(session.user).toMatchObject({ + id: persona.userId, + email: persona.email, + role: persona.expectedPlatformRole, + }) + expect(session.session?.activeOrganizationId ?? null).toBe(persona.expectedActiveOrganizationId) + const settingsResponse = await context.request.get('/api/users/me/settings') + expect(settingsResponse.status()).toBe(200) + const userSettings = (await settingsResponse.json()) as { + data?: { superUserModeEnabled?: boolean } + } + expect(userSettings.data?.superUserModeEnabled ?? false).toBe(persona.expectedSuperUserMode) + + const workspacesResponse = await context.request.get('/api/workspaces?scope=all') + expect(workspacesResponse.status()).toBe(200) + const workspacePayload = (await workspacesResponse.json()) as { + workspaces?: Array<{ id: string; permissions: string; role: string }> + } + const expectedAccessible = persona.workspaces + .filter(({ access }) => access !== 'none') + .map(({ workspaceId }) => workspaceId) + .sort() + expect(workspacePayload.workspaces?.map(({ id }) => id).sort()).toEqual(expectedAccessible) + + for (const expected of persona.workspaces) { + const response = await context.request.get( + `/api/workspaces/${encodeURIComponent(expected.workspaceId)}/host-context` + ) + if (expected.access === 'none') { + expect(response.status()).toBe(403) + continue + } + expect(response.status()).toBe(200) + const host = (await response.json()) as { + workspace?: { id?: string } + ownerBilling?: { plan?: string; isOrgScoped?: boolean } + viewer?: { + permission?: string + isHostOrganizationMember?: boolean + isHostOrganizationAdmin?: boolean + } + } + expect(host.workspace?.id).toBe(expected.workspaceId) + expect(host.viewer?.permission).toBe(expected.access) + expect(host.ownerBilling?.plan).toBe(expected.hostContext.plan) + expect(host.ownerBilling?.isOrgScoped).toBe( + expected.hostContext.payerScope === 'organization' + ) + expect(host.viewer?.isHostOrganizationMember).toBe( + expected.hostContext.payerScope === 'organization' && + expected.hostContext.hostMembership !== 'external' + ) + expect(host.viewer?.isHostOrganizationAdmin).toBe( + expected.hostContext.payerScope === 'organization' && + (expected.roleSource === 'owner' || expected.roleSource === 'org-admin') + ) + + const permissionsResponse = await context.request.get( + `/api/workspaces/${encodeURIComponent(expected.workspaceId)}/permissions` + ) + expect(permissionsResponse.status()).toBe(200) + const permissionPayload = (await permissionsResponse.json()) as { + users?: Array<{ + userId?: string + permissionType?: string + roleSource?: string + isExternal?: boolean + }> + } + const viewer = permissionPayload.users?.find(({ userId }) => userId === persona.userId) + expect(viewer).toMatchObject({ + permissionType: expected.access, + roleSource: expected.roleSource, + isExternal: expected.hostContext.hostMembership === 'external', + }) + } + + const accessibleWorkspace = persona.workspaces.find(({ access }) => access !== 'none') + expect(accessibleWorkspace).toBeTruthy() + const groupResponse = await context.request.get( + `/api/permission-groups/user?workspaceId=${encodeURIComponent( + accessibleWorkspace?.workspaceId ?? '' + )}` + ) + expect(groupResponse.status()).toBe(200) + const group = (await groupResponse.json()) as { + permissionGroupId?: string | null + config?: Record | null + } + if (persona.permissionGroupIds.length > 0) { + expect(group.permissionGroupId).toBe(persona.permissionGroupIds[0]) + expect(group.config).toMatchObject({ + hideSecretsTab: true, + hideApiKeysTab: true, + hideInboxTab: true, + disableMcpTools: true, + disableCustomTools: true, + }) + } else { + expect(group.permissionGroupId).toBeNull() + } + }) +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing persona contract environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/settings/persona-isolation.spec.ts b/apps/sim/e2e/settings/persona-isolation.spec.ts new file mode 100644 index 00000000000..a68cc52ab92 --- /dev/null +++ b/apps/sim/e2e/settings/persona-isolation.spec.ts @@ -0,0 +1,75 @@ +import { expect, requirePersona, test } from '../fixtures/persona-test' + +test.describe.configure({ mode: 'parallel' }) + +test('primary world cannot discover or access the isolation twin', async ({ + contextForPersona, + personaManifest, +}) => { + const ownPersona = requirePersona(personaManifest, 'personalFreeOwner') + await assertWorldCannotSee( + await contextForPersona('personalFreeOwner'), + ownPersona, + requirePersona(personaManifest, 'isolationTwinOwner').workspaces[0].workspaceId + ) +}) + +test('isolation twin cannot discover or access the primary world', async ({ + contextForPersona, + personaManifest, +}) => { + const ownPersona = requirePersona(personaManifest, 'isolationTwinOwner') + await assertWorldCannotSee( + await contextForPersona('isolationTwinOwner'), + ownPersona, + requirePersona(personaManifest, 'personalFreeOwner').workspaces[0].workspaceId + ) +}) + +async function assertWorldCannotSee( + context: import('@playwright/test').BrowserContext, + ownPersona: import('../fixtures/e2e-world').PersonaManifestEntry, + foreignWorkspaceId: string +): Promise { + const page = await context.newPage() + const canonicalUrl = new URL(ownPersona.canonicalRoute, requiredEnv('E2E_BASE_URL')).toString() + await page.goto(canonicalUrl) + await expect(page).toHaveURL(canonicalUrl) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + + const sessionResponse = await context.request.get('/api/auth/get-session') + expect(sessionResponse.status()).toBe(200) + const session = (await sessionResponse.json()) as { user?: { id?: string; email?: string } } + expect(session.user).toMatchObject({ id: ownPersona.userId, email: ownPersona.email }) + + const listResponse = await context.request.get('/api/workspaces?scope=all') + expect(listResponse.status()).toBe(200) + const payload = (await listResponse.json()) as { workspaces?: Array<{ id: string }> } + const listedWorkspaceIds = payload.workspaces?.map(({ id }) => id) + expect(listedWorkspaceIds).toContain(ownPersona.workspaces[0].workspaceId) + expect(listedWorkspaceIds).not.toContain(foreignWorkspaceId) + + const ownHostContext = await context.request.get( + `/api/workspaces/${encodeURIComponent(ownPersona.workspaces[0].workspaceId)}/host-context` + ) + expect(ownHostContext.status()).toBe(200) + + const response = await context.request.get( + `/api/workspaces/${encodeURIComponent(foreignWorkspaceId)}/host-context` + ) + expect(response.status()).toBe(403) + + const patchResponse = await context.request.patch( + `/api/workspaces/${encodeURIComponent(foreignWorkspaceId)}`, + { + data: { name: 'E Two E Foreign Mutation Must Fail' }, + } + ) + expect([403, 404]).toContain(patchResponse.status()) +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing persona isolation environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/settings/personas.ts b/apps/sim/e2e/settings/personas.ts new file mode 100644 index 00000000000..71e49a8ec62 --- /dev/null +++ b/apps/sim/e2e/settings/personas.ts @@ -0,0 +1,462 @@ +import { createScenarioNamespace, type ScenarioNamespace } from '../fixtures/namespace' +import { + type ExpectedWorkspaceAccess, + type PersonaWorkspaceExpectation, + type RoleSource, + SCENARIO_VERSION, + type ScenarioDefinition, + type ScenarioPersona, + type ScenarioUser, + type SubscriptionPlan, +} from '../fixtures/scenario' + +const HOSTED_BILLING = { hosted: true, billingEnabled: true } as const + +export const SETTINGS_PERSONA_KEYS = [ + 'personalFreeOwner', + 'personalPaidOwner', + 'personalMaxOwner', + 'paidOrganizationOwner', + 'workspaceReadMember', + 'workspaceWriteMember', + 'workspaceAdminMember', + 'externalWorkspaceAdmin', + 'enterpriseOrganizationAdmin', + 'freeOrganizationOwner', + 'permissionGroupRestricted', + 'platformAdmin', +] as const + +export type SettingsPersonaKey = (typeof SETTINGS_PERSONA_KEYS)[number] + +export interface SettingsPersonaScenarios { + primary: ScenarioDefinition + isolationTwin: ScenarioDefinition +} + +export function createSettingsPersonaScenarios(run: string): SettingsPersonaScenarios { + return { + primary: createPrimarySettingsScenario(createScenarioNamespace(run, 'settings-primary')), + isolationTwin: createIsolationTwinScenario( + createScenarioNamespace(run, 'settings-isolation-twin') + ), + } +} + +export function createPrimarySettingsScenario(namespace: ScenarioNamespace): ScenarioDefinition { + const userKeys = [ + 'personal-free-owner', + 'personal-paid-owner', + 'personal-max-owner', + 'paid-organization-owner', + 'workspace-read-member', + 'workspace-write-member', + 'workspace-admin-member', + 'external-workspace-admin', + 'enterprise-organization-owner', + 'enterprise-organization-admin', + 'free-organization-owner', + 'permission-group-restricted', + 'platform-admin', + ] as const + const users: ScenarioUser[] = userKeys.map((key) => user(namespace, key)) + const platformAdmin = users.find(({ key }) => key === 'platform-admin') + if (!platformAdmin) throw new Error('Platform admin declaration is missing') + platformAdmin.platformRole = 'admin' + platformAdmin.superUserModeEnabled = true + + const organizations = [ + { + key: 'team-organization', + name: namespace.name('team-organization'), + slug: namespace.slug('team-organization'), + ownerUserKey: 'paid-organization-owner', + ...HOSTED_BILLING, + }, + { + key: 'enterprise-organization', + name: namespace.name('enterprise-organization'), + slug: namespace.slug('enterprise-organization'), + ownerUserKey: 'enterprise-organization-owner', + ...HOSTED_BILLING, + }, + { + key: 'lapsed-organization', + name: namespace.name('lapsed-organization'), + slug: namespace.slug('lapsed-organization'), + ownerUserKey: 'free-organization-owner', + ...HOSTED_BILLING, + }, + ] as const + + const organizationMemberships = [ + membership('team-organization', 'paid-organization-owner', 'owner'), + membership('team-organization', 'workspace-read-member', 'member'), + membership('team-organization', 'workspace-write-member', 'member'), + membership('team-organization', 'workspace-admin-member', 'member'), + membership('enterprise-organization', 'enterprise-organization-owner', 'owner'), + membership('enterprise-organization', 'enterprise-organization-admin', 'admin'), + membership('enterprise-organization', 'permission-group-restricted', 'member'), + membership('lapsed-organization', 'free-organization-owner', 'owner'), + ] as const + + const subscriptions = [ + { + key: 'personal-paid-subscription', + plan: 'pro_6000', + status: 'active', + billingReference: { kind: 'user', userKey: 'personal-paid-owner' }, + ...HOSTED_BILLING, + }, + { + key: 'personal-max-subscription', + plan: 'pro_25000', + status: 'active', + billingReference: { kind: 'user', userKey: 'personal-max-owner' }, + ...HOSTED_BILLING, + }, + { + key: 'team-subscription', + plan: 'team_6000', + status: 'active', + billingReference: { kind: 'organization', organizationKey: 'team-organization' }, + seats: 4, + ...HOSTED_BILLING, + }, + { + key: 'enterprise-subscription', + plan: 'enterprise', + status: 'active', + billingReference: { + kind: 'organization', + organizationKey: 'enterprise-organization', + }, + seats: 3, + enterprise: { + plan: 'enterprise', + monthlyPrice: 12_000, + seats: 3, + }, + ...HOSTED_BILLING, + }, + { + key: 'lapsed-team-subscription', + plan: 'team_6000', + status: 'lapsed', + billingReference: { kind: 'organization', organizationKey: 'lapsed-organization' }, + seats: 1, + ...HOSTED_BILLING, + }, + ] as const + + const workspaces = [ + personalWorkspace(namespace, 'personal-free-workspace', 'personal-free-owner'), + personalWorkspace( + namespace, + 'personal-paid-workspace', + 'personal-paid-owner', + 'personal-paid-subscription' + ), + personalWorkspace( + namespace, + 'personal-max-workspace', + 'personal-max-owner', + 'personal-max-subscription' + ), + { + key: 'team-workspace', + name: namespace.name('team-workspace'), + ownerUserKey: 'paid-organization-owner', + organizationKey: 'team-organization', + payer: { kind: 'organization', organizationKey: 'team-organization' }, + subscriptionKey: 'team-subscription', + ...HOSTED_BILLING, + }, + { + key: 'team-invitation-workspace', + name: namespace.name('team-invitation-workspace'), + ownerUserKey: 'paid-organization-owner', + organizationKey: 'team-organization', + payer: { kind: 'organization', organizationKey: 'team-organization' }, + subscriptionKey: 'team-subscription', + ...HOSTED_BILLING, + }, + { + key: 'enterprise-workspace', + name: namespace.name('enterprise-workspace'), + ownerUserKey: 'enterprise-organization-owner', + organizationKey: 'enterprise-organization', + payer: { kind: 'organization', organizationKey: 'enterprise-organization' }, + subscriptionKey: 'enterprise-subscription', + ...HOSTED_BILLING, + }, + { + key: 'lapsed-organization-workspace', + name: namespace.name('lapsed-organization-workspace'), + ownerUserKey: 'free-organization-owner', + organizationKey: 'lapsed-organization', + payer: { kind: 'organization', organizationKey: 'lapsed-organization' }, + subscriptionKey: 'lapsed-team-subscription', + ...HOSTED_BILLING, + }, + personalWorkspace(namespace, 'platform-admin-workspace', 'platform-admin'), + ] as const + + const workspaceGrants = [ + grant('team-workspace', 'workspace-read-member', 'read'), + grant('team-workspace', 'workspace-write-member', 'write'), + grant('team-workspace', 'workspace-admin-member', 'admin'), + grant('team-workspace', 'external-workspace-admin', 'admin'), + grant('enterprise-workspace', 'permission-group-restricted', 'read'), + ] as const + + const permissionGroups = [ + { + key: 'restricted-enterprise-group', + name: namespace.name('restricted-enterprise-group'), + organizationKey: 'enterprise-organization', + workspaceKeys: ['enterprise-workspace'], + memberUserKeys: ['permission-group-restricted'], + restrictions: { + hiddenSettings: ['secrets', 'api-keys', 'inbox'], + disabledFeatures: ['mcp', 'custom-tools'], + }, + }, + ] as const + + const invitations = [ + { + key: 'pending-team-invitation', + organizationKey: 'team-organization', + email: namespace.email('pending-team-invitee'), + token: namespace.invitationToken('pending-team-invitation'), + role: 'member', + expiresAt: '2099-01-01T00:00:00.000Z', + workspaceGrants: [ + { workspaceKey: 'team-workspace', access: 'read' }, + { workspaceKey: 'team-invitation-workspace', access: 'write' }, + ], + }, + ] as const + + const personas: ScenarioPersona[] = [ + persona(namespace, 'personalFreeOwner', 'personal-free-owner', [ + expected('personal-free-workspace', 'admin', 'owner', 'owner', 'user', 'free', true), + expected('team-workspace', 'none', 'none', 'external', 'organization', 'team_6000', false), + ]), + persona(namespace, 'personalPaidOwner', 'personal-paid-owner', [ + expected('personal-paid-workspace', 'admin', 'owner', 'owner', 'user', 'pro_6000', true), + ]), + persona(namespace, 'personalMaxOwner', 'personal-max-owner', [ + expected('personal-max-workspace', 'admin', 'owner', 'owner', 'user', 'pro_25000', true), + ]), + persona(namespace, 'paidOrganizationOwner', 'paid-organization-owner', [ + expected('team-workspace', 'admin', 'owner', 'owner', 'organization', 'team_6000', true), + expected( + 'team-invitation-workspace', + 'admin', + 'owner', + 'owner', + 'organization', + 'team_6000', + true + ), + ]), + persona(namespace, 'workspaceReadMember', 'workspace-read-member', [ + expected('team-workspace', 'read', 'explicit', 'member', 'organization', 'team_6000', false), + ]), + persona(namespace, 'workspaceWriteMember', 'workspace-write-member', [ + expected('team-workspace', 'write', 'explicit', 'member', 'organization', 'team_6000', false), + ]), + persona(namespace, 'workspaceAdminMember', 'workspace-admin-member', [ + expected('team-workspace', 'admin', 'explicit', 'member', 'organization', 'team_6000', false), + ]), + persona(namespace, 'externalWorkspaceAdmin', 'external-workspace-admin', [ + expected( + 'team-workspace', + 'admin', + 'explicit', + 'external', + 'organization', + 'team_6000', + false + ), + ]), + persona(namespace, 'enterpriseOrganizationAdmin', 'enterprise-organization-admin', [ + expected( + 'enterprise-workspace', + 'admin', + 'org-admin', + 'member', + 'organization', + 'enterprise', + false + ), + ]), + persona(namespace, 'freeOrganizationOwner', 'free-organization-owner', [ + expected( + 'lapsed-organization-workspace', + 'admin', + 'owner', + 'owner', + 'organization', + 'free', + true + ), + ]), + persona( + namespace, + 'permissionGroupRestricted', + 'permission-group-restricted', + [ + expected( + 'enterprise-workspace', + 'read', + 'explicit', + 'member', + 'organization', + 'enterprise', + false + ), + ], + ['restricted-enterprise-group'] + ), + persona( + namespace, + 'platformAdmin', + 'platform-admin', + [expected('platform-admin-workspace', 'admin', 'owner', 'owner', 'user', 'free', true)], + [], + 'admin', + true + ), + ] + + return { + version: SCENARIO_VERSION, + namespace: namespaceDescriptor(namespace), + deployment: HOSTED_BILLING, + users, + organizations, + organizationMemberships, + subscriptions, + workspaces, + workspaceGrants, + permissionGroups, + invitations, + personas, + } +} + +export function createIsolationTwinScenario(namespace: ScenarioNamespace): ScenarioDefinition { + return { + version: SCENARIO_VERSION, + namespace: namespaceDescriptor(namespace), + deployment: HOSTED_BILLING, + users: [user(namespace, 'isolation-twin-owner')], + organizations: [], + organizationMemberships: [], + subscriptions: [], + workspaces: [personalWorkspace(namespace, 'isolation-twin-workspace', 'isolation-twin-owner')], + workspaceGrants: [], + permissionGroups: [], + invitations: [], + personas: [ + persona(namespace, 'isolationTwinOwner', 'isolation-twin-owner', [ + expected('isolation-twin-workspace', 'admin', 'owner', 'owner', 'user', 'free', true), + ]), + ], + } +} + +function user(namespace: ScenarioNamespace, key: string): ScenarioUser { + return { + key, + email: namespace.email(key), + name: namespace.name(key), + platformRole: 'user', + superUserModeEnabled: false, + ...HOSTED_BILLING, + } +} + +function namespaceDescriptor(namespace: ScenarioNamespace) { + return { + run: namespace.run, + world: namespace.world, + prefix: namespace.prefix, + } +} + +function membership(organizationKey: string, userKey: string, role: 'owner' | 'admin' | 'member') { + return { organizationKey, userKey, role } as const +} + +function grant(workspaceKey: string, userKey: string, access: 'read' | 'write' | 'admin') { + return { workspaceKey, userKey, access } as const +} + +function personalWorkspace( + namespace: ScenarioNamespace, + key: string, + ownerUserKey: string, + subscriptionKey?: string +) { + return { + key, + name: namespace.name(key), + ownerUserKey, + payer: { kind: 'user', userKey: ownerUserKey }, + ...(subscriptionKey ? { subscriptionKey } : {}), + ...HOSTED_BILLING, + } as const +} + +function persona( + namespace: ScenarioNamespace, + key: string, + userKey: string, + workspaces: readonly PersonaWorkspaceExpectation[], + permissionGroupKeys: readonly string[] = [], + expectedPlatformRole: 'user' | 'admin' = 'user', + expectedSuperUserMode = false +): ScenarioPersona { + const canonicalWorkspace = workspaces.find(({ access }) => access !== 'none') + if (!canonicalWorkspace) throw new Error(`Persona "${key}" needs an accessible workspace`) + return { + key, + userKey, + storageStateFilename: namespace.storageStateFilename(key), + workspaces, + permissionGroupKeys, + canonicalRoute: { + workspaceKey: canonicalWorkspace.workspaceKey, + settingsSection: 'general', + }, + expectedPlatformRole, + expectedSuperUserMode, + } +} + +function expected( + workspaceKey: string, + access: ExpectedWorkspaceAccess, + roleSource: RoleSource, + hostMembership: 'owner' | 'member' | 'external', + payerScope: 'user' | 'organization', + plan: 'free' | SubscriptionPlan, + isOwner: boolean +): PersonaWorkspaceExpectation { + return { + workspaceKey, + access, + roleSource, + hostContext: { + isOwner, + hostMembership, + payerScope, + plan, + ...HOSTED_BILLING, + }, + } +} diff --git a/apps/sim/e2e/settings/smoke/authenticated.spec.ts b/apps/sim/e2e/settings/smoke/authenticated.spec.ts index a64c7e9a1db..51392426fb1 100644 --- a/apps/sim/e2e/settings/smoke/authenticated.spec.ts +++ b/apps/sim/e2e/settings/smoke/authenticated.spec.ts @@ -2,9 +2,11 @@ import { createHash } from 'node:crypto' import { writeFileSync } from 'node:fs' import path from 'node:path' import { expect, test } from '@playwright/test' +import { FOUNDATION_TEST_PASSWORD } from '../../support/runtime-secrets' test('billing-enabled signup, login, and settings use real Sim boundaries', async ({ browser, + contextOptions, page, }, testInfo) => { test.slow() @@ -15,7 +17,7 @@ test('billing-enabled signup, login, and settings use real Sim boundaries', asyn .digest('hex') .slice(0, 16) const email = `e2e-foundation-${runId}-${testIdentity}@example.com` - const password = 'E2eFoundation1!' + const password = FOUNDATION_TEST_PASSWORD const storageStatePath = path.join(requiredEnv('E2E_STORAGE_STATE_DIR'), `${testIdentity}.json`) await page.goto('/signup') @@ -44,7 +46,11 @@ test('billing-enabled signup, login, and settings use real Sim boundaries', asyn await expect(page).toHaveURL(/\/workspace(?:\/|$)/) await page.context().storageState({ path: storageStatePath }) - const restoredContext = await browser.newContext({ storageState: storageStatePath }) + const restoredContext = await browser.newContext({ + ...contextOptions, + baseURL: requiredEnv('E2E_BASE_URL'), + storageState: storageStatePath, + }) try { const restoredPage = await restoredContext.newPage() await restoredPage.goto(`${requiredEnv('E2E_BASE_URL')}${settingsPath}`) diff --git a/apps/sim/e2e/support/build-manifest.ts b/apps/sim/e2e/support/build-manifest.ts new file mode 100644 index 00000000000..b7859beb013 --- /dev/null +++ b/apps/sim/e2e/support/build-manifest.ts @@ -0,0 +1,386 @@ +import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + cpSync, + existsSync, + lstatSync, + mkdirSync, + readdirSync, + readFileSync, + readlinkSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs' +import path from 'node:path' +import type { ChildEnvironment } from './env' +import { E2E_BUILD_CACHE_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' + +const BUILD_MANIFEST_VERSION = 1 +const NEXT_DIR = path.join(SIM_APP_DIR, '.next') +const ROOT_BUILD_FILES = new Set([ + 'bun.lock', + 'bunfig.toml', + 'package.json', + 'turbo.json', + 'tsconfig.json', +]) +const UNHASHED_OS_ENV_KEYS = new Set([ + 'PATH', + 'USER', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SYSTEMROOT', + 'GITHUB_ACTIONS', +]) + +export interface BuildIdentity { + schemaVersion: typeof BUILD_MANIFEST_VERSION + nextBuildHash: string + sourceHash: string + profileHash: string + nodeVersion: string + bunVersion: string + nextVersion: string + platform: NodeJS.Platform + architecture: string +} + +interface BuildManifest extends BuildIdentity { + completed: true + createdAt: string + buildId: string + artifactHash: string +} + +export interface BuildReuseDecision { + reused: boolean + nextBuildHash: string + reason: string + cacheDirectory: string +} + +export interface BuildArtifactPaths { + activeNextDirectory: string + buildCacheDirectory: string +} + +const DEFAULT_ARTIFACT_PATHS: BuildArtifactPaths = { + activeNextDirectory: NEXT_DIR, + buildCacheDirectory: E2E_BUILD_CACHE_DIR, +} + +export function computeBuildIdentity(options: { + buildEnvironment: ChildEnvironment + nodeExecutable: string +}): BuildIdentity { + const sourceHash = hashRepositoryFiles(listRepositoryFiles().filter(isNextBuildInput)) + const profileHash = hashJson( + Object.fromEntries( + Object.entries(options.buildEnvironment.env) + .filter(([key]) => !UNHASHED_OS_ENV_KEYS.has(key)) + .sort(([left], [right]) => left.localeCompare(right)) + ) + ) + const nodeVersion = getExecutableVersion(options.nodeExecutable) + const bunVersion = process.versions.bun ?? 'not-bun' + const nextVersion = readPackageVersion(path.join(REPO_ROOT, 'node_modules/next/package.json')) + const stableIdentity = { + schemaVersion: BUILD_MANIFEST_VERSION, + sourceHash, + profileHash, + nodeVersion, + bunVersion, + nextVersion, + platform: process.platform, + architecture: process.arch, + } as const + + return { + ...stableIdentity, + nextBuildHash: hashJson(stableIdentity), + } +} + +export function restoreCachedBuild( + identity: BuildIdentity, + paths: BuildArtifactPaths = DEFAULT_ARTIFACT_PATHS +): BuildReuseDecision { + clearInterruptedBuildStores(paths.buildCacheDirectory) + const cacheDirectory = getCacheDirectory(identity.nextBuildHash, paths) + const manifestPath = path.join(cacheDirectory, 'manifest.json') + const artifactDirectory = path.join(cacheDirectory, '.next') + const miss = (reason: string): BuildReuseDecision => ({ + reused: false, + nextBuildHash: identity.nextBuildHash, + reason, + cacheDirectory, + }) + + if (!existsSync(manifestPath) || !existsSync(artifactDirectory)) { + return miss('cache artifact or completed manifest is missing') + } + + let manifest: BuildManifest + try { + manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as BuildManifest + } catch { + return miss('cache manifest is unreadable') + } + if (!isMatchingIdentity(manifest, identity) || manifest.completed !== true) { + return miss('cache manifest identity does not match') + } + const buildIdPath = path.join(artifactDirectory, 'BUILD_ID') + if (!existsSync(buildIdPath) || readFileSync(buildIdPath, 'utf8').trim() !== manifest.buildId) { + return miss('cached BUILD_ID does not match the manifest') + } + if (hashDirectory(artifactDirectory) !== manifest.artifactHash) { + return miss('cached artifact checksum does not match the manifest') + } + + activateDirectory(artifactDirectory, paths.activeNextDirectory) + return { + reused: true, + nextBuildHash: identity.nextBuildHash, + reason: 'verified cache hit', + cacheDirectory, + } +} + +export function storeCompletedBuild( + identity: BuildIdentity, + paths: BuildArtifactPaths = DEFAULT_ARTIFACT_PATHS +): BuildReuseDecision { + const buildIdPath = path.join(paths.activeNextDirectory, 'BUILD_ID') + if (!existsSync(buildIdPath)) { + throw new Error('Next build completed without .next/BUILD_ID') + } + + mkdirSync(paths.buildCacheDirectory, { recursive: true }) + const cacheDirectory = getCacheDirectory(identity.nextBuildHash, paths) + const temporaryDirectory = `${cacheDirectory}.tmp-${process.pid}-${Date.now()}` + rmSync(temporaryDirectory, { recursive: true, force: true }) + mkdirSync(temporaryDirectory, { recursive: true }) + const artifactDirectory = path.join(temporaryDirectory, '.next') + cpSync(paths.activeNextDirectory, artifactDirectory, { + recursive: true, + verbatimSymlinks: true, + }) + + const manifest: BuildManifest = { + ...identity, + completed: true, + createdAt: new Date().toISOString(), + buildId: readFileSync(buildIdPath, 'utf8').trim(), + artifactHash: hashDirectory(artifactDirectory), + } + writeFileSync( + path.join(temporaryDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + { mode: 0o600 } + ) + rmSync(cacheDirectory, { recursive: true, force: true }) + renameSync(temporaryDirectory, cacheDirectory) + + return { + reused: false, + nextBuildHash: identity.nextBuildHash, + reason: 'fresh build stored in verified cache', + cacheDirectory, + } +} + +export function pruneBuildCache( + retainedHash: string, + maxEntries = 1, + buildCacheDirectory = E2E_BUILD_CACHE_DIR +): string[] { + if (!existsSync(buildCacheDirectory)) return [] + clearInterruptedBuildStores(buildCacheDirectory) + const entries = readdirSync(buildCacheDirectory) + const candidates = entries + .filter((name) => !name.includes('.tmp-')) + .map((name) => { + const directory = path.join(buildCacheDirectory, name) + const stats = lstatSync(directory) + return stats.isDirectory() && !name.includes('.tmp-') + ? { name, directory, modifiedAt: stats.mtimeMs } + : null + }) + .filter( + (candidate): candidate is { name: string; directory: string; modifiedAt: number } => + candidate !== null + ) + .sort((left, right) => { + if (left.name === retainedHash) return -1 + if (right.name === retainedHash) return 1 + return right.modifiedAt - left.modifiedAt + }) + const removed = candidates.slice(Math.max(1, maxEntries)) + for (const candidate of removed) { + rmSync(candidate.directory, { recursive: true, force: true }) + } + return removed.map(({ name }) => name) +} + +function clearInterruptedBuildStores(buildCacheDirectory: string): void { + if (!existsSync(buildCacheDirectory)) return + for (const name of readdirSync(buildCacheDirectory).filter((entry) => entry.includes('.tmp-'))) { + rmSync(path.join(buildCacheDirectory, name), { recursive: true, force: true }) + } +} + +export function clearActiveNextBuild( + activeNextDirectory = DEFAULT_ARTIFACT_PATHS.activeNextDirectory +): void { + rmSync(activeNextDirectory, { recursive: true, force: true }) + clearStaleActivationDirectories(activeNextDirectory) +} + +function listRepositoryFiles(): string[] { + const result = spawnSync( + 'git', + ['-C', REPO_ROOT, 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], + { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + } + ) + if (result.status !== 0) { + throw new Error(`Unable to enumerate E2E hash inputs: ${result.stderr}`) + } + return [...new Set(result.stdout.split('\0').filter(Boolean))].sort() +} + +function isNextBuildInput(relativePath: string): boolean { + const normalized = relativePath.replaceAll(path.sep, '/') + if (normalized.startsWith('packages/')) return true + if (ROOT_BUILD_FILES.has(normalized)) return true + if (!normalized.startsWith('apps/sim/')) return false + return !( + normalized.startsWith('apps/sim/e2e/') || + normalized.startsWith('apps/sim/.next/') || + normalized.startsWith('apps/sim/playwright-report/') || + normalized.startsWith('apps/sim/test-results/') + ) +} + +function hashRepositoryFiles(files: string[]): string { + const hash = createHash('sha256') + for (const file of files) { + hash.update(file) + hash.update('\0') + hash.update(hashRepositoryPath(file)) + hash.update('\0') + } + return hash.digest('hex') +} + +function hashRepositoryPath(relativePath: string): string { + const absolutePath = path.join(REPO_ROOT, relativePath) + if (!existsSync(absolutePath)) return 'missing' + const stats = lstatSync(absolutePath) + if (stats.isSymbolicLink()) return hashText(`symlink:${readlinkSync(absolutePath)}`) + if (!stats.isFile()) return `unsupported:${stats.mode}` + return hashText(readFileSync(absolutePath)) +} + +function hashDirectory(directory: string): string { + const hash = createHash('sha256') + const visit = (current: string): void => { + for (const name of readdirSync(current).sort()) { + const absolutePath = path.join(current, name) + const relativePath = path.relative(directory, absolutePath).replaceAll(path.sep, '/') + const stats = lstatSync(absolutePath) + hash.update(relativePath) + hash.update('\0') + if (stats.isDirectory()) { + hash.update('directory\0') + visit(absolutePath) + } else if (stats.isSymbolicLink()) { + hash.update(`symlink:${readlinkSync(absolutePath)}\0`) + } else if (stats.isFile()) { + hash.update(readFileSync(absolutePath)) + hash.update('\0') + } else { + throw new Error(`Unsupported file in cached Next build: ${absolutePath}`) + } + } + } + visit(directory) + return hash.digest('hex') +} + +function activateDirectory(source: string, destination: string): void { + clearStaleActivationDirectories(destination) + const temporary = `${destination}.e2e-restore-${process.pid}` + const backup = `${destination}.e2e-backup-${process.pid}` + rmSync(temporary, { recursive: true, force: true }) + rmSync(backup, { recursive: true, force: true }) + cpSync(source, temporary, { recursive: true, verbatimSymlinks: true }) + try { + if (existsSync(destination)) renameSync(destination, backup) + renameSync(temporary, destination) + rmSync(backup, { recursive: true, force: true }) + } catch (error) { + rmSync(temporary, { recursive: true, force: true }) + if (!existsSync(destination) && existsSync(backup)) renameSync(backup, destination) + throw error + } +} + +function clearStaleActivationDirectories(destination: string): void { + const parent = path.dirname(destination) + if (!existsSync(parent)) return + const base = path.basename(destination) + for (const name of readdirSync(parent)) { + if (name.startsWith(`${base}.e2e-restore-`) || name.startsWith(`${base}.e2e-backup-`)) { + rmSync(path.join(parent, name), { recursive: true, force: true }) + } + } +} + +function isMatchingIdentity(manifest: BuildManifest, identity: BuildIdentity): boolean { + return ( + manifest.schemaVersion === identity.schemaVersion && + manifest.nextBuildHash === identity.nextBuildHash && + manifest.sourceHash === identity.sourceHash && + manifest.profileHash === identity.profileHash && + manifest.nodeVersion === identity.nodeVersion && + manifest.bunVersion === identity.bunVersion && + manifest.nextVersion === identity.nextVersion && + manifest.platform === identity.platform && + manifest.architecture === identity.architecture + ) +} + +function getCacheDirectory(nextBuildHash: string, paths: BuildArtifactPaths): string { + return path.join(paths.buildCacheDirectory, nextBuildHash) +} + +function getExecutableVersion(executable: string): string { + const result = spawnSync(executable, ['--version'], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (result.status !== 0) { + throw new Error(`Unable to read ${executable} version: ${result.stderr}`) + } + return result.stdout.trim() +} + +function readPackageVersion(packageJsonPath: string): string { + const parsed = JSON.parse(readFileSync(packageJsonPath, 'utf8')) as { version?: string } + if (!parsed.version) throw new Error(`Package has no version: ${packageJsonPath}`) + return parsed.version +} + +function hashJson(value: unknown): string { + return hashText(JSON.stringify(value)) +} + +function hashText(value: string | Buffer): string { + return createHash('sha256').update(value).digest('hex') +} diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts index badcd7bb054..2dc238edb42 100644 --- a/apps/sim/e2e/support/deployment-profile.ts +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -1,17 +1,28 @@ import { buildChildEnvironment, type ChildEnvironment } from './env' +import { E2E_CACHE_DIR } from './paths' +import type { E2eRuntimeSecrets } from './runtime-secrets' export const E2E_PROFILE = 'hosted-billing-chromium' export const E2E_HOST = 'e2e.sim.ai' export const E2E_ORIGIN = `http://${E2E_HOST}:3000` export const E2E_SOCKET_ORIGIN = `http://${E2E_HOST}:3002` -const REQUIRED_KEYS = [ +const BUILD_SECRET_SENTINELS: E2eRuntimeSecrets = { + betterAuthSecret: 'build-sentinel-better-auth-secret-00000000000000000000000000000000', + encryptionKey: 'aa'.repeat(32), + apiEncryptionKey: 'bb'.repeat(32), + internalApiSecret: 'build-sentinel-internal-api-secret-000000000000000000000000000000', + adminApiKey: 'build-sentinel-admin-api-key-00000000000000000000000000000000', + stripeSecretKey: 'sk_test_build_sentinel', + stripeWebhookSecret: 'whsec_build_sentinel', +} + +const APP_REQUIRED_KEYS = [ 'NODE_ENV', 'NEXT_PUBLIC_APP_URL', 'BETTER_AUTH_URL', 'BETTER_AUTH_SECRET', 'DATABASE_URL', - 'MIGRATION_DATABASE_URL', 'ENCRYPTION_KEY', 'API_ENCRYPTION_KEY', 'INTERNAL_API_SECRET', @@ -20,10 +31,35 @@ const REQUIRED_KEYS = [ 'NEXT_PUBLIC_BILLING_ENABLED', 'STRIPE_SECRET_KEY', 'STRIPE_API_BASE_URL', + 'TELEMETRY_ENDPOINT', 'E2E_PROFILE', 'E2E_RUN_ID', 'HOME', - 'PLAYWRIGHT_BROWSERS_PATH', +] as const +const APP_ENVIRONMENT_KEYS = [ + ...APP_REQUIRED_KEYS, + 'NODE_OPTIONS', + 'NEXT_TELEMETRY_DISABLED', + 'XDG_CONFIG_HOME', + 'AWS_EC2_METADATA_DISABLED', + 'AWS_SHARED_CREDENTIALS_FILE', + 'AWS_CONFIG_FILE', + 'CLOUDSDK_CONFIG', + 'AZURE_CONFIG_DIR', + 'E2E_BASE_URL', + 'EMAIL_VERIFICATION_ENABLED', + 'EMAIL_PASSWORD_SIGNUP_ENABLED', + 'NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED', + 'DISABLE_REGISTRATION', + 'DISABLE_EMAIL_SIGNUP', + 'SIGNUP_MX_VALIDATION_ENABLED', + 'NEXT_PUBLIC_POSTHOG_ENABLED', + 'BLACKLISTED_PROVIDERS', + 'STRIPE_WEBHOOK_SECRET', + 'STRIPE_FREE_PRICE_ID', + 'SOCKET_SERVER_URL', + 'NEXT_PUBLIC_SOCKET_URL', + 'CI', ] as const const ALLOWED_SENSITIVE_KEYS = new Set([ @@ -42,49 +78,65 @@ export interface HostedBillingProfileOptions { runId: string databaseUrl: string stripeApiBaseUrl: string - homeDirectory: string + runtimeHomeDirectory: string + setupHomeDirectory: string + authCaptureHomeDirectory: string + playwrightHomeDirectory: string playwrightBrowsersPath: string + runtimeSecrets: E2eRuntimeSecrets ci: boolean } export interface HostedBillingProfile { id: typeof E2E_PROFILE origin: typeof E2E_ORIGIN - childEnvironment: ChildEnvironment + environments: { + build: ChildEnvironment + app: ChildEnvironment + realtime: ChildEnvironment + migration: ChildEnvironment + seed: ChildEnvironment + authCapture: ChildEnvironment + playwright: ChildEnvironment + } } export function createHostedBillingProfile({ runId, databaseUrl, stripeApiBaseUrl, - homeDirectory, + runtimeHomeDirectory, + setupHomeDirectory, + authCaptureHomeDirectory, + playwrightHomeDirectory, playwrightBrowsersPath, + runtimeSecrets, ci, }: HostedBillingProfileOptions): HostedBillingProfile { const values: Record = { NODE_ENV: 'production', NODE_OPTIONS: '--no-warnings --max-old-space-size=8192 --dns-result-order=ipv4first', NEXT_TELEMETRY_DISABLED: '1', - HOME: homeDirectory, - XDG_CONFIG_HOME: `${homeDirectory}/xdg`, + HOME: runtimeHomeDirectory, + XDG_CONFIG_HOME: `${runtimeHomeDirectory}/xdg`, AWS_EC2_METADATA_DISABLED: 'true', AWS_SHARED_CREDENTIALS_FILE: '/dev/null', AWS_CONFIG_FILE: '/dev/null', - CLOUDSDK_CONFIG: `${homeDirectory}/gcloud`, - AZURE_CONFIG_DIR: `${homeDirectory}/azure`, + CLOUDSDK_CONFIG: `${runtimeHomeDirectory}/gcloud`, + AZURE_CONFIG_DIR: `${runtimeHomeDirectory}/azure`, PLAYWRIGHT_BROWSERS_PATH: playwrightBrowsersPath, E2E_PROFILE, E2E_RUN_ID: runId, E2E_BASE_URL: E2E_ORIGIN, NEXT_PUBLIC_APP_URL: E2E_ORIGIN, BETTER_AUTH_URL: E2E_ORIGIN, - BETTER_AUTH_SECRET: 'e2e-better-auth-secret-at-least-32-characters-long', + BETTER_AUTH_SECRET: runtimeSecrets.betterAuthSecret, DATABASE_URL: databaseUrl, MIGRATION_DATABASE_URL: databaseUrl, - ENCRYPTION_KEY: '11'.repeat(32), - API_ENCRYPTION_KEY: '22'.repeat(32), - INTERNAL_API_SECRET: 'e2e-internal-api-secret-at-least-32-characters', - ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', + ENCRYPTION_KEY: runtimeSecrets.encryptionKey, + API_ENCRYPTION_KEY: runtimeSecrets.apiEncryptionKey, + INTERNAL_API_SECRET: runtimeSecrets.internalApiSecret, + ADMIN_API_KEY: runtimeSecrets.adminApiKey, BILLING_ENABLED: 'true', NEXT_PUBLIC_BILLING_ENABLED: 'true', EMAIL_VERIFICATION_ENABLED: 'false', @@ -95,28 +147,189 @@ export function createHostedBillingProfile({ SIGNUP_MX_VALIDATION_ENABLED: 'false', NEXT_PUBLIC_POSTHOG_ENABLED: 'false', BLACKLISTED_PROVIDERS: 'ollama,ollama-cloud,vllm,litellm,openrouter,together,fireworks,baseten', - STRIPE_SECRET_KEY: 'sk_test_sim_e2e_foundation', - STRIPE_WEBHOOK_SECRET: 'whsec_sim_e2e_foundation', + STRIPE_SECRET_KEY: runtimeSecrets.stripeSecretKey, + STRIPE_WEBHOOK_SECRET: runtimeSecrets.stripeWebhookSecret, STRIPE_FREE_PRICE_ID: 'price_e2e_free', STRIPE_API_BASE_URL: stripeApiBaseUrl, + TELEMETRY_ENDPOINT: `${stripeApiBaseUrl}/v1/traces`, SOCKET_SERVER_URL: 'http://127.0.0.1:3002', NEXT_PUBLIC_SOCKET_URL: E2E_SOCKET_ORIGIN, CI: ci ? 'true' : 'false', } validateProfileValues(values) + const buildHomeDirectory = `${E2E_CACHE_DIR}/build-home` + const buildValues = { + ...pickValues(values, APP_ENVIRONMENT_KEYS), + XDG_CONFIG_HOME: `${buildHomeDirectory}/xdg`, + AWS_EC2_METADATA_DISABLED: values.AWS_EC2_METADATA_DISABLED, + AWS_SHARED_CREDENTIALS_FILE: values.AWS_SHARED_CREDENTIALS_FILE, + AWS_CONFIG_FILE: values.AWS_CONFIG_FILE, + CLOUDSDK_CONFIG: `${buildHomeDirectory}/gcloud`, + AZURE_CONFIG_DIR: `${buildHomeDirectory}/azure`, + E2E_RUN_ID: 'build_sentinel', + HOME: buildHomeDirectory, + BETTER_AUTH_SECRET: BUILD_SECRET_SENTINELS.betterAuthSecret, + ENCRYPTION_KEY: BUILD_SECRET_SENTINELS.encryptionKey, + API_ENCRYPTION_KEY: BUILD_SECRET_SENTINELS.apiEncryptionKey, + INTERNAL_API_SECRET: BUILD_SECRET_SENTINELS.internalApiSecret, + ADMIN_API_KEY: BUILD_SECRET_SENTINELS.adminApiKey, + DATABASE_URL: 'postgresql://e2e_build:e2e_build@127.0.0.1:1/sim_e2e_build_sentinel', + STRIPE_SECRET_KEY: BUILD_SECRET_SENTINELS.stripeSecretKey, + STRIPE_WEBHOOK_SECRET: BUILD_SECRET_SENTINELS.stripeWebhookSecret, + STRIPE_API_BASE_URL: 'http://127.0.0.1:1', + TELEMETRY_ENDPOINT: 'http://127.0.0.1:1/v1/traces', + CI: 'false', + } + validateProfileValues(buildValues) return { id: E2E_PROFILE, origin: E2E_ORIGIN, - childEnvironment: buildChildEnvironment({ - values, - required: REQUIRED_KEYS, - allowedSensitiveKeys: ALLOWED_SENSITIVE_KEYS, - }), + environments: { + build: createEnvironment(buildValues, APP_REQUIRED_KEYS), + app: createEnvironment(pickValues(values, APP_ENVIRONMENT_KEYS), APP_REQUIRED_KEYS), + realtime: createEnvironment( + pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'HOME', + 'DATABASE_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'INTERNAL_API_SECRET', + 'NEXT_PUBLIC_APP_URL', + 'E2E_RUN_ID', + 'CI', + ]), + [ + 'NODE_ENV', + 'HOME', + 'DATABASE_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'INTERNAL_API_SECRET', + 'NEXT_PUBLIC_APP_URL', + 'E2E_RUN_ID', + ], + false + ), + migration: createEnvironment( + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'MIGRATION_DATABASE_URL', + 'DATABASE_URL', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'CI', + ]), + HOME: setupHomeDirectory, + }, + ['NODE_ENV', 'HOME', 'MIGRATION_DATABASE_URL', 'DATABASE_URL', 'E2E_PROFILE', 'E2E_RUN_ID'], + false + ), + seed: createEnvironment( + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'DATABASE_URL', + 'ADMIN_API_KEY', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + HOME: setupHomeDirectory, + }, + [ + 'NODE_ENV', + 'HOME', + 'DATABASE_URL', + 'ADMIN_API_KEY', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + ], + false + ), + authCapture: createEnvironment( + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + HOME: authCaptureHomeDirectory, + }, + [ + 'NODE_ENV', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + ], + false + ), + playwright: createEnvironment( + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + HOME: playwrightHomeDirectory, + }, + [ + 'NODE_ENV', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + ], + false + ), + }, } } +function createEnvironment( + values: Record, + required: readonly string[], + shadowDiscovered = true +): ChildEnvironment { + return buildChildEnvironment({ + values, + required, + allowedSensitiveKeys: ALLOWED_SENSITIVE_KEYS, + shadowDiscovered, + }) +} + +function pickValues( + values: Record, + keys: readonly string[] +): Record { + return Object.fromEntries( + keys.flatMap((key) => { + const value = values[key] + return value === undefined ? [] : [[key, value]] + }) + ) +} + function validateProfileValues(values: Record): void { if (values.NEXT_PUBLIC_APP_URL !== E2E_ORIGIN || values.BETTER_AUTH_URL !== E2E_ORIGIN) { throw new Error('E2E app and Better Auth origins must exactly match') diff --git a/apps/sim/e2e/support/env.ts b/apps/sim/e2e/support/env.ts index 7527596b41e..227e2ce40ec 100644 --- a/apps/sim/e2e/support/env.ts +++ b/apps/sim/e2e/support/env.ts @@ -29,6 +29,7 @@ export interface BuildChildEnvironmentOptions { required: readonly string[] allowedSensitiveKeys: ReadonlySet envDirectory?: string + shadowDiscovered?: boolean } export function discoverEnvFileKeys(directory = SIM_APP_DIR): string[] { @@ -55,6 +56,7 @@ export function buildChildEnvironment({ required, allowedSensitiveKeys, envDirectory = SIM_APP_DIR, + shadowDiscovered = true, }: BuildChildEnvironmentOptions): ChildEnvironment { const missing = required.filter((key) => !values[key]?.trim()) if (missing.length > 0) { @@ -76,10 +78,12 @@ export function buildChildEnvironment({ const discoveredKeys = discoverEnvFileKeys(envDirectory) const shadowedKeys: string[] = [] - for (const key of discoveredKeys) { - if (key in env) continue - env[key] = '' - shadowedKeys.push(key) + if (shadowDiscovered) { + for (const key of discoveredKeys) { + if (key in env) continue + env[key] = '' + shadowedKeys.push(key) + } } return { env, discoveredKeys, shadowedKeys } @@ -98,5 +102,3 @@ export function formatRedactedEnvironmentSummary( `Shadowed local keys: ${childEnvironment.shadowedKeys.join(', ') || '(none)'}`, ].join('\n') } - -export const E2E_OS_PASSTHROUGH_KEYS = OS_PASSTHROUGH_KEYS diff --git a/apps/sim/e2e/support/leak-canary.ts b/apps/sim/e2e/support/leak-canary.ts new file mode 100644 index 00000000000..4c4f48c1864 --- /dev/null +++ b/apps/sim/e2e/support/leak-canary.ts @@ -0,0 +1,132 @@ +import { existsSync, lstatSync, readdirSync, readFileSync, rmSync } from 'node:fs' +import path from 'node:path' +import JSZip from 'jszip' +import { writeJsonAtomic } from '../fixtures/e2e-world' + +const BINARY_EXTENSIONS = new Set(['.gif', '.jpeg', '.jpg', '.mp4', '.png', '.webp']) + +interface SyntheticSecretCanary { + schemaVersion: 1 + runId: string + secrets: string[] +} + +export function writeSyntheticSecretCanary( + filePath: string, + runId: string, + secrets: string[] +): void { + if (secrets.length === 0 || secrets.some((secret) => !secret)) { + throw new Error('Synthetic secret canary requires non-empty secrets') + } + writeJsonAtomic(filePath, { + schemaVersion: 1, + runId, + secrets: [...new Set(secrets)], + } satisfies SyntheticSecretCanary) +} + +export function readSyntheticSecretCanarySecrets(filePath: string): string[] { + return readSyntheticSecretCanary(filePath).secrets +} + +export function loadSyntheticSecretCanaryForScan( + inMemorySecrets: string[], + filePath: string +): string[] { + if (!existsSync(filePath)) return inMemorySecrets + return [...new Set([...inMemorySecrets, ...readSyntheticSecretCanarySecrets(filePath)])] +} + +export function scrubUnscannableArtifacts(roots: string[]): void { + const failures: unknown[] = [] + for (const root of roots) { + try { + rmPath(root) + } catch (error) { + failures.push(error) + } + } + if (failures.length > 0) { + throw new AggregateError(failures, 'Unable to scrub unscannable E2E artifacts') + } +} + +export async function assertNoSyntheticSecretLeaks(options: { + secrets: string[] + roots: string[] + excludedPaths?: string[] +}): Promise { + if (options.secrets.length === 0 || options.secrets.some((secret) => !secret)) { + throw new Error('Synthetic secret leak scan requires non-empty secrets') + } + const secrets = [...new Set(options.secrets)] + const excluded = new Set((options.excludedPaths ?? []).map((value) => path.resolve(value))) + const violations: string[] = [] + + for (const root of options.roots) { + if (!existsSync(root)) continue + for (const filePath of listFiles(path.resolve(root), excluded)) { + if (BINARY_EXTENSIONS.has(path.extname(filePath).toLowerCase())) continue + const contents = + path.extname(filePath).toLowerCase() === '.zip' + ? await readZipContents(filePath) + : readFileSync(filePath) + if (secrets.some((secret) => contents.includes(Buffer.from(secret)))) { + violations.push(filePath) + } + } + } + + if (violations.length > 0) { + throw new Error( + `Synthetic E2E secret leaked outside private artifacts:\n${violations + .map((filePath) => `- ${filePath}`) + .join('\n')}` + ) + } +} + +function readSyntheticSecretCanary(filePath: string): SyntheticSecretCanary { + const parsed = JSON.parse(readFileSync(filePath, 'utf8')) as Partial + if ( + parsed.schemaVersion !== 1 || + typeof parsed.runId !== 'string' || + !Array.isArray(parsed.secrets) || + parsed.secrets.length === 0 || + parsed.secrets.some((secret) => typeof secret !== 'string' || !secret) + ) { + throw new Error('Invalid synthetic secret canary artifact') + } + return parsed as SyntheticSecretCanary +} + +function listFiles(currentPath: string, excluded: ReadonlySet = new Set()): string[] { + if (excluded.has(currentPath)) return [] + const stats = lstatSync(currentPath) + if (stats.isSymbolicLink()) return [] + if (stats.isFile()) return [currentPath] + if (!stats.isDirectory()) return [] + return readdirSync(currentPath).flatMap((name) => + listFiles(path.join(currentPath, name), excluded) + ) +} + +function rmPath(filePath: string): void { + rmSync(filePath, { recursive: true, force: true }) + if (existsSync(filePath)) throw new Error(`Unscannable E2E artifact still exists: ${filePath}`) +} + +async function readZipContents(filePath: string): Promise { + try { + const archive = await JSZip.loadAsync(readFileSync(filePath)) + const contents = await Promise.all( + Object.values(archive.files) + .filter((entry) => !entry.dir) + .map(async (entry) => Buffer.from(await entry.async('uint8array'))) + ) + return Buffer.concat(contents) + } catch { + throw new Error(`Unable to inspect E2E diagnostic archive: ${filePath}`) + } +} diff --git a/apps/sim/e2e/support/paths.ts b/apps/sim/e2e/support/paths.ts index 44ec033e0e4..2fe4fbe299a 100644 --- a/apps/sim/e2e/support/paths.ts +++ b/apps/sim/e2e/support/paths.ts @@ -1,10 +1,12 @@ import path from 'node:path' -export const SIM_APP_DIR = path.resolve(process.cwd()) +export const SIM_APP_DIR = path.resolve(__dirname, '../..') export const REPO_ROOT = path.resolve(SIM_APP_DIR, '../..') export const DB_PACKAGE_DIR = path.join(REPO_ROOT, 'packages/db') export const REALTIME_APP_DIR = path.join(REPO_ROOT, 'apps/realtime') export const PLAYWRIGHT_CLI = path.join(REPO_ROOT, 'node_modules/@playwright/test/cli.js') +export const E2E_CACHE_DIR = path.join(SIM_APP_DIR, 'e2e/.cache') +export const E2E_BUILD_CACHE_DIR = path.join(E2E_CACHE_DIR, 'builds') export function getRunDirectory(runId: string): string { return path.join(SIM_APP_DIR, 'e2e/.runs', runId) diff --git a/apps/sim/e2e/support/probes.ts b/apps/sim/e2e/support/probes.ts index 40f4e42bae1..4dbc0b7140c 100644 --- a/apps/sim/e2e/support/probes.ts +++ b/apps/sim/e2e/support/probes.ts @@ -1,4 +1,5 @@ import postgres from 'postgres' +import { readScenarioManifest } from '../fixtures/e2e-world' export async function assertAdminApiBoundary(origin: string, adminKey: string): Promise { const endpoint = `${origin}/api/v1/admin/users?limit=1&offset=0` @@ -45,3 +46,36 @@ export async function inspectFoundationUsers( await sql.end() } } + +export async function assertManifestWorkspaceIdentities( + databaseUrl: string, + manifestPath: string +): Promise { + const manifest = readScenarioManifest(manifestPath) + const expected = Object.values(manifest.worlds).flatMap((world) => + Object.values(world.workspaceIdentities) + ) + const sql = postgres(databaseUrl, { max: 1, connect_timeout: 10 }) + try { + const ids = expected.map(({ id }) => id) + const rows = + ids.length === 0 + ? [] + : await sql>` + SELECT id, name + FROM workspace + WHERE id = ANY(${ids}) + ` + const actualById = new Map(rows.map((row) => [row.id, row.name])) + for (const workspace of expected) { + if (actualById.get(workspace.id) !== workspace.name) { + throw new Error(`Manifest workspace changed or disappeared: ${workspace.id}`) + } + } + if (rows.length !== expected.length) { + throw new Error('Manifest workspace inventory contains unexpected duplicates') + } + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index 406502275af..bf270e15e39 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -1,7 +1,9 @@ -import { type ChildProcess, spawn, spawnSync } from 'node:child_process' +import { type ChildProcess, spawn } from 'node:child_process' import { closeSync, mkdirSync, openSync } from 'node:fs' import { createServer } from 'node:net' import path from 'node:path' +import { sleep } from '@sim/utils/helpers' +import { isProcessGroupAlive } from './signal-cleanup' export interface CommandOptions { name: string @@ -53,7 +55,7 @@ export function spawnManagedProcess(options: CommandOptions): ManagedProcess { const finalize = (result: ProcessCompletion): void => { if (finalized) return finalized = true - activeProcesses.delete(managed) + if (!child.pid || !isProcessGroupAlive(child.pid)) activeProcesses.delete(managed) closeSync(logFd) resolveCompletion(result) } @@ -66,6 +68,7 @@ export function spawnManagedProcess(options: CommandOptions): ManagedProcess { export async function runCommand(options: CommandOptions): Promise { const managed = spawnManagedProcess(options) const result = await managed.completion + await managed.stop() if (result.error) { throw new Error(`${options.name} failed to spawn. See ${managed.logPath}`, { cause: result.error, @@ -144,26 +147,35 @@ async function stopProcess(managed: ManagedProcess): Promise { const { child } = managed if (!child.pid) { await managed.completion + activeProcesses.delete(managed) + return + } + if (!isProcessGroupAlive(child.pid)) { + activeProcesses.delete(managed) return } - if (child.exitCode !== null || child.signalCode !== null) return const processIds = [child.pid] sendSignalToPids(processIds, 'SIGTERM') - const stopped = await Promise.race([ - managed.completion.then(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), - ]) - if (stopped) return - - sendSignalToPids(processIds.filter(isPidRunning), 'SIGKILL') - const killed = await Promise.race([ - managed.completion.then(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 2_000)), - ]) - if (!killed) { - throw new Error(`Managed process ${managed.name} did not exit after SIGKILL`) + if (await waitForProcessGroupExit(child.pid, 5_000)) { + activeProcesses.delete(managed) + return + } + + sendSignalToPids(processIds, 'SIGKILL') + if (!(await waitForProcessGroupExit(child.pid, 2_000))) { + throw new Error(`Managed process group ${managed.name} survived SIGKILL`) + } + activeProcesses.delete(managed) +} + +async function waitForProcessGroupExit(groupId: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (!isProcessGroupAlive(groupId)) return true + await sleep(50) } + return !isProcessGroupAlive(groupId) } function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { @@ -188,35 +200,3 @@ function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { } } } - -function isPidRunning(processId: number): boolean { - return getRunningPids([processId]).length > 0 -} - -function getRunningPids(processIds: number[]): number[] { - if (processIds.length === 0) return [] - if (process.platform === 'win32') { - return processIds.filter((processId) => { - try { - process.kill(processId, 0) - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false - throw error - } - }) - } - - const status = spawnSync('ps', ['-o', 'pid=,stat=', '-p', processIds.join(',')], { - encoding: 'utf8', - env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, - }) - if (status.status !== 0 && status.status !== 1) { - throw new Error(`Unable to inspect managed E2E processes: ${status.stderr}`) - } - return status.stdout.split('\n').flatMap((line) => { - const [pidText, processStatus] = line.trim().split(/\s+/) - const pid = Number(pidText) - return Number.isInteger(pid) && !processStatus?.startsWith('Z') ? [pid] : [] - }) -} diff --git a/apps/sim/e2e/support/run-lock.ts b/apps/sim/e2e/support/run-lock.ts new file mode 100644 index 00000000000..4b67424e988 --- /dev/null +++ b/apps/sim/e2e/support/run-lock.ts @@ -0,0 +1,168 @@ +import { spawnSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs' +import path from 'node:path' +import { E2E_CACHE_DIR } from './paths' + +const OWNER_FILE = 'owner.json' +const ACQUISITION_GRACE_MS = 2_000 +const ACQUISITION_RETRIES = 10 +const ACQUISITION_RETRY_MS = 10 + +interface RunLockDescriptor { + pid: number + token: string + startedAt: string + processStartIdentity: string | null + retainedFailure?: string +} + +export interface E2eRunLock { + path: string + token: string + transfer(pid: number): void + retain(reason: string): void + release(): void +} + +export function acquireE2eRunLock( + lockPath = path.join(E2E_CACHE_DIR, 'orchestrator.lock') +): E2eRunLock { + mkdirSync(path.dirname(lockPath), { recursive: true }) + const descriptor: RunLockDescriptor = { + pid: process.pid, + token: randomUUID(), + startedAt: new Date().toISOString(), + processStartIdentity: readProcessStartIdentity(process.pid), + } + + for (let attempt = 0; attempt < ACQUISITION_RETRIES; attempt += 1) { + try { + mkdirSync(lockPath, { mode: 0o700 }) + writeDescriptor(lockPath, descriptor) + return { + path: lockPath, + token: descriptor.token, + transfer(pid: number): void { + const current = readDescriptor(lockPath) + if (current?.token !== descriptor.token) return + writeDescriptor(lockPath, { + ...current, + pid, + processStartIdentity: readProcessStartIdentity(pid), + }) + }, + retain(reason: string): void { + retainE2eRunLock(lockPath, descriptor.token, reason) + }, + release(): void { + releaseE2eRunLock(lockPath, descriptor.token) + }, + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + const existing = readDescriptor(lockPath) + if (existing?.retainedFailure) { + throw new Error( + `Previous E2E cleanup failed and retained ${lockPath}: ${existing.retainedFailure}. Remove the lock only after manual cleanup.` + ) + } + if (existing && isLockOwnerAlive(existing)) { + throw new Error( + `Another E2E orchestrator owns ${lockPath} (PID ${existing.pid}, started ${existing.startedAt})` + ) + } + if (!existing && lockAgeMs(lockPath) < ACQUISITION_GRACE_MS) { + if (attempt === ACQUISITION_RETRIES - 1) { + throw new Error(`Another E2E orchestrator is acquiring ${lockPath}`) + } + sleepSync(ACQUISITION_RETRY_MS) + continue + } + rmSync(lockPath, { recursive: true, force: true }) + } + } + throw new Error(`Unable to acquire E2E orchestrator lock: ${lockPath}`) +} + +export function releaseE2eRunLock(lockPath: string, token: string): void { + if (!existsSync(lockPath)) return + const current = readDescriptor(lockPath) + if (current?.token === token) { + rmSync(lockPath, { recursive: true, force: true }) + } +} + +export function retainE2eRunLock(lockPath: string, token: string, reason: string): void { + const current = readDescriptor(lockPath) + if (current?.token !== token) return + writeDescriptor(lockPath, { ...current, retainedFailure: reason }) +} + +function readDescriptor(lockPath: string): RunLockDescriptor | null { + try { + const parsed = JSON.parse( + readFileSync(path.join(lockPath, OWNER_FILE), 'utf8') + ) as Partial + return typeof parsed.pid === 'number' && + typeof parsed.token === 'string' && + typeof parsed.startedAt === 'string' && + (typeof parsed.processStartIdentity === 'string' || parsed.processStartIdentity === null) && + (parsed.retainedFailure === undefined || typeof parsed.retainedFailure === 'string') + ? (parsed as RunLockDescriptor) + : null + } catch { + return null + } +} + +function writeDescriptor(lockPath: string, descriptor: RunLockDescriptor): void { + const temporary = path.join(lockPath, `${OWNER_FILE}.tmp-${process.pid}`) + writeFileSync(temporary, `${JSON.stringify(descriptor)}\n`, { mode: 0o600 }) + renameSync(temporary, path.join(lockPath, OWNER_FILE)) +} + +function lockAgeMs(lockPath: string): number { + try { + return Date.now() - statSync(lockPath).mtimeMs + } catch { + return 0 + } +} + +function sleepSync(milliseconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds) +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM' + } +} + +function isLockOwnerAlive(descriptor: RunLockDescriptor): boolean { + if (!isProcessAlive(descriptor.pid)) return false + if (!descriptor.processStartIdentity) return true + return readProcessStartIdentity(descriptor.pid) === descriptor.processStartIdentity +} + +function readProcessStartIdentity(pid: number): string | null { + const result = spawnSync('ps', ['-o', 'lstart=', '-p', String(pid)], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (result.status !== 0) return null + const identity = result.stdout.trim() + return identity || null +} diff --git a/apps/sim/e2e/support/runtime-secrets.ts b/apps/sim/e2e/support/runtime-secrets.ts new file mode 100644 index 00000000000..9cc5aeaed8e --- /dev/null +++ b/apps/sim/e2e/support/runtime-secrets.ts @@ -0,0 +1,29 @@ +import { randomBytes } from 'node:crypto' + +export const FOUNDATION_TEST_PASSWORD = 'E2eFoundation1!' + +export interface E2eRuntimeSecrets { + betterAuthSecret: string + encryptionKey: string + apiEncryptionKey: string + internalApiSecret: string + adminApiKey: string + stripeSecretKey: string + stripeWebhookSecret: string +} + +export function createE2eRuntimeSecrets(): E2eRuntimeSecrets { + return { + betterAuthSecret: randomBytes(32).toString('hex'), + encryptionKey: randomBytes(32).toString('hex'), + apiEncryptionKey: randomBytes(32).toString('hex'), + internalApiSecret: randomBytes(32).toString('hex'), + adminApiKey: randomBytes(32).toString('hex'), + stripeSecretKey: `sk_test_sim_e2e_${randomBytes(24).toString('hex')}`, + stripeWebhookSecret: `whsec_sim_e2e_${randomBytes(24).toString('hex')}`, + } +} + +export function runtimeSecretValues(secrets: E2eRuntimeSecrets): string[] { + return Object.values(secrets) +} diff --git a/apps/sim/e2e/support/sandbox-bundles.ts b/apps/sim/e2e/support/sandbox-bundles.ts new file mode 100644 index 00000000000..eef0abf023d --- /dev/null +++ b/apps/sim/e2e/support/sandbox-bundles.ts @@ -0,0 +1,67 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import path from 'node:path' +import { REPO_ROOT } from './paths' + +const INTEGRITY_PATH = 'apps/sim/lib/execution/sandbox/bundles/integrity.json' + +interface SandboxBundleIntegrity { + schemaVersion: 1 + bunVersion: string + sources: Record + dependencies: Record + outputs: Record +} + +export function verifySandboxBundleIntegrity(options?: { runningBunVersion?: string }): void { + const integrity = readJson(path.join(REPO_ROOT, INTEGRITY_PATH)) + if (integrity.schemaVersion !== 1) { + throw new Error(`Unsupported sandbox bundle integrity version: ${integrity.schemaVersion}`) + } + const repositoryPackage = readJson<{ packageManager?: string }>( + path.join(REPO_ROOT, 'package.json') + ) + if (repositoryPackage.packageManager !== `bun@${integrity.bunVersion}`) { + throw new Error( + `Sandbox bundle Bun fingerprint is ${integrity.bunVersion}, but package.json declares ${repositoryPackage.packageManager ?? 'nothing'}` + ) + } + const runningBunVersion = options?.runningBunVersion ?? process.versions.bun + if (runningBunVersion !== integrity.bunVersion) { + throw new Error( + `Sandbox bundles require Bun ${integrity.bunVersion}, but verification is running under ${runningBunVersion ?? 'Node'}` + ) + } + + verifyHashes('source', integrity.sources) + verifyHashes('output', integrity.outputs) + for (const [packageName, expectedVersion] of Object.entries(integrity.dependencies)) { + const packageJson = readJson<{ version?: string }>( + path.join(REPO_ROOT, 'node_modules', packageName, 'package.json') + ) + if (packageJson.version !== expectedVersion) { + throw new Error( + `Sandbox bundle dependency ${packageName} changed: expected ${expectedVersion}, received ${packageJson.version ?? 'missing'}` + ) + } + } +} + +function verifyHashes(kind: string, expected: Record): void { + for (const [relativePath, expectedHash] of Object.entries(expected)) { + const actualHash = hashFile(path.join(REPO_ROOT, relativePath)) + if (actualHash !== expectedHash) { + throw new Error( + `Sandbox bundle ${kind} fingerprint changed for ${relativePath}; regenerate bundles and review integrity.json` + ) + } + } +} + +function hashFile(filePath: string): string { + return createHash('sha256').update(readFileSync(filePath)).digest('hex') +} + +function readJson(filePath: string): T { + return JSON.parse(readFileSync(filePath, 'utf8')) as T +} diff --git a/apps/sim/e2e/support/seed-safety.ts b/apps/sim/e2e/support/seed-safety.ts new file mode 100644 index 00000000000..a5edf8d43ba --- /dev/null +++ b/apps/sim/e2e/support/seed-safety.ts @@ -0,0 +1,33 @@ +import { + assertLoopbackPostgresUrl, + assertSafeDatabaseName, + createRunDatabaseName, +} from './database' +import { E2E_ORIGIN, E2E_PROFILE } from './deployment-profile' + +export function assertSafeSeedEnvironment(environment: { + E2E_ORCHESTRATED: string + E2E_PROFILE: string + E2E_RUN_ID: string + E2E_BASE_URL: string + DATABASE_URL: string +}): void { + if (environment.E2E_ORCHESTRATED !== '1') { + throw new Error('seed-world must run under the guarded E2E orchestrator') + } + if (environment.E2E_PROFILE !== E2E_PROFILE) { + throw new Error(`seed-world requires profile ${E2E_PROFILE}`) + } + if (environment.E2E_BASE_URL !== E2E_ORIGIN) { + throw new Error(`seed-world requires origin ${E2E_ORIGIN}`) + } + const databaseUrl = assertLoopbackPostgresUrl(environment.DATABASE_URL) + const databaseName = decodeURIComponent(databaseUrl.pathname.replace(/^\//, '')) + assertSafeDatabaseName(databaseName) + const expectedName = createRunDatabaseName(environment.E2E_RUN_ID) + if (databaseName !== expectedName) { + throw new Error( + `seed-world database ${databaseName} does not match guarded run ${environment.E2E_RUN_ID}` + ) + } +} diff --git a/apps/sim/e2e/support/signal-cleanup.ts b/apps/sim/e2e/support/signal-cleanup.ts index 18fcfe1e0d8..d98e08b105c 100644 --- a/apps/sim/e2e/support/signal-cleanup.ts +++ b/apps/sim/e2e/support/signal-cleanup.ts @@ -6,3 +6,14 @@ export function parseProcessGroupIds(rawValue: string | undefined): number[] { .map(Number) .filter((value) => Number.isInteger(value) && value > 0) } + +export function isProcessGroupAlive(groupId: number): boolean { + if (!Number.isInteger(groupId) || groupId <= 0) return false + try { + if (process.platform !== 'win32') process.kill(-groupId, 0) + else process.kill(groupId, 0) + return true + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM' + } +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts index d1ce203d6b6..222aafd4eb7 100644 --- a/apps/sim/e2e/support/stack.ts +++ b/apps/sim/e2e/support/stack.ts @@ -1,4 +1,13 @@ +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' import path from 'node:path' +import { + clearActiveNextBuild, + computeBuildIdentity, + pruneBuildCache, + restoreCachedBuild, + storeCompletedBuild, +} from './build-manifest' +import type { ChildEnvironment } from './env' import { DB_PACKAGE_DIR, PLAYWRIGHT_CLI, REALTIME_APP_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' import { type ManagedProcess, @@ -7,6 +16,7 @@ import { waitForManagedProcessReady, } from './process' import { waitForHttpReady } from './readiness' +import { verifySandboxBundleIntegrity } from './sandbox-bundles' export interface StackCommandOptions { bunExecutable: string @@ -15,6 +25,12 @@ export interface StackCommandOptions { logsDirectory: string } +export interface BuildAppOptions extends Omit { + buildEnvironment: ChildEnvironment + reuseBuild: boolean + ci: boolean +} + export async function runMigrations(options: StackCommandOptions): Promise { await runCommand({ name: 'migrate', @@ -26,23 +42,74 @@ export async function runMigrations(options: StackCommandOptions): Promise }) } -export async function buildApp(options: StackCommandOptions): Promise { +export async function seedE2eWorld(options: StackCommandOptions): Promise { await runCommand({ - name: 'sandbox-bundles', + name: 'seed-world', command: options.bunExecutable, - args: ['--no-env-file', 'run', 'build:sandbox-bundles'], + args: ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/seed-world.ts')], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function capturePersonaAuthStates(options: StackCommandOptions): Promise { + await runCommand({ + name: 'capture-auth-states', + command: options.nodeExecutable, + args: ['--import', 'tsx', path.join(SIM_APP_DIR, 'e2e/scripts/capture-auth-states.ts')], cwd: SIM_APP_DIR, env: options.env, logsDirectory: options.logsDirectory, }) +} + +export async function buildApp(options: BuildAppOptions): Promise { + verifySandboxBundleIntegrity() + const identity = + options.ci || !options.reuseBuild + ? null + : computeBuildIdentity({ + buildEnvironment: options.buildEnvironment, + nodeExecutable: options.nodeExecutable, + }) + if (options.reuseBuild) { + if (!identity) throw new Error('CI cannot restore a local E2E build cache') + const reuseDecision = restoreCachedBuild(identity) + writeBuildDecision(options.logsDirectory, reuseDecision) + if (reuseDecision.reused) { + console.info(`Reused verified Next build ${identity.nextBuildHash}`) + return + } + console.info(`Next build cache miss: ${reuseDecision.reason}`) + } + + clearActiveNextBuild() + const buildHome = options.buildEnvironment.env.HOME + if (buildHome) { + rmSync(buildHome, { recursive: true, force: true }) + mkdirSync(buildHome, { recursive: true, mode: 0o700 }) + } await runCommand({ name: 'next-build', command: options.nodeExecutable, args: [path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), 'build'], cwd: SIM_APP_DIR, - env: options.env, + env: options.buildEnvironment.env, logsDirectory: options.logsDirectory, }) + if (identity) { + const storedDecision = storeCompletedBuild(identity) + pruneBuildCache(identity.nextBuildHash) + writeBuildDecision(options.logsDirectory, storedDecision) + } else { + writeBuildDecision(options.logsDirectory, { + reused: false, + reason: options.ci + ? 'CI build cache population is disabled' + : 'local cache population requires --reuse-build', + }) + } } export async function startRealtime(options: StackCommandOptions): Promise { @@ -134,3 +201,10 @@ export async function runPlaywright( logsDirectory: options.logsDirectory, }) } + +function writeBuildDecision(logsDirectory: string, decision: object): void { + writeFileSync( + path.join(logsDirectory, 'build-reuse-decision.json'), + `${JSON.stringify(decision, null, 2)}\n` + ) +} diff --git a/apps/sim/lib/execution/sandbox/bundles/build.ts b/apps/sim/lib/execution/sandbox/bundles/build.ts index 5e6ab81645d..b10a90296dc 100644 --- a/apps/sim/lib/execution/sandbox/bundles/build.ts +++ b/apps/sim/lib/execution/sandbox/bundles/build.ts @@ -10,8 +10,9 @@ * Run via: `bun run build:sandbox-bundles`. */ -import { mkdirSync, rmSync, writeFileSync } from 'node:fs' -import { dirname, join } from 'node:path' +import { createHash } from 'node:crypto' +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, isAbsolute, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { createLogger } from '@sim/logger' @@ -33,9 +34,11 @@ interface BunBuildOptions { declare const Bun: { build: (opts: BunBuildOptions) => Promise } const HERE = dirname(fileURLToPath(import.meta.url)) -const BUNDLES_DIR = HERE -const ENTRIES_DIR = join(HERE, '.entries') const APP_SIM_ROOT = join(HERE, '..', '..', '..', '..') +const REPO_ROOT = join(APP_SIM_ROOT, '..', '..') +const INTEGRITY_PATH = join(HERE, 'integrity.json') +const INTEGRITY_SOURCES = [join(HERE, 'build.ts'), join(HERE, '_polyfills.ts')] as const +const INTEGRITY_DEPENDENCIES = ['pdf-lib', 'docx', 'pptxgenjs', 'buffer', 'process'] as const interface BundleSpec { /** Key on `globalThis.__bundles`. */ @@ -91,12 +94,15 @@ globalThis.__bundles['pptxgenjs'] = PptxGenJS ] async function main(): Promise { - rmSync(ENTRIES_DIR, { recursive: true, force: true }) - mkdirSync(ENTRIES_DIR, { recursive: true }) - mkdirSync(BUNDLES_DIR, { recursive: true }) + const { outputDirectory, writeIntegrity } = resolveOptions(process.argv.slice(2)) + const entriesDirectory = join(outputDirectory, '.entries') + + rmSync(entriesDirectory, { recursive: true, force: true }) + mkdirSync(entriesDirectory, { recursive: true }) + mkdirSync(outputDirectory, { recursive: true }) for (const spec of BUNDLES) { - const entryPath = join(ENTRIES_DIR, `${spec.name}.entry.ts`) + const entryPath = join(entriesDirectory, `${spec.name}.entry.ts`) writeFileSync(entryPath, spec.entry, 'utf-8') const result = await Bun.build({ @@ -121,11 +127,86 @@ async function main(): Promise { const code = await result.outputs[0].text() const banner = `// sandbox bundle: ${spec.name}\n// generated by apps/sim/lib/execution/sandbox/bundles/build.ts\n// do not edit by hand. run \`bun run build:sandbox-bundles\` to regenerate.\n` - writeFileSync(join(BUNDLES_DIR, spec.outFile), banner + code, 'utf-8') + writeFileSync(join(outputDirectory, spec.outFile), banner + code, 'utf-8') logger.info(`built ${spec.outFile} (${code.length.toLocaleString()} chars)`) } - rmSync(ENTRIES_DIR, { recursive: true, force: true }) + rmSync(entriesDirectory, { recursive: true, force: true }) + if (writeIntegrity) { + if (outputDirectory !== HERE) { + throw new Error('--write-integrity requires the committed bundle output directory') + } + writeIntegrityManifest() + logger.info('updated integrity.json') + } +} + +function resolveOptions(args: string[]): { + outputDirectory: string + writeIntegrity: boolean +} { + const outputOption = args.find((arg) => arg.startsWith('--output-dir=')) + const unknown = args.filter( + (arg) => arg !== '--write-integrity' && !arg.startsWith('--output-dir=') + ) + if ( + unknown.length > 0 || + args.filter((arg) => arg === '--write-integrity').length > 1 || + args.filter((arg) => arg.startsWith('--output-dir=')).length > 1 + ) { + throw new Error('Usage: build.ts [--output-dir=/absolute/path] [--write-integrity]') + } + const requested = outputOption?.slice('--output-dir='.length) + if (requested !== undefined && (!requested || !isAbsolute(requested))) { + throw new Error('--output-dir must be an absolute path') + } + return { + outputDirectory: requested ? resolve(requested) : HERE, + writeIntegrity: args.includes('--write-integrity'), + } +} + +function writeIntegrityManifest(): void { + const repositoryPackage = readJson<{ packageManager?: string }>(join(REPO_ROOT, 'package.json')) + const declaredBunVersion = repositoryPackage.packageManager?.match(/^bun@(.+)$/)?.[1] + if (!declaredBunVersion) throw new Error('package.json must declare a Bun packageManager version') + if (process.versions.bun !== declaredBunVersion) { + throw new Error( + `Sandbox bundle integrity must be generated with Bun ${declaredBunVersion}; running ${process.versions.bun ?? 'Node'}` + ) + } + + const relative = (filePath: string): string => + filePath.slice(REPO_ROOT.length + 1).replaceAll('\\', '/') + const manifest = { + schemaVersion: 1, + bunVersion: declaredBunVersion, + sources: Object.fromEntries( + INTEGRITY_SOURCES.map((filePath) => [relative(filePath), hashFile(filePath)]) + ), + dependencies: Object.fromEntries( + INTEGRITY_DEPENDENCIES.map((packageName) => [ + packageName, + readJson<{ version: string }>(join(REPO_ROOT, 'node_modules', packageName, 'package.json')) + .version, + ]) + ), + outputs: Object.fromEntries( + BUNDLES.map(({ outFile }) => { + const filePath = join(HERE, outFile) + return [relative(filePath), hashFile(filePath)] + }) + ), + } + writeFileSync(INTEGRITY_PATH, `${JSON.stringify(manifest, null, 2)}\n`) +} + +function hashFile(filePath: string): string { + return createHash('sha256').update(readFileSync(filePath)).digest('hex') +} + +function readJson(filePath: string): T { + return JSON.parse(readFileSync(filePath, 'utf8')) as T } main().catch((err) => { diff --git a/apps/sim/lib/execution/sandbox/bundles/integrity.json b/apps/sim/lib/execution/sandbox/bundles/integrity.json new file mode 100644 index 00000000000..c2c5d055dd1 --- /dev/null +++ b/apps/sim/lib/execution/sandbox/bundles/integrity.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "bunVersion": "1.3.13", + "sources": { + "apps/sim/lib/execution/sandbox/bundles/build.ts": "5ac7773ab65451f908fe54ba609978469117fcaa61cff47be973d7ba528a9982", + "apps/sim/lib/execution/sandbox/bundles/_polyfills.ts": "131dd6e7aec37bff3acf6507d6fe7bc95aeabae07fa2e97bbf2e32aa7251d491" + }, + "dependencies": { + "pdf-lib": "1.17.1", + "docx": "9.7.1", + "pptxgenjs": "4.0.1", + "buffer": "6.0.3", + "process": "0.11.10" + }, + "outputs": { + "apps/sim/lib/execution/sandbox/bundles/pdf-lib.cjs": "936750faa220af85bf9b4edfe82bcab64ad38ee637b7f584322598f229d24036", + "apps/sim/lib/execution/sandbox/bundles/docx.cjs": "4c2aaf691ab633f272cd654df92be557e460874ace581e3fdbca4ae44c1c62ce", + "apps/sim/lib/execution/sandbox/bundles/pptxgenjs.cjs": "f83df52b4577edce2ae0808e0f6da97cd119f1cbb475374b2e4ea13571fa024b" + } +} diff --git a/apps/sim/package.json b/apps/sim/package.json index 288bf38b3c7..8e4308f079e 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -19,6 +19,7 @@ "load:workflow:isolation": "BASE_URL=${BASE_URL:-http://localhost:3000} ISOLATION_DURATION=${ISOLATION_DURATION:-30} TOTAL_RATE=${TOTAL_RATE:-9} WORKSPACE_A_WEIGHT=${WORKSPACE_A_WEIGHT:-8} WORKSPACE_B_WEIGHT=${WORKSPACE_B_WEIGHT:-1} bunx artillery run scripts/load/workflow-isolation.yml", "build": "bun run build:sandbox-bundles && NODE_OPTIONS='--max-old-space-size=8192' next build", "build:sandbox-bundles": "bun run ./lib/execution/sandbox/bundles/build.ts", + "build:sandbox-bundles:integrity": "bun run ./lib/execution/sandbox/bundles/build.ts --write-integrity", "start": "next start", "prepare": "cd ../.. && bun husky", "test": "vitest run", @@ -250,6 +251,7 @@ "postcss": "^8", "react-email": "4.3.2", "tailwindcss": "^3.4.1", + "tsx": "4.23.1", "typescript": "^7.0.2", "vite-tsconfig-paths": "^5.1.4", "vitest": "^4.1.0" diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts index 70cb2641644..8ea60e16de9 100644 --- a/apps/sim/playwright.config.ts +++ b/apps/sim/playwright.config.ts @@ -15,7 +15,7 @@ export default defineConfig({ fullyParallel: true, forbidOnly: isCI, retries: 0, - workers: 1, + workers: 2, timeout: 60_000, expect: { timeout: 10_000, @@ -51,8 +51,23 @@ export default defineConfig({ { name: 'hosted-billing-chromium-workflows', testMatch: ['**/settings/smoke/authenticated.spec.ts', '**/settings/workflows/**/*.spec.ts'], + dependencies: ['hosted-billing-chromium-navigation'], fullyParallel: false, workers: 1, }, + { + name: 'hosted-billing-chromium-personas', + testMatch: '**/settings/persona-contracts.spec.ts', + dependencies: ['hosted-billing-chromium-workflows'], + fullyParallel: false, + workers: 1, + }, + { + name: 'hosted-billing-chromium-persona-isolation', + testMatch: '**/settings/persona-isolation.spec.ts', + dependencies: ['hosted-billing-chromium-personas'], + fullyParallel: true, + workers: 2, + }, ], }) diff --git a/bun.lock b/bun.lock index 51ef90840cb..8e9da57756b 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", @@ -316,6 +317,7 @@ "postcss": "^8", "react-email": "4.3.2", "tailwindcss": "^3.4.1", + "tsx": "4.23.1", "typescript": "^7.0.2", "vite-tsconfig-paths": "^5.1.4", "vitest": "^4.1.0", @@ -3088,7 +3090,7 @@ "lru.min": ["lru.min@1.1.4", "", {}, "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA=="], - "lucide-react": ["lucide-react@1.23.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw=="], + "lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], @@ -3982,7 +3984,7 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "tsx": ["tsx@4.22.4", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg=="], + "tsx": ["tsx@4.23.1", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ=="], "tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="], @@ -4468,10 +4470,6 @@ "@sim/db/@types/node": ["@types/node@22.19.21", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-VMeFBSCKQKmm2swI2kW51SFusDqekC6q9trBCvJ/JliDchFSuoYYKN7yVNjPthP1HKZcx3U1gI/wTcEBjEFKTA=="], - "@sim/emcn/lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], - - "@sim/workflow-renderer/lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], - "@smithy/middleware-compression/fflate": ["fflate@0.8.1", "", {}, "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ=="], "@socket.io/redis-adapter/debug": ["debug@4.3.7", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ=="], @@ -4626,6 +4624,8 @@ "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "drizzle-kit/tsx": ["tsx@4.22.4", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg=="], + "duplexify/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], "e2b/glob": ["glob@11.1.0", "", { "dependencies": { "foreground-child": "^3.3.1", "jackspeak": "^4.1.1", "minimatch": "^10.1.1", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^2.0.0" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw=="], @@ -4670,10 +4670,14 @@ "fumadocs-openapi/ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], + "fumadocs-openapi/lucide-react": ["lucide-react@1.23.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw=="], + "fumadocs-openapi/tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="], "fumadocs-ui/@radix-ui/react-slot": ["@radix-ui/react-slot@1.3.0", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-MojKku4U/miO8Av4Dkb+ctMAQx7JmY96LmtDQlAarCRtd7rN52QCSzBF+XAvr5S6coSVj9HEPBgHAHKEJVk/WA=="], + "fumadocs-ui/lucide-react": ["lucide-react@1.23.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw=="], + "fumadocs-ui/tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="], "gaxios/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], @@ -4830,8 +4834,6 @@ "sharp/semver": ["semver@7.8.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA=="], - "sim/lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], - "sim/tailwindcss": ["tailwindcss@3.4.19", "", { "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", "chokidar": "^3.6.0", "didyoumean": "^1.2.2", "dlv": "^1.1.3", "fast-glob": "^3.3.2", "glob-parent": "^6.0.2", "is-glob": "^4.0.3", "jiti": "^1.21.7", "lilconfig": "^3.1.3", "micromatch": "^4.0.8", "normalize-path": "^3.0.0", "object-hash": "^3.0.0", "picocolors": "^1.1.1", "postcss": "^8.4.47", "postcss-import": "^15.1.0", "postcss-js": "^4.0.1", "postcss-load-config": "^4.0.2 || ^5.0 || ^6.0", "postcss-nested": "^6.2.0", "postcss-selector-parser": "^6.1.2", "resolve": "^1.22.8", "sucrase": "^3.35.0" }, "bin": { "tailwind": "lib/cli.js", "tailwindcss": "lib/cli.js" } }, "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ=="], "simstudio/@types/node": ["@types/node@20.19.43", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA=="], @@ -5096,6 +5098,8 @@ "docx/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + "drizzle-kit/tsx/esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], + "duplexify/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], "engine.io/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], @@ -5392,6 +5396,58 @@ "cli-truncate/string-width/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + "drizzle-kit/tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], + + "drizzle-kit/tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], + + "drizzle-kit/tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], + + "drizzle-kit/tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], + + "drizzle-kit/tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], + + "drizzle-kit/tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], + + "drizzle-kit/tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], + + "drizzle-kit/tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], + + "drizzle-kit/tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], + "gaxios/node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], "gaxios/node-fetch/whatwg-url/webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], From fc8a7c2eb4daadad119f651323a74ec39c44748c Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos <157128530+BillLeoutsakosvl346@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:43:25 -0700 Subject: [PATCH 4/8] Harden Playwright foundation and persona fidelity (#5820) * Harden Playwright lifecycle and persona fidelity Close production-state and cleanup gaps so the settings E2E platform fails closed without sacrificing useful diagnostics. * Serialize E2E signal cleanup ownership Prevent opposite signals and normal finalization from racing detached cleanup or releasing its run lock. * Preserve browser fixture failures Report network isolation violations without masking the original Playwright test or fixture error. * Make E2E signal handoff resilient Keep cleanup single-flight across repeated signals and retain lock ownership when supervisor logging or startup fails. * fix(e2e): preserve replacement org coverage Co-authored-by: Cursor * fix(e2e): derive usage from entitled coverage Co-authored-by: Cursor * fix(e2e): serialize cleanup lock ownership Co-authored-by: Cursor * fix(ci): detect safe E2E diagnostics directly Co-authored-by: Cursor --------- Co-authored-by: Bill Leoutsakos Co-authored-by: Cursor --- .github/workflows/test-build.yml | 15 +- apps/sim/app/(landing)/layout.tsx | 3 +- apps/sim/app/api/users/me/settings/route.ts | 1 + apps/sim/app/layout.tsx | 7 +- apps/sim/e2e/README.md | 30 +- apps/sim/e2e/fixtures/browser-test.ts | 32 + apps/sim/e2e/fixtures/factories/billing.ts | 12 + apps/sim/e2e/fixtures/persona-test.ts | 22 +- apps/sim/e2e/fixtures/scenario-billing.ts | 58 + apps/sim/e2e/fixtures/validate-scenario.ts | 36 +- .../sim/e2e/foundation/build-manifest.spec.ts | 104 +- apps/sim/e2e/foundation/safety.spec.ts | 41 +- .../foundation/scenario-validation.spec.ts | 55 +- apps/sim/e2e/scripts/capture-auth-states.ts | 19 +- apps/sim/e2e/scripts/run.ts | 253 +- apps/sim/e2e/scripts/seed-world.ts | 73 +- apps/sim/e2e/scripts/signal-cleanup.ts | 4 +- apps/sim/e2e/settings/personas.ts | 10 +- .../e2e/settings/smoke/authenticated.spec.ts | 24 +- .../settings/smoke/unauthenticated.spec.ts | 2 +- apps/sim/e2e/support/browser-network.ts | 50 + apps/sim/e2e/support/build-manifest.ts | 10 +- apps/sim/e2e/support/process.ts | 22 +- apps/sim/e2e/support/run-lock.ts | 108 +- apps/sim/e2e/support/runtime-secrets.ts | 2 + apps/sim/e2e/support/sandbox-bundles.ts | 13 + apps/sim/e2e/support/signal-cleanup.ts | 27 + apps/sim/e2e/support/stack.ts | 25 +- .../lib/billing/stripe-client-config.test.ts | 5 + apps/sim/lib/billing/stripe-client-config.ts | 16 + .../subscriptions/organization-coverage.ts | 29 + apps/sim/lib/billing/webhooks/subscription.ts | 31 +- .../lib/execution/sandbox/bundles/docx.cjs | 26 +- .../execution/sandbox/bundles/integrity.json | 4 +- .../execution/sandbox/bundles/pptxgenjs.cjs | 34 +- .../0264_settings_super_user_default.sql | 1 + .../db/migrations/meta/0264_snapshot.json | 17365 ++++++++++++++++ packages/db/migrations/meta/_journal.json | 7 + packages/db/schema.ts | 2 +- 39 files changed, 18322 insertions(+), 256 deletions(-) create mode 100644 apps/sim/e2e/fixtures/browser-test.ts create mode 100644 apps/sim/e2e/fixtures/scenario-billing.ts create mode 100644 apps/sim/e2e/support/browser-network.ts create mode 100644 apps/sim/lib/billing/subscriptions/organization-coverage.ts create mode 100644 packages/db/migrations/0264_settings_super_user_default.sql create mode 100644 packages/db/migrations/meta/0264_snapshot.json diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 579c4fc0d48..4046cef2756 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -318,8 +318,21 @@ jobs: E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' run: bun run test:e2e - - name: Upload E2E diagnostics + - name: Check E2E diagnostics eligibility + id: e2e_diagnostics if: failure() + shell: bash + run: | + shopt -s nullglob + markers=(apps/sim/e2e/.runs/*/markers/leak-scan-complete.json) + if (( ${#markers[@]} > 0 )); then + echo "safe=true" >> "$GITHUB_OUTPUT" + else + echo "safe=false" >> "$GITHUB_OUTPUT" + fi + + - name: Upload E2E diagnostics + if: failure() && steps.e2e_diagnostics.outputs.safe == 'true' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: settings-e2e-${{ github.run_id }} diff --git a/apps/sim/app/(landing)/layout.tsx b/apps/sim/app/(landing)/layout.tsx index 9fef6524507..f9c31e12780 100644 --- a/apps/sim/app/(landing)/layout.tsx +++ b/apps/sim/app/(landing)/layout.tsx @@ -11,6 +11,7 @@ import { XPageViewTracker } from '@/app/(landing)/x-page-view-tracker' const HUBSPOT_SCRIPT_SRC = 'https://js-na2.hs-scripts.com/246720681.js' as const const X_PIXEL_ID = 'q5xbl' as const +const isMarketingAnalyticsEnabled = isHosted && !process.env.E2E_PROFILE /** X (Twitter) conversion tracking base code — loads uwt.js and fires the initial PageView. */ const X_PIXEL_BASE_CODE = `!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments); @@ -43,7 +44,7 @@ export default function LandingLayout({ children }: { children: ReactNode }) { {children} {/* HubSpot + X pixel tracking — hosted only */} - {isHosted && ( + {isMarketingAnalyticsEnabled && ( <>