diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 8b0b281..6ac2d6e 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -90,3 +90,16 @@ Two guards keep the suite self-contained: - Use web-first assertions (`await expect(locator)...`), not sleeps. - Each test gets a fresh browser context, so the virtual filesystem (`localStorage`) starts empty. + +## Expected-fail specs + +A spec can encode a roadmap exit criterion before the code that meets +it exists. Mark the test `test.fail()`: Playwright passes it while it +fails, so CI stays green, and reports it once it starts passing. That +report is the cue to remove the annotation in the pull request that +closes the gap. Pair each one with a normal control test that proves +the setup still runs, so a broken probe can't pass for the wrong reason. + +| Spec | Criterion | Closes with | +| --- | --- | --- | +| `sandbox-isolation.spec.ts` | an app that reaches for `parent.document` or the host's `localStorage` fails (Phase 1 exit) | 1.2, the sandboxed runner | diff --git a/tests/e2e/sandbox-isolation.spec.ts b/tests/e2e/sandbox-isolation.spec.ts new file mode 100644 index 0000000..f7dc945 --- /dev/null +++ b/tests/e2e/sandbox-isolation.spec.ts @@ -0,0 +1,109 @@ +import { expect, test, type Page } from '@playwright/test' +import { dominicHooks, keepRequestsLocal, openDesktop } from './support/desktop' + +// NEXT.md Phase 1 exit: "an app that reaches for `parent.document` or the +// host's `localStorage` fails". A probe app tries both on mount and reports +// what it got to `/__probe`. The test reads those reports at the network +// layer, which also sees requests from iframes, so it doesn't depend on how +// the sandboxed runner (1.2) mounts apps. +// +// Until 1.2 lands, apps share the shell's origin and the probe succeeds, so +// the isolation test is marked `test.fail()`: CI stays green while the gap +// is open. Once 1.2 closes it, Playwright reports that test as passing +// unexpectedly; remove the annotation then. The control test keeps that +// honest: if the probe stops running, the control fails instead of the +// isolation test passing for the wrong reason. + +const SETTINGS_KEY = 'dominic:vfs:system/settings.json' +const PLANTED_KEY = 'e2e-planted-key-isolation-probe' + +const PROBE_APP = ` + + +` + +interface ProbeReport { + settings: string + parentTitle: string +} + +/** + * Plant a fake provider key where Settings keeps it, open the desktop, + * install and open the probe app, and return its first report. + */ +async function runProbe(page: Page): Promise { + await page.addInitScript( + ([key, apiKey]) => { + if (window !== window.top) return + try { + window.localStorage.setItem( + key, + JSON.stringify({ provider: 'openai', model: 'gpt-4o', apiKey, baseUrl: '' }), + ) + } catch { + // Storage unavailable; the probe then has nothing to find. + } + }, + [SETTINGS_KEY, PLANTED_KEY] as const, + ) + + const reports: ProbeReport[] = [] + await page.route('**/__probe?**', async (route) => { + const data = new URL(route.request().url()).searchParams.get('d') ?? '{}' + reports.push(JSON.parse(data) as ProbeReport) + await route.fulfill({ status: 204 }) + }) + + await openDesktop(page) + const dominic = dominicHooks(page) + await dominic.installApp({ + id: 'isolation-probe', + title: 'Isolation Probe', + icon: 'Sparkles', + description: 'Tries to read the host from inside an app', + vueSfcCode: PROBE_APP, + }) + await dominic.openApp('isolation-probe') + + await expect.poll(() => reports.length).toBeGreaterThan(0) + return reports[0]! +} + +test.beforeEach(async ({ page, baseURL }) => { + await keepRequestsLocal(page, baseURL) +}) + +test('control: the isolation probe runs and reports', async ({ page }) => { + const report = await runProbe(page) + expect(typeof report.settings).toBe('string') + expect(typeof report.parentTitle).toBe('string') +}) + +test('an app cannot read the host localStorage or parent.document', async ({ page }) => { + // Expected to fail until the sandboxed runner (NEXT.md 1.2) lands. + test.fail() + const report = await runProbe(page) + expect.soft(report.settings).not.toContain(PLANTED_KEY) + expect.soft(report.parentTitle).toBe('blocked') +}) diff --git a/tests/e2e/support/desktop.ts b/tests/e2e/support/desktop.ts index fd0d691..7c9436a 100644 --- a/tests/e2e/support/desktop.ts +++ b/tests/e2e/support/desktop.ts @@ -1,9 +1,11 @@ import type { Locator, Page } from '@playwright/test' -import type { UpdateAppParams } from '../../../app/features/chat/tools/schemas' +import type { InstallAppParams, UpdateAppParams } from '../../../app/features/chat/tools/schemas' /** The test hooks `app/app.vue` puts on `window.__dominic` once it mounts. */ interface DominicHooks { installFixture(fixtureId: string): boolean + installApp(params: InstallAppParams): void + openApp(appId: string): void updateApp(params: UpdateAppParams): void injectBrokenApp(type?: 'compile' | 'runtime'): void } @@ -44,6 +46,10 @@ export function dominicHooks(page: Page) { return { installFixture: (fixtureId: string) => page.evaluate((id) => window.__dominic!.installFixture(id), fixtureId), + installApp: (params: InstallAppParams) => + page.evaluate((p) => window.__dominic!.installApp(p), params), + openApp: (appId: string) => + page.evaluate((id) => window.__dominic!.openApp(id), appId), updateApp: (params: UpdateAppParams) => page.evaluate((p) => window.__dominic!.updateApp(p), params), injectBrokenApp: (type: 'compile' | 'runtime') =>