From 2ea288581becbca675836ac67a9b87b3eb5d4863 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Sat, 1 Aug 2026 10:02:40 +0200 Subject: [PATCH] feat(playground): run the engine in the browser, on a public repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds site/playground/: paste a public GitHub repository and index it entirely client-side. Not a mockup and not a precomputed index — the same walk, the same tree-sitter grammars, the same link-graph, compiled for the browser and run in a Web Worker. The engine source is untouched. Node builtins are redirected at resolution time to shims in src/browser/ (an in-memory VFS, a synchronous SHA-1, a Buffer covering readText's four decode paths), and everything spawn-based degrades along the fallbacks the engine already ships and already tests: sh() reports its binary missing, so git.ts returns no commit and grep.ts uses its pure-JS backend. scripts/engine.mjs, engine.d.mts and cli.mjs are byte-identical to before. WALK DECIDES WHAT IS DOWNLOADED. A file list carries paths and sizes, and sizes alone satisfy lstatSync — so the tree is mounted with no contents, the real walk() runs on it, and its keep-list IS the download list. No ignore rule, size cap or `capped` flag is reimplemented in the UI, and pruneUnfetched() drops whatever was listed but never fetched so a file can never reach the index as a phantom empty one. Source ordering is measured, not assumed. jsDelivr's branch manifest is a snapshot rather than the branch head: on t3-oss/create-t3-turbo it listed 125 files of which 37 no longer existed (404 on raw too, so genuinely deleted). The GitHub trees API listed 141 blobs and all 141 fetched. So GitHub is primary and jsDelivr is the fallback for when the 60/hour API limit runs out, with the caveat shown rather than hidden — 88 -> 132 files indexed on that repo. The palette exposes the 24 commands that mean something here and lists the seven that do not with their reason, because a command that needs git history returning an empty result reads like a bug. Built by scripts/build-browser.mjs rather than a third tsup target: tsup registers its own node-protocol plugin ahead of user plugins and resolves node:fs to an external bare "fs", which no browser can load and which Node quietly satisfies from the real disk — a silent failure under test. Verified: tests/browser-build.test.ts indexes three fixtures through the VFS and asserts graph.json and symbols.json are byte-identical to the Node build, with the grammars asserted loaded so the comparison cannot pass vacuously. Shim behaviour is checked against the real node: modules, the whole palette runs against a real index, and the fetch pipeline has an opt-in live test (CODEINDEX_PLAYGROUND_E2E=1). check:build holds the committed bundle byte-reproducible. --- .github/workflows/site.yml | 28 +- .gitignore | 6 + package.json | 5 +- pnpm-lock.yaml | 3 + scripts/build-browser.mjs | 84 +++++ site/index.html | 1 + site/playground/commands.js | 244 ++++++++++++++ site/playground/engine.browser.mjs | 232 +++++++++++++ site/playground/index.html | 363 ++++++++++++++++++++ site/playground/playground.js | 517 +++++++++++++++++++++++++++++ site/playground/report.js | 52 +++ site/playground/sources.js | 144 ++++++++ site/playground/worker.js | 203 +++++++++++ src/browser/child_process.ts | 37 +++ src/browser/crypto.ts | 229 +++++++++++++ src/browser/entry.ts | 25 ++ src/browser/fs.ts | 348 +++++++++++++++++++ src/browser/globals.ts | 200 +++++++++++ src/browser/grammars.ts | 53 +++ src/browser/os.ts | 30 ++ src/browser/path.ts | 262 +++++++++++++++ src/browser/readline.ts | 9 + src/browser/url.ts | 26 ++ src/browser/worker_threads.ts | 24 ++ src/browser/zlib.ts | 18 + tests/browser-build.test.ts | 187 +++++++++++ tests/browser-shims.test.ts | 246 ++++++++++++++ tests/playground-commands.test.ts | 241 ++++++++++++++ tests/playground-e2e.test.ts | 155 +++++++++ tests/playground-sources.test.ts | 160 +++++++++ tsup.config.ts | 6 + 31 files changed, 4132 insertions(+), 6 deletions(-) create mode 100644 scripts/build-browser.mjs create mode 100644 site/playground/commands.js create mode 100644 site/playground/engine.browser.mjs create mode 100644 site/playground/index.html create mode 100644 site/playground/playground.js create mode 100644 site/playground/report.js create mode 100644 site/playground/sources.js create mode 100644 site/playground/worker.js create mode 100644 src/browser/child_process.ts create mode 100644 src/browser/crypto.ts create mode 100644 src/browser/entry.ts create mode 100644 src/browser/fs.ts create mode 100644 src/browser/globals.ts create mode 100644 src/browser/grammars.ts create mode 100644 src/browser/os.ts create mode 100644 src/browser/path.ts create mode 100644 src/browser/readline.ts create mode 100644 src/browser/url.ts create mode 100644 src/browser/worker_threads.ts create mode 100644 src/browser/zlib.ts create mode 100644 tests/browser-build.test.ts create mode 100644 tests/browser-shims.test.ts create mode 100644 tests/playground-commands.test.ts create mode 100644 tests/playground-e2e.test.ts create mode 100644 tests/playground-sources.test.ts diff --git a/.github/workflows/site.yml b/.github/workflows/site.yml index 1255329..5b32fad 100644 --- a/.github/workflows/site.yml +++ b/.github/workflows/site.yml @@ -1,14 +1,25 @@ name: Site -# One-page GitHub Pages site (site/index.html). Self-contained: zero external -# requests at runtime, so the deploy step is a plain static-asset publish — -# no build step, no dependency install. Triggers only on changes to the site -# itself (or this workflow), plus manual dispatch. +# GitHub Pages site. Still a plain static-asset publish — no build step and no +# dependency install — because every generated artifact it serves is committed: +# site/index.html is hand-written, and site/playground/engine.browser.mjs is +# built by scripts/build-browser.mjs and held byte-reproducible by +# `pnpm run check:build`. +# +# The one thing not committed is the tree-sitter wasm the playground loads. It +# already lives in scripts/grammars/ (22 MB), so it is COPIED into the artifact +# here rather than duplicated in git. +# +# Note on the site's "zero external requests" property: that holds for +# site/index.html, which remains fully self-contained. site/playground/ is +# deliberately different — it fetches repositories from jsDelivr at runtime, +# which is the entire point of a playground. on: push: branches: [main] paths: - "site/**" + - "scripts/grammars/**" - ".github/workflows/site.yml" workflow_dispatch: @@ -32,6 +43,15 @@ jobs: - uses: actions/configure-pages@v5 + # The AST tier the playground demonstrates is these files. Without them it + # still works, but silently degrades to the regex tier — so a missing copy + # would quietly weaken the one thing the page exists to show. + - name: Stage the tree-sitter grammars for the playground + run: | + cp -R scripts/grammars site/playground/grammars + test -f site/playground/grammars/web-tree-sitter.wasm + test -f site/playground/engine.browser.mjs + - uses: actions/upload-pages-artifact@v4 with: path: site diff --git a/.gitignore b/.gitignore index 71d525b..78cc453 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ scripts/embed-asset/.venv/ # never in git: `node scripts/fetch-grammars.mjs --extended` regenerates it, and # `codeindex grammars pull` is how a consumer gets it. scripts/grammars-extended/ + +# The playground serves the committed grammars from its own directory. They are +# COPIED there at deploy time (.github/workflows/site.yml) rather than committed +# twice — 22 MB of wasm already lives in scripts/grammars/. Populate it locally +# with: cp -R scripts/grammars site/playground/grammars +site/playground/grammars/ diff --git a/package.json b/package.json index a9ab77e..23f6f5c 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "node": ">=18" }, "scripts": { - "build": "tsup && node scripts/postbuild.mjs", + "build": "tsup && node scripts/postbuild.mjs && node scripts/build-browser.mjs", "grammars": "node scripts/fetch-grammars.mjs", "test": "node scripts/fetch-grammars.mjs --extended && vitest run", "test:watch": "node scripts/fetch-grammars.mjs --extended && vitest", @@ -38,7 +38,7 @@ "quality": "node scripts/fetch-grammars.mjs --extended && CODEINDEX_QUALITY_WRITE=1 vitest run tests/quality.test.ts", "quality:report": "node scripts/fetch-grammars.mjs --extended && CODEINDEX_QUALITY_REPORT=1 vitest run tests/quality.test.ts", "typecheck": "tsc --noEmit", - "check:build": "tsup && node scripts/postbuild.mjs && git diff --exit-code -- scripts/engine.mjs scripts/engine.d.mts scripts/cli.mjs scripts/grammars", + "check:build": "tsup && node scripts/postbuild.mjs && node scripts/build-browser.mjs && git diff --exit-code -- scripts/engine.mjs scripts/engine.d.mts scripts/cli.mjs scripts/grammars site/playground/engine.browser.mjs", "demo": "node scripts/cli.mjs graph --repo tests/fixtures/mini-repo", "release": "semantic-release", "grammars:extended": "node scripts/fetch-grammars.mjs --extended" @@ -66,6 +66,7 @@ "@tree-sitter-grammars/tree-sitter-lua": "0.4.1", "@tree-sitter-grammars/tree-sitter-zig": "1.1.2", "@types/node": "^20.14.0", + "esbuild": "0.27.7", "semantic-release": "^25.0.8", "tree-sitter-bash": "0.25.1", "tree-sitter-c": "^0.24.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 205cbd5..bb1484a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -32,6 +32,9 @@ importers: '@types/node': specifier: ^20.14.0 version: 20.19.43 + esbuild: + specifier: 0.27.7 + version: 0.27.7 semantic-release: specifier: ^25.0.8 version: 25.0.8(typescript@5.9.3) diff --git a/scripts/build-browser.mjs b/scripts/build-browser.mjs new file mode 100644 index 0000000..25792e9 --- /dev/null +++ b/scripts/build-browser.mjs @@ -0,0 +1,84 @@ +#!/usr/bin/env node +// Builds site/playground/engine.browser.mjs — the SAME engine as +// scripts/engine.mjs, resolved against browser shims instead of node builtins. +// Committed like the other two artifacts and held byte-reproducible by +// `pnpm run check:build`, so GitHub Pages stays a plain static publish with no +// build step and no dependency install. +// +// WHY NOT A THIRD tsup TARGET. This build's whole job is to intercept the +// resolution of node builtins, and tsup registers its own node-protocol plugin +// ahead of any user plugin. esbuild takes the FIRST onResolve result, so tsup's +// plugin wins and resolves `node:fs` to an external bare "fs". The build still +// succeeds; the output just carries `import { readFileSync } from "fs"` +// statements that no browser can load — and that Node satisfies from the real +// disk, so it fails silently rather than loudly. Owning the plugin list here +// removes that ordering hazard entirely. +// +// Determinism: esbuild is pinned in package.json and its output is a pure +// function of (input, options), so two builds of an unchanged tree are +// byte-identical — the same property check:build already enforces for the Node +// artifacts. + +import { build } from "esbuild"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const shim = (name) => join(root, "src", "browser", `${name}.ts`); + +// Node builtins the engine imports. Matched under both spellings because the +// source is not uniform about the `node:` prefix and third-party code inside +// the bundle (web-tree-sitter) uses the bare form. +const BUILTINS = "fs|path|crypto|os|url|child_process|worker_threads|readline|zlib"; + +const browserShims = { + name: "browser-shims", + setup(build) { + // `namespace: "file"` is load-bearing, not decoration. A plugin-resolved + // path with no namespace gets a DIFFERENT module identity from the same + // file reached by ordinary relative resolution — so src/browser/fs.ts would + // be instantiated twice: once for `node:fs` (what walk.ts reads) and once + // for `./fs.js` (what the playground's mount API writes). Two instances + // means two VFS maps, and every mount would land in a filesystem the engine + // cannot see. + build.onResolve({ filter: new RegExp(`^(node:)?(${BUILTINS})$`) }, (args) => ({ + path: shim(args.path.replace(/^node:/, "")), + namespace: "file", + })); + + // web-tree-sitter probes for a Node environment before falling back to its + // fetch/XHR loaders. That branch is dead here — ENVIRONMENT_IS_NODE is + // false in a browser, and ensureGrammars hands the runtime an explicit + // wasmBinary so it never reaches for a loader at all — but esbuild still + // has to resolve the specifiers statically. Empty modules satisfy it + // without pulling any of Node's machinery into the bundle. + build.onResolve({ filter: /^(node:)?(fs[/]promises|module)$/ }, () => ({ path: "node-only", namespace: "browser-empty" })); + build.onLoad({ filter: /.*/, namespace: "browser-empty" }, () => ({ + contents: "export const createRequire = () => { throw new Error('node-only API in the browser build'); };\nexport default {};\n", + loader: "js", + })); + }, +}; + +const outfile = join(root, "site", "playground", "engine.browser.mjs"); + +await build({ + entryPoints: [join(root, "src", "browser", "entry.ts")], + outfile, + bundle: true, + format: "esm", + platform: "browser", + target: "es2022", + // Minified because this artifact is served over the wire to a browser rather + // than vendored into a repo, where the Node bundle's readability matters. + minify: true, + sourcemap: false, + legalComments: "none", + plugins: [browserShims], + // Buffer and process are free identifiers in the engine source (walk.ts's + // decoder, loader.ts's env reads). Injection substitutes both without editing + // a line of that source. + inject: [shim("globals")], +}); + +console.log(`build-browser: ${outfile}`); diff --git a/site/index.html b/site/index.html index 9a10e36..6b02ab8 100644 --- a/site/index.html +++ b/site/index.html @@ -932,6 +932,7 @@