Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/e2e-cli.yml
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,7 @@ jobs:
.llm/tmp/e2e-report-scaffold-runtime*.ndjson
.llm/tmp/e2e-image-pulls.json
.llm/tmp/cli-e2e/*/.netscript/e2e/listener-unreachable-receipt.json
.llm/tmp/e2e-invocation-diagnostics/*-diagnostic.json
.llm/tmp/gate-receipts/scaffold-runtime/agent.aspire-mcp-smoke*
include-hidden-files: true
if-no-files-found: ignore
Expand Down Expand Up @@ -507,6 +508,7 @@ jobs:
.llm/tmp/e2e-report-scaffold-runtime-sqlite*.ndjson
.llm/tmp/e2e-image-pulls.json
.llm/tmp/cli-e2e/*/.netscript/e2e/listener-unreachable-receipt.json
.llm/tmp/e2e-invocation-diagnostics/*-diagnostic.json
.llm/tmp/gate-receipts/scaffold-runtime-sqlite/agent.aspire-mcp-smoke*
.llm/tmp/showcase-ai-browser/
include-hidden-files: true
Expand Down
22 changes: 22 additions & 0 deletions .llm/tools/docs/assert-corpus-bytes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/** Exact byte comparison with bounded failure diagnostics, never an expanded corpus diff. */
import { AssertionError } from '@std/assert';

async function digest(value: Uint8Array): Promise<string> {
const copy = new Uint8Array(value.byteLength);
copy.set(value);
return [...new Uint8Array(await crypto.subtle.digest('SHA-256', copy))]
.map((byte) => byte.toString(16).padStart(2, '0')).join('');
}

export async function assertExactCorpusBytes(
actual: Uint8Array,
expected: Uint8Array,
): Promise<void> {
let equal = actual.byteLength === expected.byteLength;
for (let i = 0; equal && i < actual.byteLength; i++) equal = actual[i] === expected[i];
if (equal) return;
throw new AssertionError(
`Corpus bytes differ: length ${actual.byteLength}/${expected.byteLength}; ` +
`SHA256 ${await digest(actual)}/${await digest(expected)}`,
);
}
20 changes: 20 additions & 0 deletions .llm/tools/docs/assert-corpus-bytes_test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { assert, AssertionError, assertRejects } from '@std/assert';
import { assertExactCorpusBytes } from './assert-corpus-bytes.ts';

Deno.test('exact corpus bytes accept identical and empty payloads', async () => {
await assertExactCorpusBytes(new Uint8Array(), new Uint8Array());
await assertExactCorpusBytes(new Uint8Array([1, 2, 3]), new Uint8Array([1, 2, 3]));
});
Deno.test('length or same-length last-byte mismatch stays red with bounded payload-free diagnostics', async () => {
const actual = new Uint8Array(2 << 20).fill(65), expected = actual.slice();
expected[expected.length - 1] = 66;
for (const pair of [[actual, expected], [actual, expected.subarray(1)]]) {
const error = await assertRejects(
() => assertExactCorpusBytes(pair[0], pair[1]),
AssertionError,
);
assert(error.message.length < 256);
assert(error.message.includes('SHA256'));
assert(!error.message.includes('AAAA'));
}
});
73 changes: 73 additions & 0 deletions .llm/tools/docs/export-corpus-doc-command.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/** Explicit static-documentation child policy; ordinary callers keep their existing loader. */
import { isAbsolute, join, resolve } from '@std/path';

export interface ExportCorpusDocOptions {
readonly lockfile?: string;
readonly importHosts?: readonly string[];
readonly cacheDir?: string;
readonly compilerFlags?: string;
readonly onChild?: (notice: {
readonly packageName: string;
readonly ordinal: number;
readonly total: number;
readonly phase: 'attempt' | 'complete';
readonly exitCode?: number;
}) => void;
}

/** Construct only supported Deno doc flags and a sterile strict-mode environment. */
export function exportCorpusDocCommand(
root: string,
entrypoints: readonly string[],
options: ExportCorpusDocOptions,
ordinaryEnvironment: () => Record<string, string>,
): { args: string[]; env: Record<string, string> } {
const strict = options.lockfile !== undefined || options.importHosts !== undefined ||
options.cacheDir !== undefined;
if (!strict) return { args: ['doc', '--json', ...entrypoints], env: ordinaryEnvironment() };
const lockfile = options.lockfile && resolve(root, options.lockfile);
const cache = options.cacheDir && resolve(options.cacheDir);
const hosts = options.importHosts;
if (
lockfile !== resolve(root, 'deno.lock') || !cache || !isAbsolute(options.cacheDir!) ||
cache === resolve(root) || !hosts?.length || hosts.length > 16 ||
new Set(hosts).size !== hosts.length
) {
throw new Error(
'Locked documentation requires the repository lock, explicit cache and public import hosts',
);
}
for (const host of hosts) {
if (
!/^[a-z0-9]+(?:[.-][a-z0-9]+)*\.[a-z]{2,}$/.test(host) ||
/(?:^|\.)(?:localhost|local|internal|invalid|test)$/.test(host)
) {
throw new Error('Documentation import host is not an explicit public hostname');
}
}
if (
options.compilerFlags && !/^--max-old-space-size=[1-9][0-9]{1,4}$/.test(options.compilerFlags)
) {
throw new Error('Unsupported documentation compiler flags');
}
const home = join(cache, 'home');
return {
args: [
'doc',
'--json',
'--lock',
lockfile,
'--frozen-lockfile',
'--allow-import=' + hosts.join(','),
'--node-modules-dir=none',
...entrypoints,
],
env: {
DENO_DIR: cache,
HOME: home,
USERPROFILE: home,
NO_COLOR: '1',
...(options.compilerFlags ? { DENO_V8_FLAGS: options.compilerFlags } : {}),
},
};
}
98 changes: 98 additions & 0 deletions .llm/tools/docs/export-corpus-doc-command_test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import { assertEquals, assertThrows } from '@std/assert';
import { exportCorpusDocCommand } from './export-corpus-doc-command.ts';
import { parseExportCorpusOptions } from './export-corpus-git-read.ts';

const options = {
lockfile: '/synthetic/repository/deno.lock',
cacheDir: '/synthetic/cache',
importHosts: ['jsr.io', 'registry.npmjs.org'],
compilerFlags: '--max-old-space-size=1152',
};
Deno.test('locked documentation composes supported exact child flags and never copies caller auth environment', () => {
let environmentReads = 0;
const neverRead = () => {
environmentReads++;
return { SYNTHETIC_AUTH_TOKEN: 'never-copy-fixture' };
};
const plan = exportCorpusDocCommand(
'/synthetic/repository',
['first.ts', 'second.ts'],
options,
neverRead,
);
assertEquals(plan.args, [
'doc',
'--json',
'--lock',
options.lockfile,
'--frozen-lockfile',
'--allow-import=jsr.io,registry.npmjs.org',
'--node-modules-dir=none',
'first.ts',
'second.ts',
]);
assertEquals(plan.env, {
DENO_DIR: '/synthetic/cache',
HOME: '/synthetic/cache/home',
USERPROFILE: '/synthetic/cache/home',
NO_COLOR: '1',
DENO_V8_FLAGS: '--max-old-space-size=1152',
});
assertEquals(
Object.keys(plan.env).some((key) => /token|auth|proxy|session|force_color/i.test(key)),
false,
);
assertEquals(plan.args.includes('--cached-only'), false);
assertEquals(environmentReads, 0);
});
Deno.test('strict documentation refuses incomplete locks, private hosts, cache overrides and arbitrary compiler flags', () => {
const ordinary = () => ({});
for (
const invalid of [
{ ...options, lockfile: '/elsewhere/deno.lock' },
{ ...options, cacheDir: '/synthetic/repository' },
{ ...options, cacheDir: 'relative-cache' },
{ ...options, importHosts: ['localhost'] },
{ ...options, importHosts: ['service.internal'] },
{ ...options, importHosts: ['127.0.0.1'] },
{ ...options, importHosts: ['https://jsr.io'] },
{ ...options, importHosts: ['jsr.io', 'jsr.io'] },
{ ...options, compilerFlags: '--allow-natives-syntax' },
{ cacheDir: '/synthetic/cache' },
{ lockfile: options.lockfile },
]
) {
assertThrows(() =>
exportCorpusDocCommand('/synthetic/repository', ['first.ts'], invalid, ordinary)
);
}
});
Deno.test('ordinary documentation command and colour-normalized environment remain unchanged', () => {
const env = { NO_COLOR: '1', EXAMPLE: 'unchanged' };
assertEquals(exportCorpusDocCommand('/synthetic/repository', ['first.ts'], {}, () => env), {
args: ['doc', '--json', 'first.ts'],
env,
});
});
Deno.test('documented strict CLI flags map to typed policy without accepting unsupported native cached-only', () => {
assertEquals(
parseExportCorpusOptions([
'--doc-lockfile',
options.lockfile,
'--doc-cache-dir',
options.cacheDir,
'--doc-import-hosts',
'jsr.io,registry.npmjs.org',
]),
{
check: false,
allowDirty: false,
gitReadWrapper: undefined,
docLockfile: options.lockfile,
docCacheDir: options.cacheDir,
docImportHosts: options.importHosts,
},
);
assertThrows(() => parseExportCorpusOptions(['--cached-only']));
assertThrows(() => parseExportCorpusOptions(['--doc-lockfile']));
});
133 changes: 133 additions & 0 deletions .llm/tools/docs/export-corpus-git-read.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
/** Explicit Git-read delegation for the generator's existing clean-source guard. */
import { resolve } from '@std/path';

export interface GeneratorGitCommand {
readonly executable: string;
readonly args: string[];
readonly cwd: string;
readonly strict: boolean;
}
export interface GeneratorGitOutput {
readonly success: boolean;
readonly code: number;
readonly stdout: Uint8Array;
readonly stderr: Uint8Array;
}
export type GeneratorGitExecutor = (command: GeneratorGitCommand) => Promise<GeneratorGitOutput>;

/** Parse supported generator options without treating wrapper values as new flags. */
export function parseExportCorpusOptions(args: readonly string[]): {
check: boolean;
allowDirty: boolean;
gitReadWrapper?: string;
docLockfile?: string;
docCacheDir?: string;
docImportHosts?: string[];
} {
const check = args.includes('--check'), allowDirty = args.includes('--allow-dirty');
const index = args.indexOf('--git-read-wrapper');
const gitReadWrapper = index < 0 ? undefined : args[index + 1];
if (
index >= 0 && (!gitReadWrapper || gitReadWrapper.startsWith('--') ||
args.lastIndexOf('--git-read-wrapper') !== index)
) throw new Error('--git-read-wrapper requires one executable');
const docFlags = ['--doc-lockfile', '--doc-cache-dir', '--doc-import-hosts'];
const docValues: (string | undefined)[] = [];
const consumed = new Set<number>();
for (const flag of docFlags) {
const at = args.indexOf(flag), value = at < 0 ? undefined : args[at + 1];
if (at >= 0) {
if (!value || value.startsWith('--') || args.lastIndexOf(flag) !== at) {
throw new Error(`${flag} requires one value`);
}
consumed.add(at);
consumed.add(at + 1);
}
docValues.push(value);
}
const unknown = args.filter((value, position) =>
!consumed.has(position) &&
value !== '--check' && value !== '--allow-dirty' &&
!(index >= 0 && (position === index || position === index + 1))
);
if (unknown.length) throw new Error(`unknown argument: ${unknown[0]}`);
if (check && allowDirty) {
throw new Error('--allow-dirty is only valid when writing the generated corpus');
}
if (gitReadWrapper && allowDirty) throw new Error('--git-read-wrapper cannot use --allow-dirty');
return {
check,
allowDirty,
gitReadWrapper,
...(docValues[0] !== undefined ? { docLockfile: docValues[0] } : {}),
...(docValues[1] !== undefined ? { docCacheDir: docValues[1] } : {}),
...(docValues[2] !== undefined ? { docImportHosts: docValues[2].split(',') } : {}),
};
}

/** The wrapper receives the actual resolved workspace and the exact source read set. */
export function generatorGitCommand(root: string, wrapper?: string): GeneratorGitCommand {
const cwd = resolve(root);
const args = ['status', '--porcelain', '--', 'packages', 'plugins'];
return wrapper
? { executable: wrapper, args: [cwd, '--', ...args], cwd, strict: true }
: { executable: 'git', args, cwd, strict: false };
}

async function execute(command: GeneratorGitCommand): Promise<GeneratorGitOutput> {
return await new Deno.Command(command.executable, {
args: command.args,
cwd: command.cwd,
stdout: 'piped',
stderr: 'piped',
...(command.strict ? { signal: AbortSignal.timeout(20_000) } : {}),
}).output();
}

/**
* Preserve the ordinary generator guard; explicit delegation refuses closed.
* A wrapper failure never falls back to Git or a separate earlier clean proof.
*/
export async function guardGeneratorWrite(
root: string,
options: { readonly allowDirty: boolean; readonly gitReadWrapper?: string },
run: GeneratorGitExecutor = execute,
): Promise<void> {
if (options.gitReadWrapper && options.allowDirty) {
throw new Error('An approved Git read wrapper cannot be combined with --allow-dirty');
}
const command = generatorGitCommand(root, options.gitReadWrapper);
let output: GeneratorGitOutput;
try {
output = await run(command);
} catch (error) {
if (command.strict) {
throw new Error('Approved Git read did not confirm clean generator sources');
}
console.warn(
'WARNING: Git is unavailable; continuing MCP export-corpus generation without ' +
`clean-tree verification (${error instanceof Error ? error.message : String(error)}).`,
);
return;
}
if (!output.success) {
if (command.strict) throw new Error('Approved Git read refused generator source verification');
const detail = new TextDecoder().decode(output.stderr).trim() || `exit ${output.code}`;
console.warn(
'WARNING: Git status is unavailable; continuing MCP export-corpus generation without ' +
`clean-tree verification (${detail}).`,
);
return;
}
const dirty = new TextDecoder().decode(output.stdout).trimEnd();
if (!dirty) return;
const summary = `MCP export-corpus generator read set is dirty:\n${dirty}`;
if (options.allowDirty) {
console.warn(`WARNING: ${summary}\nProceeding because --allow-dirty was explicitly supplied.`);
return;
}
throw new Error(
`${summary}\nRefusing to generate from uncommitted package/plugin sources. ` +
'Commit or stash them, or deliberately rerun with --allow-dirty.',
);
}
Loading
Loading