From 7b71312afb1a1ea630c27d811bb80f23203f408d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 17 Nov 2026 07:03:20 -0500 Subject: [PATCH] feat: Run Background Commands Without Holding Their Worktree --- docker-compose.yaml | 2 + docs/remote-bridge/README.md | 7 + packages/code/README.md | 50 +- packages/code/src/cli.ts | 27 +- packages/code/src/linked-worktrees.test.ts | 66 ++- packages/code/src/native-pool.test.ts | 280 +++++++++- packages/code/src/native-pool.ts | 107 +++- packages/code/src/protocol.test.ts | 69 +++ packages/code/src/protocol.ts | 34 ++ packages/code/src/storage.ts | 65 +++ packages/code/src/worker-detached.test.ts | 496 ++++++++++++++++++ packages/code/src/worker.ts | 120 ++++- packages/code/src/workspace-cli.test.ts | 91 ++++ packages/code/src/workspace-guards.test.ts | 168 ++++++ packages/code/src/workspace-guards.ts | 85 ++- packages/code/src/workspace-instances.test.ts | 88 +++- packages/code/src/workspace.ts | 8 + service/src/bridge/detached.test.ts | 309 +++++++++++ service/src/bridge/index.ts | 1 + service/src/bridge/router.test.ts | 64 +++ service/src/bridge/router.ts | 3 + service/src/bridge/slots.test.ts | 62 ++- service/src/bridge/slots.ts | 14 +- service/src/bridge/store.ts | 134 ++++- service/src/config.ts | 6 + service/src/workspace-tools/index.ts | 3 + service/src/workspace-tools/requests.test.ts | 60 ++- service/src/workspace-tools/requests.ts | 4 +- tests/compose-bridge-config.cjs | 3 + 29 files changed, 2341 insertions(+), 85 deletions(-) create mode 100644 packages/code/src/worker-detached.test.ts create mode 100644 packages/code/src/workspace-guards.test.ts create mode 100644 service/src/bridge/detached.test.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index a4a43cdd..9e221083 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,7 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} + - CODEAPI_BRIDGE_DETACHED_COMMANDS=${CODEAPI_BRIDGE_DETACHED_COMMANDS:-true} - CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES=${CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES:-104857600} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} @@ -72,6 +73,7 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} + - CODEAPI_BRIDGE_DETACHED_COMMANDS=${CODEAPI_BRIDGE_DETACHED_COMMANDS:-true} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 85a8e097..f16b76a1 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -366,6 +366,13 @@ execution. (`worktree: `) nests beneath its checkout's key: sibling lanes run concurrently, while a lane and its checkout exclude each other, and a lane cannot start while its checkout is quarantined. +- An `execute_command` request marked `detached` takes a slot under a key of + its own instead of its worktree's, so other operations there are admitted + while it runs and it is admitted while they run. Detached commands may hold + at most one slot fewer than the negotiated count. Code API offers the feature + only above one slot (`CODEAPI_BRIDGE_DETACHED_COMMANDS=false` opts out) and + refuses the field for a worker that does not advertise + `commandFeatures: ['detached']`. - Workspace tool admission waits for capacity up to 30 seconds without a `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can advertise a longer per-request allowance, bounded by five minutes and any server queue ceiling. diff --git a/packages/code/README.md b/packages/code/README.md index 94da7d0f..960e633c 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -866,7 +866,8 @@ librechat-code run \ Slots are per machine, not a fleet-wide execution limit. A busy machine does not consume another machine's slots. Requests for the same root remain serialized, -including commands started through background tools. Independent checkouts can +except a command the caller marks detached (see +[Detached commands](#detached-commands)). Independent checkouts can use different slots; selecting subdirectories beneath one registered parent root does not create separate scheduling boundaries. @@ -1080,6 +1081,51 @@ retain the selected root's sandbox boundary, not a shared parent-directory grant The `LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE` single-file override is rejected when multiple roots are configured; unset it to use separate root-derived markers. +#### Detached commands + +A long test or build started in the background would otherwise hold its root, +checkout or lane for its whole run, and every later read, edit or command there +would queue behind it. A caller can instead mark an `execute_command` request +`detached: true`. A detached command still takes one slot, but it does not hold +its worktree: other operations in that worktree, its checkout and its lanes are +admitted and run while it runs, and it is admitted while they run. Its timeout, +output, cancellation, settlement and result are those of any other command. + +Nothing serializes a detached command with the work beside it, so the caller is +responsible for choosing commands that tolerate concurrent edits, such as a test +run or a build, rather than a formatter or a `git` operation that rewrites the +tree. + +The feature is on by default wherever it can hold: Code API offers it when +`CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS` is above 1, and a worker advertises +`commandFeatures: ['detached']` when it runs native-srt commands with +`--workspace-lease-slots` above 1 and Code API offered it. LibreChat reads the +advertisement from the worker status and sends `detached` only then; a request +that carries it for a worker that does not advertise it is refused, never run +serialized. Set `CODEAPI_BRIDGE_DETACHED_COMMANDS=false` on every Code API +replica, or start the worker with `--no-detached-commands` +(`LIBRECHAT_CODE_DETACHED_COMMANDS=false`), to opt out. An older Code API never +offers the feature and an older worker never advertises it, so either side can +be upgraded first. + +- Detached commands may hold at most one slot fewer than the negotiated count, + so one slot always remains for other work. Further detached commands wait + without holding back the requests queued behind them. +- Each detached command runs in its own SRT executor for the same root, with + the root's sandbox policy, created for the command and closed after it. It + counts toward the executor bound, and it never shares the root's own executor, + scratch directory or credentials. +- Each detached command arms its own durable marker in a `.detached` directory + beside its worktree's quarantine marker, and clears it after settlement. If the + worker stops while one runs, or one ends ambiguously, the marker stays and the + worktree is quarantined exactly as for any other uncertain mutation. A detached + command also refuses to start in a quarantined worktree. + `clear-workspace-quarantine` removes these markers together with the worktree's + own, and `--reset-workspace-quarantine` refuses while any remain. +- A detached command has no server-side fence on its worktree. Its own fence, and + the receipt left if it is quarantined, expire with the request's outcome instead + of waiting for a reset. + `LIBRECHAT_CODE_WORKSPACE_LEASE_SLOTS` is the equivalent worker setting. Both ceilings must be integers from 1 to 8; the lower ceiling wins. An older Code API without the negotiation receipt keeps the worker on the serial protocol. Deploy @@ -1088,7 +1134,7 @@ change while work is active fails closed; stop and drain the worker before changing it. Different roots can run concurrently; requests targeting the **same root remain -serialized**, even across chats or agents. This is root-level exclusion, not +serialized**, even across chats or agents, apart from detached commands. This is root-level exclusion, not file-level locking. Assign separate project/worktree roots for independent work. The admission queue remains bounded at 32 requests per worker. An idle SRT process cache is bounded by the local slot setting and evicts only idle executors. Runtime diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index a5eff92f..f1c46f2d 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -31,9 +31,10 @@ import { defaultBridgeIdentityPath, defaultWorkspaceQuarantinePath, defaultWorkspacePath, + detachedWorkspaceMutationQuarantineDirectory, ensurePrivateWorkspaceDirectory, loadBridgeIdentity, - loadWorkspaceMutationQuarantine, + loadWorkspaceMutationQuarantines, saveBridgeIdentity, saveWorkspaceMutationQuarantine, prepareEnvironmentPreparationDirectory, @@ -47,7 +48,7 @@ import { } from './runtime.js'; import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; -import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { NativeWorkspaceCommandPool, detachedCommandsEnabled } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; import { LaneGitWorkspaceTools, ownsGitMetadata, ownsLinkedWorktreeMetadata } from './lane-git.js'; import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools, verifyLinkedWorktree } from './linked-worktrees.js'; @@ -856,6 +857,10 @@ async function run( }), ]), ); + const rootQuarantineStatePaths = [...rootQuarantinePaths.values()].flatMap((path) => [ + path, + detachedWorkspaceMutationQuarantineDirectory(path), + ]); for (const path of setupProtectedPaths) { const namespace = await lstat(path).catch((error: NodeJS.ErrnoException) => { if (error.code === 'ENOENT') return undefined; @@ -885,13 +890,13 @@ async function run( }))); await assertEnvironmentResourceIsolation(environments.flatMap(environment => environment.resources ?? []), roots.map(root => root.root), [ identityPath, ...setupProtectedPaths, preparationDirectory, github.privateKeyPath, - ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), + ...environments.map(environment => environment.path), ...rootQuarantineStatePaths, ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); const snapshotStores = environments.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); await assertEnvironmentResourceIsolation(snapshotStores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), roots.map(root => root.root), [ identityPath, ...setupProtectedPaths, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), - ...rootQuarantinePaths.values(), ...environments.flatMap(environment => (environment.resources ?? []).map(resource => resource.path)), + ...rootQuarantineStatePaths, ...environments.flatMap(environment => (environment.resources ?? []).map(resource => resource.path)), ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(snapshotStores.map(store => ({ path: store.store, sourceParents: store.controlPaths, definition: { name: 'dependency-store', root: store.store }, fingerprint: '' })), roots); @@ -1139,7 +1144,7 @@ async function run( ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), ...snapshotStores.map(store => store.store), ...environments.map(environment => environment.path), - ...rootQuarantinePaths.values(), + ...rootQuarantineStatePaths, github.privateKeyPath, ].filter((path): path is string => path != null), allowedDomains: commandAllowedDomains, @@ -1204,6 +1209,12 @@ async function run( ) : new NativeProcessWorkspaceCommandSandbox(nativeOptionsForWorkspace(roots[0].id)) : undefined; + const detachedCommands = detachedCommandsEnabled({ + optOut: args.includes('--no-detached-commands'), + enabled: process.env.LIBRECHAT_CODE_DETACHED_COMMANDS, + pooled: nativeCommandSandbox instanceof NativeWorkspaceCommandPool, + workspaceLeaseSlots, + }); if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ workspaceTools, @@ -1211,6 +1222,7 @@ async function run( ...(nativeProgrammaticEnabled ? { programmaticLanguages: ['bash'] } : {}), + ...(detachedCommands ? { commandFeatures: ['detached'] } : {}), commandSandbox: nativeCommandSandbox ?? new RuntimeWorkspaceCommandSandbox({ @@ -1408,7 +1420,7 @@ async function run( workerId, workspaceRoot: join(source.root, LINKED_WORKTREE_DIRECTORY, worktree), }); - return (await loadWorkspaceMutationQuarantine(path)) != null; + return (await loadWorkspaceMutationQuarantines(path)).length > 0; }, log(level, message) { if (level === 'debug' && !debugLogs) return; @@ -1633,8 +1645,7 @@ async function run( roots.length === 1 ? { async assertAvailable() { - const record = - await loadWorkspaceMutationQuarantine( + const [record] = await loadWorkspaceMutationQuarantines( mutationQuarantinePath, ); if (record != null) { diff --git a/packages/code/src/linked-worktrees.test.ts b/packages/code/src/linked-worktrees.test.ts index 30c5b7e6..75d1d49d 100644 --- a/packages/code/src/linked-worktrees.test.ts +++ b/packages/code/src/linked-worktrees.test.ts @@ -20,11 +20,12 @@ import { linkedWorktreeWorkspaceId, verifyLinkedWorktree, } from './linked-worktrees.js'; +import { NativeWorkspaceCommandPool as CommandPool } from './native-pool.js'; import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; import type { NativeWorkspaceCommandPool } from './native-pool.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; -import type { WorkspaceToolRequest } from './protocol.js'; +import type { WorkspaceExecuteCommandRequest, WorkspaceToolRequest } from './protocol.js'; const execFileAsync = promisify(execFile); @@ -345,3 +346,66 @@ test('a removed lane releases its command root and credential route', async (t) ); assert.deepEqual(released, [join(root, '.worktrees', 'task-a')]); }); + +test('a detached lane command runs in its own executor while the lane keeps working', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const executors: Array<{ root: string; requests: WorkspaceExecuteCommandRequest[]; closed: boolean }> = []; + let releaseBackground!: () => void; + const backgroundFinished = new Promise((resolve) => { releaseBackground = resolve; }); + let backgroundStarted!: () => void; + const backgroundRunning = new Promise((resolve) => { backgroundStarted = resolve; }); + const pool = new CommandPool(new Map([['repo', { workspaceRoot: root }]]), 2, (options) => { + const executor = { root: options.workspaceRoot, requests: [] as WorkspaceExecuteCommandRequest[], closed: false }; + executors.push(executor); + return { + async prepare() {}, + async close() { + executor.closed = true; + }, + async execute(request) { + executor.requests.push(request); + if (request.command === 'npm test') { + backgroundStarted(); + await backgroundFinished; + } + return { ...request, stdout: request.command, stderr: '', exitCode: 0, timedOut: false, truncated: false }; + }, + }; + }); + const tools = await laneTools(root, pool); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree: 'task-a', + }; + const id = linkedWorktreeWorkspaceId('repo', 'task-a'); + const laneRoot = join(root, '.worktrees', 'task-a'); + + const background = tools.execute({ ...command, command: 'npm test', detached: true }); + await backgroundRunning; + const ordinary = await tools.execute({ ...command, command: 'git status' }); + assert.equal(ordinary.workspaceId, 'repo'); + assert.deepEqual(await tools.whileIdle('repo', 'task-a', async () => 'retired'), { ran: false }); + assert.deepEqual(executors, [ + { + root: laneRoot, + requests: [{ protocolVersion: 1, operation: 'execute_command', workspaceId: id, command: 'npm test' }], + closed: false, + }, + { + root: laneRoot, + requests: [{ protocolVersion: 1, operation: 'execute_command', workspaceId: id, command: 'git status' }], + closed: false, + }, + ]); + + releaseBackground(); + const result = await background; + assert.equal(result.workspaceId, 'repo'); + assert.equal(result.operation === 'execute_command' && result.stdout, 'npm test'); + assert.deepEqual(await tools.whileIdle('repo', 'task-a', async () => 'retired'), { ran: true, value: 'retired' }); + assert.deepEqual(executors.map((executor) => executor.closed), [true, true]); + await pool.close(); +}); diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index 2a339d59..a7a9d1d7 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -1,7 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { NativeWorkspaceCommandPool } from './native-pool.js'; -import { WorkspaceToolError } from './workspace.js'; +import { NativeWorkspaceCommandPool, detachedCommandsEnabled } from './native-pool.js'; +import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; import type { WorkspaceExecuteCommandRequest } from './protocol.js'; const roots = new Map( @@ -322,3 +322,279 @@ test('idle eviction failure stays mutation-atomic for the new root', async () => }); await assert.rejects(pool.close(), AggregateError); }); + +interface RecordedExecutor { + root: string; + requests: WorkspaceExecuteCommandRequest[]; + closed: number; +} + +/** Executors that hold each command until the test releases it by command text. */ +function gatedExecutors(options: { failClose?: (executor: RecordedExecutor) => boolean } = {}) { + const executors: RecordedExecutor[] = []; + const gates = new Map; enter(): void; finish: Promise; release(): void }>(); + const gate = (command: string) => { + let existing = gates.get(command); + if (!existing) { + let enter!: () => void; + let release!: () => void; + existing = { + entered: new Promise((resolve) => { enter = resolve; }), + enter: () => enter(), + finish: new Promise((resolve) => { release = resolve; }), + release: () => release(), + }; + gates.set(command, existing); + } + return existing; + }; + const failures = new Map(); + return { + executors, + failures, + entered: (command: string) => gate(command).entered, + release: (command: string) => gate(command).release(), + create: (sandbox: { workspaceRoot: string }) => { + const executor: RecordedExecutor = { root: sandbox.workspaceRoot, requests: [], closed: 0 }; + executors.push(executor); + return { + async prepare() {}, + async close() { + executor.closed += 1; + if (options.failClose?.(executor)) throw new Error('fixture cleanup failure'); + }, + async execute(req: WorkspaceExecuteCommandRequest) { + executor.requests.push(req); + const running = gate(req.command); + running.enter(); + await running.finish; + const failure = failures.get(req.command); + if (failure) throw failure; + return { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: req.workspaceId, + stdout: req.command, + stderr: '', + exitCode: 0, + timedOut: false, + truncated: false, + }; + }, + }; + }, + }; +} + +const command = (workspaceId: string, text: string, detached = false): WorkspaceExecuteCommandRequest => ({ + ...request(workspaceId), + command: text, + ...(detached ? { detached: true as const } : {}), +}); + +for (const detachedFirst of [true, false]) { + test(`a detached command runs in its own executor beside its root's (detached ${detachedFirst ? 'first' : 'second'})`, async () => { + const fixture = gatedExecutors(); + const pool = new NativeWorkspaceCommandPool(roots, 2, fixture.create); + const order = detachedFirst ? ['background', 'ordinary'] : ['ordinary', 'background']; + const pending = new Map>(); + for (const text of order) { + pending.set(text, pool.execute(command('a', text, text === 'background'))); + await fixture.entered(text); + } + assert.deepEqual(fixture.executors.map((executor) => executor.root), ['/fixture/a', '/fixture/a']); + const dedicated = fixture.executors[detachedFirst ? 0 : 1]!; + const own = fixture.executors[detachedFirst ? 1 : 0]!; + assert.deepEqual(dedicated.requests, [command('a', 'background')]); + assert.equal('detached' in dedicated.requests[0]!, false); + assert.deepEqual(own.requests, [command('a', 'ordinary')]); + + fixture.release('background'); + assert.deepEqual(await pending.get('background'), { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'a', + stdout: 'background', + stderr: '', + exitCode: 0, + timedOut: false, + truncated: false, + }); + fixture.release('ordinary'); + await pending.get('ordinary'); + + fixture.release('again'); + await pool.execute(command('a', 'again')); + assert.equal(dedicated.closed, 1); + assert.equal(own.closed, 0); + assert.deepEqual(own.requests.map((req) => req.command), ['ordinary', 'again']); + assert.equal(fixture.executors.length, 2); + await pool.close(); + assert.equal(dedicated.closed, 1); + assert.equal(own.closed, 1); + }); +} + +test('a dedicated executor counts toward capacity and evicts only idle executors', async () => { + const single = gatedExecutors(); + const serial = new NativeWorkspaceCommandPool(roots, 1, single.create); + const ordinary = serial.execute(command('a', 'ordinary')); + await single.entered('ordinary'); + await assert.rejects(serial.execute(command('a', 'background', true)), (error) => { + assert.ok(error instanceof WorkspaceToolError); + assert.deepEqual(error.commandDiagnostic, { stage: 'allocation', reason: 'capacity_exhausted' }); + assert.equal(error.mutationMayHaveCommitted, false); + return true; + }); + single.release('ordinary'); + await ordinary; + await serial.close(); + + const fixture = gatedExecutors(); + const pool = new NativeWorkspaceCommandPool(roots, 2, fixture.create); + for (const root of ['a', 'b']) { + fixture.release(`warm-${root}`); + await pool.execute(command(root, `warm-${root}`)); + } + const background = pool.execute(command('b', 'background', true)); + await fixture.entered('background'); + assert.deepEqual(fixture.executors.map((executor) => [executor.root, executor.closed]), [ + ['/fixture/a', 1], + ['/fixture/b', 0], + ['/fixture/b', 0], + ]); + const busy = pool.execute(command('b', 'busy')); + await fixture.entered('busy'); + await assert.rejects(pool.execute(command('c', 'excess')), { + code: 'COMMAND_UNAVAILABLE', + mutationMayHaveCommitted: false, + }); + for (const text of ['background', 'busy']) fixture.release(text); + await Promise.all([background, busy]); + fixture.release('next'); + await pool.execute(command('c', 'next')); + assert.deepEqual(fixture.executors.map((executor) => [executor.root, executor.closed]), [ + ['/fixture/a', 1], + ['/fixture/b', 0], + ['/fixture/b', 1], + ['/fixture/c', 0], + ]); + await pool.close(); +}); + +test('a root is neither removed nor replaced while its detached command runs', async () => { + const fixture = gatedExecutors(); + const pool = new NativeWorkspaceCommandPool(roots, 2, fixture.create); + const identity = (ino: string) => ({ + workspaceRoot: '/fixture/lane', + workspaceIdentity: { path: '/fixture/lane', dev: '1', ino }, + }); + await pool.registerRoot('lane', identity('1')); + const background = pool.execute(command('lane', 'background', true)); + await fixture.entered('background'); + await assert.rejects(pool.unregisterRoot('lane'), /still executing/); + await assert.rejects(pool.registerRoot('lane', identity('2')), /changed during execution/); + await pool.registerRoot('lane', identity('1')); + fixture.release('ordinary'); + await pool.execute(command('lane', 'ordinary')); + fixture.release('background'); + await background; + await pool.unregisterRoot('lane'); + assert.deepEqual(fixture.executors.map((executor) => executor.closed), [1, 1]); + await assert.rejects(pool.execute(command('lane', 'gone', true)), /unavailable/); + await pool.close(); +}); + +test('a failed or cancelled detached command still closes its executor and spares the root', async () => { + const fixture = gatedExecutors(); + const pool = new NativeWorkspaceCommandPool(roots, 2, fixture.create); + fixture.failures.set('ambiguous', new WorkspaceToolError('executor lost', 'COMMAND_UNAVAILABLE', true, true)); + fixture.release('ambiguous'); + await assert.rejects(pool.execute(command('a', 'ambiguous', true)), { + code: 'COMMAND_UNAVAILABLE', + mutationMayHaveCommitted: true, + requiresQuarantine: true, + }); + const cancelled = new AbortController(); + cancelled.abort(); + await assert.rejects(pool.execute(command('a', 'never', true), cancelled.signal), { + code: 'EXECUTION_ABORTED', + mutationMayHaveCommitted: false, + }); + fixture.release('ordinary'); + await pool.execute(command('a', 'ordinary')); + assert.deepEqual(fixture.executors.map((executor) => [executor.requests.length, executor.closed]), [ + [1, 1], + [0, 1], + [1, 0], + ]); + await pool.close(); +}); + +test('a dedicated executor that cannot confirm cleanup is kept for eviction or shutdown', async () => { + const fixture = gatedExecutors({ failClose: (executor) => executor.requests[0]?.command === 'background' }); + const pool = new NativeWorkspaceCommandPool(roots, 2, fixture.create); + fixture.release('background'); + await pool.execute(command('a', 'background', true)); + fixture.release('ordinary'); + await pool.execute(command('a', 'ordinary')); + await assert.rejects(pool.execute(command('b', 'excess')), { + code: 'COMMAND_UNAVAILABLE', + mutationMayHaveCommitted: false, + }); + await assert.rejects(pool.close(), AggregateError); +}); + +test('detached commands are offered only by a pool with more than one lease slot', () => { + const base = { optOut: false, pooled: true, workspaceLeaseSlots: 2 }; + assert.equal(detachedCommandsEnabled(base), true); + assert.equal(detachedCommandsEnabled({ ...base, enabled: ' TRUE ' }), true); + assert.equal(detachedCommandsEnabled({ ...base, workspaceLeaseSlots: 8 }), true); + assert.equal(detachedCommandsEnabled({ ...base, workspaceLeaseSlots: 1 }), false); + assert.equal(detachedCommandsEnabled({ ...base, pooled: false }), false); + assert.equal(detachedCommandsEnabled({ ...base, optOut: true }), false); + assert.equal(detachedCommandsEnabled({ ...base, enabled: 'false' }), false); + assert.throws( + () => detachedCommandsEnabled({ ...base, enabled: 'sometimes' }), + /LIBRECHAT_CODE_DETACHED_COMMANDS must be true or false/, + ); +}); + +test('checkout commands reach the pool with detached intact only when the sandbox offers it', async () => { + const fixture = gatedExecutors(); + const pool = new NativeWorkspaceCommandPool(roots, 2, fixture.create); + const files = { + capabilities: { protocolVersion: 1 as const, operations: ['read_file' as const], workspaces: [{ id: 'a' }] }, + async execute(): Promise { + throw new Error('not a file request'); + }, + }; + const offering = new SandboxWorkspaceTools({ + workspaceTools: files, + commandSandbox: pool, + commandWorkspaces: ['a'], + commandFeatures: ['detached'], + }); + assert.deepEqual(offering.capabilities.commandFeatures, ['detached']); + const ordinary = offering.execute(command('a', 'ordinary')); + await fixture.entered('ordinary'); + fixture.release('background'); + const result = await offering.execute(command('a', 'background', true)); + assert.equal(result.operation === 'execute_command' && result.stdout, 'background'); + assert.deepEqual(fixture.executors.map((executor) => executor.requests), [ + [command('a', 'ordinary')], + [command('a', 'background')], + ]); + fixture.release('ordinary'); + await ordinary; + + const withholding = new SandboxWorkspaceTools({ + workspaceTools: files, + commandSandbox: pool, + commandWorkspaces: ['a'], + }); + assert.equal('commandFeatures' in withholding.capabilities, false); + await assert.rejects(withholding.execute(command('a', 'refused', true)), { code: 'INVALID_REQUEST' }); + assert.equal(fixture.executors.length, 2); + await pool.close(); +}); diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 60ba4b43..d7f2df63 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -15,16 +15,49 @@ interface Entry { > & Partial>; busy: boolean; + /** Set on a dedicated executor: the root whose detached command it runs. */ + root?: string; +} + +/** Dedicated executors are keyed apart from every root, whose IDs never contain NUL. */ +const DETACHED_EXECUTOR_KEY_PREFIX = '\0detached\0'; + +/** + * Whether this worker offers detached commands. On by default wherever it holds: + * a native executor pool with more than one lease slot, so a dedicated executor + * can run beside its root's own. `--no-detached-commands` or + * `LIBRECHAT_CODE_DETACHED_COMMANDS=false` opts out; any other value of that + * variable fails startup rather than silently choosing either way. + */ +export function detachedCommandsEnabled(input: { + optOut: boolean; + enabled?: string; + pooled: boolean; + workspaceLeaseSlots: number; +}): boolean { + const enabled = input.enabled?.trim().toLowerCase() ?? ''; + if (enabled !== '' && enabled !== 'true' && enabled !== 'false') { + throw new Error('LIBRECHAT_CODE_DETACHED_COMMANDS must be true or false'); + } + return ( + !input.optOut && + enabled !== 'false' && + input.pooled && + input.workspaceLeaseSlots > 1 + ); } /** Bounded persistent executor cache. Each child owns one root's SRT policy; * only idle children may be evicted, and ambiguous failures are never retried. + * A detached command runs in a dedicated executor for its root, created for the + * command and closed after it, so the root's own executor stays free for other work. */ export class NativeWorkspaceCommandPool { readonly mutationFailuresAreAtomic = true as const; private readonly entries = new Map(); private allocation: Promise = Promise.resolve(); private closing = false; + private detachedSequence = 0; constructor( roots: ReadonlyMap, private readonly capacity: number, @@ -49,15 +82,14 @@ export class NativeWorkspaceCommandPool { /** Confirm executor cleanup before a failed provisioning attempt removes its root. */ async unregisterRoot(id: string): Promise { const pending = this.allocation.then(async () => { - const entry = this.entries.get(id); - if (entry?.busy) { + const owned = this.executors(id); + if (owned.some(([, entry]) => entry.busy)) { throw new WorkspaceToolError( 'Native workspace still executing', 'COMMAND_UNAVAILABLE' ); } - if (entry) await entry.sandbox.close(); - this.entries.delete(id); + await this.closeExecutors(owned); this.roots.delete(id); }); this.allocation = pending.catch(() => undefined); @@ -94,24 +126,36 @@ export class NativeWorkspaceCommandPool { ) { return; } - const entry = this.entries.get(id); - if (entry?.busy) { + const owned = this.executors(id); + if (owned.some(([, entry]) => entry.busy)) { throw new WorkspaceToolError( 'Native workspace changed during execution', 'REGISTRATION_INVALID' ); } - if (entry) { - await entry.sandbox.close(); - this.entries.delete(id); - } + await this.closeExecutors(owned); this.roots.set(id, options); }); this.allocation = pending.catch(() => undefined); await pending; } - private allocate(root: string): Promise { + /** A root's own executor and any dedicated executor still open for it. */ + private executors(root: string): [string, Entry][] { + return [...this.entries].filter( + ([key, entry]) => key === root || entry.root === root + ); + } + + private async closeExecutors(owned: [string, Entry][]): Promise { + for (const [key, entry] of owned) { + await entry.sandbox.close(); + this.entries.delete(key); + } + } + + /** `key` defaults to the root's own executor; any other key names a dedicated one. */ + private allocate(root: string, key = root): Promise { const pending = this.allocation.then(async () => { const options = this.roots.get(root); if (this.closing || !options) @@ -119,7 +163,7 @@ export class NativeWorkspaceCommandPool { 'Native workspace unavailable', 'REGISTRATION_INVALID' ); - let entry = this.entries.get(root); + let entry = this.entries.get(key); if (entry?.busy) throw new WorkspaceToolError( 'Native workspace already executing', @@ -151,12 +195,13 @@ export class NativeWorkspaceCommandPool { entry = { sandbox: this.createSandbox(options), busy: false, + ...(key === root ? {} : { root }), }; } entry.busy = true; // Map insertion order is the idle eviction order. - this.entries.delete(root); - this.entries.set(root, entry); + this.entries.delete(key); + this.entries.set(key, entry); return entry; }); const checked = pending.catch((error) => { @@ -199,10 +244,44 @@ export class NativeWorkspaceCommandPool { ); } + /** + * The executor never sees `detached`: it is this pool's routing decision. The + * result is returned as soon as the command ends; closing the executor queues + * ahead of later allocations, which therefore still find capacity accounted for. + */ + private async executeDetached( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal + ): Promise { + const { detached: _detached, ...command } = request; + const key = `${DETACHED_EXECUTOR_KEY_PREFIX}${++this.detachedSequence}`; + const entry = await this.allocate(request.workspaceId, key); + try { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Command cancelled before dispatch', + 'EXECUTION_ABORTED' + ); + return await entry.sandbox.execute(command, signal); + } finally { + this.allocation = this.allocation.then(async () => { + try { + await entry.sandbox.close(); + this.entries.delete(key); + } catch { + /* Retain ownership for subsequent cleanup/shutdown. */ + } finally { + entry.busy = false; + } + }); + } + } + async execute( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal ): Promise { + if (request.detached === true) return this.executeDetached(request, signal); const entry = await this.allocate(request.workspaceId); let enteredExecutor = false; try { diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 25c00cf3..b6a12fe3 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -11,7 +11,9 @@ import { isWorkspaceToolErrorCode, isWorkspaceToolRequest, isWorkspaceToolResult, + isWorkspaceDetachedCommandKey, WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, + workspaceDetachedCommandKey, workspaceIsolationKey, workspaceIsolationKeysConflict, workspaceIsolationParent, @@ -1048,3 +1050,70 @@ test('workspace capabilities advertise linked-worktree scopes exactly', () => { assert.equal(isValidBridgeWorkerCapabilities(capabilities([])), false); assert.equal(isValidBridgeWorkerCapabilities(capabilities('git_linked_worktree')), false); }); + +test('command requests accept detached only as true and never with an environment action', () => { + const base = { protocolVersion: 1, operation: 'execute_command', workspaceId: 'repo', command: 'npm test' }; + assert.equal(isWorkspaceToolRequest(base), true); + assert.equal(isWorkspaceToolRequest({ ...base, detached: true }), true); + assert.equal(isWorkspaceToolRequest({ ...base, detached: true, worktree: 'task-a', timeoutMs: 90_000 }), true); + assert.equal(isWorkspaceToolRequest({ ...base, detached: true, workspaceInstanceId: 'a'.repeat(64) }), true); + for (const detached of [false, 'true', 1, 0, null, {}, []]) { + assert.equal(isWorkspaceToolRequest({ ...base, detached }), false, JSON.stringify(detached)); + } + const environmentAction = { name: 'test', fingerprint: 'f'.repeat(64) }; + assert.equal(isWorkspaceToolRequest({ ...base, environmentAction }), true); + assert.equal(isWorkspaceToolRequest({ ...base, environmentAction, detached: true }), false); + assert.equal( + isWorkspaceToolRequest({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md', detached: true }), + false, + ); +}); + +test('workspace capabilities advertise the detached command feature exactly', () => { + const capabilities = (operations: string[], commandFeatures: unknown) => ({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations, + workspaces: [{ id: 'repo' }], + commandFeatures, + }, + }); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['execute_command'], ['detached'])), true); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['read_file', 'execute_command'], undefined)), true); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['read_file'], ['detached'])), false); + for (const commandFeatures of [[], ['lane_git'], ['detached', 'detached'], 'detached', null, [true]]) { + assert.equal( + isValidBridgeWorkerCapabilities(capabilities(['execute_command'], commandFeatures)), + false, + JSON.stringify(commandFeatures), + ); + } +}); + +test('a detached command key never equals or nests with a checkout or lane key', () => { + const instanceId = 'a'.repeat(64); + const detached = workspaceDetachedCommandKey('assignment-1'); + const others = [ + 'repo', + 'assignment-1', + workspaceIsolationKey('repo', instanceId), + workspaceIsolationKey('repo', undefined, 'task-a'), + workspaceIsolationKey('repo', instanceId, 'task-a'), + workspaceIsolationKey('repo', undefined, 'assignment-1'), + workspaceDetachedCommandKey('assignment-2'), + ]; + assert.equal(isWorkspaceDetachedCommandKey(detached), true); + assert.equal(workspaceIsolationParent(detached), undefined); + assert.equal(workspaceIsolationKeysConflict(detached, detached), true); + for (const other of others) { + assert.notEqual(detached, other); + assert.equal(workspaceIsolationKeysConflict(detached, other), false, JSON.stringify(other)); + assert.equal(workspaceIsolationKeysConflict(other, detached), false, JSON.stringify(other)); + } + for (const other of others.slice(0, -1)) { + assert.equal(isWorkspaceDetachedCommandKey(other), false, JSON.stringify(other)); + } +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 0fcb6090..89dee672 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -312,6 +312,18 @@ export function workspaceIsolationKeysConflict(left: string, right: string): boo ); } +const DETACHED_COMMAND_KEY_PREFIX = '\0detached-command\0'; + +/** Scheduling identity of one detached command. Unique per assignment, it never equals + * or nests with a checkout or lane key, so it neither waits for nor holds a worktree. */ +export function workspaceDetachedCommandKey(assignmentId: string): string { + return `${DETACHED_COMMAND_KEY_PREFIX}${assignmentId}`; +} + +export function isWorkspaceDetachedCommandKey(key: string): boolean { + return key.startsWith(DETACHED_COMMAND_KEY_PREFIX); +} + export type BridgeWorkspaceToolOperation = | 'read_file' | 'search_text' @@ -382,6 +394,8 @@ export function boundedHead(value: unknown): string | null { export type WorkspaceListFileFeature = 'after_path'; /** `lane_git`: `execute_command` results may carry `laneGit`. Sent only when Code API echoes it. */ export type WorkspaceCommandResultFeature = 'lane_git'; +/** `detached`: `execute_command` requests may carry `detached`. Sent only when Code API offers it. */ +export type WorkspaceCommandFeature = 'detached'; /** Git state of the lane a command ran in. Never carries paths, remotes or repository content. */ export interface WorkspaceLaneGit { @@ -438,6 +452,8 @@ export interface BridgeWorkspaceToolCapabilities { listFileFeatures?: WorkspaceListFileFeature[]; /** Omitted by workers that do not report lane Git state on command results. */ commandResultFeatures?: WorkspaceCommandResultFeature[]; + /** Omitted by workers that cannot run a command without holding its worktree. */ + commandFeatures?: WorkspaceCommandFeature[]; /** Languages that can execute PTC replay inside a selected workspace. */ programmaticLanguages?: WorkspaceProgrammaticLanguage[]; } @@ -677,6 +693,9 @@ export interface WorkspaceExecuteCommandRequest { /** Aggregate UTF-8 stdout and stderr budget. */ maxOutputBytes?: number; environmentAction?: { name: string; fingerprint: string }; + /** Occupies a worker slot without holding its worktree: other operations there are + * admitted while it runs. Sent only to workers advertising the `detached` command feature. */ + detached?: true; } export interface WorkspaceExecuteCommandResult { @@ -839,6 +858,7 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'cwd', 'timeoutMs', 'maxOutputBytes', + 'detached', ]); const WORKSPACE_IMPORT_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -979,6 +999,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceScopes?: ['git_linked_worktree']; /** Command result fields this Code API accepts in settlements. */ supportedWorkspaceCommandResultFeatures?: WorkspaceCommandResultFeature[]; + /** Command request features this Code API can schedule for a capability-aware worker. */ + supportedWorkspaceCommandFeatures?: WorkspaceCommandFeature[]; /** Added workspace tool error codes this Code API accepts in settlements. */ supportedWorkspaceToolErrorCodes?: WorkspaceToolErrorCode[]; } @@ -1734,6 +1756,8 @@ export function isWorkspaceToolRequest( .fingerprint, ))) && hasOnlyKeys(request, WORKSPACE_COMMAND_REQUEST_KEYS) && + (request.detached === undefined || + (request.detached === true && request.environmentAction === undefined)) && typeof request.command === 'string' && request.command.trim().length > 0 && Buffer.from(request.command).toString('utf8') === request.command && @@ -2105,6 +2129,16 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.commandFeatures !== undefined && + (!Array.isArray(capabilities.commandFeatures) || + capabilities.commandFeatures.length !== 1 || + !capabilities.operations.includes('execute_command') || + capabilities.commandFeatures[0] !== 'detached') + ) { + return false; + } + if ( capabilities.programmaticLanguages !== undefined && (!Array.isArray(capabilities.programmaticLanguages) || diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 578f9fb6..cd436ffd 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -5,6 +5,7 @@ import { lstat, mkdir, open, + readdir, rename, rm, stat, @@ -612,6 +613,68 @@ export async function loadWorkspaceMutationQuarantine( return record; } +const DETACHED_QUARANTINE_ASSIGNMENT_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; + +/** Beside a workspace's own marker: one marker per detached command running in it. */ +export function detachedWorkspaceMutationQuarantineDirectory(path: string): string { + return `${path}.detached`; +} + +/** A detached command arms this marker, never the workspace's own, which an ordinary mutation may hold. */ +export function detachedWorkspaceMutationQuarantinePath( + path: string, + assignmentId: string, +): string { + if (!DETACHED_QUARANTINE_ASSIGNMENT_PATTERN.test(assignmentId)) { + throw new BridgeProtocolError( + 'Detached command quarantine requires a filename-safe assignment ID', + ); + } + return join(detachedWorkspaceMutationQuarantineDirectory(path), `${assignmentId}.json`); +} + +async function detachedWorkspaceMutationQuarantinePaths(path: string): Promise { + const directory = detachedWorkspaceMutationQuarantineDirectory(path); + let names: string[]; + try { + names = await readdir(directory); + } catch (error) { + if (isMissingPathError(error)) return []; + throw error; + } + /* An emptied directory is evidence too: another account able to write it could have removed a marker. */ + await assertReadPathPrivate(directory); + return names.map((name) => join(directory, name)); +} + +/** The workspace's own marker followed by the marker of every detached command in it. */ +export async function loadWorkspaceMutationQuarantines( + path: string, +): Promise { + assertPrivateStorageSupported(); + const [own, detachedPaths] = await Promise.all([ + loadWorkspaceMutationQuarantine(path), + detachedWorkspaceMutationQuarantinePaths(path), + ]); + const detached = await Promise.all(detachedPaths.map(loadWorkspaceMutationQuarantine)); + return [own, ...detached].filter( + (record): record is WorkspaceMutationQuarantineRecord => record != null, + ); +} + +async function clearDetachedWorkspaceMutationQuarantines(path: string): Promise { + const directory = detachedWorkspaceMutationQuarantineDirectory(path); + try { + await lstat(directory); + } catch (error) { + if (isMissingPathError(error)) return; + throw error; + } + await rm(directory, { recursive: true, force: true }); + await syncParentDirectory(directory); +} + +/** Without an owner this is an operator's reset, which also removes every detached command marker. */ export async function clearWorkspaceMutationQuarantine( path: string, ownerId?: string, @@ -625,6 +688,8 @@ export async function clearWorkspaceMutationQuarantine( 'Workspace quarantine is owned by another worker incarnation', ); } + } else { + await clearDetachedWorkspaceMutationQuarantines(path); } try { await lstat(path); diff --git a/packages/code/src/worker-detached.test.ts b/packages/code/src/worker-detached.test.ts new file mode 100644 index 00000000..e93284a5 --- /dev/null +++ b/packages/code/src/worker-detached.test.ts @@ -0,0 +1,496 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readdir, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; +import { workspaceIsolationKey } from './protocol.js'; +import { + clearWorkspaceMutationQuarantine, + detachedWorkspaceMutationQuarantineDirectory, +} from './storage.js'; +import { workspaceMutationGuard } from './workspace-guards.js'; +import type { TestContext } from 'node:test'; +import type { + BridgeAssignment, + BridgeWorkspaceToolCapabilities, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; + +const lane = 'task-a'; +const laneKey = workspaceIsolationKey('repo', undefined, lane); +const capabilities: BridgeWorkspaceToolCapabilities = { + protocolVersion: 1, + operations: ['read_file', 'execute_command'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }], + commandFeatures: ['detached'], +}; + +interface Gate { + started: Promise; + markStarted(): void; + finished: Promise; + release(): void; +} + +interface Call { + action: string; + assignmentId?: string; + body: Record; +} + +interface Harness { + worker: BridgeWorker; + calls: Call[]; + executed: string[]; + errors: unknown[]; + failures: Map; + leases: BridgeAssignment[]; + unacknowledged: Set; + marker(worktree?: string): string; + detachedMarkers(worktree?: string): Promise; + started(command: string): Promise; + release(command: string): void; + quarantined(): Set; +} + +async function scratch(t: TestContext): Promise { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-detached-')); + t.after(() => rm(directory, { recursive: true, force: true })); + return directory; +} + +/** A regression must fail its test, not leave it waiting on a command that never runs. */ +async function within(pending: Promise, expectation: string): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + pending, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(`Timed out waiting for ${expectation}`)), 5_000); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +function harness( + t: TestContext, + directory: string, + options: { incarnationId?: string; offered?: boolean } = {}, +): Harness { + const incarnationId = options.incarnationId ?? 'incarnation-detached'; + const gates = new Map(); + let finished = false; + t.after(() => { + finished = true; + for (const running of gates.values()) running.release(); + }); + const gate = (command: string): Gate => { + let existing = gates.get(command); + if (existing == null) { + let markStarted!: () => void; + let release!: () => void; + existing = { + started: new Promise((resolve) => { markStarted = resolve; }), + markStarted: () => markStarted(), + finished: new Promise((resolve) => { release = resolve; }), + release: () => release(), + }; + gates.set(command, existing); + } + return existing; + }; + const calls: Call[] = []; + const executed: string[] = []; + const errors: unknown[] = []; + const failures = new Map(); + const leases: BridgeAssignment[] = []; + const unacknowledged = new Set(); + const marker = (worktree?: string): string => + join(directory, worktree == null ? 'repo.json' : `lane-${worktree}.json`); + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId, + sandboxEndpoint: 'http://localhost:2', + reconnectDelayMs: 1, + onError: (error) => { + errors.push(error); + }, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 3, + requiresReadyConfirmation: true, + workspaceTools: capabilities, + }, + workspaceTools: { + capabilities, + mutationFailuresAreAtomic: true, + async execute(request: WorkspaceToolRequest): Promise { + if (request.operation !== 'execute_command') throw new Error('unexpected operation'); + executed.push(request.command); + const running = gate(request.command); + running.markStarted(); + if (!finished) await running.finished; + const failure = failures.get(request.command); + if (failure != null) throw failure; + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: request.command, + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + workspaceQuarantines: new Map([ + ['repo', workspaceMutationGuard(marker(), 'worker', 'repo', incarnationId)], + ]), + linkedWorktreeQuarantineResolver: (workspaceId, worktree) => + workspaceMutationGuard( + marker(worktree), + 'worker', + workspaceIsolationKey(workspaceId, undefined, worktree), + incarnationId, + ), + linkedWorktreeNames: async () => [lane], + fetchImpl: async (url, init) => { + const path = new URL(String(url)).pathname; + const body = JSON.parse(String(init?.body ?? '{}')) as Record; + const assignment = /\/assignments\/([^/]+)\/([^/]+)$/.exec(path); + const action = assignment?.[2] ?? path.slice(path.lastIndexOf('/') + 1); + const assignmentId = assignment == null ? undefined : decodeURIComponent(assignment[1]!); + calls.push({ action, assignmentId, body }); + if (action === 'register') { + return Response.json({ + protocolVersion: 1, + workerId: 'worker', + incarnationId, + registrationGeneration: 1, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + workspaceLeaseSlots: 3, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + supportedWorkspaceScopes: ['git_linked_worktree'], + ...(options.offered === false ? {} : { supportedWorkspaceCommandFeatures: ['detached'] }), + }); + } + if (action === 'ready') return Response.json({ protocolVersion: 1, ready: true }); + if (action === 'cancellation') return Response.json({ protocolVersion: 1, cancelled: false }); + if (action === 'workspace-cleanup' && assignmentId != null && unacknowledged.has(assignmentId)) { + return Response.json({ error: 'receipt outage' }, { status: 503 }); + } + if (action === 'lease') { + const index = leases.findIndex((lease) => lease.workspaceLeaseSlot === body.workspaceLeaseSlot); + if (index >= 0) { + return Response.json({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment: leases.splice(index, 1)[0], + }); + } + await new Promise((resolve) => { + const timer = setTimeout(resolve, 10); + init?.signal?.addEventListener('abort', () => { clearTimeout(timer); resolve(); }, { once: true }); + }); + return Response.json({ protocolVersion: 1, serverElapsedMs: 0 }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + return { + worker, + calls, + executed, + errors, + failures, + leases, + unacknowledged, + marker, + async detachedMarkers(worktree) { + try { + return await readdir(detachedWorkspaceMutationQuarantineDirectory(marker(worktree))); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + }, + started: (command) => within(gate(command).started, `${command} to start`), + release: (command) => gate(command).release(), + quarantined: () => + (worker as unknown as { quarantinedWorkspaces: Set }).quarantinedWorkspaces, + }; +} + +function assignment( + assignmentId: string, + slot: number, + request: Partial = {}, + incarnationId = 'incarnation-detached', +): BridgeAssignment { + return { + protocolVersion: 1, + assignmentId, + workerId: 'worker', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + remainingMs: 10_000, + executionKind: 'workspace_tool', + workspaceLeaseSlot: slot, + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'repo', + worktree: lane, + command: assignmentId, + ...request, + }, + }; +} + +const exists = async (path: string): Promise => + stat(path).then(() => true, () => false); + +const settlements = (calls: Call[], assignmentId: string): Record[] => + calls + .filter((call) => call.action === 'settle' && call.assignmentId === assignmentId) + .map((call) => call.body); + +for (const detachedFirst of [true, false]) { + test(`a detached command overlaps ordinary work in its lane and checkout (detached ${detachedFirst ? 'first' : 'second'})`, async (t) => { + const fixture = harness(t, await scratch(t)); + await fixture.worker.register(); + const run = (id: string, slot: number, request?: Partial) => + fixture.worker.executeAndSettle(assignment(id, slot, request)); + const order = detachedFirst ? ['background', 'ordinary'] : ['ordinary', 'background']; + const pending = new Map>(); + for (const [slot, id] of order.entries()) { + pending.set(id, run(id, slot, id === 'background' ? { detached: true } : {})); + await fixture.started(id); + } + assert.deepEqual(fixture.executed, order); + assert.equal(await exists(fixture.marker(lane)), true); + assert.deepEqual(await fixture.detachedMarkers(lane), ['background.json']); + + fixture.release('ordinary'); + await pending.get('ordinary'); + assert.equal(await exists(fixture.marker(lane)), false); + assert.deepEqual(await fixture.detachedMarkers(lane), ['background.json']); + + const next = run('next', detachedFirst ? 1 : 0); + await fixture.started('next'); + fixture.release('next'); + await next; + const checkout = run('checkout', detachedFirst ? 1 : 0, { worktree: undefined }); + await fixture.started('checkout'); + assert.equal(await exists(fixture.marker()), true); + fixture.release('checkout'); + await checkout; + + fixture.release('background'); + await pending.get('background'); + assert.deepEqual(await fixture.detachedMarkers(lane), []); + assert.equal(await exists(fixture.marker()), false); + assert.equal(fixture.quarantined().size, 0); + for (const id of ['background', 'ordinary', 'next', 'checkout']) { + assert.deepEqual( + settlements(fixture.calls, id).map((body) => [body.status, (body.result as { stdout: string }).stdout]), + [['fulfilled', id]], + ); + assert.equal( + fixture.calls.filter((call) => call.action === 'workspace-cleanup' && call.assignmentId === id).length, + 1, + ); + } + assert.equal(fixture.calls.some((call) => call.action === 'quarantine'), false); + }); +} + +test('a detached marker left by another incarnation quarantines the worktree until an operator clears it', async (t) => { + const directory = await scratch(t); + const previous = harness(t, directory, { incarnationId: 'incarnation-previous' }); + const crashed = previous.worker.executeAndSettle( + assignment('crashed', 0, { detached: true }, 'incarnation-previous'), + ); + await previous.started('crashed'); + assert.deepEqual(await previous.detachedMarkers(lane), ['crashed.json']); + assert.equal(await exists(previous.marker(lane)), false); + + const restarted = harness(t, directory); + await restarted.worker.register(); + for (const [slot, request] of [{ worktree: undefined }, {}, { detached: true as const }].entries()) { + await assert.rejects( + restarted.worker.executeAndSettle(assignment(`refused-${slot}`, slot, request)), + BridgeWorkspaceQuarantinedError, + ); + } + assert.deepEqual(restarted.executed, []); + assert.deepEqual([...restarted.quarantined()].sort(), ['repo', laneKey].sort()); + + const maintenance = harness(t, directory, { incarnationId: 'incarnation-maintenance' }); + await assert.rejects( + maintenance.worker.resetNativeWorkspace('repo', undefined, undefined, lane), + { code: 'WORKSPACE_QUARANTINED' }, + ); + assert.equal(maintenance.calls.some((call) => call.action === 'reset'), false); + await clearWorkspaceMutationQuarantine(previous.marker(lane)); + assert.equal(await exists(detachedWorkspaceMutationQuarantineDirectory(previous.marker(lane))), false); + await maintenance.worker.resetNativeWorkspace('repo', undefined, undefined, lane); + assert.deepEqual( + maintenance.calls.filter((call) => call.action === 'reset').map((call) => call.body.runtimeSessionId), + [`native-workspace:${laneKey}`], + ); + + const recovered = harness(t, directory, { incarnationId: 'incarnation-recovered' }); + await recovered.worker.register(); + const ordinary = recovered.worker.executeAndSettle( + assignment('recovered', 0, {}, 'incarnation-recovered'), + ); + await recovered.started('recovered'); + recovered.release('recovered'); + await ordinary; + assert.deepEqual(recovered.executed, ['recovered']); + + previous.release('crashed'); + await crashed.catch(() => undefined); +}); + +test('an ambiguous detached failure quarantines its worktree, keeps its marker and reports the assignment', async (t) => { + const fixture = harness(t, await scratch(t)); + fixture.failures.set('background', new Error('executor state unknown')); + fixture.leases.push(assignment('background', 0, { detached: true })); + const controller = new AbortController(); + const running = fixture.worker.run(controller.signal); + const reported = async (assignmentId: string): Promise => { + const deadline = Date.now() + 5_000; + while (!fixture.calls.some((call) => call.action === 'quarantine' && call.assignmentId === assignmentId)) { + assert.ok(Date.now() < deadline, `Timed out waiting for the quarantine receipt of ${assignmentId}`); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }; + try { + await fixture.started('background'); + assert.deepEqual(await fixture.detachedMarkers(lane), ['background.json']); + fixture.release('background'); + await reported('background'); + assert.deepEqual(await fixture.detachedMarkers(lane), ['background.json']); + assert.equal(await exists(fixture.marker(lane)), false); + assert.deepEqual([...fixture.quarantined()], [laneKey]); + assert.deepEqual(settlements(fixture.calls, 'background'), []); + + fixture.leases.push(assignment('ordinary', 1)); + await reported('ordinary'); + fixture.leases.push(assignment('checkout', 2, { worktree: undefined })); + await reported('checkout'); + assert.deepEqual(fixture.executed, ['background']); + assert.deepEqual( + fixture.errors.map((error) => [(error as Error).name, (error as Error).message]), + [ + ['Error', 'executor state unknown'], + [ + 'BridgeWorkspaceQuarantinedError', + 'Worker stopped after a workspace mutation completed without a fulfilled settlement', + ], + ['BridgeWorkspaceQuarantinedError', 'Workspace is quarantined'], + ['BridgeWorkspaceQuarantinedError', 'Workspace is quarantined'], + ], + ); + } finally { + controller.abort(); + await running; + } +}); + +test('a lost cleanup acknowledgement for a detached command leaves its worktree usable', async (t) => { + const fixture = harness(t, await scratch(t)); + await fixture.worker.register(); + fixture.unacknowledged.add('background'); + const background = fixture.worker.executeAndSettle(assignment('background', 0, { detached: true })); + await fixture.started('background'); + fixture.release('background'); + await assert.rejects(background, (error: unknown) => + error instanceof BridgeWorkspaceQuarantinedError && error.acknowledgementOnly, + ); + assert.deepEqual(await fixture.detachedMarkers(lane), []); + assert.equal(fixture.quarantined().size, 0); + assert.equal(settlements(fixture.calls, 'background')[0]?.status, 'fulfilled'); + + fixture.unacknowledged.add('ordinary'); + const ordinary = fixture.worker.executeAndSettle(assignment('ordinary', 1)); + await fixture.started('ordinary'); + fixture.release('ordinary'); + await assert.rejects(ordinary, BridgeWorkspaceQuarantinedError); + assert.deepEqual([...fixture.quarantined()], [laneKey]); +}); + +test('a detached request is refused unless Code API negotiated the feature', async (t) => { + const withheld = harness(t, await scratch(t), { offered: false }); + await withheld.worker.register(); + const registrations = (calls: Call[]): unknown[] => + calls + .filter((call) => call.action === 'register') + .map((call) => (call.body.capabilities as { workspaceTools?: BridgeWorkspaceToolCapabilities }) + .workspaceTools?.commandFeatures); + assert.deepEqual(registrations(withheld.calls).filter((features) => features !== undefined), []); + await withheld.worker.executeAndSettle(assignment('background', 0, { detached: true })); + assert.deepEqual(withheld.executed, []); + assert.deepEqual(await withheld.detachedMarkers(lane), []); + assert.equal(withheld.quarantined().size, 0); + assert.deepEqual( + settlements(withheld.calls, 'background').map((body) => [body.status, body.error]), + [['rejected', 'Workspace command feature is not advertised']], + ); + + const offered = harness(t, await scratch(t)); + await offered.worker.register(); + assert.deepEqual(registrations(offered.calls).at(-1), ['detached']); +}); + +test('detached commands cannot be advertised with a single lease slot', () => { + const capabilities: BridgeWorkspaceToolCapabilities = { + protocolVersion: 1, + operations: ['execute_command'], + workspaces: [{ id: 'repo' }], + commandFeatures: ['detached'], + }; + assert.throws( + () => + new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: capabilities, + }, + workspaceTools: { + capabilities, + async execute() { + throw new Error('must not execute'); + }, + }, + workspaceMutationQuarantine: { + async assertAvailable() {}, + async arm() {}, + async clear() {}, + async quarantine() {}, + }, + }), + /Detached commands require at least two lease slots/, + ); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index fb7bb1ed..2f2f4efd 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -9,6 +9,7 @@ import { isValidLinkedWorktreeName, isWorkspaceMutationOperation, isWorkspaceToolResult, + workspaceDetachedCommandKey, workspaceIsolationKey, workspaceIsolationKeysConflict, workspaceIsolationParent, @@ -109,7 +110,11 @@ export interface BridgeWorkerOptions { } export interface WorkspaceMutationQuarantine { - assertAvailable(): Promise; + /** + * `isLive` names assignments still running in this worker. A marker one of them + * owns is in flight, not left behind, and does not make the workspace unavailable. + */ + assertAvailable(isLive?: (assignmentId: string) => boolean): Promise; arm(reason: string, assignmentId?: string): Promise; clear(assignmentId?: string): Promise; quarantine( @@ -117,6 +122,12 @@ export interface WorkspaceMutationQuarantine { cause?: unknown, assignmentId?: string, ): Promise; + /** + * Markers of detached commands, one per assignment and apart from the workspace's + * own, which an ordinary mutation may hold at the same time. `assertAvailable` + * covers them too. + */ + detached?: Pick; } export interface BridgeWorkerIdentity { @@ -234,6 +245,11 @@ function workspaceCapabilitiesMatch( (feature, index) => feature === executor.commandResultFeatures?.[index], ) ?? executor.commandResultFeatures == null) && + advertised.commandFeatures?.length === executor.commandFeatures?.length && + (advertised.commandFeatures?.every( + (feature, index) => feature === executor.commandFeatures?.[index], + ) ?? + executor.commandFeatures == null) && advertised.programmaticLanguages?.length === executor.programmaticLanguages?.length && (advertised.programmaticLanguages?.every( @@ -331,6 +347,7 @@ function registrationCompatibleCapabilities( editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, commandResultFeatures: _commandResultFeatures, + commandFeatures: _commandFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleWorkspaceTools } = workspaceTools; @@ -429,6 +446,9 @@ function supportedWorkspaceCapabilities( const commandResultFeatures = desired.commandResultFeatures?.filter((feature) => registration.supportedWorkspaceCommandResultFeatures?.includes(feature), ); + const commandFeatures = desired.commandFeatures?.filter((feature) => + registration.supportedWorkspaceCommandFeatures?.includes(feature), + ); const programmaticLanguages = desired.programmaticLanguages?.filter( (language) => registration.supportedWorkspaceProgrammaticLanguages?.includes(language), @@ -439,6 +459,7 @@ function supportedWorkspaceCapabilities( editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, commandResultFeatures: _commandResultFeatures, + commandFeatures: _commandFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleDesired } = desired; @@ -463,6 +484,9 @@ function supportedWorkspaceCapabilities( ...(operations.includes('execute_command') && commandResultFeatures?.length ? { commandResultFeatures } : {}), + ...(operations.includes('execute_command') && commandFeatures?.length + ? { commandFeatures } + : {}), ...(operations.includes('execute_command') && programmaticLanguages?.length ? { programmaticLanguages } @@ -475,6 +499,8 @@ export class BridgeWorkspaceQuarantinedError extends Error { constructor( message: string, public readonly cause?: unknown, + /** Only an acknowledgement to Code API failed: the assignment left no durable marker behind. */ + public readonly acknowledgementOnly = false, ) { super(message); this.name = 'BridgeWorkspaceQuarantinedError'; @@ -508,6 +534,9 @@ export class BridgeWorker { string, { id: string; done: Promise } >(); + private readonly activeAssignmentIds = new Set(); + private readonly isLiveAssignment = (assignmentId: string): boolean => + this.activeAssignmentIds.has(assignmentId); private readonly armedWorkspaces = new Set(); private negotiatedWorkspaceSlots = 1; private concurrentRunning = false; @@ -605,6 +634,14 @@ export class BridgeWorker { 'Linked worktree lanes require a durable quarantine resolver and at least two lease slots', ); } + if ( + options.capabilities.workspaceTools?.commandFeatures?.includes('detached') === true && + (options.capabilities.workspaceLeaseSlots ?? 1) < 2 + ) { + throw new BridgeProtocolError( + 'Detached commands require at least two lease slots', + ); + } if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { if ( options.capabilities.requiresReadyConfirmation !== true || @@ -1454,7 +1491,12 @@ export class BridgeWorker { assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { - const root = this.assignmentWorkspaceId(assignment); + const worktree = this.assignmentWorkspaceId(assignment); + const detached = worktree != null && this.assignmentDetached(assignment); + /** A detached command occupies only itself: it neither waits for its worktree nor holds it. */ + const root = detached + ? workspaceDetachedCommandKey(assignment.assignmentId) + : worktree; const waitingAt = Date.now(); for ( let active = root == null ? undefined : this.conflictingActiveAssignment(root); @@ -1498,13 +1540,15 @@ export class BridgeWorker { } } let release!: () => void; - if (root != null) + if (root != null) { this.activeWorkspaceAssignments.set(root, { id: assignment.assignmentId, done: new Promise((resolve) => { release = resolve; }), }); + this.activeAssignmentIds.add(assignment.assignmentId); + } const adjusted = assignment.remainingMs === undefined ? assignment @@ -1518,15 +1562,21 @@ export class BridgeWorker { try { await this.executeOwned(adjusted, signal); } catch (error) { - if (root != null && error instanceof BridgeWorkspaceQuarantinedError) { + if ( + worktree != null && + error instanceof BridgeWorkspaceQuarantinedError && + !(detached && error.acknowledgementOnly) + ) { // A failed unlink/fsync may have removed the durable marker already. // Fence locally before releasing the handoff to the next root assignment. - this.quarantinedWorkspaces.add(root); + // A detached command fences the worktree it ran in, which it never occupied. + this.quarantinedWorkspaces.add(worktree); } throw error; } finally { if (root != null) { this.activeWorkspaceAssignments.delete(root); + this.activeAssignmentIds.delete(assignment.assignmentId); release(); } } @@ -1559,7 +1609,9 @@ export class BridgeWorker { throw new Error('Parent workspace requires an explicit quarantine reset'); } if (workspaceId != null && parent === workspaceId) { - await this.options.workspaceQuarantines?.get(workspaceId)?.assertAvailable(); + await this.options.workspaceQuarantines + ?.get(workspaceId) + ?.assertAvailable(this.isLiveAssignment); } return; } @@ -1572,7 +1624,9 @@ export class BridgeWorker { } const names = (await this.options.linkedWorktreeNames?.(workspaceId)) ?? []; await Promise.all( - names.map(async (name) => (await resolveLaneGuard(workspaceId, name)).assertAvailable()), + names.map(async (name) => + (await resolveLaneGuard(workspaceId, name)).assertAvailable(this.isLiveAssignment), + ), ); } @@ -1613,6 +1667,15 @@ export class BridgeWorker { ); } + private assignmentDetached(assignment: BridgeAssignment): boolean { + return ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + assignment.request.operation === 'execute_command' && + assignment.request.detached === true + ); + } + private assignmentWorktree(assignment: BridgeAssignment): string | undefined { if ( assignment.executionKind === 'workspace_tool' && @@ -1674,6 +1737,10 @@ export class BridgeWorker { unresolvedGuard instanceof Promise ? await unresolvedGuard : unresolvedGuard; + const detached = this.assignmentDetached(assignment); + /** A detached command arms its own marker, never the one its worktree's mutations use. */ + const mutationGuard = detached ? guard?.detached : guard; + let armedWorkspace: string | undefined; if (signal?.aborted === true) { throw signal.reason instanceof Error ? signal.reason @@ -1797,7 +1864,7 @@ export class BridgeWorker { this.options.workspaceQuarantineResolver != null || this.options.linkedWorktreeQuarantineResolver != null ) { - await guard?.assertAvailable(); + await guard?.assertAvailable(this.isLiveAssignment); await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { @@ -1898,14 +1965,27 @@ export class BridgeWorker { 'Workspace listing feature is not advertised', ); } + if (detached && !advertised.commandFeatures?.includes('detached')) { + throw new BridgeProtocolError( + 'Workspace command feature is not advertised', + ); + } + if (detached && mutationGuard == null) { + throw new BridgeProtocolError( + 'Detached commands require a per-command durable guard', + ); + } const isMutation = isWorkspaceMutationOperation( workspaceRequest.operation, ); if (isMutation) { this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceKey); - await guard!.arm( + armedWorkspace = detached + ? workspaceDetachedCommandKey(assignment.assignmentId) + : workspaceKey; + this.armedWorkspaces.add(armedWorkspace); + await mutationGuard!.arm( `Workspace mutation ${workspaceRequest.operation} is pending settlement`, assignment.assignmentId, ); @@ -1978,7 +2058,7 @@ export class BridgeWorker { this.options.workspaceQuarantineResolver != null || this.options.linkedWorktreeQuarantineResolver != null ) { - await guard?.assertAvailable(); + await guard?.assertAvailable(this.isLiveAssignment); await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { @@ -2020,7 +2100,8 @@ export class BridgeWorker { } this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceKey); + armedWorkspace = workspaceKey; + this.armedWorkspaces.add(armedWorkspace); await guard!.arm( 'Workspace programmatic execution is pending settlement', assignment.assignmentId, @@ -2308,12 +2389,12 @@ export class BridgeWorker { if (workspaceMutationArmed) { try { if (assignment.workspaceLeaseSlot === undefined) { - await guard!.clear(assignment.assignmentId); + await mutationGuard!.clear(assignment.assignmentId); } else { let timer!: ReturnType; try { await Promise.race([ - guard!.clear(assignment.assignmentId), + mutationGuard!.clear(assignment.assignmentId), new Promise((_resolve, reject) => { timer = setTimeout( () => @@ -2332,8 +2413,7 @@ export class BridgeWorker { clearTimeout(timer); } } - const workspaceId = this.assignmentWorkspaceId(assignment); - if (workspaceId != null) this.armedWorkspaces.delete(workspaceId); + if (armedWorkspace != null) this.armedWorkspaces.delete(armedWorkspace); this.mutationGuardArmed = false; } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -2350,6 +2430,7 @@ export class BridgeWorker { throw new BridgeWorkspaceQuarantinedError( 'Workspace cleanup acknowledgement could not be confirmed', error, + true, ); } } @@ -2501,7 +2582,11 @@ export class BridgeWorker { unresolvedGuard instanceof Promise ? await unresolvedGuard : unresolvedGuard; - await guard?.quarantine(message, cause, assignment?.assignmentId); + const mutationGuard = + assignment != null && this.assignmentDetached(assignment) + ? guard?.detached + : guard; + await mutationGuard?.quarantine(message, cause, assignment?.assignmentId); return new BridgeWorkspaceQuarantinedError(message, cause); } catch (error) { return new BridgeWorkspaceQuarantinedError( @@ -2595,6 +2680,7 @@ export class BridgeWorker { throw new BridgeWorkspaceQuarantinedError( 'Unexecuted workspace cleanup acknowledgement could not be confirmed', error, + true, ); } } diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 52743bae..3874cbb5 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -11,6 +11,8 @@ import test from 'node:test'; import { defaultWorkspaceQuarantinePath, defaultWorkspacePath, + detachedWorkspaceMutationQuarantineDirectory, + detachedWorkspaceMutationQuarantinePath, saveWorkspaceMutationQuarantine, } from './storage.js'; @@ -408,6 +410,95 @@ test('CLI refuses quarantined mutation registration until an operator clears it' } }); +test('CLI refuses a detached command marker left behind until an operator clears it', async () => { + const directory = await mkdtemp( + join(tmpdir(), 'librechat-code-cli-detached-'), + ); + const workspace = join(directory, 'workspace'); + const quarantine = join(directory, 'quarantine.json'); + await mkdir(workspace); + try { + await saveWorkspaceMutationQuarantine( + detachedWorkspaceMutationQuarantinePath(quarantine, 'assignment-1'), + { + version: 1, + workerId: 'engineering-vm', + workspaceId: 'primary', + ownerId: 'incarnation-previous:assignment-1', + quarantinedAt: new Date().toISOString(), + reason: 'Workspace mutation execute_command is pending settlement', + }, + ); + await assert.rejects(access(quarantine)); + const env = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE: quarantine, + }; + const run = () => + spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspace, + '--allow-workspace-writes', + ], + { encoding: 'utf8', env, timeout: 2_000 }, + ); + const blocked = run(); + assert.notEqual(blocked.status, 0); + assert.match( + blocked.stderr, + /workspace mutations are quarantined.*execute_command is pending settlement/i, + ); + + const cleared = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'clear-workspace-quarantine', + ], + { encoding: 'utf8', env }, + ); + assert.equal(cleared.status, 0, cleared.stderr); + await assert.rejects(access(detachedWorkspaceMutationQuarantineDirectory(quarantine))); + assert.doesNotMatch(run().stderr, /quarantined/i); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('CLI rejects a malformed detached command setting at startup', async (t) => { + const workspace = await mkdtemp(join(tmpdir(), 'librechat-code-cli-detached-')); + t.after(() => rm(workspace, { recursive: true, force: true })); + const run = (value: string, ...extra: string[]) => + spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url)), 'run', '--worker-dir', workspace, ...extra], + { + encoding: 'utf8', + timeout: 2_000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_DETACHED_COMMANDS: value, + }, + }, + ); + const malformed = run('sometimes'); + assert.notEqual(malformed.status, 0); + assert.match(malformed.stderr, /LIBRECHAT_CODE_DETACHED_COMMANDS must be true or false/); + for (const accepted of [run('false'), run('true', '--no-detached-commands')]) { + assert.doesNotMatch(accepted.stderr, /LIBRECHAT_CODE_DETACHED_COMMANDS/); + } +}); + test('CLI cannot bypass quarantine by renaming the same physical workspace', async () => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-cli-root-')); const workspace = join(directory, 'workspace'); diff --git a/packages/code/src/workspace-guards.test.ts b/packages/code/src/workspace-guards.test.ts new file mode 100644 index 00000000..cc643aed --- /dev/null +++ b/packages/code/src/workspace-guards.test.ts @@ -0,0 +1,168 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, open, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { + clearWorkspaceMutationQuarantine, + detachedWorkspaceMutationQuarantineDirectory, + detachedWorkspaceMutationQuarantinePath, + loadWorkspaceMutationQuarantine, + loadWorkspaceMutationQuarantines, +} from './storage.js'; +import { workspaceMutationGuard } from './workspace-guards.js'; + +import type { FileHandle } from 'node:fs/promises'; +import type { TestContext } from 'node:test'; + +async function markerPath(t: TestContext): Promise { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-guards-')); + t.after(() => rm(directory, { recursive: true, force: true })); + return join(directory, 'state', 'workspace.json'); +} + +const guard = (path: string, incarnationId = 'incarnation-1') => + workspaceMutationGuard(path, 'worker', 'repo', incarnationId); + +const live = (...assignmentIds: string[]) => (assignmentId: string): boolean => + assignmentIds.includes(assignmentId); + +const quarantined = { code: 'WORKSPACE_QUARANTINED' }; + +test('a detached command keeps its own private marker beside the workspace marker', async (t) => { + const path = await markerPath(t); + const directory = detachedWorkspaceMutationQuarantineDirectory(path); + const workspace = guard(path); + await workspace.arm('ordinary mutation', 'ordinary'); + await workspace.detached!.arm('detached command', 'background-1'); + await workspace.detached!.arm('detached command', 'durable-background_2'); + await assert.rejects(workspace.arm('second ordinary mutation', 'other'), { code: 'EEXIST' }); + await assert.rejects(workspace.detached!.arm('replayed', 'background-1'), { code: 'EEXIST' }); + + assert.equal(directory, `${path}.detached`); + assert.deepEqual((await readdir(directory)).sort(), ['background-1.json', 'durable-background_2.json']); + assert.equal((await stat(directory)).mode & 0o777, 0o700); + const marker = detachedWorkspaceMutationQuarantinePath(path, 'background-1'); + assert.equal((await stat(marker)).mode & 0o777, 0o600); + const records = await loadWorkspaceMutationQuarantines(path); + assert.deepEqual([records[0]?.ownerId, records[0]?.reason], ['incarnation-1:ordinary', 'ordinary mutation']); + assert.deepEqual( + records.slice(1).map((record) => [record.ownerId, record.reason]).sort(), + [ + ['incarnation-1:background-1', 'detached command'], + ['incarnation-1:durable-background_2', 'detached command'], + ], + ); + + await workspace.detached!.quarantine('ambiguous', undefined, 'background-1'); + await assert.rejects( + guard(path, 'incarnation-2').detached!.quarantine('ambiguous', undefined, 'background-1'), + /owned by another worker incarnation/, + ); + await assert.rejects( + guard(path, 'incarnation-2').detached!.clear('background-1'), + /owned by another worker incarnation/, + ); + await workspace.detached!.clear('background-1'); + await workspace.clear('ordinary'); + assert.equal(await loadWorkspaceMutationQuarantine(path), undefined); + assert.deepEqual(await readdir(directory), ['durable-background_2.json']); + await assert.rejects(workspace.detached!.clear('background-1'), /owned by another worker incarnation/); +}); + +test('only markers of assignments still running in this incarnation leave a workspace available', async (t) => { + const path = await markerPath(t); + const workspace = guard(path); + await workspace.assertAvailable(); + await workspace.assertAvailable(live()); + + await workspace.detached!.arm('detached command', 'background'); + await assert.rejects(workspace.assertAvailable(), quarantined); + await assert.rejects(workspace.assertAvailable(live('ordinary')), quarantined); + await workspace.assertAvailable(live('background')); + + await workspace.arm('ordinary mutation', 'ordinary'); + await assert.rejects(workspace.assertAvailable(live('background')), quarantined); + await workspace.assertAvailable(live('background', 'ordinary')); + await assert.rejects( + guard(path, 'incarnation-2').assertAvailable(live('background', 'ordinary')), + quarantined, + ); + + await workspace.clear('ordinary'); + await workspace.detached!.clear('background'); + await guard(path, 'incarnation-2').assertAvailable(); +}); + +test('a detached marker requires a filename-safe assignment', async (t) => { + const path = await markerPath(t); + const workspace = guard(path); + for (const assignmentId of ['../escape', 'a/b', '', '.', 'x'.repeat(129), 'with space']) { + await assert.rejects(workspace.detached!.arm('detached command', assignmentId)); + } + assert.throws( + () => detachedWorkspaceMutationQuarantinePath(path, '../escape'), + /filename-safe assignment ID/, + ); + await assert.rejects(stat(join(path, '..')), { code: 'ENOENT' }); +}); + +test('an operator clear removes every detached marker with the workspace marker', async (t) => { + const path = await markerPath(t); + const directory = detachedWorkspaceMutationQuarantineDirectory(path); + const previous = guard(path, 'incarnation-previous'); + await previous.detached!.arm('detached command', 'background-1'); + await previous.detached!.arm('detached command', 'background-2'); + await writeFile(join(directory, 'torn.json'), '{"version":', { mode: 0o600 }); + await assert.rejects(guard(path).assertAvailable(), /invalid workspace quarantine file/i); + + await clearWorkspaceMutationQuarantine(path); + await assert.rejects(stat(directory), { code: 'ENOENT' }); + await guard(path).assertAvailable(); + + await previous.arm('ordinary mutation', 'ordinary'); + await previous.detached!.arm('detached command', 'background-3'); + await clearWorkspaceMutationQuarantine(path); + assert.deepEqual(await loadWorkspaceMutationQuarantines(path), []); + await assert.rejects(stat(directory), { code: 'ENOENT' }); +}); + +test('an availability check waits for a marker that is still being armed', async (t) => { + const path = await markerPath(t); + const probe = await open(tmpdir(), 'r'); + const fileHandlePrototype = Object.getPrototypeOf(probe) as { + chmod(mode: number): Promise; + }; + await probe.close(); + const originalChmod = fileHandlePrototype.chmod; + let created!: () => void; + let finish!: () => void; + let finished = Promise.resolve(); + t.mock.method(fileHandlePrototype, 'chmod', async function (this: FileHandle, mode: number) { + created(); + await finished; + return originalChmod.call(this, mode); + }); + const arms: Array<(workspace: ReturnType) => Promise> = [ + (workspace) => workspace.arm('ordinary mutation', 'assignment'), + (workspace) => workspace.detached!.arm('detached command', 'assignment'), + ]; + for (const arm of arms) { + const creating = new Promise((resolve) => { created = resolve; }); + finished = new Promise((resolve) => { finish = resolve; }); + const arming = arm(guard(path)); + await creating; + let settled = false; + const check = guard(path).assertAvailable(live('assignment')).finally(() => { settled = true; }); + void check.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 25)); + assert.equal(settled, false); + finish(); + await arming; + await check; + } + await guard(path).clear('assignment'); + await guard(path).detached!.clear('assignment'); + await guard(path).assertAvailable(); +}); diff --git a/packages/code/src/workspace-guards.ts b/packages/code/src/workspace-guards.ts index 98b6bb02..93651e54 100644 --- a/packages/code/src/workspace-guards.ts +++ b/packages/code/src/workspace-guards.ts @@ -2,11 +2,34 @@ import { BridgeProtocolError } from './protocol.js'; import { assertWorkspaceMutationQuarantineOwner, clearWorkspaceMutationQuarantine, - loadWorkspaceMutationQuarantine, + detachedWorkspaceMutationQuarantinePath, + loadWorkspaceMutationQuarantines, saveWorkspaceMutationQuarantine, } from './storage.js'; +import type { WorkspaceMutationQuarantineRecord } from './storage.js'; import type { WorkspaceMutationQuarantine } from './worker.js'; +const markerOperations = new Map>(); + +/** + * Detached commands let one workspace's markers be armed, cleared and checked at + * the same time. Each workspace's marker accesses run one after another, so a + * check never reads a marker this process is still writing. + */ +async function exclusive(path: string, operation: () => Promise): Promise { + const result = (markerOperations.get(path) ?? Promise.resolve()).then(operation); + const settled = result.then( + () => undefined, + () => undefined, + ); + markerOperations.set(path, settled); + try { + return await result; + } finally { + if (markerOperations.get(path) === settled) markerOperations.delete(path); + } +} + /** Each path is outside sandbox roots and each pending write has a unique owner. */ export function workspaceMutationGuard( path: string, @@ -14,15 +37,32 @@ export function workspaceMutationGuard( workspaceId: string, incarnationId: string, ): WorkspaceMutationQuarantine { - const owner = (assignmentId?: string): string => { + const required = (assignmentId?: string): string => { if (!assignmentId) throw new Error('Workspace mutation requires an assignment owner'); - return `${incarnationId}:${assignmentId}`; + return assignmentId; }; + const owner = (assignmentId?: string): string => + `${incarnationId}:${required(assignmentId)}`; + const record = (reason: string, assignmentId?: string): WorkspaceMutationQuarantineRecord => ({ + version: 1, + workerId, + workspaceId, + ownerId: owner(assignmentId), + quarantinedAt: new Date().toISOString(), + reason, + }); + const detachedPath = (assignmentId?: string): string => + detachedWorkspaceMutationQuarantinePath(path, required(assignmentId)); + const liveOwnerPrefix = `${incarnationId}:`; return { - async assertAvailable() { - const record = await loadWorkspaceMutationQuarantine(path); - if (record != null) + async assertAvailable(isLive) { + const records = await exclusive(path, () => loadWorkspaceMutationQuarantines(path)); + const inFlight = ({ ownerId }: WorkspaceMutationQuarantineRecord): boolean => + isLive != null && + ownerId?.startsWith(liveOwnerPrefix) === true && + isLive(ownerId.slice(liveOwnerPrefix.length)); + if (!records.every(inFlight)) throw new BridgeProtocolError( `Workspace ${workspaceId} is quarantined; inspect it and clear its quarantine before restarting the worker`, undefined, @@ -30,20 +70,33 @@ export function workspaceMutationGuard( ); }, async arm(reason, assignmentId) { - await saveWorkspaceMutationQuarantine(path, { - version: 1, - workerId, - workspaceId, - ownerId: owner(assignmentId), - quarantinedAt: new Date().toISOString(), - reason, - }); + const pending = record(reason, assignmentId); + await exclusive(path, () => saveWorkspaceMutationQuarantine(path, pending)); }, async clear(assignmentId) { - await clearWorkspaceMutationQuarantine(path, owner(assignmentId)); + const ownerId = owner(assignmentId); + await exclusive(path, () => clearWorkspaceMutationQuarantine(path, ownerId)); }, async quarantine(_reason, _cause, assignmentId) { - await assertWorkspaceMutationQuarantineOwner(path, owner(assignmentId)); + const ownerId = owner(assignmentId); + await exclusive(path, () => assertWorkspaceMutationQuarantineOwner(path, ownerId)); + }, + detached: { + async arm(reason, assignmentId) { + const marker = detachedPath(assignmentId); + const pending = record(reason, assignmentId); + await exclusive(path, () => saveWorkspaceMutationQuarantine(marker, pending)); + }, + async clear(assignmentId) { + const marker = detachedPath(assignmentId); + const ownerId = owner(assignmentId); + await exclusive(path, () => clearWorkspaceMutationQuarantine(marker, ownerId)); + }, + async quarantine(_reason, _cause, assignmentId) { + const marker = detachedPath(assignmentId); + const ownerId = owner(assignmentId); + await exclusive(path, () => assertWorkspaceMutationQuarantineOwner(marker, ownerId)); + }, }, }; } diff --git a/packages/code/src/workspace-instances.test.ts b/packages/code/src/workspace-instances.test.ts index a1cd6859..6c062974 100644 --- a/packages/code/src/workspace-instances.test.ts +++ b/packages/code/src/workspace-instances.test.ts @@ -6,12 +6,15 @@ import { join } from 'node:path'; import { promisify } from 'node:util'; import test from 'node:test'; -import { GitWorktreeWorkspaceTools } from './workspace-instances.js'; +import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; import { LocalWorkspaceTools } from './workspace.js'; import { GitWorktreeManager } from './worktrees.js'; import { readRepositoryInstructions } from './instructions.js'; import { captureWorkspaceRootIdentity } from './root-identity.js'; +import type { WorkspaceExecuteCommandRequest } from './protocol.js'; + const execFileAsync = promisify(execFile); async function repository(): Promise<{ parent: string; root: string }> { @@ -232,3 +235,86 @@ test('leaves legacy requests on the selected source workspace', async (t) => { assert.equal(result.operation, 'read_file'); assert.equal(result.content, 'source'); }); + +test('a detached conversation command runs in its own executor for the conversation worktree', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const executors: Array<{ root: string; requests: WorkspaceExecuteCommandRequest[]; closed: boolean }> = []; + let releaseBackground!: () => void; + const backgroundFinished = new Promise((resolve) => { releaseBackground = resolve; }); + let backgroundStarted!: () => void; + const backgroundRunning = new Promise((resolve) => { backgroundStarted = resolve; }); + const commandPool = new NativeWorkspaceCommandPool( + new Map([['primary', { workspaceRoot: fixture.root }]]), + 2, + (options) => { + const executor = { root: options.workspaceRoot, requests: [] as WorkspaceExecuteCommandRequest[], closed: false }; + executors.push(executor); + return { + async prepare() {}, + async close() { + executor.closed = true; + }, + async execute(request) { + executor.requests.push(request); + if (request.command === 'npm test') { + backgroundStarted(); + await backgroundFinished; + } + return { ...request, stdout: request.command, stderr: '', exitCode: 0, timedOut: false, truncated: false }; + }, + }; + }, + ); + const tools = new GitWorktreeWorkspaceTools({ + commandPool, + delegate, + manager, + sources: new Map([ + ['primary', { command: { workspaceRoot: fixture.root }, repositoryInstructions: false, writable: true }], + ]), + }); + const instanceId = 'c'.repeat(64); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + workspaceInstanceId: instanceId, + }; + const id = internalWorkspaceId('primary', instanceId); + + const background = tools.execute({ ...command, command: 'npm test', detached: true }); + await backgroundRunning; + const ordinary = await tools.execute({ ...command, command: 'git status' }); + assert.equal(ordinary.workspaceId, 'primary'); + const instance = await manager.resolve('primary', instanceId); + assert.deepEqual(executors, [ + { + root: instance.root, + requests: [{ protocolVersion: 1, operation: 'execute_command', workspaceId: id, command: 'npm test' }], + closed: false, + }, + { + root: instance.root, + requests: [{ protocolVersion: 1, operation: 'execute_command', workspaceId: id, command: 'git status' }], + closed: false, + }, + ]); + await assert.rejects(commandPool.unregisterRoot(id), /still executing/); + + releaseBackground(); + const result = await background; + assert.equal(result.workspaceId, 'primary'); + await commandPool.unregisterRoot(id); + assert.deepEqual(executors.map((executor) => executor.closed), [true, true]); + await commandPool.close(); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index ec5a4218..a535da22 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -153,6 +153,8 @@ export interface SandboxWorkspaceToolsOptions { commandWorkspaces: string[]; /** Optional execution-scoped languages supplied by the same command sandbox. */ programmaticLanguages?: BridgeWorkspaceToolCapabilities['programmaticLanguages']; + /** Command request features the command sandbox itself can honor. */ + commandFeatures?: BridgeWorkspaceToolCapabilities['commandFeatures']; } const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; @@ -2618,6 +2620,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { ...(base.listFileFeatures != null ? { listFileFeatures: base.listFileFeatures } : {}), + ...(options.commandFeatures?.length + ? { commandFeatures: [...options.commandFeatures] } + : {}), ...(options.programmaticLanguages?.length ? { programmaticLanguages: [...options.programmaticLanguages] } : {}), @@ -2655,6 +2660,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { if (request.environmentAction) { throw new WorkspaceToolError('Environment action was not resolved by this worker', 'INVALID_REQUEST'); } + if (request.detached === true && !this.capabilities.commandFeatures?.includes('detached')) { + throw new WorkspaceToolError('Detached commands are unavailable in this command sandbox', 'INVALID_REQUEST'); + } if (!this.commandWorkspaces.has(request.workspaceId)) { throw new WorkspaceToolError( 'Command execution is disabled for this workspace', diff --git a/service/src/bridge/detached.test.ts b/service/src/bridge/detached.test.ts new file mode 100644 index 00000000..6f4b7a55 --- /dev/null +++ b/service/src/bridge/detached.test.ts @@ -0,0 +1,309 @@ +import { createHash } from 'node:crypto'; +import { afterEach, expect, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; +import Redis from 'ioredis'; +import { RedisBridgeStore } from './store'; +import { + BRIDGE_PROTOCOL_VERSION, + workspaceDetachedCommandKey, + workspaceIsolationKey, +} from '../../../packages/code/src/protocol'; +import type { CodeBridgeAssignment } from './store'; +import type { + BridgeRejectedSettlement, + WorkspaceCommandFeature, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, +} from '../../../packages/code/src/protocol'; + +const testRedisUrl = process.env.BRIDGE_TEST_REDIS_URL; +const redis = testRedisUrl !== undefined && testRedisUrl.length > 0 + ? new Redis(testRedisUrl) + : new RedisMock() as unknown as Redis; +const store = new RedisBridgeStore(redis, 60, 1000, 3); +const workerId = 'detached-worker'; +const incarnationId = 'detached-incarnation'; +const controllers: AbortController[] = []; +const pending: Promise[] = []; + +// Only a disposable test Redis may be supplied. +afterEach(async () => { + for (const controller of controllers.splice(0)) controller.abort(); + await Promise.allSettled(pending.splice(0)); + await redis.flushall(); +}); + +async function register( + commandFeatures: WorkspaceCommandFeature[] = ['detached'], + bridge = store, + workspaceLeaseSlots = 3, +): Promise { + const generation = await bridge.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + runtimes: [], + sandboxProfile: 'native-srt', + ...(workspaceLeaseSlots > 1 ? { requiresReadyConfirmation: true, workspaceLeaseSlots } : {}), + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'execute_command'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }], + ...(commandFeatures.length === 0 ? {} : { commandFeatures }), + }, + }, + }); + if (workspaceLeaseSlots > 1) await bridge.confirmReady(workerId, incarnationId, generation); +} + +const command = ( + request: Partial = {}, +): WorkspaceExecuteCommandRequest => ({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId: 'repo', + worktree: 'task-a', + command: 'npm test', + ...request, +}); + +function dispatch(request: WorkspaceToolRequest, bridge = store): Promise { + const controller = new AbortController(); + controllers.push(controller); + const result = bridge.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + 3000, + executionTimeoutMs: 5000, + request, + }); + void result.catch(() => undefined); + pending.push(result); + return result; +} + +async function lease(slot: number, waitMs = 2000): Promise { + const assignment = await store.lease(workerId, incarnationId, waitMs, undefined, undefined, slot); + if (assignment != null) { + await store.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + } + return assignment; +} + +function rejection(assignment: CodeBridgeAssignment, error: string): BridgeRejectedSettlement { + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error, + }; +} + +async function fulfil(assignment: CodeBridgeAssignment): Promise { + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'fulfilled', + result: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId: 'repo', + exitCode: 0, + stdout: assignment.assignmentId, + stderr: '', + truncated: false, + timedOut: false, + }, + }); +} + +const confirmCleanup = (assignment: CodeBridgeAssignment): Promise => + store.confirmWorkspaceCleanup( + workerId, + assignment.assignmentId, + rejection(assignment, 'local cleanup confirmed'), + ); + +function fenceKey(isolationKey: string): string { + const hash = createHash('sha256').update(`native-workspace:${isolationKey}`).digest('hex'); + return `codeapi:bridge:v1:worker:${workerId}:workspace:${hash}:quarantined`; +} + +const laneFence = fenceKey(workspaceIsolationKey('repo', undefined, 'task-a')); +const checkoutFence = fenceKey('repo'); +const detachedFence = (assignment: CodeBridgeAssignment): string => + fenceKey(workspaceDetachedCommandKey(assignment.assignmentId)); +const isDetached = (assignment: CodeBridgeAssignment | undefined): boolean => + (assignment?.request as WorkspaceExecuteCommandRequest | undefined)?.detached === true; + +for (const detachedFirst of [true, false]) { + test(`a detached command and its lane's ordinary work overlap and settle independently (detached ${detachedFirst ? 'first' : 'second'})`, async () => { + await register(); + const requests = [command({ detached: true }), command({ command: 'git status' })]; + const [first, second] = detachedFirst ? requests : [...requests].reverse(); + const firstResult = dispatch(first); + const firstAssignment = (await lease(0))!; + const secondResult = dispatch(second); + const secondAssignment = (await lease(1))!; + expect(isDetached(firstAssignment)).toBe(detachedFirst); + expect(isDetached(secondAssignment)).toBe(!detachedFirst); + const background = detachedFirst ? firstAssignment : secondAssignment; + const ordinary = detachedFirst ? secondAssignment : firstAssignment; + + expect(await redis.get(laneFence)).toBe(ordinary.assignmentId); + expect(await redis.get(detachedFence(background))).toBe(background.assignmentId); + expect(await redis.ttl(laneFence)).toBe(-1); + const fenceTtl = await redis.ttl(detachedFence(background)); + expect(fenceTtl).toBeGreaterThan(0); + expect(fenceTtl).toBeLessThanOrEqual(35); + + await fulfil(ordinary); + await confirmCleanup(ordinary); + await expect(detachedFirst ? secondResult : firstResult).resolves.toMatchObject({ + status: 'fulfilled', + result: { stdout: ordinary.assignmentId }, + }); + expect(await redis.get(detachedFence(background))).toBe(background.assignmentId); + + const queued = dispatch(command({ command: 'git diff' })); + const next = (await lease(detachedFirst ? 1 : 0))!; + expect(isDetached(next)).toBe(false); + expect(await redis.get(laneFence)).toBe(next.assignmentId); + + await fulfil(background); + await confirmCleanup(background); + await expect(detachedFirst ? firstResult : secondResult).resolves.toMatchObject({ + status: 'fulfilled', + result: { stdout: background.assignmentId }, + }); + expect(await redis.exists(detachedFence(background))).toBe(0); + expect(await redis.get(laneFence)).toBe(next.assignmentId); + + await fulfil(next); + await confirmCleanup(next); + await expect(queued).resolves.toMatchObject({ status: 'fulfilled' }); + expect(await redis.exists(laneFence)).toBe(0); + }); +} + +test('a detached command also overlaps work in its parent checkout', async () => { + await register(); + const checkout = dispatch(command({ worktree: undefined, command: 'git worktree list' })); + const held = (await lease(0))!; + const background = dispatch(command({ detached: true })); + const admitted = (await lease(1))!; + expect(isDetached(admitted)).toBe(true); + expect(await redis.get(checkoutFence)).toBe(held.assignmentId); + expect(await redis.exists(laneFence)).toBe(0); + await fulfil(admitted); + await confirmCleanup(admitted); + await fulfil(held); + await confirmCleanup(held); + await expect(background).resolves.toMatchObject({ status: 'fulfilled' }); + await expect(checkout).resolves.toMatchObject({ status: 'fulfilled' }); +}); + +test('a detached request is refused unless the worker advertises the feature with spare slots', async () => { + await register([]); + await expect(dispatch(command({ detached: true }))).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + const ordinary = dispatch(command()); + const assignment = (await lease(0))!; + await fulfil(assignment); + await confirmCleanup(assignment); + await expect(ordinary).resolves.toMatchObject({ status: 'fulfilled' }); + + await redis.flushall(); + await register(['detached'], store, 1); + await expect(dispatch(command({ detached: true }))).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); +}); + +test('a replica that opted out refuses detached requests for an advertising worker', async () => { + await register(); + const optedOut = new RedisBridgeStore(redis, 60, 1000, 3, false, false); + expect(optedOut.detachedCommandsOffered).toBe(false); + expect(store.detachedCommandsOffered).toBe(true); + expect(new RedisBridgeStore(redis, 60, 1000, 1).detachedCommandsOffered).toBe(false); + await expect(dispatch(command({ detached: true }), optedOut)).rejects.toMatchObject({ + code: 'WORKER_MISMATCH', + }); +}); + +test('a quarantine receipt for a detached command expires and leaves its lane usable', async () => { + await register(); + const background = dispatch(command({ detached: true })); + const assignment = (await lease(0))!; + await store.settle( + workerId, + assignment.assignmentId, + rejection(assignment, 'Workspace requires inspection before reset.'), + undefined, + undefined, + true, + ); + await expect(background).resolves.toMatchObject({ status: 'rejected' }); + expect(await redis.get(detachedFence(assignment))).toBe(`quarantined:${assignment.assignmentId}`); + const receiptTtl = await redis.ttl(detachedFence(assignment)); + expect(receiptTtl).toBeGreaterThan(0); + expect(receiptTtl).toBeLessThanOrEqual(35); + expect(await redis.exists(laneFence)).toBe(0); + + const ordinary = dispatch(command({ command: 'git status' })); + const next = (await lease(0))!; + expect(isDetached(next)).toBe(false); + await store.settle( + workerId, + next.assignmentId, + rejection(next, 'Workspace requires inspection before reset.'), + undefined, + undefined, + true, + ); + await expect(ordinary).resolves.toMatchObject({ status: 'rejected' }); + expect(await redis.get(laneFence)).toBe(`quarantined:${next.assignmentId}`); + expect(await redis.ttl(laneFence)).toBe(-1); +}); + +for (const quarantined of ['lane', 'checkout'] as const) { + test(`a detached command is refused while its ${quarantined} is quarantined`, async () => { + await register(); + await redis.set(quarantined === 'lane' ? laneFence : checkoutFence, 'quarantined:earlier'); + await expect(dispatch(command({ detached: true }))).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + expect(await redis.keys(`codeapi:bridge:v1:worker:${workerId}:workspace:*:quarantined`)).toHaveLength(1); + }); +} + +test('detached commands never take the last slot', async () => { + await register(); + const results = ['one', 'two', 'three'].map((name) => + dispatch(command({ detached: true, command: name })), + ); + const first = (await lease(0))!; + const second = (await lease(1))!; + expect(await lease(2, 300)).toBeUndefined(); + const ordinary = dispatch(command({ command: 'git status' })); + const third = (await lease(2))!; + expect(isDetached(third)).toBe(false); + for (const assignment of [first, second, third]) { + await fulfil(assignment); + await confirmCleanup(assignment); + } + const capped = (await lease(0))!; + expect(isDetached(capped)).toBe(true); + await fulfil(capped); + await confirmCleanup(capped); + await expect(Promise.all([...results, ordinary])).resolves.toHaveLength(4); +}); diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index 93370dfa..3a386ccb 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -11,6 +11,7 @@ export const bridgeStore = new RedisBridgeStore( undefined, env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, env.BRIDGE_LANE_GIT, + env.BRIDGE_DETACHED_COMMANDS, ); export const bridgePairings = new RedisBridgePairingStore( connection, diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index a478f623..3a62f7c8 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -216,6 +216,70 @@ describe('paired bridge HTTP API', () => { }); } + for (const [ceiling, enabled] of [[1, true], [2, true], [2, false]] as const) { + const offered = ceiling > 1 && enabled; + test(`registration ${offered ? 'offers' : 'omits'} detached commands with a ceiling of ${ceiling} and the setting ${enabled ? 'on' : 'off'}`, async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis, undefined, undefined, ceiling, false, enabled), + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const headers = { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }; + const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/register`, { + method: 'POST', + headers, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + requiresReadyConfirmation: true, + workspaceLeaseSlots: 2, + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'execute_command'], + workspaces: [{ id: 'primary' }], + commandFeatures: ['detached'], + }, + }, + }), + }); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.workspaceLeaseSlots).toBe(ceiling); + if (offered) expect(body.supportedWorkspaceCommandFeatures).toEqual(['detached']); + else expect('supportedWorkspaceCommandFeatures' in body).toBe(false); + + const status = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/vm-1/status`, { headers }); + expect(status.status).toBe(200); + await expect(status.json()).resolves.toMatchObject({ + workerId: 'vm-1', + online: true, + capabilities: { + workspaceLeaseSlots: ceiling, + workspaceTools: { commandFeatures: ['detached'] }, + }, + }); + }); + } + for (const workspaceImportFile of [false, true]) { test(`registration ${workspaceImportFile ? 'offers' : 'omits'} import_file when grants ${workspaceImportFile ? 'can' : 'cannot'} be minted`, async () => { const app = express(); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index c3b5aead..a40eeb01 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -617,6 +617,9 @@ router.post( ...(options.store.laneGitEnabled ? { supportedWorkspaceCommandResultFeatures: ['lane_git'] } : {}), + ...(options.store.detachedCommandsOffered + ? { supportedWorkspaceCommandFeatures: ['detached'] } + : {}), supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], supportedWorkspaceScopes: ['git_linked_worktree'], diff --git a/service/src/bridge/slots.test.ts b/service/src/bridge/slots.test.ts index 1efe3b97..8a4a3ef5 100644 --- a/service/src/bridge/slots.test.ts +++ b/service/src/bridge/slots.test.ts @@ -3,7 +3,7 @@ import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { BridgeAdmissionQueue } from './admission'; import { BridgeWorkspaceSlots } from './slots'; -import { workspaceIsolationKey } from '../../../packages/code/src/protocol'; +import { workspaceDetachedCommandKey, workspaceIsolationKey } from '../../../packages/code/src/protocol'; const redis = new RedisMock() as unknown as Redis; const admission = new BridgeAdmissionQueue(redis); @@ -34,6 +34,8 @@ async function finish(assignmentId: string) { } const lane = (name: string, parent = 'repo') => workspaceIsolationKey(parent, undefined, name); +const detached = (assignmentId: string, capacity = 3): ReturnType => + enqueue(assignmentId, workspaceDetachedCommandKey(assignmentId), capacity); test('slots admit independent workspaces, skip a busy root, and bound capacity', async () => { const a = await enqueue('a', 'root-a'); @@ -147,3 +149,61 @@ test('lanes nest beneath a conversation checkout, not the source root', async () expect(await slots.reserve(nested)).toBeUndefined(); expect(await slots.reserve(source)).toBe(1); }); + +test('a detached command runs beside its busy lane and checkout, and they beside it', async () => { + const laneWork = await enqueue('lane', lane('task-a'), 3); + expect(await slots.reserve(laneWork)).toBe(0); + expect(await slots.reserve(await detached('beside-lane'))).toBe(1); + await finish('lane'); + const checkout = await enqueue('checkout', 'repo', 3); + expect(await slots.reserve(checkout)).toBe(0); + await finish('beside-lane'); + expect(await slots.reserve(await detached('beside-checkout'))).toBe(1); + await finish('checkout'); + const laneAgain = await enqueue('lane-again', lane('task-a'), 3); + expect(await slots.reserve(laneAgain)).toBe(0); + const sibling = await enqueue('sibling', lane('task-b'), 3); + expect(await slots.reserve(sibling)).toBe(2); +}); + +test('a queued detached command is not held behind a held lane or a waiting checkout', async () => { + const running = await enqueue('running', lane('task-a'), 3); + expect(await slots.reserve(running)).toBe(0); + const heldLane = await enqueue('held-lane', lane('task-a'), 3); + const waitingCheckout = await enqueue('waiting-checkout', 'repo', 3); + const background = await detached('background'); + expect(await slots.reserve(heldLane)).toBeUndefined(); + expect(await slots.reserve(waitingCheckout)).toBeUndefined(); + expect(await slots.reserve(background)).toBe(1); + expect(await slots.reserve(heldLane)).toBeUndefined(); + await finish('running'); + expect(await slots.reserve(heldLane)).toBe(0); +}); + +test('detached commands leave one slot for other work and never act as a barrier', async () => { + expect(await slots.reserve(await detached('d1'))).toBe(0); + expect(await slots.reserve(await detached('d2'))).toBe(1); + const capped = await detached('d3'); + const laneWork = await enqueue('lane', lane('task-a'), 3); + expect(await slots.reserve(capped)).toBeUndefined(); + expect(await slots.reserve(laneWork)).toBe(2); + await finish('d1'); + expect(await slots.reserve(capped)).toBe(0); + await finish('d2'); + await finish('d3'); + await finish('lane'); + + const only = await detached('only', 2); + expect(await slots.reserve(only)).toBe(0); + const second = await detached('second', 2); + const checkout = await enqueue('checkout', 'repo', 2); + expect(await slots.reserve(second)).toBeUndefined(); + expect(await slots.reserve(checkout)).toBe(1); +}); + +test('a single slot never admits a detached command', async () => { + const lone = await detached('lone', 1); + const checkout = await enqueue('checkout', 'repo', 1); + expect(await slots.reserve(lone)).toBeUndefined(); + expect(await slots.reserve(checkout)).toBe(0); +}); diff --git a/service/src/bridge/slots.ts b/service/src/bridge/slots.ts index f748054b..01bdd790 100644 --- a/service/src/bridge/slots.ts +++ b/service/src/bridge/slots.ts @@ -13,6 +13,10 @@ export const MAX_WORKSPACE_LEASE_SLOTS = 8; * with every lane beneath it. Sibling lanes run concurrently. A waiting checkout * request also holds back newer lanes beneath it, so root operations such as * `git worktree add` cannot be starved by a steady stream of lane work. + * + * A detached command key (`\0detached-command\0`) conflicts with + * nothing and is held back by nothing but capacity. Detached commands may hold + * at most `capacity - 1` slots, so one slot always remains for other work. */ export class BridgeWorkspaceSlots { constructor(private readonly redis: Redis) {} @@ -74,11 +78,16 @@ export class BridgeWorkspaceSlots { ' if last == nil or last <= #lanePrefix + 1 then return nil end', ' return string.sub(key, #lanePrefix + 1, last - 1)', 'end', + 'local detachedPrefix = "\\0detached-command\\0"', + 'local function isDetached(key)', + ' return string.sub(key, 1, #detachedPrefix) == detachedPrefix', + 'end', "local lock = redis.call('GET', KEYS[2])", "local owner = 'workspace-slots:' .. ARGV[1]", 'if lock and lock ~= owner then return -1 end', 'local busy = {}', 'local busyChildren = {}', + 'local detachedHeld = 0', 'local free = nil', 'local latest = tonumber(ARGV[5])', `for slot = 0, ${MAX_WORKSPACE_LEASE_SLOTS - 1} do`, @@ -101,6 +110,7 @@ export class BridgeWorkspaceSlots { ' busy[entry[3]] = true', ' local busyParent = parentOf(entry[3])', ' if busyParent then busyChildren[busyParent] = true end', + ' if isDetached(entry[3]) then detachedHeld = detachedHeld + 1 end', ' latest = math.max(latest, tonumber(entry[4]))', ' end', ' end', @@ -108,6 +118,8 @@ export class BridgeWorkspaceSlots { 'end', 'local function conflicts(key)', ' if busy[key] then return true end', + // Beyond its own reservation, only the cap on detached holders holds a detached command back. + ' if isDetached(key) then return detachedHeld >= tonumber(ARGV[4]) - 1 end', ' local parent = parentOf(key)', ' if parent then return busy[parent] == true end', ' return busyChildren[key] == true', @@ -130,7 +142,7 @@ export class BridgeWorkspaceSlots { ' local parent = parentOf(workspace)', ' local held = conflicts(workspace) or (parent ~= nil and waitingCheckouts[parent] == true)', ' if not held then selected = id; break end', - ' if parent == nil then waitingCheckouts[workspace] = true end', + ' if parent == nil and not isDetached(workspace) then waitingCheckouts[workspace] = true end', 'end', 'if selected ~= ARGV[2] then return -1 end', "redis.call('HSET', KEYS[1], 'a:' .. free, ARGV[2], 'i:' .. free, ARGV[1], 'w:' .. free, ARGV[3], 'e:' .. free, ARGV[5])", diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 4dbd7d66..5075d3dc 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -15,9 +15,11 @@ import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceDetachedCommandKey, isWorkspaceMutationOperation, isWorkspaceToolRequest, isWorkspaceToolResult, + workspaceDetachedCommandKey, workspaceIsolationKey, workspaceIsolationParent, } from '../../../packages/code/src/protocol'; @@ -146,6 +148,26 @@ function workspaceFenceParent(fence: string): string | undefined { : `${NATIVE_WORKSPACE_FENCE_PREFIX}${parent}`; } +/** A detached command's fence belongs to its assignment alone and is never reset by an operator. */ +function isDetachedCommandFence(fence: string | undefined): boolean { + return ( + fence?.startsWith(NATIVE_WORKSPACE_FENCE_PREFIX) === true && + isWorkspaceDetachedCommandKey(fence.slice(NATIVE_WORKSPACE_FENCE_PREFIX.length)) + ); +} + +/** Fences of the worktree a detached command runs in and of the checkout that worktree nests beneath. */ +function detachedCommandWorktreeFences(assignment: StoredAssignment): string[] { + const request = assignment.request as WorkspaceToolRequest; + const worktree = `${NATIVE_WORKSPACE_FENCE_PREFIX}${workspaceAdmissionId( + request.workspaceId, + request.workspaceInstanceId, + request.worktree, + )}`; + const parent = workspaceFenceParent(worktree); + return parent === undefined ? [worktree] : [worktree, parent]; +} + function workspaceFenceReceiptKey(assignmentId: string): string { return `${assignmentKey(assignmentId)}:workspace-fence-owner`; } @@ -173,9 +195,14 @@ function isWorkspaceMutation(request: WorkspaceToolRequest): boolean { return isWorkspaceMutationOperation(request.operation); } +function isDetachedCommand(request: WorkspaceToolRequest): boolean { + return request.operation === 'execute_command' && request.detached === true; +} + function supportsWorkspaceTool( registration: RegisteredBridgeWorker, request: WorkspaceToolRequest, + detachedCommands: boolean, ): boolean { const capabilities = registration.capabilities.workspaceTools; const workspace = capabilities?.workspaces.find( @@ -205,6 +232,13 @@ function supportsWorkspaceTool( if (request.operation === 'list_files' && request.afterPath !== undefined) { return capabilities?.listFileFeatures?.includes('after_path') === true; } + if (isDetachedCommand(request)) { + return ( + detachedCommands && + (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 && + capabilities.commandFeatures?.includes('detached') === true + ); + } if (request.operation === 'write_file') { const mode = request.overwrite === false ? 'create' : 'replace'; const modes = capabilities?.writeFileModes; @@ -297,6 +331,17 @@ export function workspaceAdmissionId( return workspaceIsolationKey(workspaceId, instanceId, worktree); } +/** The key a workspace tool request is admitted and fenced under. A detached command is + * keyed by its own assignment, so it neither waits for nor holds its worktree. */ +export function workspaceToolAdmissionId( + request: WorkspaceToolRequest, + assignmentId: string, +): string { + return isDetachedCommand(request) + ? workspaceDetachedCommandKey(assignmentId) + : workspaceAdmissionId(request.workspaceId, request.workspaceInstanceId, request.worktree); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -483,6 +528,8 @@ export class RedisBridgeStore { private readonly maxWorkspaceLeaseSlots = 1, /** Accept `laneGit` on command results and offer `lane_git` at registration. Off by default. */ public readonly laneGitEnabled = false, + /** Schedule `detached` commands apart from their worktree. On by default; needs more than one slot. */ + private readonly detachedCommandsEnabled = true, ) { if ( !Number.isSafeInteger(maxWorkspaceLeaseSlots) || @@ -499,6 +546,11 @@ export class RedisBridgeStore { return Math.min(this.maxWorkspaceLeaseSlots, requested); } + /** Whether registration offers `detached`: one slot always stays free of detached commands. */ + get detachedCommandsOffered(): boolean { + return this.detachedCommandsEnabled && this.maxWorkspaceLeaseSlots > 1; + } + private async dispatchCommand( command: () => Promise, args: { deadlineAtMs: number; signal: AbortSignal }, @@ -831,7 +883,7 @@ export class RedisBridgeStore { throw new BridgeStoreError('WORKER_UNAUTHORIZED', 'Code environment is not authorized for this tenant'); } if ((registration.capabilities.workspaceLeaseSlots ?? 1) > this.maxWorkspaceLeaseSlots || - !supportsWorkspaceTool(registration, args.request)) { + !supportsWorkspaceTool(registration, args.request, this.detachedCommandsOffered)) { throw new BridgeStoreError('WORKER_MISMATCH', 'Code environment does not support the requested workspace operation'); } return registration; @@ -855,7 +907,7 @@ export class RedisBridgeStore { throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed while the request was waiting'); } const admission = new BridgeAdmissionQueue(this.redis); - const workspace = workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree); + const workspace = workspaceToolAdmissionId(record.request, record.id); const capacity = registration.capabilities.workspaceLeaseSlots ?? 1; if (capacity === 1 && !await admission.isHead(record.workerId, record.id)) return; const expiresAtMs = Date.now() + record.executionTimeoutMs; @@ -890,7 +942,7 @@ export class RedisBridgeStore { if (ready.registration.incarnationId !== registration.incarnationId || ready.registration.identityId !== registration.identityId || ready.registration.binding?.tenantId !== registration.binding?.tenantId || - !supportsWorkspaceTool(ready.registration, record.request)) { + !supportsWorkspaceTool(ready.registration, record.request, this.detachedCommandsOffered)) { throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed before admission'); } await this.enqueueForActiveIncarnation(assignment, assignmentTtlSeconds(Date.parse(assignment.expiresAt)), ready.readyToken, guard); @@ -1059,7 +1111,7 @@ export class RedisBridgeStore { } if ( args.workspaceRequest != null && - !supportsWorkspaceTool(registration, args.workspaceRequest) + !supportsWorkspaceTool(registration, args.workspaceRequest, this.detachedCommandsOffered) ) { throw new BridgeStoreError( 'WORKER_MISMATCH', @@ -1113,14 +1165,16 @@ export class RedisBridgeStore { args.workspaceRequest?.workspaceInstanceId ?? workspaceInstanceId(args.body); const selectedWorktree = args.workspaceRequest?.worktree ?? workspaceWorktree(args.body); - const selectedWorkspaceAdmissionId = - selectedWorkspaceId == null - ? undefined - : workspaceAdmissionId( - selectedWorkspaceId, - selectedWorkspaceInstanceId, - selectedWorktree, - ); + let selectedWorkspaceAdmissionId: string | undefined; + if (args.workspaceRequest != null) { + selectedWorkspaceAdmissionId = workspaceToolAdmissionId(args.workspaceRequest, assignmentId); + } else if (selectedWorkspaceId != null) { + selectedWorkspaceAdmissionId = workspaceAdmissionId( + selectedWorkspaceId, + selectedWorkspaceInstanceId, + selectedWorktree, + ); + } const workspaceSlots = selectedWorkspaceId != null && (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 @@ -1241,7 +1295,11 @@ export class RedisBridgeStore { } if ( (args.workspaceRequest != null && - !supportsWorkspaceTool(current.registration, args.workspaceRequest)) || + !supportsWorkspaceTool( + current.registration, + args.workspaceRequest, + this.detachedCommandsOffered, + )) || (args.workspaceId != null && !supportsWorkspaceProgrammatic( current.registration, @@ -1357,6 +1415,7 @@ export class RedisBridgeStore { !supportsWorkspaceTool( replacement.registration, args.workspaceRequest, + this.detachedCommandsOffered, ) ) { throw new BridgeStoreError( @@ -1408,12 +1467,17 @@ export class RedisBridgeStore { await boundedCommand(this.redis.eval( [ "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", - "redis.call('SET', KEYS[1], 'quarantined:' .. ARGV[1])", + "if ARGV[2] ~= '' then", + " redis.call('SET', KEYS[1], 'quarantined:' .. ARGV[1], 'EX', ARGV[2])", + 'else', + " redis.call('SET', KEYS[1], 'quarantined:' .. ARGV[1])", + 'end', 'return 1', ].join('\n'), 1, workspaceQuarantineKey(args.workerId, assignment.workspaceFence), assignment.assignmentId, + this.detachedFenceTtlSeconds(assignment), ), this.redisCommandTimeoutMs, 'Bridge native workspace finalization quarantine'); } else if (assignmentWorkspace(assignment) !== undefined) { await this.quarantine( @@ -2162,7 +2226,11 @@ export class RedisBridgeStore { // A completed cleanup receipt is terminal: a lost response must // not let a late quarantine overwrite a newer root owner. " if redis.call('HGET', KEYS[1], 'localCleanup') == '1' and redis.call('HGET', KEYS[1], 'resultCommitted') == '1' then return 1 end", - " redis.call('SET', KEYS[5], 'quarantined:' .. ARGV[5])", + " if ARGV[9] ~= '' then", + " redis.call('SET', KEYS[5], 'quarantined:' .. ARGV[5], 'EX', ARGV[9])", + ' else', + " redis.call('SET', KEYS[5], 'quarantined:' .. ARGV[5])", + ' end', // Never replace a committed result. Before settlement, terminate the waiter. " redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[7], 'NX')", 'else', @@ -2200,6 +2268,7 @@ export class RedisBridgeStore { JSON.stringify(settlement), assignmentOutcomeTtlSeconds(receipt), quarantine ? '1' : '0', + this.detachedFenceTtlSeconds(receipt), ), signal, 'Workspace quarantine fence commit', @@ -2429,6 +2498,13 @@ export class RedisBridgeStore { ); } + /** Expiry of a detached command's fence or quarantine receipt; empty for a worktree fence, which never expires. */ + private detachedFenceTtlSeconds(assignment: AssignmentOwnership): string { + return isDetachedCommandFence(assignment.workspaceFence) + ? String(assignmentOutcomeTtlSeconds(assignment)) + : ''; + } + private async cancel( assignmentId: string, assignment?: AssignmentOwnership, @@ -2464,6 +2540,13 @@ export class RedisBridgeStore { "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', 'if keyCount >= 7 and redis.call(\'EXISTS\', KEYS[7]) == 1 then return -1 end', + // A detached command holds no worktree, but still refuses a quarantined one. + "if ARGV[9] == 'detached' then", + ' for i = 10, keyCount do', + " local held = redis.call('GET', KEYS[i])", + " if held and string.sub(held, 1, 12) == 'quarantined:' then return -1 end", + ' end', + 'end', // A lane refuses a quarantined checkout. A checkout reaches enqueue only once no // lane beneath it holds a slot, so any lane fence still indexed here is stuck. "if ARGV[9] == 'lane' and redis.call('EXISTS', KEYS[10]) == 1 then return -1 end", @@ -2478,7 +2561,14 @@ export class RedisBridgeStore { ? ['redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])'] : []), 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', - 'if keyCount >= 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end', + 'if keyCount >= 7 then', + // Nobody resets a detached command's fence, so it expires with the assignment outcome. + " if ARGV[13] ~= '' then", + " redis.call('SET', KEYS[7], ARGV[4], 'EX', ARGV[13])", + ' else', + " redis.call('SET', KEYS[7], ARGV[4])", + ' end', + 'end', "if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end", 'if keyCount >= 9 then', " local epoch = redis.call('GET', KEYS[9])", @@ -2524,14 +2614,21 @@ export class RedisBridgeStore { expiresAt: assignment.expiresAt, durableRequestKey: assignment.durableRequestKey, }; - let fenceScope: '' | 'lane' | 'checkout' = ''; + let fenceScope: '' | 'lane' | 'checkout' | 'detached' = ''; if (assignment.workspaceLeaseSlot !== undefined) { keys.push( workspaceFenceReceiptKey(assignment.assignmentId), `${workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!)}:epoch`, ); const parent = workspaceFenceParent(assignment.workspaceFence!); - if (parent !== undefined) { + if (isDetachedCommandFence(assignment.workspaceFence)) { + keys.push( + ...detachedCommandWorktreeFences(assignment).map((fence) => + workspaceQuarantineKey(assignment.workerId, fence), + ), + ); + fenceScope = 'detached'; + } else if (parent !== undefined) { const parentFence = workspaceQuarantineKey(assignment.workerId, parent); keys.push(parentFence, workspaceLaneFenceIndexKey(parentFence)); fenceScope = 'lane'; @@ -2568,6 +2665,7 @@ export class RedisBridgeStore { keyCount, guard?.token ?? '', guard == null ? ttlSeconds : assignmentOutcomeTtlSeconds(assignment), + this.detachedFenceTtlSeconds(assignment), ); if (Number(result) === -1) { throw new BridgeStoreError( diff --git a/service/src/config.ts b/service/src/config.ts index 9a94897e..876aca2b 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -428,6 +428,12 @@ export const env = { * Off by default: enable only after the LibreChat talking to this service accepts the field. */ BRIDGE_LANE_GIT: process.env.CODEAPI_BRIDGE_LANE_GIT === 'true', + /** + * Offer `detached` commands to workers: an `execute_command` marked `detached` holds a + * worker slot but not its worktree. On by default wherever more than one workspace lease + * slot is allowed; set to `false` to keep every command serialized with its worktree. + */ + BRIDGE_DETACHED_COMMANDS: process.env.CODEAPI_BRIDGE_DETACHED_COMMANDS !== 'false', /** * Largest stored object relayed into a worker by `import_file`. The operation * is offered only when EGRESS_GATEWAY_URL can mint its one-object grant. diff --git a/service/src/workspace-tools/index.ts b/service/src/workspace-tools/index.ts index 58c043d9..b8a00cfe 100644 --- a/service/src/workspace-tools/index.ts +++ b/service/src/workspace-tools/index.ts @@ -29,6 +29,9 @@ const requests = coordinationScope === undefined ? undefined : new RedisWorkspac 'codeapi.workspace.id': record.request.workspaceId, 'codeapi.workspace.instance_id': record.request.workspaceInstanceId ?? '', 'codeapi.workspace.worktree': record.request.worktree ?? '', + ...(record.request.operation === 'execute_command' && record.request.detached === true + ? { 'codeapi.workspace.detached': true } + : {}), 'codeapi.admission.wait_ms': Math.max(0, (record.admittedAtMs ?? Math.min(record.finishedAtMs ?? Date.now(), record.queueDeadlineAtMs)) - record.createdAtMs), 'codeapi.admission.error_code': record.error?.code ?? '', }; diff --git a/service/src/workspace-tools/requests.test.ts b/service/src/workspace-tools/requests.test.ts index 458950f8..5968ccb6 100644 --- a/service/src/workspace-tools/requests.test.ts +++ b/service/src/workspace-tools/requests.test.ts @@ -5,6 +5,7 @@ import Redis from 'ioredis'; import { RedisBridgeStore } from '../bridge/store'; import { BridgeAdmissionQueue } from '../bridge/admission'; import type { CodeBridgeAssignment } from '../bridge/store'; +import { workspaceDetachedCommandKey } from '../../../packages/code/src/protocol'; import type { BridgeRejectedSettlement, WorkspaceToolRequest } from '../../../packages/code/src/protocol'; import { RedisWorkspaceRequests } from './requests'; import { workspaceRequestCoordinationScope } from './coordination'; @@ -25,7 +26,7 @@ const queueKey = `codeapi:bridge:v1:worker:${workerId}:admission`; // Only a disposable test Redis may be supplied. afterEach(async () => { await redis.flushall(); }); -async function setup(slots = 1, lanes = false): Promise { +async function setup(slots = 1, lanes = false, detached = false): Promise { const generation = await bridge.register({ protocolVersion: 1, workerId, incarnationId, capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: ['bash'], @@ -34,7 +35,8 @@ async function setup(slots = 1, lanes = false): Promise workspaces: [ { id: 'primary', ...(lanes ? { workspaceScopes: ['git_linked_worktree' as const] } : {}) }, { id: 'independent' }, - ] }, + ], + ...(detached ? { commandFeatures: ['detached' as const] } : {}) }, }, }); if (slots > 1) await bridge.confirmReady(workerId, incarnationId, generation); @@ -763,3 +765,57 @@ test.each(['quarantined after rejection', 'interrupted'])('an uncertain lane out state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' }, }); }); + +async function completeCommand(assignment: CodeBridgeAssignment): Promise { + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, + assignment.generation, assignment.leaseToken); + await bridge.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, incarnationId, generation: assignment.generation, leaseToken: assignment.leaseToken, + status: 'fulfilled', result: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', + exitCode: 0, stdout: assignment.assignmentId, stderr: '', truncated: false, timedOut: false }, + }); + await bridge.confirmWorkspaceCleanup(workerId, assignment.assignmentId, rejection(assignment, 'local cleanup confirmed')); +} + +test('a durable detached command runs beside its lane and settles independently', async () => { + const requests = await setup(2, true, true); + await submitLane(requests, requestId, { operation: 'execute_command', command: 'git status' }); + await submitLane(requests, 'request-00000000002', { operation: 'execute_command', command: 'npm test', detached: true }); + await submitLane(requests, 'request-00000000003', { operation: 'list_files' }); + await tick(requests); await tick(requests); + const ordinary = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + const background = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1))!; + expect(ordinary.request).toMatchObject({ command: 'git status', worktree: 'review' }); + expect(background.request).toMatchObject({ command: 'npm test', worktree: 'review', detached: true }); + expect(background.assignmentId.startsWith('durable-')).toBe(true); + expect((await requests.get(owner, 'request-00000000003'))?.state).toBe('queued'); + + const detachedFence = fenceKey(`native-workspace:${workspaceDetachedCommandKey(background.assignmentId)}`); + expect(await redis.get(laneFence)).toBe(ordinary.assignmentId); + expect(await redis.get(detachedFence)).toBe(background.assignmentId); + expect(await redis.ttl(laneFence)).toBe(-1); + const fenceTtl = await redis.ttl(detachedFence); + expect(fenceTtl).toBeGreaterThan(23 * 60 * 60); + expect(fenceTtl).toBeLessThanOrEqual(24 * 60 * 60); + + await completeCommand(ordinary); await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'completed', result: { stdout: ordinary.assignmentId } }); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('admitted'); + await tick(requests); + const queued = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + expect(queued.request).toMatchObject({ operation: 'list_files', worktree: 'review' }); + expect(await redis.get(laneFence)).toBe(queued.assignmentId); + expect(await redis.get(detachedFence)).toBe(background.assignmentId); + + await completeCommand(background); await tick(requests); + expect(await requests.get(owner, 'request-00000000002')).toMatchObject({ state: 'completed', result: { stdout: background.assignmentId } }); + expect(await redis.exists(detachedFence)).toBe(0); + expect(await redis.get(laneFence)).toBe(queued.assignmentId); +}); + +test('a durable detached command is refused for a worker that does not advertise the feature', async () => { + const requests = await setup(2, true); + await expect(submitLane(requests, requestId, { operation: 'execute_command', command: 'npm test', detached: true })) + .rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.zcard(queueKey)).toBe(0); +}); diff --git a/service/src/workspace-tools/requests.ts b/service/src/workspace-tools/requests.ts index 902651fc..27ee27a0 100644 --- a/service/src/workspace-tools/requests.ts +++ b/service/src/workspace-tools/requests.ts @@ -3,7 +3,7 @@ import type Redis from 'ioredis'; import type { WorkspaceToolRequest, WorkspaceToolResult } from '../../../packages/code/src/protocol'; import type { RedisBridgeStore, RegisteredBridgeWorker, DurableWorkspaceAssignment, CodeBridgeWorkspaceSettlement } from '../bridge/store'; import { BridgeAdmissionQueue } from '../bridge/admission'; -import { BridgeStoreError, workspaceAdmissionId } from '../bridge/store'; +import { BridgeStoreError, workspaceToolAdmissionId } from '../bridge/store'; const PREFIX = 'codeapi:workspace-requests:v1'; const ACTIVE = `${PREFIX}:active`; @@ -141,7 +141,7 @@ export class RedisWorkspaceRequests { try { accepted = await queue.submit({ workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, - workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), + workspaceId: workspaceToolAdmissionId(record.request, record.id), key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, }); } catch (error) { diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index 79268b52..88682ebf 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -16,6 +16,7 @@ function render(overrides) { CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', CODEAPI_BRIDGE_LANE_GIT: '', + CODEAPI_BRIDGE_DETACHED_COMMANDS: '', CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES: '', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', @@ -37,6 +38,7 @@ for (const overrides of [ CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', CODEAPI_BRIDGE_LANE_GIT: 'true', + CODEAPI_BRIDGE_DETACHED_COMMANDS: 'false', CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES: '1048576', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', @@ -54,6 +56,7 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_TOKEN, token); assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); assert.equal(env.CODEAPI_BRIDGE_LANE_GIT, overrides.CODEAPI_BRIDGE_LANE_GIT ?? 'false'); + assert.equal(env.CODEAPI_BRIDGE_DETACHED_COMMANDS, overrides.CODEAPI_BRIDGE_DETACHED_COMMANDS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); assert.equal(env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, overrides.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? '');