From 61bdc9970656cd56bfefcc245e5da3526c36155b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 16 Nov 2026 18:39:10 -0500 Subject: [PATCH] fix: Release a Rejected Lane's Fence Without Its Cleanup Acknowledgement A worker rejects only work that left the root unchanged, yet a slot-scoped fence was released only after the worker also acknowledged local cleanup. When a worker's concurrent run stopped between a committed rejection and that acknowledgement, nothing ever re-sent it: the lane kept a plain fence with no TTL, and its parent checkout refused every request as quarantined. The result commit now releases the fence for a rejected settlement, still by compare-and-delete on the assignment's own id, so a quarantine receipt or a newer owner is never released. Fulfilled results keep waiting for cleanup, and interrupted or reported-uncertain outcomes still quarantine. --- service/src/bridge/concurrent-store.test.ts | 20 +++- service/src/bridge/store.ts | 8 +- service/src/workspace-tools/requests.test.ts | 100 ++++++++++++++++++- 3 files changed, 121 insertions(+), 7 deletions(-) diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index 6bdfde88..fc3d2c41 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -110,8 +110,24 @@ test('committed results retain the root fence until cleanup, including receipt e status: 'rejected' as const, error: 'fixture', }; - await store.settle(workerId, assignment.assignmentId, intent); - await pending; + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'fulfilled', + result: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: 'a', + path: 'file.txt', + content: 'fixture', + startLine: 1, + endLine: 1, + truncated: false, + }, + }); + await expect(pending).resolves.toMatchObject({ status: 'fulfilled' }); const fenceKey = ( await redis.keys( `codeapi:bridge:v1:worker:${workerId}:workspace:*:quarantined`, diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index f9919f49..4dbd7d66 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -1989,7 +1989,7 @@ export class RedisBridgeStore { settlement.status === 'rejected' ) { // The dispatcher may already have timed out and finished its cleanup. - // Release only this settled reservation, retaining a quarantine marker. + // Release this settled reservation and its fence; a quarantine marker stays. await this.commitPendingWorkspace(assignment, settlement); await this.cleanupWithRetry(workerId, assignmentId, assignment); } @@ -2664,7 +2664,10 @@ export class RedisBridgeStore { [ "if redis.call('EXISTS', KEYS[2]) == 0 then return 0 end", "redis.call('HSET', KEYS[2], 'resultCommitted', '1')", - "if redis.call('HGET', KEYS[2], 'localCleanup') == '1' and redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]) end", + // A worker rejects only work that left the root unchanged, so its fence need not + // wait for a cleanup acknowledgement that a stopping worker may never deliver. + // A quarantine receipt has already replaced the fence and is never released here. + "if (ARGV[2] == 'rejected' or redis.call('HGET', KEYS[2], 'localCleanup') == '1') and redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]) end", 'return 1', ].join('\n'), 2, @@ -2674,6 +2677,7 @@ export class RedisBridgeStore { ), workspaceFenceReceiptKey(assignment.assignmentId), assignment.assignmentId, + settlement.status, ), this.redisCommandTimeoutMs, 'Bridge native workspace result commit', diff --git a/service/src/workspace-tools/requests.test.ts b/service/src/workspace-tools/requests.test.ts index a0f7d99f..458950f8 100644 --- a/service/src/workspace-tools/requests.test.ts +++ b/service/src/workspace-tools/requests.test.ts @@ -5,7 +5,7 @@ import Redis from 'ioredis'; import { RedisBridgeStore } from '../bridge/store'; import { BridgeAdmissionQueue } from '../bridge/admission'; import type { CodeBridgeAssignment } from '../bridge/store'; -import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import type { BridgeRejectedSettlement, WorkspaceToolRequest } from '../../../packages/code/src/protocol'; import { RedisWorkspaceRequests } from './requests'; import { workspaceRequestCoordinationScope } from './coordination'; import type { WorkspaceRequestCoordinationPolicy } from './coordination'; @@ -25,13 +25,16 @@ const queueKey = `codeapi:bridge:v1:worker:${workerId}:admission`; // Only a disposable test Redis may be supplied. afterEach(async () => { await redis.flushall(); }); -async function setup(slots = 1): Promise { +async function setup(slots = 1, lanes = false): Promise { const generation = await bridge.register({ protocolVersion: 1, workerId, incarnationId, capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: ['bash'], ...(slots > 1 ? { workspaceLeaseSlots: slots, requiresReadyConfirmation: true } : {}), workspaceTools: { protocolVersion: 1, operations: ['read_file', 'execute_command', 'list_files', 'search_text', 'preview_edit'], - workspaces: [{ id: 'primary' }, { id: 'independent' }] }, + workspaces: [ + { id: 'primary', ...(lanes ? { workspaceScopes: ['git_linked_worktree' as const] } : {}) }, + { id: 'independent' }, + ] }, }, }); if (slots > 1) await bridge.confirmReady(workerId, incarnationId, generation); @@ -669,3 +672,94 @@ test('unconfirmed durable mutation cancellation still reports an unknown outcome const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`; expect(await redis.get(fence)).toBe(assignment.assignmentId); }, 10_000); + +const lane = { workspaceId: 'primary', worktree: 'review' }; + +function fenceKey(fence: string): string { + return `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update(fence).digest('hex')}:quarantined`; +} + +const laneFence = fenceKey('native-workspace:\0linked-worktree\0primary\0review'); + +function submitLane(requests: RedisWorkspaceRequests, id: string, request: Partial): ReturnType { + return requests.submit({ owner, requestId: id, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, ...lane, ...request } as WorkspaceToolRequest, + queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); +} + +async function leaseAnySlot(): Promise { + for (const slot of [0, 1]) { + const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot); + if (assignment != null) return assignment; + } + return undefined; +} + +function rejection(assignment: CodeBridgeAssignment, error: string): BridgeRejectedSettlement { + return { protocolVersion: 1 as const, incarnationId, generation: assignment.generation, + leaseToken: assignment.leaseToken, status: 'rejected' as const, error }; +} + +test('a rejected lane releases its fence even when its cleanup acknowledgement is lost', async () => { + const requests = await setup(2, true); + await submitLane(requests, requestId, { operation: 'execute_command', command: 'git status' }); + await submitLane(requests, 'request-00000000002', { operation: 'list_files' }); + await tick(requests); + const refused = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1)).toBeUndefined(); + await bridge.acknowledgeLease(workerId, incarnationId, refused.assignmentId, refused.generation, refused.leaseToken); + // The worker refuses a lane that does not exist, then stops before acknowledging local cleanup. + await bridge.settle(workerId, refused.assignmentId, { + ...rejection(refused, 'No linked worktree named review in .worktrees'), errorCode: 'INVALID_REQUEST', + }); + expect(await redis.get(laneFence)).toBeNull(); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'INVALID_REQUEST' } }); + + await tick(requests); + const queued = (await leaseAnySlot())!; + expect(queued.request).toMatchObject({ operation: 'list_files', worktree: 'review' }); + // A late acknowledgement for the refused assignment never releases the lane's next owner. + await bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId, rejection(refused, 'local cleanup confirmed')); + expect(await redis.get(laneFence)).toBe(queued.assignmentId); + await settle(queued, false); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('failed'); + + await submit(requests, 'request-00000000003'); await tick(requests); + const checkout = (await leaseAnySlot())!; + expect(checkout.request).toMatchObject({ operation: 'read_file', workspaceId: 'primary' }); + expect((checkout.request as WorkspaceToolRequest).worktree).toBeUndefined(); +}); + +test.each(['quarantined after rejection', 'interrupted'])('an uncertain lane outcome still fences the lane and its checkout (%s)', async outcome => { + const requests = await setup(2, true); + await submitLane(requests, requestId, { operation: 'execute_command', command: 'git status' }); + await tick(requests); + const assignment = (await leaseAnySlot())!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + if (outcome === 'interrupted') { + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + stored.expiresAt = new Date(Date.now() - 1).toISOString(); + await redis.hset(key, 'assignment', JSON.stringify(stored)); + } else { + // The settlement commits, but the worker cannot clear its local guard and reports the lane instead. + await bridge.settle(workerId, assignment.assignmentId, rejection(assignment, 'Command failed atomically')); + await bridge.settle(workerId, assignment.assignmentId, rejection(assignment, 'Workspace requires inspection before reset.'), + undefined, undefined, true); + } + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('failed'); + expect(await redis.get(laneFence)).toBe( + outcome === 'interrupted' ? assignment.assignmentId : `quarantined:${assignment.assignmentId}`, + ); + + // Simulate execution-ownership TTL expiry. The lane fence outlives the reservation. + await redis.del(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`, + `codeapi:bridge:v1:worker:${workerId}:lock`, `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`); + await submit(requests, 'request-00000000002'); await tick(requests); + expect(await requests.get(owner, 'request-00000000002')).toMatchObject({ + state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' }, + }); +});