Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions service/src/bridge/concurrent-store.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,24 @@ test('committed results retain the root fence until cleanup, including receipt e
status: 'rejected' as const,
error: 'fixture',
};
await store.settle(workerId, assignment.assignmentId, intent);
await pending;
await store.settle(workerId, assignment.assignmentId, {
protocolVersion: BRIDGE_PROTOCOL_VERSION,
incarnationId,
generation: assignment.generation,
leaseToken: assignment.leaseToken,
status: 'fulfilled',
result: {
protocolVersion: BRIDGE_PROTOCOL_VERSION,
operation: 'read_file',
workspaceId: 'a',
path: 'file.txt',
content: 'fixture',
startLine: 1,
endLine: 1,
truncated: false,
},
});
await expect(pending).resolves.toMatchObject({ status: 'fulfilled' });
const fenceKey = (
await redis.keys(
`codeapi:bridge:v1:worker:${workerId}:workspace:*:quarantined`,
Expand Down
8 changes: 6 additions & 2 deletions service/src/bridge/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1989,7 +1989,7 @@ export class RedisBridgeStore {
settlement.status === 'rejected'
) {
// The dispatcher may already have timed out and finished its cleanup.
// Release only this settled reservation, retaining a quarantine marker.
// Release this settled reservation and its fence; a quarantine marker stays.
await this.commitPendingWorkspace(assignment, settlement);
await this.cleanupWithRetry(workerId, assignmentId, assignment);
}
Expand Down Expand Up @@ -2664,7 +2664,10 @@ export class RedisBridgeStore {
[
"if redis.call('EXISTS', KEYS[2]) == 0 then return 0 end",
"redis.call('HSET', KEYS[2], 'resultCommitted', '1')",
"if redis.call('HGET', KEYS[2], 'localCleanup') == '1' and redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]) end",
// A worker rejects only work that left the root unchanged, so its fence need not
// wait for a cleanup acknowledgement that a stopping worker may never deliver.
// A quarantine receipt has already replaced the fence and is never released here.
"if (ARGV[2] == 'rejected' or redis.call('HGET', KEYS[2], 'localCleanup') == '1') and redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]) end",
'return 1',
].join('\n'),
2,
Expand All @@ -2674,6 +2677,7 @@ export class RedisBridgeStore {
),
workspaceFenceReceiptKey(assignment.assignmentId),
assignment.assignmentId,
settlement.status,
),
this.redisCommandTimeoutMs,
'Bridge native workspace result commit',
Expand Down
100 changes: 97 additions & 3 deletions service/src/workspace-tools/requests.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import Redis from 'ioredis';
import { RedisBridgeStore } from '../bridge/store';
import { BridgeAdmissionQueue } from '../bridge/admission';
import type { CodeBridgeAssignment } from '../bridge/store';
import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol';
import type { BridgeRejectedSettlement, WorkspaceToolRequest } from '../../../packages/code/src/protocol';
import { RedisWorkspaceRequests } from './requests';
import { workspaceRequestCoordinationScope } from './coordination';
import type { WorkspaceRequestCoordinationPolicy } from './coordination';
Expand All @@ -25,13 +25,16 @@ const queueKey = `codeapi:bridge:v1:worker:${workerId}:admission`;
// Only a disposable test Redis may be supplied.
afterEach(async () => { await redis.flushall(); });

async function setup(slots = 1): Promise<RedisWorkspaceRequests> {
async function setup(slots = 1, lanes = false): Promise<RedisWorkspaceRequests> {
const generation = await bridge.register({ protocolVersion: 1, workerId, incarnationId,
capabilities: {
sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: ['bash'],
...(slots > 1 ? { workspaceLeaseSlots: slots, requiresReadyConfirmation: true } : {}),
workspaceTools: { protocolVersion: 1, operations: ['read_file', 'execute_command', 'list_files', 'search_text', 'preview_edit'],
workspaces: [{ id: 'primary' }, { id: 'independent' }] },
workspaces: [
{ id: 'primary', ...(lanes ? { workspaceScopes: ['git_linked_worktree' as const] } : {}) },
{ id: 'independent' },
] },
},
});
if (slots > 1) await bridge.confirmReady(workerId, incarnationId, generation);
Expand Down Expand Up @@ -669,3 +672,94 @@ test('unconfirmed durable mutation cancellation still reports an unknown outcome
const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`;
expect(await redis.get(fence)).toBe(assignment.assignmentId);
}, 10_000);

const lane = { workspaceId: 'primary', worktree: 'review' };

function fenceKey(fence: string): string {
return `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update(fence).digest('hex')}:quarantined`;
}

const laneFence = fenceKey('native-workspace:\0linked-worktree\0primary\0review');

function submitLane(requests: RedisWorkspaceRequests, id: string, request: Partial<WorkspaceToolRequest>): ReturnType<RedisWorkspaceRequests['submit']> {
return requests.submit({ owner, requestId: id, workerId, requireTenantBinding: false,
request: { protocolVersion: 1, ...lane, ...request } as WorkspaceToolRequest,
queueWaitMs: 300_000, executionTimeoutMs: 30_000,
});
}

async function leaseAnySlot(): Promise<CodeBridgeAssignment | undefined> {
for (const slot of [0, 1]) {
const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot);
if (assignment != null) return assignment;
}
return undefined;
}

function rejection(assignment: CodeBridgeAssignment, error: string): BridgeRejectedSettlement {
return { protocolVersion: 1 as const, incarnationId, generation: assignment.generation,
leaseToken: assignment.leaseToken, status: 'rejected' as const, error };
}

test('a rejected lane releases its fence even when its cleanup acknowledgement is lost', async () => {
const requests = await setup(2, true);
await submitLane(requests, requestId, { operation: 'execute_command', command: 'git status' });
await submitLane(requests, 'request-00000000002', { operation: 'list_files' });
await tick(requests);
const refused = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!;
expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1)).toBeUndefined();
await bridge.acknowledgeLease(workerId, incarnationId, refused.assignmentId, refused.generation, refused.leaseToken);
// The worker refuses a lane that does not exist, then stops before acknowledging local cleanup.
await bridge.settle(workerId, refused.assignmentId, {
...rejection(refused, 'No linked worktree named review in .worktrees'), errorCode: 'INVALID_REQUEST',
});
expect(await redis.get(laneFence)).toBeNull();
await tick(requests);
expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'INVALID_REQUEST' } });

await tick(requests);
const queued = (await leaseAnySlot())!;
expect(queued.request).toMatchObject({ operation: 'list_files', worktree: 'review' });
// A late acknowledgement for the refused assignment never releases the lane's next owner.
await bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId, rejection(refused, 'local cleanup confirmed'));
expect(await redis.get(laneFence)).toBe(queued.assignmentId);
await settle(queued, false); await tick(requests);
expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('failed');

await submit(requests, 'request-00000000003'); await tick(requests);
const checkout = (await leaseAnySlot())!;
expect(checkout.request).toMatchObject({ operation: 'read_file', workspaceId: 'primary' });
expect((checkout.request as WorkspaceToolRequest).worktree).toBeUndefined();
});

test.each(['quarantined after rejection', 'interrupted'])('an uncertain lane outcome still fences the lane and its checkout (%s)', async outcome => {
const requests = await setup(2, true);
await submitLane(requests, requestId, { operation: 'execute_command', command: 'git status' });
await tick(requests);
const assignment = (await leaseAnySlot())!;
await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken);
if (outcome === 'interrupted') {
const key = (await redis.zrange(activeKey, 0, -1))[0];
const stored = JSON.parse((await redis.hget(key, 'assignment'))!);
stored.expiresAt = new Date(Date.now() - 1).toISOString();
await redis.hset(key, 'assignment', JSON.stringify(stored));
} else {
// The settlement commits, but the worker cannot clear its local guard and reports the lane instead.
await bridge.settle(workerId, assignment.assignmentId, rejection(assignment, 'Command failed atomically'));
await bridge.settle(workerId, assignment.assignmentId, rejection(assignment, 'Workspace requires inspection before reset.'),
undefined, undefined, true);
}
await tick(new RedisWorkspaceRequests(redis, bridge));
expect((await requests.get(owner, requestId))?.state).toBe('failed');
expect(await redis.get(laneFence)).toBe(
outcome === 'interrupted' ? assignment.assignmentId : `quarantined:${assignment.assignmentId}`,
);

// Simulate execution-ownership TTL expiry. The lane fence outlives the reservation.
await redis.del(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`,
`codeapi:bridge:v1:worker:${workerId}:lock`, `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`);
await submit(requests, 'request-00000000002'); await tick(requests);
expect(await requests.get(owner, 'request-00000000002')).toMatchObject({
state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' },
});
});
Loading