Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions service/src/bridge/concurrent-store.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ test('late quarantine after confirmed cleanup cannot fence a newer assignment',
undefined,
0,
))!;
await store.settle(
await expect(store.settle(
workerId,
assignment.assignmentId,
{
Expand All @@ -190,7 +190,7 @@ test('late quarantine after confirmed cleanup cannot fence a newer assignment',
undefined,
undefined,
true,
);
)).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' });
await settle(next);
await expect(second).resolves.toMatchObject({ status: 'rejected' });
});
Expand Down
4 changes: 3 additions & 1 deletion service/src/bridge/store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2481,8 +2481,10 @@ export class RedisBridgeStore {
'if keyCount >= 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end',
"if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end",
'if keyCount >= 9 then',
// A successor invalidates every older receipt, even if its predecessor
// rejected before cleanup and the successor later releases its fence.
" redis.call('INCR', KEYS[9])",
" local epoch = redis.call('GET', KEYS[9])",
' if type(epoch) ~= \'string\' then epoch = \'0\'; redis.call(\'SET\', KEYS[9], epoch, \'EX\', ARGV[12]) end',
' if redis.call(\'PTTL\', KEYS[9]) < tonumber(ARGV[12]) * 1000 then redis.call(\'EXPIRE\', KEYS[9], ARGV[12]) end',
" redis.call('HSET', KEYS[8], 'metadata', ARGV[8], 'epoch', epoch)",
' redis.call(\'EXPIRE\', KEYS[8], ARGV[12])',
Expand Down
44 changes: 43 additions & 1 deletion service/src/workspace-tools/requests.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -721,7 +721,8 @@ test('a rejected lane releases its fence even when its cleanup acknowledgement i
const queued = (await leaseAnySlot())!;
expect(queued.request).toMatchObject({ operation: 'list_files', worktree: 'review' });
// A late acknowledgement for the refused assignment never releases the lane's next owner.
await bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId, rejection(refused, 'local cleanup confirmed'));
await expect(bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId,
rejection(refused, 'local cleanup confirmed'))).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' });
expect(await redis.get(laneFence)).toBe(queued.assignmentId);
await settle(queued, false); await tick(requests);
expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('failed');
Expand Down Expand Up @@ -763,3 +764,44 @@ test.each(['quarantined after rejection', 'interrupted'])('an uncertain lane out
state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' },
});
});

test.each(['active', 'completed', 'quarantined'].flatMap(state =>
[false, true].map(legacyReceipt => [state, legacyReceipt] as const),
))(
'late quarantine from a rejected lane cannot replace its %s successor (legacy receipt=%s)', async (state, legacyReceipt) => {
const requests = await setup(2, true);
await submitLane(requests, requestId, { operation: 'execute_command', command: 'git status' });
await submitLane(requests, 'request-00000000002', { operation: 'list_files' });
await tick(requests);
const refused = (await leaseAnySlot())!;
await bridge.acknowledgeLease(workerId, incarnationId, refused.assignmentId, refused.generation, refused.leaseToken);
await bridge.settle(workerId, refused.assignmentId, rejection(refused, 'No linked worktree'));
if (legacyReceipt) {
// Receipts from before per-assignment epochs must also lose authority.
await redis.hset(`codeapi:bridge:v1:assignment:${refused.assignmentId}:workspace-fence-owner`, 'epoch', '0');
await redis.set(`${laneFence}:epoch`, '0', 'PX', await redis.pttl(`${laneFence}:epoch`));
}
await tick(requests); await tick(requests);
const successor = (await leaseAnySlot())!;
expect(await redis.get(laneFence)).toBe(successor.assignmentId);
if (state === 'completed') {
await settle(successor, false); await tick(requests);
} else if (state === 'quarantined') {
await bridge.settle(workerId, successor.assignmentId, rejection(successor, 'Workspace requires inspection'),
undefined, undefined, true);
}
const fence = await redis.get(laneFence);
expect(fence).toBe(state === 'completed' ? null : state === 'quarantined'
? `quarantined:${successor.assignmentId}` : successor.assignmentId);
await expect(bridge.settle(workerId, refused.assignmentId, rejection(refused, 'Late quarantine'),
undefined, undefined, true)).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' });
expect(await redis.get(laneFence)).toBe(fence);
await expect(bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId,
rejection(refused, 'Late cleanup'))).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' });
expect(await redis.get(laneFence)).toBe(fence);
if (state === 'active') {
await settle(successor, false); await tick(requests);
expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('failed');
}
},
);
Loading