From 36e1d486ee15ea292593ae8a4c0111613ceec94d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 17 Nov 2026 07:05:19 -0500 Subject: [PATCH] fix: fence stale quarantine receipts on workspace reuse --- service/src/bridge/concurrent-store.test.ts | 4 +- service/src/bridge/store.ts | 4 +- service/src/workspace-tools/requests.test.ts | 44 +++++++++++++++++++- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index fc3d2c41..7aa9cfe8 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -176,7 +176,7 @@ test('late quarantine after confirmed cleanup cannot fence a newer assignment', undefined, 0, ))!; - await store.settle( + await expect(store.settle( workerId, assignment.assignmentId, { @@ -190,7 +190,7 @@ test('late quarantine after confirmed cleanup cannot fence a newer assignment', undefined, undefined, true, - ); + )).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); await settle(next); await expect(second).resolves.toMatchObject({ status: 'rejected' }); }); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 4dbd7d66..55f4d685 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -2481,8 +2481,10 @@ export class RedisBridgeStore { 'if keyCount >= 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end', "if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end", 'if keyCount >= 9 then', + // A successor invalidates every older receipt, even if its predecessor + // rejected before cleanup and the successor later releases its fence. + " redis.call('INCR', KEYS[9])", " local epoch = redis.call('GET', KEYS[9])", - ' if type(epoch) ~= \'string\' then epoch = \'0\'; redis.call(\'SET\', KEYS[9], epoch, \'EX\', ARGV[12]) end', ' if redis.call(\'PTTL\', KEYS[9]) < tonumber(ARGV[12]) * 1000 then redis.call(\'EXPIRE\', KEYS[9], ARGV[12]) end', " redis.call('HSET', KEYS[8], 'metadata', ARGV[8], 'epoch', epoch)", ' redis.call(\'EXPIRE\', KEYS[8], ARGV[12])', diff --git a/service/src/workspace-tools/requests.test.ts b/service/src/workspace-tools/requests.test.ts index 458950f8..58eb1d4c 100644 --- a/service/src/workspace-tools/requests.test.ts +++ b/service/src/workspace-tools/requests.test.ts @@ -721,7 +721,8 @@ test('a rejected lane releases its fence even when its cleanup acknowledgement i const queued = (await leaseAnySlot())!; expect(queued.request).toMatchObject({ operation: 'list_files', worktree: 'review' }); // A late acknowledgement for the refused assignment never releases the lane's next owner. - await bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId, rejection(refused, 'local cleanup confirmed')); + await expect(bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId, + rejection(refused, 'local cleanup confirmed'))).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); expect(await redis.get(laneFence)).toBe(queued.assignmentId); await settle(queued, false); await tick(requests); expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('failed'); @@ -763,3 +764,44 @@ test.each(['quarantined after rejection', 'interrupted'])('an uncertain lane out state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' }, }); }); + +test.each(['active', 'completed', 'quarantined'].flatMap(state => + [false, true].map(legacyReceipt => [state, legacyReceipt] as const), +))( + 'late quarantine from a rejected lane cannot replace its %s successor (legacy receipt=%s)', async (state, legacyReceipt) => { + const requests = await setup(2, true); + await submitLane(requests, requestId, { operation: 'execute_command', command: 'git status' }); + await submitLane(requests, 'request-00000000002', { operation: 'list_files' }); + await tick(requests); + const refused = (await leaseAnySlot())!; + await bridge.acknowledgeLease(workerId, incarnationId, refused.assignmentId, refused.generation, refused.leaseToken); + await bridge.settle(workerId, refused.assignmentId, rejection(refused, 'No linked worktree')); + if (legacyReceipt) { + // Receipts from before per-assignment epochs must also lose authority. + await redis.hset(`codeapi:bridge:v1:assignment:${refused.assignmentId}:workspace-fence-owner`, 'epoch', '0'); + await redis.set(`${laneFence}:epoch`, '0', 'PX', await redis.pttl(`${laneFence}:epoch`)); + } + await tick(requests); await tick(requests); + const successor = (await leaseAnySlot())!; + expect(await redis.get(laneFence)).toBe(successor.assignmentId); + if (state === 'completed') { + await settle(successor, false); await tick(requests); + } else if (state === 'quarantined') { + await bridge.settle(workerId, successor.assignmentId, rejection(successor, 'Workspace requires inspection'), + undefined, undefined, true); + } + const fence = await redis.get(laneFence); + expect(fence).toBe(state === 'completed' ? null : state === 'quarantined' + ? `quarantined:${successor.assignmentId}` : successor.assignmentId); + await expect(bridge.settle(workerId, refused.assignmentId, rejection(refused, 'Late quarantine'), + undefined, undefined, true)).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + expect(await redis.get(laneFence)).toBe(fence); + await expect(bridge.confirmWorkspaceCleanup(workerId, refused.assignmentId, + rejection(refused, 'Late cleanup'))).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + expect(await redis.get(laneFence)).toBe(fence); + if (state === 'active') { + await settle(successor, false); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('failed'); + } + }, +);