From e37040b51949ad1a27049efb421dfb6096e30d9a Mon Sep 17 00:00:00 2001 From: aifanatic Date: Wed, 4 Nov 2026 16:15:19 -0600 Subject: [PATCH] fix(scheduler): scope failed dispatch rollback to its queued attempt --- .../src/post-processor-dispatcher.test.ts | 32 ++++++++++++++++++- .../src/post-processor-dispatcher.ts | 12 +++++-- docs/architecture/post-processing.md | 7 +++- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/apps/scheduler/src/post-processor-dispatcher.test.ts b/apps/scheduler/src/post-processor-dispatcher.test.ts index afe2c2219..f8a27e04d 100644 --- a/apps/scheduler/src/post-processor-dispatcher.test.ts +++ b/apps/scheduler/src/post-processor-dispatcher.test.ts @@ -116,10 +116,40 @@ describe("PostProcessorDispatcher", () => { // Should have rolled back status for req-1 expect(collection.updateOne).toHaveBeenCalledWith( - { _id: "req-1" }, + { _id: "req-1", "run._id": "run-1", "run.postProcessorStatus": "queued" }, { $unset: { "run.postProcessorStatus": "" } }, ); // Should have stopped the batch (req-2 never sent) expect(queueClient.sendMessage).toHaveBeenCalledTimes(1); }); + + it.each([ + { runId: "run-2", status: "queued", reason: "a newer attempt was queued" }, + { runId: "run-1", status: "processing", reason: "the worker already claimed the message" }, + { runId: "run-1", status: "done", reason: "the worker already completed the message" }, + ])("preserves status after a send failure when $reason", async ({ runId, status }) => { + db._findOne.mockResolvedValue({ _id: "post-processor", version: 1 }); + collection.findOneAndUpdate.mockResolvedValueOnce({ + _id: "req-1", run: { _id: "run-1", postProcessorStatus: "queued" }, + }); + + // Model a retry or delivery racing the failed send. Apply the rollback + // only if its Mongo predicates match the document that exists now. + const current: Record = { + _id: "req-1", "run._id": runId, "run.postProcessorStatus": status, + }; + collection.updateOne = vi.fn(async (filter: Record) => { + const matches = Object.entries(filter).every(([key, value]) => current[key] === value); + if (matches) delete current["run.postProcessorStatus"]; + return { matchedCount: matches ? 1 : 0 }; + }); + queueClient.sendMessage.mockRejectedValueOnce(new Error("send acknowledgement lost")); + + const dispatcher = new PostProcessorDispatcher(collection, db, queueClient, 60_000); + await (dispatcher as any).dispatch(); + + expect(collection.updateOne).toHaveBeenCalledOnce(); + expect(current["run.postProcessorStatus"]).toBe(status); + expect(queueClient.sendMessage).toHaveBeenCalledTimes(1); + }); }); diff --git a/apps/scheduler/src/post-processor-dispatcher.ts b/apps/scheduler/src/post-processor-dispatcher.ts index 20baa6c4a..f360c77c9 100644 --- a/apps/scheduler/src/post-processor-dispatcher.ts +++ b/apps/scheduler/src/post-processor-dispatcher.ts @@ -89,12 +89,18 @@ export class PostProcessorDispatcher { await this.queueClient.sendMessage(message); dispatched++; } catch (err) { - // Roll back status so the doc is picked up on next poll cycle + // Release only our still-queued attempt. A retry may have replaced + // the run, or a worker may have received the message despite a lost + // acknowledgement; neither state belongs to this rollback. await this.collection.updateOne( - { _id: claimed._id } as any, + { + _id: claimed._id, + "run._id": claimed.run?._id, + "run.postProcessorStatus": "queued", + }, { $unset: { "run.postProcessorStatus": "" } } as any, ); - console.error(`[PostProcessorDispatcher] Queue send failed for ${claimed._id}, rolled back status:`, err); + console.error(`[PostProcessorDispatcher] Queue send failed for ${claimed._id}, attempted queued-status rollback:`, err); break; // Stop batch — queue may be unavailable } console.log(`[PostProcessorDispatcher] Dispatched ${claimed._id}`); diff --git a/docs/architecture/post-processing.md b/docs/architecture/post-processing.md index 9774769e4..96a691b34 100644 --- a/docs/architecture/post-processing.md +++ b/docs/architecture/post-processing.md @@ -17,6 +17,11 @@ Two paths trigger post-processing: 1. **Event-driven (primary):** Coder workers enqueue a post-processing message immediately when a run completes, setting `run.postProcessorStatus: "queued"` atomically in the same DB write. 2. **Scheduler backfill:** The `PostProcessorDispatcher` polls every 30 seconds for runs that have `status: "done"` but no `postProcessorStatus` set (or a `postProcessorVersion` below the current target). This catches runs that completed before the event-driven dispatch was deployed, or that need re-processing after a version bump. +If the scheduler's queue send fails, it releases only the claimed run attempt +while its post-processing status is still `queued`. A concurrent retry's newer +attempt is left untouched. The rollback also preserves `processing` or terminal +status when a worker received the message despite a lost send acknowledgement. + ## ATIF Generation [ATIF](https://github.com/AISafety-ATIF/specification) (AI Tool Interaction Format) is an open specification for recording AI agent tool interactions. The post-processor converts HAR (HTTP Archive) recordings captured during benchmark runs into ATIF trajectory files using the [`atifact`](https://github.com/waldekmastykarz/atifact) package (pinned to `^0.10.1`, which extracts trajectories from both HTTP/SSE exchanges and WebSocket exchanges in a HAR). @@ -146,4 +151,4 @@ stateDiagram-v2 - **Blob Storage:** `snapshots` container (shared with HAR, tool-calls, and other artifacts) - **Database:** MongoDB `requests` collection (`run.postProcessorStatus`, `run.postProcessorVersion` fields) - **KEDA:** ScaledObject scales the worker to zero when queue is empty -- **Version registration:** Kubernetes Job runs on each deploy; Docker Compose uses an init service \ No newline at end of file +- **Version registration:** Kubernetes Job runs on each deploy; Docker Compose uses an init service