From 82a3969d88547dba81b7124dea74cae270e7a95e Mon Sep 17 00:00:00 2001 From: Codevil Agent Date: Sun, 20 Sep 2026 10:36:23 +0000 Subject: [PATCH] worker: reap expired sandbox workspace backups from R2 Cloudflare Sandbox createBackup ttl only rejects expired archives at restore time; the platform never deletes the R2 objects. The codevil-sandbox-backups bucket had no lifecycle rule and no cleaner, so every 600MB /workspace snapshot accumulated indefinitely (23.5GB and counting, 13.5GB over the R2 free tier). Add a daily scheduled sweep that deletes backups/ objects older than the 30-day WORKSPACE_CACHE_TTL_SECONDS and removes the matching workspace_snapshots rows from D1 so restores never target a missing archive. --- packages/worker/src/backup-reaper.ts | 108 +++++++++++++++++ packages/worker/src/index.ts | 25 ++++ packages/worker/test/backup-reaper.test.mjs | 127 ++++++++++++++++++++ packages/worker/wrangler.toml | 3 + 4 files changed, 263 insertions(+) create mode 100644 packages/worker/src/backup-reaper.ts create mode 100644 packages/worker/test/backup-reaper.test.mjs diff --git a/packages/worker/src/backup-reaper.ts b/packages/worker/src/backup-reaper.ts new file mode 100644 index 0000000..6c07c5d --- /dev/null +++ b/packages/worker/src/backup-reaper.ts @@ -0,0 +1,108 @@ +import { WORKSPACE_CACHE_TTL_SECONDS } from "./workspace-cache.js"; + +/** + * Expired backup sweep for the `codevil-sandbox-backups` R2 bucket. + * + * Cloudflare Sandboxes stores each directory backup as two objects under + * `backups/{backupId}/`: `data.sqsh` and `meta.json`. The `ttl` passed to + * `createBackup` only causes restore-time rejection — the platform never + * deletes the R2 objects — so a bucket can grow without bound unless + * something reaps expired archives. This module deletes objects older than + * the workspace-cache TTL (30 days) and removes the matching + * `workspace_snapshots` rows so restores never target a missing archive. + */ + +/** R2 object prefix under which sandbox backup archives are stored. */ +export const BACKUP_OBJECT_PREFIX = "backups/"; + +/** Largest batch handed to a single R2 delete call / list page. */ +export const BACKUP_REAPER_PAGE_LIMIT = 1000; + +export interface BackupReaperResult { + /** Objects inspected under the `backups/` prefix. */ + listed: number; + /** R2 objects deleted (data.sqsh + meta.json per expired backup). */ + deleted: number; + /** `workspace_snapshots` rows removed from D1. */ + expiredRows: number; +} + +/** Epoch-ms threshold before which a backup object is considered expired. */ +export function backupExpiryCutoffMs( + now: Date, + ttlSeconds = WORKSPACE_CACHE_TTL_SECONDS, +): number { + return now.getTime() - ttlSeconds * 1000; +} + +/** + * Select the keys of backup objects whose upload time has passed the TTL. + * Untimestamped objects are kept (never delete what we cannot date-check). + */ +export function selectExpiredBackupKeys( + objects: readonly { key: string; uploaded: Date }[], + now = new Date(), + ttlSeconds = WORKSPACE_CACHE_TTL_SECONDS, +): string[] { + const cutoff = backupExpiryCutoffMs(now, ttlSeconds); + return objects + .filter((object) => object.uploaded instanceof Date && object.uploaded.getTime() < cutoff) + .map((object) => object.key); +} + +/** D1 statement that removes workspace snapshot rows older than the TTL. */ +export function expiredWorkspaceSnapshotsDelete( + now = new Date(), + ttlSeconds = WORKSPACE_CACHE_TTL_SECONDS, +): { sql: string; bindings: [string] } { + return { + sql: "DELETE FROM workspace_snapshots WHERE created_at < ?", + bindings: [new Date(backupExpiryCutoffMs(now, ttlSeconds)).toISOString()], + }; +} + +/** + * Delete expired backup objects and their D1 snapshot rows. + * + * `bucket` is optional because the R2 binding is not configured in every + * local/dev environment; without it the sweep is a no-op for storage. The + * D1 row cleanup still runs so metadata cannot outlive intent. + */ +export async function sweepWorkspaceBackups( + bucket: R2Bucket | undefined, + db: D1Database, + now = new Date(), + opts: { ttlSeconds?: number; pageLimit?: number } = {}, +): Promise { + const ttlSeconds = opts.ttlSeconds ?? WORKSPACE_CACHE_TTL_SECONDS; + const pageLimit = opts.pageLimit ?? BACKUP_REAPER_PAGE_LIMIT; + + let listed = 0; + let deleted = 0; + if (bucket) { + let cursor: string | undefined; + do { + const page = await bucket.list({ + prefix: BACKUP_OBJECT_PREFIX, + limit: pageLimit, + ...(cursor ? { cursor } : {}), + }); + const objects = page.objects ?? []; + listed += objects.length; + const expiredKeys = selectExpiredBackupKeys(objects, now, ttlSeconds); + for (let offset = 0; offset < expiredKeys.length; offset += BACKUP_REAPER_PAGE_LIMIT) { + const chunk = expiredKeys.slice(offset, offset + BACKUP_REAPER_PAGE_LIMIT); + await bucket.delete(chunk); + deleted += chunk.length; + } + cursor = page.truncated ? page.cursor : undefined; + } while (cursor); + } + + let expiredRows = 0; + const statement = expiredWorkspaceSnapshotsDelete(now, ttlSeconds); + const result = await db.prepare(statement.sql).bind(...statement.bindings).run(); + expiredRows = result.meta.changes ?? 0; + + return { listed, deleted, expiredRows }; +} \ No newline at end of file diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts index f28c603..64f17ab 100644 --- a/packages/worker/src/index.ts +++ b/packages/worker/src/index.ts @@ -19,11 +19,14 @@ import { observeRoutedResponse, sandboxLifecycleLogger, withRequestId, + workerLog, + workerLogException, } from "./logging.js"; import { collectWorkerSecretValues } from "./worker-env.js"; import { redactEvent } from "./redaction.js"; import type { Env } from "./worker-env.js"; import { handleSandboxProxy } from "./sandbox-proxy.js"; +import { sweepWorkspaceBackups } from "./backup-reaper.js"; export type { Env } from "./worker-env.js"; @@ -177,6 +180,28 @@ function withCors(request: Request, env: Env, response: Response): Response { } export default { + async scheduled(controller: ScheduledController, env: Env): Promise { + const secrets = collectWorkerSecretValues(env); + try { + const result = await sweepWorkspaceBackups(env.BACKUP_BUCKET, env.DB); + workerLog( + result.deleted > 0 ? "INFO" : "DEBUG", + "workspace_cache.sweep", + { + cron: controller.cron, + listed: result.listed, + deleted: result.deleted, + expired_rows: result.expiredRows, + }, + secrets, + ); + } catch (error) { + workerLogException("workspace_cache.sweep.failed", error, { + cron: controller.cron, + }, secrets); + } + }, + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { const redactionSecrets = collectWorkerSecretValues(env); if (request.method === "OPTIONS") { diff --git a/packages/worker/test/backup-reaper.test.mjs b/packages/worker/test/backup-reaper.test.mjs new file mode 100644 index 0000000..4f460ac --- /dev/null +++ b/packages/worker/test/backup-reaper.test.mjs @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + BACKUP_OBJECT_PREFIX, + backupExpiryCutoffMs, + expiredWorkspaceSnapshotsDelete, + selectExpiredBackupKeys, + sweepWorkspaceBackups, +} from "../dist/backup-reaper.js"; + +const TTL_30D = 30 * 24 * 60 * 60; + +test("backup expiry cutoff subtracts the TTL from now", () => { + const now = new Date("2026-09-20T00:00:00.000Z"); + assert.equal(backupExpiryCutoffMs(now), now.getTime() - TTL_30D * 1000); + assert.equal(backupExpiryCutoffMs(now, 60), now.getTime() - 60_000); +}); + +test("selectExpiredBackupKeys returns keys older than the TTL only", () => { + const now = new Date("2026-09-20T00:00:00.000Z"); + const older = { key: "backups/x/data.sqsh", uploaded: new Date("2026-07-01T00:00:00.000Z") }; + const recent = { key: "backups/y/data.sqsh", uploaded: new Date("2026-09-18T00:00:00.000Z") }; + const boundary = { key: "backups/z/data.sqsh", uploaded: new Date(backupExpiryCutoffMs(now) - 1) }; + const notAged = { key: "backups/w/meta.json", uploaded: new Date(backupExpiryCutoffMs(now) + 1) }; + + assert.deepEqual( + selectExpiredBackupKeys([older, recent, boundary, notAged], now), + ["backups/x/data.sqsh", "backups/z/data.sqsh"], + ); +}); + +test("selectExpiredBackupKeys never deletes objects without an upload timestamp", () => { + const now = new Date("2026-09-20T00:00:00.000Z"); + const untracked = { key: "backups/x/data.sqsh", uploaded: null }; + assert.deepEqual(selectExpiredBackupKeys([untracked], now), []); +}); + +test("expiredWorkspaceSnapshotsDelete scopes deletion to rows older than the TTL", () => { + const now = new Date("2026-09-20T00:00:00.000Z"); + assert.deepEqual(expiredWorkspaceSnapshotsDelete(now, 60), { + sql: "DELETE FROM workspace_snapshots WHERE created_at < ?", + bindings: [new Date(now.getTime() - 60_000).toISOString()], + }); +}); + +test("sweepWorkspaceBackups lists, filters, and deletes expired objects by page", async () => { + const now = new Date("2026-09-20T00:00:00.000Z"); + const oldObject = { key: "backups/a/data.sqsh", uploaded: new Date("2026-08-01T00:00:00.000Z") }; + const oldMeta = { key: "backups/a/meta.json", uploaded: new Date("2026-08-01T00:00:00.000Z") }; + const freshObject = { key: "backups/b/data.sqsh", uploaded: new Date("2026-09-19T00:00:00.000Z") }; + + const deletedPages = []; + const bucket = { + list: async () => ({ objects: [oldObject, freshObject, oldMeta], truncated: false }), + delete: async (keys) => deletedPages.push(keys), + }; + const runs = []; + const db = { + prepare: () => ({ + bind: (...args) => { + runs.push(args); + return { run: async () => ({ meta: { changes: 2 } }) }; + }, + }), + }; + + const result = await sweepWorkspaceBackups(bucket, db, now); + + assert.deepEqual(result, { listed: 3, deleted: 2, expiredRows: 2 }); + assert.deepEqual(deletedPages, [["backups/a/data.sqsh", "backups/a/meta.json"]]); + assert.deepEqual(runs[0], [new Date(backupExpiryCutoffMs(now)).toISOString()]); + assert.equal(BACKUP_OBJECT_PREFIX, "backups/"); +}); + +test("sweepWorkspaceBackups follows the pagination cursor", async () => { + const now = new Date("2026-09-20T00:00:00.000Z"); + const pages = [ + { + objects: [ + { key: "backups/a/data.sqsh", uploaded: new Date("2026-08-01T00:00:00.000Z") }, + ], + truncated: true, + cursor: "page-2", + }, + { + objects: [ + { key: "backups/b/meta.json", uploaded: new Date("2026-08-02T00:00:00.000Z") }, + ], + truncated: false, + }, + ]; + const listCalls = []; + const deletedPages = []; + const bucket = { + list: async (options) => { + listCalls.push(options); + return pages.length ? pages.shift() : { objects: [], truncated: false }; + }, + delete: async (keys) => deletedPages.push(keys), + }; + const db = { + prepare: () => ({ + bind: () => ({ run: async () => ({ meta: { changes: 0 } }) }), + }), + }; + + const result = await sweepWorkspaceBackups(bucket, db, now); + + assert.equal(result.listed, 2); + assert.equal(result.deleted, 2); + assert.deepEqual(listCalls.map((o) => o.cursor), [undefined, "page-2"]); + assert.deepEqual(deletedPages, [["backups/a/data.sqsh"], ["backups/b/meta.json"]]); +}); + +test("sweepWorkspaceBackups tolerates a missing R2 binding (local dev)", async () => { + const now = new Date("2026-09-20T00:00:00.000Z"); + const db = { + prepare: () => ({ + bind: () => ({ run: async () => ({ meta: { changes: 0 } }) }), + }), + }; + + const result = await sweepWorkspaceBackups(undefined, db, now); + + assert.deepEqual(result, { listed: 0, deleted: 0, expiredRows: 0 }); +}); \ No newline at end of file diff --git a/packages/worker/wrangler.toml b/packages/worker/wrangler.toml index f26f77d..c132b7a 100644 --- a/packages/worker/wrangler.toml +++ b/packages/worker/wrangler.toml @@ -26,6 +26,9 @@ head_sampling_rate = 1 enabled = true head_sampling_rate = 1 +[triggers] +crons = ["0 4 * * *"] + [[containers]] class_name = "Sandbox" image = "../../Dockerfile.sandbox"