worker: reap expired sandbox workspace backups from R2

packages/worker/src/backup-reaper.ts

@@ -0,0 +1,108 @@
+import { WORKSPACE_CACHE_TTL_SECONDS } from "./workspace-cache.js";
+
+/**
+ * Expired backup sweep for the `codevil-sandbox-backups` R2 bucket.
+ *
+ * Cloudflare Sandboxes stores each directory backup as two objects under
+ * `backups/{backupId}/`: `data.sqsh` and `meta.json`. The `ttl` passed to
+ * `createBackup` only causes restore-time rejection — the platform never
+ * deletes the R2 objects — so a bucket can grow without bound unless
+ * something reaps expired archives. This module deletes objects older than
+ * the workspace-cache TTL (30 days) and removes the matching
+ * `workspace_snapshots` rows so restores never target a missing archive.
+ */
+
+/** R2 object prefix under which sandbox backup archives are stored. */
+export const BACKUP_OBJECT_PREFIX = "backups/";
+
+/** Largest batch handed to a single R2 delete call / list page. */
+export const BACKUP_REAPER_PAGE_LIMIT = 1000;
+
+export interface BackupReaperResult {
+  /** Objects inspected under the `backups/` prefix. */
+  listed: number;
+  /** R2 objects deleted (data.sqsh + meta.json per expired backup). */
+  deleted: number;
+  /** `workspace_snapshots` rows removed from D1. */
+  expiredRows: number;
+}
+
+/** Epoch-ms threshold before which a backup object is considered expired. */
+export function backupExpiryCutoffMs(
+  now: Date,
+  ttlSeconds = WORKSPACE_CACHE_TTL_SECONDS,
+): number {
+  return now.getTime() - ttlSeconds * 1000;
+}
+
+/**
+ * Select the keys of backup objects whose upload time has passed the TTL.
+ * Untimestamped objects are kept (never delete what we cannot date-check).
+ */
+export function selectExpiredBackupKeys(
+  objects: readonly { key: string; uploaded: Date }[],
+  now = new Date(),
+  ttlSeconds = WORKSPACE_CACHE_TTL_SECONDS,
+): string[] {
+  const cutoff = backupExpiryCutoffMs(now, ttlSeconds);
+  return objects
+    .filter((object) => object.uploaded instanceof Date && object.uploaded.getTime() < cutoff)
+    .map((object) => object.key);
+}
+
+/** D1 statement that removes workspace snapshot rows older than the TTL. */
+export function expiredWorkspaceSnapshotsDelete(
+  now = new Date(),
+  ttlSeconds = WORKSPACE_CACHE_TTL_SECONDS,
+): { sql: string; bindings: [string] } {
+  return {
+    sql: "DELETE FROM workspace_snapshots WHERE created_at < ?",
+    bindings: [new Date(backupExpiryCutoffMs(now, ttlSeconds)).toISOString()],
+  };
+}
+
+/**
+ * Delete expired backup objects and their D1 snapshot rows.
+ *
+ * `bucket` is optional because the R2 binding is not configured in every
+ * local/dev environment; without it the sweep is a no-op for storage. The
+ * D1 row cleanup still runs so metadata cannot outlive intent.
+ */
+export async function sweepWorkspaceBackups(
+  bucket: R2Bucket | undefined,
+  db: D1Database,
+  now = new Date(),
+  opts: { ttlSeconds?: number; pageLimit?: number } = {},
+): Promise<BackupReaperResult> {
+  const ttlSeconds = opts.ttlSeconds ?? WORKSPACE_CACHE_TTL_SECONDS;
+  const pageLimit = opts.pageLimit ?? BACKUP_REAPER_PAGE_LIMIT;
+
+  let listed = 0;
+  let deleted = 0;
+  if (bucket) {
+    let cursor: string | undefined;
+    do {
+      const page = await bucket.list({
+        prefix: BACKUP_OBJECT_PREFIX,
+        limit: pageLimit,
+        ...(cursor ? { cursor } : {}),
+      });
+      const objects = page.objects ?? [];
+      listed += objects.length;
+      const expiredKeys = selectExpiredBackupKeys(objects, now, ttlSeconds);
+      for (let offset = 0; offset < expiredKeys.length; offset += BACKUP_REAPER_PAGE_LIMIT) {
+        const chunk = expiredKeys.slice(offset, offset + BACKUP_REAPER_PAGE_LIMIT);
+        await bucket.delete(chunk);
+        deleted += chunk.length;
+      }
+      cursor = page.truncated ? page.cursor : undefined;
+    } while (cursor);
+  }
+
+  let expiredRows = 0;
+  const statement = expiredWorkspaceSnapshotsDelete(now, ttlSeconds);
+  const result = await db.prepare(statement.sql).bind(...statement.bindings).run();
+  expiredRows = result.meta.changes ?? 0;
+
+  return { listed, deleted, expiredRows };
+}

packages/worker/src/index.ts

@@ -19,11 +19,14 @@ import {
   observeRoutedResponse,
   sandboxLifecycleLogger,
   withRequestId,
+  workerLog,
+  workerLogException,
 } from "./logging.js";
 import { collectWorkerSecretValues } from "./worker-env.js";
 import { redactEvent } from "./redaction.js";
 import type { Env } from "./worker-env.js";
 import { handleSandboxProxy } from "./sandbox-proxy.js";
+import { sweepWorkspaceBackups } from "./backup-reaper.js";
 
 export type { Env } from "./worker-env.js";
 
@@ -177,6 +180,28 @@ function withCors(request: Request, env: Env, response: Response): Response {
 }
 
 export default {
+  async scheduled(controller: ScheduledController, env: Env): Promise<void> {
+    const secrets = collectWorkerSecretValues(env);
+    try {
+      const result = await sweepWorkspaceBackups(env.BACKUP_BUCKET, env.DB);
+      workerLog(
+        result.deleted > 0 ? "INFO" : "DEBUG",
+        "workspace_cache.sweep",
+        {
+          cron: controller.cron,
+          listed: result.listed,
+          deleted: result.deleted,
+          expired_rows: result.expiredRows,
+        },
+        secrets,
+      );
+    } catch (error) {
+      workerLogException("workspace_cache.sweep.failed", error, {
+        cron: controller.cron,
+      }, secrets);
+    }
+  },
+
   async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
     const redactionSecrets = collectWorkerSecretValues(env);
     if (request.method === "OPTIONS") {

packages/worker/test/backup-reaper.test.mjs

@@ -0,0 +1,127 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  BACKUP_OBJECT_PREFIX,
+  backupExpiryCutoffMs,
+  expiredWorkspaceSnapshotsDelete,
+  selectExpiredBackupKeys,
+  sweepWorkspaceBackups,
+} from "../dist/backup-reaper.js";
+
+const TTL_30D = 30 * 24 * 60 * 60;
+
+test("backup expiry cutoff subtracts the TTL from now", () => {
+  const now = new Date("2026-09-20T00:00:00.000Z");
+  assert.equal(backupExpiryCutoffMs(now), now.getTime() - TTL_30D * 1000);
+  assert.equal(backupExpiryCutoffMs(now, 60), now.getTime() - 60_000);
+});
+
+test("selectExpiredBackupKeys returns keys older than the TTL only", () => {
+  const now = new Date("2026-09-20T00:00:00.000Z");
+  const older = { key: "backups/x/data.sqsh", uploaded: new Date("2026-07-01T00:00:00.000Z") };
+  const recent = { key: "backups/y/data.sqsh", uploaded: new Date("2026-09-18T00:00:00.000Z") };
+  const boundary = { key: "backups/z/data.sqsh", uploaded: new Date(backupExpiryCutoffMs(now) - 1) };
+  const notAged = { key: "backups/w/meta.json", uploaded: new Date(backupExpiryCutoffMs(now) + 1) };
+
+  assert.deepEqual(
+    selectExpiredBackupKeys([older, recent, boundary, notAged], now),
+    ["backups/x/data.sqsh", "backups/z/data.sqsh"],
+  );
+});
+
+test("selectExpiredBackupKeys never deletes objects without an upload timestamp", () => {
+  const now = new Date("2026-09-20T00:00:00.000Z");
+  const untracked = { key: "backups/x/data.sqsh", uploaded: null };
+  assert.deepEqual(selectExpiredBackupKeys([untracked], now), []);
+});
+
+test("expiredWorkspaceSnapshotsDelete scopes deletion to rows older than the TTL", () => {
+  const now = new Date("2026-09-20T00:00:00.000Z");
+  assert.deepEqual(expiredWorkspaceSnapshotsDelete(now, 60), {
+    sql: "DELETE FROM workspace_snapshots WHERE created_at < ?",
+    bindings: [new Date(now.getTime() - 60_000).toISOString()],
+  });
+});
+
+test("sweepWorkspaceBackups lists, filters, and deletes expired objects by page", async () => {
+  const now = new Date("2026-09-20T00:00:00.000Z");
+  const oldObject = { key: "backups/a/data.sqsh", uploaded: new Date("2026-08-01T00:00:00.000Z") };
+  const oldMeta = { key: "backups/a/meta.json", uploaded: new Date("2026-08-01T00:00:00.000Z") };
+  const freshObject = { key: "backups/b/data.sqsh", uploaded: new Date("2026-09-19T00:00:00.000Z") };
+
+  const deletedPages = [];
+  const bucket = {
+    list: async () => ({ objects: [oldObject, freshObject, oldMeta], truncated: false }),
+    delete: async (keys) => deletedPages.push(keys),
+  };
+  const runs = [];
+  const db = {
+    prepare: () => ({
+      bind: (...args) => {
+        runs.push(args);
+        return { run: async () => ({ meta: { changes: 2 } }) };
+      },
+    }),
+  };
+
+  const result = await sweepWorkspaceBackups(bucket, db, now);
+
+  assert.deepEqual(result, { listed: 3, deleted: 2, expiredRows: 2 });
+  assert.deepEqual(deletedPages, [["backups/a/data.sqsh", "backups/a/meta.json"]]);
+  assert.deepEqual(runs[0], [new Date(backupExpiryCutoffMs(now)).toISOString()]);
+  assert.equal(BACKUP_OBJECT_PREFIX, "backups/");
+});
+
+test("sweepWorkspaceBackups follows the pagination cursor", async () => {
+  const now = new Date("2026-09-20T00:00:00.000Z");
+  const pages = [
+    {
+      objects: [
+        { key: "backups/a/data.sqsh", uploaded: new Date("2026-08-01T00:00:00.000Z") },
+      ],
+      truncated: true,
+      cursor: "page-2",
+    },
+    {
+      objects: [
+        { key: "backups/b/meta.json", uploaded: new Date("2026-08-02T00:00:00.000Z") },
+      ],
+      truncated: false,
+    },
+  ];
+  const listCalls = [];
+  const deletedPages = [];
+  const bucket = {
+    list: async (options) => {
+      listCalls.push(options);
+      return pages.length ? pages.shift() : { objects: [], truncated: false };
+    },
+    delete: async (keys) => deletedPages.push(keys),
+  };
+  const db = {
+    prepare: () => ({
+      bind: () => ({ run: async () => ({ meta: { changes: 0 } }) }),
+    }),
+  };
+
+  const result = await sweepWorkspaceBackups(bucket, db, now);
+
+  assert.equal(result.listed, 2);
+  assert.equal(result.deleted, 2);
+  assert.deepEqual(listCalls.map((o) => o.cursor), [undefined, "page-2"]);
+  assert.deepEqual(deletedPages, [["backups/a/data.sqsh"], ["backups/b/meta.json"]]);
+});
+
+test("sweepWorkspaceBackups tolerates a missing R2 binding (local dev)", async () => {
+  const now = new Date("2026-09-20T00:00:00.000Z");
+  const db = {
+    prepare: () => ({
+      bind: () => ({ run: async () => ({ meta: { changes: 0 } }) }),
+    }),
+  };
+
+  const result = await sweepWorkspaceBackups(undefined, db, now);
+
+  assert.deepEqual(result, { listed: 0, deleted: 0, expiredRows: 0 });
+});

packages/worker/wrangler.toml

@@ -26,6 +26,9 @@ head_sampling_rate = 1
 enabled = true
 head_sampling_rate = 1
 
+[triggers]
+crons = ["0 4 * * *"]
+
 [[containers]]
 class_name = "Sandbox"
 image = "../../Dockerfile.sandbox"