Add the tiled setting to detectFaces (contracts 0.3)

app/src/pipeline/detect.ts

@@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-web/wasm';
 import wasmUrl from 'onnxruntime-web/ort-wasm-simd-threaded.wasm?url';
 import yunetUrl from '../../models/face_detection_yunet_2023mar.onnx?url';
 import { YUNET } from './models.ts';
-import { decodeOutputs, letterbox, nms, STRIDES } from './yunet.ts';
+import { type Candidate, decodeOutputs, letterbox, nms, resizeArea, STRIDES, tileCandidates, tiledSize } from './yunet.ts';
 
 export type Box = {
   x: number;     // left edge in pixels of the bitmap passed in (origin top-left, x to the right)
@@ -16,7 +16,8 @@ export type Box = {
 export type Tiling =
   | { kind: 'none' }                                      // long side resized to inputSize, one pass
   | { kind: 'tiles'; longSide: number; overlap: number }; // long side resized to longSide (never enlarged), cut into
-                                                          // inputSize tiles overlapping by `overlap` (0 to 1), merged with NMS
+                                                          // inputSize tiles overlapping by `overlap` (0 to under 1),
+                                                          // merged with NMS
 
 export type DetectOptions = {
   model: 'yunet' | 'yolo5face';
@@ -97,16 +98,8 @@ function readPixels(image: ImageBitmap): Uint8ClampedArray {
   return pixels;
 }
 
-export async function detectFaces(image: ImageBitmap, options: DetectOptions): Promise<Box[]> {
-  if (options.model !== 'yunet') throw new Error(`the ${options.model} model is not supported yet`);
-  if (options.tiling.kind !== 'none') throw new Error('tiling is not supported yet');
-  if (options.inputSize !== YUNET.inputSize) throw new Error(`${YUNET.file} takes ${YUNET.inputSize} px input only`);
-  const session = await loadDetector(options.modelSource);
-
-  const pixels = readPixels(image);
-  const size = options.inputSize;
-  const { tensor, scaleX, scaleY } = letterbox(pixels, image.width, image.height, size);
-
+/** Runs the model on one input tensor: every candidate scoring at least minScore, in the tensor's pixels. */
+async function detectTensor(session: ort.InferenceSession, tensor: Float32Array, size: number, minScore: number): Promise<Candidate[]> {
   const results = await session.run({ input: new ort.Tensor('float32', tensor, [1, 3, size, size]) });
   const outputs: Record<string, Float32Array> = {};
   for (const s of STRIDES) {
@@ -116,8 +109,49 @@ export async function detectFaces(image: ImageBitmap, options: DetectOptions): P
       outputs[name] = data;
     }
   }
+  return decodeOutputs(outputs, size, minScore);
+}
+
+export async function detectFaces(image: ImageBitmap, options: DetectOptions): Promise<Box[]> {
+  if (options.model !== 'yunet') throw new Error(`the ${options.model} model is not supported yet`);
+  if (options.inputSize !== YUNET.inputSize) throw new Error(`${YUNET.file} takes ${YUNET.inputSize} px input only`);
+  // The options come from the eval runner as untyped JSON, so check them before any work.
+  const { tiling } = options;
+  const kind: string = tiling.kind;
+  if (kind !== 'none' && kind !== 'tiles') throw new Error(`unknown tiling kind ${kind}`);
+  const size = options.inputSize;
+  let stride = 0;
+  if (tiling.kind === 'tiles') {
+    if (!(Number.isInteger(tiling.longSide) && tiling.longSide > 0)) throw new Error(`longSide ${tiling.longSide} is not a positive whole number of pixels`);
+    if (!(tiling.overlap >= 0 && tiling.overlap < 1)) throw new Error(`overlap ${tiling.overlap} is not from 0 to under 1`);
+    stride = Math.round(size * (1 - tiling.overlap));
+    if (stride < 1) throw new Error(`overlap ${tiling.overlap} puts tiles under 1 px apart`);
+  }
+  const session = await loadDetector(options.modelSource);
+
+  // The photo's full-size pixels are read inside each branch and kept by neither, so they can be
+  // freed before the model runs.
+  const minScore = Math.min(NMS_FLOOR, options.scoreThreshold);
+  let candidates: Candidate[];
+  let scaleX: number;
+  let scaleY: number;
+  if (tiling.kind === 'none') {
+    const boxed = letterbox(readPixels(image), image.width, image.height, size);
+    candidates = await detectTensor(session, boxed.tensor, size, minScore);
+    ({ scaleX, scaleY } = boxed);
+  } else {
+    // The long side shrunk to longSide, never enlarged, then tiles of `size` that overlap by
+    // `overlap` (docs/contracts.md, section 1). NMS runs once, over every tile's candidates in the
+    // shrunk image's pixels: boxes that different tiles give for one face merge when they overlap
+    // by more than nmsIoU, as those of a face lying whole in two tiles do. A face that no tile holds
+    // whole, which can happen when it is larger than the overlap, can come back as several boxes.
+    const { width, height } = tiledSize(image.width, image.height, tiling.longSide);
+    const planes = resizeArea(readPixels(image), image.width, image.height, width, height);
+    candidates = await tileCandidates(planes, width, height, size, stride, (tensor) => detectTensor(session, tensor, size, minScore));
+    scaleX = width / image.width;
+    scaleY = height / image.height;
+  }
 
-  const candidates = decodeOutputs(outputs, size, Math.min(NMS_FLOOR, options.scoreThreshold));
   return nms(candidates, options.nmsIoU)
     .filter((c) => c.score >= options.scoreThreshold)
     .map((c) => ({ x: c.x / scaleX, y: c.y / scaleY, w: c.w / scaleX, h: c.h / scaleY, score: c.score }));

app/src/pipeline/yunet.ts

@@ -1,6 +1,6 @@
-// The parts of YuNet face detection that need no ONNX Runtime: building the input tensor,
-// turning the outputs into boxes, and non-maximum suppression. Kept apart so they can be
-// unit-tested in Node (tests/unit/).
+// The parts of YuNet face detection that need no ONNX Runtime: building the input tensor (one
+// for the whole image, or one per tile), turning the outputs into boxes, and non-maximum
+// suppression. Kept apart so they can be unit-tested in Node (tests/unit/).
 //
 // face_detection_yunet_2023mar.onnx takes "input", 1 x 3 x 640 x 640 float32 in BGR order with
 // values 0 to 255 and no normalisation, the layout OpenCV's FaceDetectorYN gives it. For each
@@ -15,7 +15,9 @@ export const STRIDES = [8, 16, 32] as const;
 
 /**
  * For each output index along one axis, the first input index it covers and the weight of each
- * input index it covers: area averaging, which is what OpenCV's INTER_AREA does when shrinking.
+ * input index it covers: area averaging, the method of OpenCV's INTER_AREA when shrinking. The
+ * results agree pixel for pixel except where an average falls exactly halfway between two whole
+ * numbers: resizeArea rounds those up, while OpenCV's float32 sums can land on either side.
  */
 function areaWeights(inSize: number, outSize: number): { first: Int32Array; weights: Float32Array[] } {
   const scale = inSize / outSize;
@@ -35,18 +37,22 @@ function areaWeights(inSize: number, outSize: number): { first: Int32Array; weig
 }
 
 /**
- * Resizes RGBA pixels so the long side is `size`, by area averaging, rounds to whole values as an
- * 8-bit image would, and writes them as BGR planes into the top-left of a `size` x `size` tensor
- * whose other values are 0 (letterboxing). The result does not depend on the browser.
+ * Resizes RGBA pixels to outW x outH by area averaging and rounds to whole values, halves up, as
+ * an 8-bit image would. Returns the B, G and R planes, outW x outH each. The result does not
+ * depend on the browser.
  */
-export function letterbox(
-  rgba: Uint8ClampedArray,
-  width: number,
-  height: number,
-  size: number,
-): { tensor: Float32Array; scaleX: number; scaleY: number } {
-  const outW = Math.max(1, Math.round((width * size) / Math.max(width, height)));
-  const outH = Math.max(1, Math.round((height * size) / Math.max(width, height)));
+export function resizeArea(rgba: Uint8ClampedArray, width: number, height: number, outW: number, outH: number): Uint8ClampedArray {
+  const plane = outW * outH;
+  const planes = new Uint8ClampedArray(3 * plane);
+  if (outW === width && outH === height) {
+    // Area averaging at the same size copies every pixel; this does the same, faster.
+    for (let i = 0; i < plane; i++) {
+      planes[i] = rgba[4 * i + 2];
+      planes[plane + i] = rgba[4 * i + 1];
+      planes[2 * plane + i] = rgba[4 * i];
+    }
+    return planes;
+  }
   const cols = areaWeights(width, outW);
   const rows = areaWeights(height, outH);
 
@@ -72,8 +78,6 @@ export function letterbox(
   }
 
   // Then every column to outH pixels, into the B, G and R planes.
-  const plane = size * size;
-  const tensor = new Float32Array(3 * plane);
   for (let y = 0; y < outH; y++) {
     const w = rows.weights[y];
     const top = rows.first[y];
@@ -87,15 +91,101 @@ export function letterbox(
         g += w[k] * narrow[n + 1];
         b += w[k] * narrow[n + 2];
       }
-      const o = y * size + x;
-      tensor[o] = Math.round(b);
-      tensor[plane + o] = Math.round(g);
-      tensor[2 * plane + o] = Math.round(r);
+      const o = y * outW + x;
+      planes[o] = Math.round(b);
+      planes[plane + o] = Math.round(g);
+      planes[2 * plane + o] = Math.round(r);
     }
   }
+  return planes;
+}
+
+/**
+ * A `size` x `size` input tensor holding the pixels of B, G and R planes (width x height each)
+ * from (left, top) at its top-left, and 0 wherever it reaches past them.
+ */
+export function tile(planes: Uint8ClampedArray, width: number, height: number, left: number, top: number, size: number): Float32Array {
+  const tensor = new Float32Array(3 * size * size);
+  const w = Math.min(size, width - left);
+  const h = Math.min(size, height - top);
+  for (let c = 0; c < 3; c++) {
+    for (let y = 0; y < h; y++) {
+      const from = c * width * height + (top + y) * width + left;
+      tensor.set(planes.subarray(from, from + w), c * size * size + y * size);
+    }
+  }
+  return tensor;
+}
+
+/**
+ * Resizes RGBA pixels so the long side is `size`, by area averaging, rounds to whole values as an
+ * 8-bit image would, and writes them as BGR planes into the top-left of a `size` x `size` tensor
+ * whose other values are 0 (letterboxing). The result does not depend on the browser.
+ */
+export function letterbox(
+  rgba: Uint8ClampedArray,
+  width: number,
+  height: number,
+  size: number,
+): { tensor: Float32Array; scaleX: number; scaleY: number } {
+  const outW = Math.max(1, Math.round((width * size) / Math.max(width, height)));
+  const outH = Math.max(1, Math.round((height * size) / Math.max(width, height)));
+  const tensor = tile(resizeArea(rgba, width, height, outW, outH), outW, outH, 0, 0, size);
   return { tensor, scaleX: outW / width, scaleY: outH / height };
 }
 
+/**
+ * The size the tiled setting shrinks an image to: its long side at most `longSide`, the other side
+ * in proportion. An image already no longer keeps its size: the tiled setting never enlarges.
+ */
+export function tiledSize(width: number, height: number, longSide: number): { width: number; height: number } {
+  const long = Math.max(width, height);
+  if (long <= longSide) return { width, height };
+  return {
+    width: Math.max(1, Math.round((width * longSide) / long)),
+    height: Math.max(1, Math.round((height * longSide) / long)),
+  };
+}
+
+/**
+ * Where the tiles start along a side of `length` pixels: every `stride` pixels while a tile still
+ * ends before the edge, then once more so that the last tile ends exactly at the edge. No tile
+ * reaches past the edge, and neighbours overlap by at least size - stride. A side no longer than
+ * a tile gets one tile.
+ */
+export function tileStarts(length: number, size: number, stride: number): number[] {
+  if (!(stride >= 1)) throw new Error(`a tile stride of ${stride} px would never reach the edge`);
+  if (length <= size) return [0];
+  const starts: number[] = [];
+  for (let start = 0; start + size < length; start += stride) starts.push(start);
+  starts.push(length - size);
+  return starts;
+}
+
+/**
+ * Cuts B, G and R planes (width x height each) into `size` x `size` tiles that start every
+ * `stride` pixels (tileStarts), row by row, runs `detect` on each, and returns every tile's
+ * candidates in the planes' pixels, ready for one NMS over all of them.
+ */
+export async function tileCandidates(
+  planes: Uint8ClampedArray,
+  width: number,
+  height: number,
+  size: number,
+  stride: number,
+  detect: (tensor: Float32Array) => Promise<Candidate[]>,
+): Promise<Candidate[]> {
+  const found: Candidate[] = [];
+  for (const top of tileStarts(height, size, stride)) {
+    for (const left of tileStarts(width, size, stride)) {
+      for (const c of await detect(tile(planes, width, height, left, top, size))) {
+        found.push({ ...c, x: c.x + left, y: c.y + top });
+      }
+    }
+  }
+  return found;
+}
+
 const clamp01 = (v: number): number => Math.min(1, Math.max(0, v));
 
 /** Every grid cell scoring at least `minScore`, as a box in the pixels of the input tensor. */

docs/ai-use.md

@@ -25,3 +25,4 @@ Both authors work with Claude Code (Anthropic). This log records what Claude did
 | 2026-10-28 | Shashwat Deep | Claude Opus 5.5 made `eval/build_manifest.py` leave out images whose file carries an EXIF orientation other than 1 or EXIF data it cannot read, each replaced by the next image in its split's ranking, and print how many drawn files are larger than 1,024 px or carry EXIF data or an ICC profile; made `eval/run_detect.py`, `eval/render_copies.py`, `eval/check_colour.py` and `transforms.upright()` refuse such files, and the runner replace the EXIF data of an image stored turned with its own orientation tag; checked in Chromium that it turns an image by the orientation in its first EXIF segment only, not by a second one or by XMP's; corrected the image-size and metadata wording in `docs/eval-protocol.md` and `eval/README.md`; with tests. This follows Pratyush's review of #10. | Not yet. |
 | 2026-10-28 | Shashwat Deep | Claude Opus 5.5 made `eval/yaw_sheet.py` refuse to run until the review keeps every image in the manifest, delete the crops and page of its last build, ignore a key held down or pressed with Ctrl, Alt or Meta, let go of the focus of a clicked button, and count only a copy of the whole box as saved; defined the yaw classes by the turn of the head (under about 30°, 30° to 60°, over 60°) in the page, the scripts and `docs/eval-protocol.md`; kept the label files in `data/yaw/` until both authors are done, then in `results/yaw/`; made `eval/yaw_agreement.py` write `results/yaw/agreement.json` by default, with the SHA-256 of both label files, the disagreements file and the manifest, the git commit and the versions; with tests. This follows Pratyush's review of #13. | Not yet. |
 | 2026-10-28 | Pratyush | Claude Opus 5.5 wrote the eval page (`app/eval.html`, `app/src/eval.ts`: `window.solidMask` as in `docs/contracts.md`, section 3), moved the app's drawing into a shared `paint()` so the page's `render()` encodes exactly as the app saves, and added `tests/app/eval-page.spec.ts`, which pins detection to OpenCV's FaceDetectorYN on the test photo. It checked that the test fails when the colour channels are swapped, and ran Shashwat's `run_detect.py` and `score.py` against the page on a two-image stand-in manifest made from the public-domain test photo (one copy stored turned). | Not yet: Pratyush reviews the pull request before asking Shashwat. |
+| 2026-10-29 | Pratyush | Claude Opus 5.5 added the tiled setting to `app/src/pipeline/detect.ts` (the photo shrunk so its long side is at most 1,920 px, cut into 640 px tiles every 480 px with the last one at the edge, and one NMS over the boxes of every tile), with unit tests, two eval-page tests in three browsers (the tiled boxes against OpenCV's, and that a photo no longer than `longSide` is never enlarged), and the tile layout in section 1 of `docs/contracts.md` (version 0.3). A separate Claude agent wrote `tests/reference/opencv_yunet_tiles.py` from a step-by-step description of the setting, without the app's code; in its environment the existing reference reproduces byte for byte, and its 38 boxes match the eval page's within 0.0011 px in Chromium. `/code-review` and four more agents reviewed the change, one of them by mutation testing in its own worktree, and each finding was checked by an agent trying to refute it; the fixes are in the second commit. | Not yet: Pratyush reviews the pull request before asking Shashwat. |

docs/contracts.md

@@ -1,8 +1,8 @@
 # Contracts
 
-Version 0.2. The app and the evaluation both depend on what is written here. Change it only through a PR that both authors approve, and raise the version number.
+Version 0.3. The app and the evaluation both depend on what is written here. Change it only through a PR that both authors approve, and raise the version number.
 
-Changes: 0.1 was the draft merged at M0. 0.2 rounds each mask outward to whole pixels (section 2).
+Changes: 0.1 was the draft merged at M0. 0.2 rounds each mask outward to whole pixels (section 2). 0.3 says where the tiles of the tiled setting lie and how their boxes are merged (section 1).
 
 ## 1. Face detection
 
@@ -20,7 +20,8 @@ export type Box = {
 export type Tiling =
   | { kind: "none" }                                      // long side resized to inputSize, one pass
   | { kind: "tiles"; longSide: number; overlap: number }; // long side resized to longSide (never enlarged), cut into
-                                                          // inputSize tiles overlapping by `overlap` (0 to 1), merged with NMS
+                                                          // inputSize tiles overlapping by `overlap` (0 to under 1),
+                                                          // merged with NMS
 
 export type DetectOptions = {
   model: "yunet" | "yolo5face";
@@ -36,6 +37,7 @@ export function detectFaces(image: ImageBitmap, options: DetectOptions): Promise
 
 - The app passes the upright bitmap from `createImageBitmap(file, { imageOrientation: "from-image" })`.
 - The score filter runs after NMS. Filtering the runner's boxes at a higher threshold therefore gives the same boxes as running with that threshold, so the evaluation can sweep thresholds without rerunning detection.
+- Tiles: along each side of the shrunk image, a tile starts every `round(inputSize × (1 − overlap))` px (480 for 640 and 0.25) for as long as it ends before the edge, and one more is moved back to end exactly at the edge. No tile reaches past the image, and neighbours overlap by at least `overlap`. A side no longer than `inputSize` gets one tile, padded with zeros. NMS runs once, over the boxes of all tiles in the shrunk image's pixels, so boxes that different tiles give for one face merge only when they overlap by more than `nmsIoU`; a face that no tile holds whole can come back as several boxes.
 - Every run records the ONNX Runtime Web version, the backend, the thread count and the model file's SHA-256.
 
 ## 2. Masks and the saved file