diff --git a/CLAUDE.md b/CLAUDE.md index b32fcff..1c291db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,6 +78,8 @@ uv venv --python 3.13 .venv && uv pip sync --python .venv/bin/python eval/requir .venv/bin/python eval/run_detect.py --page URL # detections into results/detections/ (--page stub: data/stub-runs/) .venv/bin/python eval/run_detect.py --page URL --transforms none,rot45,dark2 # also rotated and darkened copies .venv/bin/python eval/check_colour.py # the transforms' sRGB images match Chromium's +.venv/bin/python eval/yaw_sheet.py --labeller NAME # page for labelling the yaw sample +.venv/bin/python eval/yaw_agreement.py [--write] # Cohen's kappa; --write fills faces.csv yaw .venv/bin/python eval/score.py RUN.jsonl --operating-point # pad and threshold, on the tune split .venv/bin/python eval/score.py RUN.jsonl --pad P --threshold T # score the report split ``` diff --git a/docs/ai-use.md b/docs/ai-use.md index a16b97f..cef60f8 100644 --- a/docs/ai-use.md +++ b/docs/ai-use.md @@ -17,3 +17,4 @@ Both authors work with Claude Code (Anthropic). This log records what Claude did | 2026-10-28 | Shashwat Deep | Claude Opus 5.5 wrote `eval/run_detect.py`, the stand-in eval page in `eval/stub/` and their tests (4, in headless Chromium); confirmed in Chromium that the EXIF orientation it adds turns images stored turned upright (a mutation check that swaps two orientation codes makes the tests fail); ran the stub over all 1,000 test images to check every decoded size; read Pratyush's `app/` branches without changing them, to match `detect.ts` and `mask.ts`; added the runner's details to part B and F of `docs/eval-protocol.md`. | Not yet. | | 2026-10-28 | Shashwat Deep | Claude Opus 5.5 reviewed Pratyush's pull requests #3 to #8: read the app, the tests and the workflow, reran the build, the unit tests and the privacy suite in Chromium, and checked the model's SHA-256 against the OpenCV Zoo and each action's pin against its release. At Shashwat's request it then approved and merged them in order, and resolved the merge conflicts with this pull request in `CLAUDE.md` and this file. | Shashwat read the review and decided to merge. | | 2026-10-28 | Shashwat Deep | Claude Opus 5.5 wrote the rotation and darkening transforms in `eval/transforms.py`, `--transforms` in `eval/run_detect.py`, `eval/check_colour.py` and their tests; proposed the parameters written into part B of `docs/eval-protocol.md` (bicubic rotation with mid-grey corners; sensor noise from the middle of the range Brooks et al. fitted, checked against their paper on arXiv); measured that converting with each image's ICC profile matches Chromium's decoded pixels within 1 on 90 test images of 29 kinds of profile, after finding that Chromium applies an RGB profile to a greyscale image. | Not yet. | +| 2026-10-28 | Shashwat Deep | Claude Opus 5.5 wrote `eval/yaw_sheet.py` (the 600-face yaw sample and a labelling page with face crops and no detector output), `eval/yaw_agreement.py` (Cohen's κ with a bootstrap interval over images, the disagreements to settle, and writing the settled labels into `faces.csv`) and their tests, and added the procedure to part B of `docs/eval-protocol.md`. It checked the κ formula against a worked example and built the page for the real sample without looking at the crops. | Not yet. | diff --git a/docs/eval-protocol.md b/docs/eval-protocol.md index b2ad7e3..c7a67df 100644 --- a/docs/eval-protocol.md +++ b/docs/eval-protocol.md @@ -51,6 +51,10 @@ Each is reported as a miss rate with its 95% CI on the report split. - Occlusion and truncation: Open Images' IsOccluded and IsTruncated flags. - Depictions (faces on posters, paintings or statues, flagged IsDepiction): reported separately and left out of the headline rate. - Yaw, the face's turn away from the camera: a random sample of 600 faces from the report split, each labelled frontal (both eyes visible, face mostly towards the camera), three-quarter (both eyes visible, face clearly turned), profile (one eye or none visible) or unclear. Both authors label independently, from face crops shown without any detector output. We report Cohen's κ, then settle disagreements together. Unclear faces are left out of this breakdown. + - The sample is the first 600 report-split faces that are not depictions, in the order of the SHA-256 of `solid-mask:yaw:0::`. Labelling starts once the test set is frozen, so the sample no longer changes. + - Each crop is a square twice the longer side of the face's box, centred on it, cut from the upright image, with ticks at the corners of the box (`eval/yaw_sheet.py`). + - Each author's labels go in a file of their own in `eval/labels/`. Neither author opens the other's file until both are complete. + - κ is Cohen's unweighted κ over the four labels, with a 95% confidence interval from the bootstrap over images; we also report κ over the faces neither author called unclear (`eval/yaw_agreement.py`). The settled labels go into the `yaw` column of `eval/manifest/faces.csv`. - In-plane rotation: every report-split image rotated counterclockwise by 0°, 15°, 30°, 45°, 90°, 135° and 180° on a canvas enlarged to fit, with the annotated boxes rotated as polygons. The rotation is Pillow's `Image.rotate` with bicubic resampling; the corners it uncovers are mid-grey (128, 128, 128); 90° and 180° are exact transposes. The 0° copy goes through the same steps and is the control for the series. - Light, synthetic: every report-split image darkened by 1, 2, 3 and 4 EV: converted to linear RGB (IEC 61966-2-1), scaled by 2^−EV, given Poisson–Gaussian noise, clipped to [0, 1] and converted back to 8-bit sRGB. For a linear value x, the noise is a Poisson count with mean x / λ_shot, times λ_shot, plus Gaussian noise with variance λ_read, so its variance is λ_read + λ_shot·x, the sensor model of Brooks et al. (CVPR 2019, Eq. 1). They fitted log λ_shot to a uniform distribution on [log 0.0001, log 0.012] and log λ_read = 2.18 log λ_shot + 1.2 on the Darmstadt Noise Dataset; we take the middle of that range, λ_shot = 0.0011, and the fitted line, λ_read = 1.17 × 10⁻⁶. The noise is drawn per pixel and channel, with a random generator for each image and level seeded from its ID and the level. - Both transforms start from the image as Chromium decodes it: the runner converts it to sRGB with its ICC profile, as Chromium does (`eval/check_colour.py` checks that the pixels agree within 1 for test images with each kind of profile), turns it upright, transforms it and gives it to the page as a PNG. diff --git a/eval/README.md b/eval/README.md index 92f48f4..13166a7 100644 --- a/eval/README.md +++ b/eval/README.md @@ -14,6 +14,9 @@ The scripts here measure how often the tool leaves a face uncovered, following [ - `stub/`: a stand-in for the app's eval page with the same interface, used until that page exists and by the tests. - `score.py`: scores a detections file against the manifest: missed faces, faces covered completely, detection recall and false masks, each with a 95% confidence interval, and the operating point. - `transforms.py`: the rotation and darkening transforms, their parameters, and the geometry of a rotation, which `score.py` uses. +- `yaw_sheet.py`: makes a page for labelling the yaw of the sample faces. +- `yaw_agreement.py`: compares the two authors' yaw labels (Cohen's κ) and writes the settled labels into `manifest/faces.csv`. +- `labels/`: each author's yaw labels, and the faces they disagreed on. - `check_colour.py`: checks that the images the transforms start from have the pixels Chromium decodes. - `tests/`: tests for these scripts (pytest). @@ -59,6 +62,15 @@ The runner gives each image to the page itself, on the page's own address, so no The first finds the padding and threshold on the tune split by the protocol's rule; the second scores the report split at those values. Each transform and setting in the file is scored on its own. A mask covers whole pixels, rounded outward as the app paints it, and a face's covered share is computed exactly from those pixels, so a face turned by a rotation transform needs no special case. The summary is JSON (`--out`); `--faces` also writes one row per face for further breakdowns. +## Yaw labels + +Each of us labels the same 600 faces alone, once the test set is frozen ([`docs/eval-protocol.md`](../docs/eval-protocol.md), part B, "Breakdowns"): + +1. `.venv/bin/python eval/yaw_sheet.py --labeller ` writes `data/yaw//index.html`. Open it, label each face with the keys 1 to 4, and save the box at the bottom as `labels/yaw-.csv`. You can stop and go on later: the page shows only the faces the file doesn't label yet. Don't open the other author's file until you have both finished. +2. `.venv/bin/python eval/yaw_agreement.py --out results/yaw-agreement.json` reports Cohen's κ with its confidence interval and writes the faces you disagree on to `labels/yaw-disagreements.csv`. +3. Settle each of those together and write the agreed label in its `settled` column. +4. `.venv/bin/python eval/yaw_agreement.py --write` puts the final labels into the `yaw` column of `manifest/faces.csv`, which `score.py` breaks the miss rate down by. + ## How the test set is drawn `build_manifest.py` downloads Open Images' image lists and box annotations for its validation and test splits and checks them against fixed SHA-256s. It keeps the images that have at least one "Human face" box, no face box marked IsGroupOf, the CC BY 2.0 licence and a known orientation. Each image's split and its place in the draw come from the SHA-256 of its ID, so they don't depend on which other images are eligible. The set is the first 200 tune and the first 800 report images that the review has not removed. diff --git a/eval/tests/test_yaw.py b/eval/tests/test_yaw.py new file mode 100644 index 0000000..3c85404 --- /dev/null +++ b/eval/tests/test_yaw.py @@ -0,0 +1,158 @@ +"""Tests of yaw_sheet.py and yaw_agreement.py on made-up faces and labels.""" +import json +import numpy as np +import pytest +from PIL import Image +import build_manifest as bm +import yaw_agreement +import yaw_sheet + +LABELLERS = ['ann', 'bob'] + + +def make_manifest(folder, n_images=30, faces_per_image=3): + """Made-up images (grey, with a white square per face) and faces; every third face is a depiction.""" + (folder / 'manifest').mkdir() + (folder / 'images').mkdir() + images, faces = [], [] + for k in range(n_images): + i = f'{k:016x}' + images.append({'imageId': i, 'split': bm.split_of(i), 'url': f'https://example.org/{i}.jpg', 'width': 120, + 'height': 90, 'rotation': 0}) + image = Image.new('RGB', (120, 90), (90, 90, 90)) + for n in range(faces_per_image): + image.paste((255, 255, 255), (10 + 35 * n, 30, 30 + 35 * n, 55)) + faces.append({'imageId': i, 'faceId': n, 'x': 10 + 35 * n, 'y': 30, 'w': 20, 'h': 25, + 'isOccluded': 0, 'isTruncated': 0, 'isDepiction': int(n == 2), 'yaw': ''}) + image.save(folder / 'images' / f'{i}.jpg', quality=95) + bm.write_csv(folder / 'manifest/images.csv', bm.IMAGE_COLUMNS, images) + bm.write_csv(folder / 'manifest/faces.csv', bm.FACE_COLUMNS, faces) + return ['--manifest', str(folder / 'manifest'), '--labels', str(folder / 'labels')] + + +def manifest_rows(folder): + return ({r['imageId']: r for r in bm.read_csv(folder / 'manifest/images.csv')}, + bm.read_csv(folder / 'manifest/faces.csv')) + + +def write_labels(folder, name, labels): + (folder / 'labels').mkdir(exist_ok=True) + bm.write_csv(folder / 'labels' / f'yaw-{name}.csv', yaw_sheet.LABEL_COLUMNS, + [{'imageId': i, 'faceId': n, 'yaw': label} for (i, n), label in sorted(labels.items())]) + + +def test_sample_takes_report_faces_that_are_not_depictions_in_a_fixed_order(tmp_path): + make_manifest(tmp_path) + images, faces = manifest_rows(tmp_path) + chosen = yaw_sheet.sample(images, faces, n=10) + assert len(chosen) == 10 and chosen == yaw_sheet.sample(images, faces, n=10) + assert all(images[f['imageId']]['split'] == 'report' and f['isDepiction'] == '0' for f in chosen) + gone = chosen[0]['imageId'] # removing an image keeps every other face's place in the order + rest = yaw_sheet.sample({i: r for i, r in images.items() if i != gone}, [f for f in faces if f['imageId'] != gone], n=10) + kept = [f for f in chosen if f['imageId'] != gone] + assert rest[:len(kept)] == kept + + +def test_crop_centres_the_face_and_pads_beyond_the_image(): + image = Image.new('RGB', (100, 80), (0, 0, 0)) + image.paste((255, 255, 255), (0, 0, 20, 20)) # a face in the top-left corner + out = yaw_sheet.crop(image, {'x': 0, 'y': 0, 'w': 20, 'h': 20}) + pixels = np.asarray(out).astype(int) + assert out.size == (yaw_sheet.CROP, yaw_sheet.CROP) + assert (pixels[5, 5] == yaw_sheet.transforms.FILL).all() # beyond the image: mid-grey + assert (pixels[128, 128] > 240).all() # the face is in the middle + assert (np.abs(pixels[61, 61:70] - yaw_sheet.TICK).max(axis=1) < 40).any() # a tick at the top-left corner + + +def test_sheet_shows_only_faces_not_yet_labelled(tmp_path): + args = make_manifest(tmp_path) + ['--images', str(tmp_path / 'images'), '--out', str(tmp_path / 'yaw')] + images, faces = manifest_rows(tmp_path) + chosen = yaw_sheet.sample(images, faces) + done = {(f['imageId'], f['faceId']): 'frontal' for f in chosen[:5]} + write_labels(tmp_path, 'ann', done) + yaw_sheet.main(args + ['--labeller', 'ann']) + page = (tmp_path / 'yaw/ann/index.html').read_text(encoding='utf-8') + shown = json.loads(page.split('const faces = ', 1)[1].split(';\n', 1)[0]) + assert len(shown) == len(chosen) - 5 and not {(f['imageId'], f['faceId']) for f in shown} & set(done) + assert all((tmp_path / 'yaw/ann' / f['src']).exists() for f in shown) + assert 'bob' not in page + with pytest.raises(SystemExit): + yaw_sheet.main(args + ['--labeller', 'Ann Smith']) + write_labels(tmp_path, 'ann', {**done, ('x', '0'): 'sideways'}) + with pytest.raises(SystemExit): + yaw_sheet.main(args + ['--labeller', 'ann']) + + +def test_kappa_matches_a_worked_example(): + # 50 faces: both frontal 20, first frontal and second profile 5, the reverse 10, both profile 15. Agreement 0.7, + # chance agreement 0.5, so kappa = 0.4. + table = np.zeros((4, 4)) + table[0, 0], table[0, 2], table[2, 0], table[2, 2] = 20, 5, 10, 15 + assert yaw_agreement.kappa(table) == pytest.approx(0.4) + assert yaw_agreement.kappa(np.diag([3, 0, 2, 0])) == 1 + assert yaw_agreement.kappa(np.zeros((4, 4))) is None and yaw_agreement.kappa(np.diag([5, 0, 0, 0])) is None + + +def test_agreement_reports_kappa_and_keeps_settled_labels(tmp_path, capsys): + args = make_manifest(tmp_path) + ['--labellers', *LABELLERS] + images, faces = manifest_rows(tmp_path) + chosen = [(f['imageId'], f['faceId']) for f in yaw_sheet.sample(images, faces)] + rng = np.random.default_rng(0) + ann = {face: yaw_sheet.LABELS[k] for face, k in zip(chosen, rng.integers(0, 3, len(chosen)))} + bob = dict(ann) + for face in chosen[:6]: + bob[face] = 'unclear' + write_labels(tmp_path, 'ann', ann) + write_labels(tmp_path, 'bob', {face: label for face, label in bob.items() if face != chosen[-1]}) + with pytest.raises(SystemExit, match='1 faces of the sample unlabelled'): + yaw_agreement.main(args) + write_labels(tmp_path, 'bob', {**bob, ('removed-in-review', '0'): 'frontal'}) + + yaw_agreement.main(args + ['--out', str(tmp_path / 'agreement.json')]) + summary = json.loads((tmp_path / 'agreement.json').read_text()) + assert summary['labelsOutsideSample'] == {'ann': 0, 'bob': 1} + assert summary['faces'] == len(chosen) and summary['disagreements'] == 6 == summary['unsettled'] + assert summary['observedAgreement'] == pytest.approx(1 - 6 / len(chosen)) + assert 0 < summary['kappa']['low'] <= summary['kappa']['value'] <= summary['kappa']['high'] <= 1 + assert summary['kappaWithoutUnclear']['value'] == 1 and summary['kappaWithoutUnclear']['faces'] == len(chosen) - 6 + assert summary['counts']['bob']['unclear'] == 6 + list(ann.values()).count('unclear') + with pytest.raises(SystemExit, match='not settled'): + yaw_agreement.main(args + ['--write']) + + disagreements = bm.read_csv(tmp_path / 'labels/yaw-disagreements.csv') + assert list(disagreements[0]) == ['imageId', 'faceId', 'ann', 'bob', 'settled'] + for row in disagreements: + row['settled'] = row['ann'] + bm.write_csv(tmp_path / 'labels/yaw-disagreements.csv', list(disagreements[0]), disagreements) + yaw_agreement.main(args[:-3] + ['--labellers', 'bob', 'ann']) # a rerun, in either order, keeps what was settled + assert all(r['settled'] for r in bm.read_csv(tmp_path / 'labels/yaw-disagreements.csv')) + yaw_agreement.main(args + ['--write']) + written = {(f['imageId'], f['faceId']): f['yaw'] for f in bm.read_csv(tmp_path / 'manifest/faces.csv')} + assert {face: written[face] for face in chosen} == ann + assert sum(1 for label in written.values() if label) == len(chosen) + capsys.readouterr() + + +def test_sheet_page_records_keys_and_backspace(chromium, tmp_path): + from playwright.sync_api import sync_playwright + args = make_manifest(tmp_path) + ['--images', str(tmp_path / 'images'), '--out', str(tmp_path / 'yaw')] + images, faces = manifest_rows(tmp_path) + chosen = yaw_sheet.sample(images, faces) + write_labels(tmp_path, 'ann', {(chosen[0]['imageId'], chosen[0]['faceId']): 'profile'}) + yaw_sheet.main(args + ['--labeller', 'ann']) + with sync_playwright() as pw: + browser = pw.chromium.launch() + page = browser.new_page() + page.goto((tmp_path / 'yaw/ann/index.html').as_uri()) + for key in ('1', '3', 'Backspace', '2', '4'): + page.keyboard.press(key) + text = page.locator('#csv').input_value() + progress = page.locator('#progress').inner_text() + browser.close() + rows = text.strip().split('\n') + expected = {f'{chosen[0]["imageId"]},{chosen[0]["faceId"]},profile', + f'{chosen[1]["imageId"]},{chosen[1]["faceId"]},frontal', + f'{chosen[2]["imageId"]},{chosen[2]["faceId"]},three-quarter', + f'{chosen[3]["imageId"]},{chosen[3]["faceId"]},unclear'} + assert rows[0] == 'imageId,faceId,yaw' and set(rows[1:]) == expected and rows[1:] == sorted(rows[1:]) + assert progress.startswith('Face 4 of') diff --git a/eval/yaw_agreement.py b/eval/yaw_agreement.py new file mode 100644 index 0000000..83fce8c --- /dev/null +++ b/eval/yaw_agreement.py @@ -0,0 +1,125 @@ +"""Compare two authors' yaw labels: Cohen's kappa, the confusion table, and the faces to settle together. + + python eval/yaw_agreement.py [--labellers shashwat pratyush] [--out results/yaw-agreement.json] [--write] + +Reads eval/labels/yaw-.csv (yaw_sheet.py) for both authors; each must label every face of the current +sample. Labels of faces outside it (their image was removed in the review) are counted and left out. +Reports Cohen's kappa over the four labels, with a 95% confidence interval from the bootstrap over images that +score.py uses (10,000 resamples of the images, numpy's default_rng(0), percentile interval); kappa over the faces +neither author called unclear; the observed agreement; each author's counts; and the confusion table. Writes the +faces they disagree on to eval/labels/yaw-disagreements.csv, with a column of labels named after each author (in +alphabetical order) and a `settled` column, keeping any settled labels already filled in. Once +the authors have filled the `settled` column together, --write puts the final labels into the yaw column of +eval/manifest/faces.csv: the shared label where they agree, the settled one where they did not. +""" +import argparse, json, sys +from pathlib import Path +import numpy as np +import build_manifest +import score +import yaw_sheet + +ROOT = Path(__file__).resolve().parents[1] + + +def kappa(table): + """Cohen's kappa of a square table of counts (rows: one rater, columns: the other); None if undefined.""" + table = np.asarray(table, dtype=float) + n = table.sum() + if n == 0: + return None + observed = np.trace(table) / n + expected = (table.sum(axis=1) @ table.sum(axis=0)) / n ** 2 + return None if expected == 1 else float((observed - expected) / (1 - expected)) + + +def kappa_interval(per_image): + """kappa with its 95% bootstrap interval, from one table per image (an array of shape images x k x k).""" + per_image = np.asarray(per_image, dtype=float) + result = {'value': kappa(per_image.sum(axis=0)), 'faces': int(per_image.sum())} + boot = score.Bootstrap(len(per_image)) + if boot.counts is None or result['value'] is None: + return result + k = per_image.shape[1] + tables = (boot.counts @ per_image.reshape(len(per_image), -1)).reshape(-1, k, k) + n = tables.sum(axis=(1, 2)) + observed = np.trace(tables, axis1=1, axis2=2) / n + expected = np.einsum('bi,bi->b', tables.sum(axis=2), tables.sum(axis=1)) / n ** 2 + defined = expected < 1 + values = (observed[defined] - expected[defined]) / (1 - expected[defined]) + result['low'], result['high'] = (float(v) for v in np.percentile(values, [2.5, 97.5])) + if (~defined).any(): + result['resamplesUndefined'] = int((~defined).sum()) + return result + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--labellers', nargs=2, default=['shashwat', 'pratyush']) + ap.add_argument('--manifest', type=Path, default=ROOT / 'eval/manifest') + ap.add_argument('--labels', type=Path, default=ROOT / 'eval/labels') + ap.add_argument('--out', type=Path, help='write the summary as JSON here') + ap.add_argument('--write', action='store_true', help='put the final labels into faces.csv') + args = ap.parse_args(argv) + + images = {r['imageId']: r for r in build_manifest.read_csv(args.manifest / 'images.csv')} + faces = build_manifest.read_csv(args.manifest / 'faces.csv') + chosen = [(f['imageId'], f['faceId']) for f in yaw_sheet.sample(images, faces)] + first, second = (yaw_sheet.read_labels(args.labels / f'yaw-{name}.csv') for name in args.labellers) + outside = {} + for name, labels in zip(args.labellers, (first, second)): + missing = set(chosen) - set(labels) + if missing: + sys.exit(f'{name} has {len(missing)} faces of the sample unlabelled; run yaw_sheet.py --labeller {name}') + outside[name] = len(set(labels) - set(chosen)) + + index = {label: k for k, label in enumerate(yaw_sheet.LABELS)} + image_ids = sorted({i for i, _ in chosen}) + row_of = {i: r for r, i in enumerate(image_ids)} + per_image = np.zeros((len(image_ids), len(index), len(index))) + for face in chosen: + per_image[row_of[face[0]], index[first[face]], index[second[face]]] += 1 + clear = [index[label] for label in yaw_sheet.LABELS if label != 'unclear'] + total = per_image.sum(axis=0) + summary = { + 'labellers': args.labellers, 'faces': len(chosen), 'images': len(image_ids), 'labelsOutsideSample': outside, + 'kappa': kappa_interval(per_image), + 'kappaWithoutUnclear': kappa_interval(per_image[:, clear][:, :, clear]), + 'observedAgreement': float(np.trace(total) / total.sum()), + 'counts': {name: {label: int(total.sum(axis=1 - side)[index[label]]) for label in yaw_sheet.LABELS} + for side, name in enumerate(args.labellers)}, + 'confusion': {a: {b: int(total[index[a], index[b]]) for b in yaw_sheet.LABELS} for a in yaw_sheet.LABELS}, + } + + path = args.labels / 'yaw-disagreements.csv' + settled = {(r['imageId'], r['faceId']): r['settled'] for r in build_manifest.read_csv(path)} if path.exists() else {} + by_name = dict(zip(args.labellers, (first, second))) + names = sorted(args.labellers) + rows = [{'imageId': i, 'faceId': n, **{name: by_name[name][(i, n)] for name in names}, + 'settled': settled.get((i, n), '')} for i, n in sorted(chosen) if first[(i, n)] != second[(i, n)]] + bad = [r for r in rows if r['settled'] not in ('', *yaw_sheet.LABELS)] + if bad: + sys.exit(f'{path}: settled must be one of {", ".join(yaw_sheet.LABELS)}, for example {bad[0]["imageId"]}') + build_manifest.write_csv(path, ['imageId', 'faceId', *names, 'settled'], rows) + summary['disagreements'] = len(rows) + summary['unsettled'] = sum(not r['settled'] for r in rows) + + text = json.dumps(summary, indent=1) + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(text + '\n', encoding='utf-8') + print(text) + + if args.write: + if summary['unsettled']: + sys.exit(f'{summary["unsettled"]} disagreements in {path} are not settled yet') + final = {face: first[face] for face in chosen} + final.update({(r['imageId'], r['faceId']): r['settled'] for r in rows}) + for face in faces: + face['yaw'] = final.get((face['imageId'], face['faceId']), '') + build_manifest.write_csv(args.manifest / 'faces.csv', build_manifest.FACE_COLUMNS, faces) + print(f'wrote the yaw of {len(final)} faces into {args.manifest / "faces.csv"}', file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/eval/yaw_sheet.py b/eval/yaw_sheet.py new file mode 100644 index 0000000..ace4a37 --- /dev/null +++ b/eval/yaw_sheet.py @@ -0,0 +1,181 @@ +"""Make a page on this computer for labelling the yaw of the sample faces (docs/eval-protocol.md, part B). + + python eval/yaw_sheet.py --labeller shashwat [--out data/yaw] + +The sample is the first 600 faces, in the order of the SHA-256 of "solid-mask:yaw:0::", among the +faces of the report split that are not depictions. The page shows one face at a time: a square crop twice the +longer side of its box, centred on it, from the upright image (transforms.upright()), with ticks at the corners of +the box and no detector output. It asks for one label per face: frontal (both eyes visible, face mostly towards the +camera), three-quarter (both eyes visible, face clearly turned), profile (one eye or none visible) or unclear. +Keys 1 to 4 choose a label; Backspace goes back one face. + +The box at the bottom of the page holds the whole of eval/labels/yaw-.csv, the labels already in that file +and the new ones; save it in place of the file, or use the download button. The page shows only faces the file does +not label yet, so labelling can stop and go on later. Each author labels alone: the page never shows the other +author's labels, and neither author opens the other's file until both are complete (yaw_agreement.py compares them). +The page loads nothing from the network and stores nothing. +""" +import argparse, html, json, re, sys +from pathlib import Path +from PIL import Image, ImageDraw +import build_manifest +import download +import transforms + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = 600 +LABELS = ('frontal', 'three-quarter', 'profile', 'unclear') +LABEL_COLUMNS = ['imageId', 'faceId', 'yaw'] +CROP = 256 # pixels on a side +TICK = (255, 214, 0) +PAGE = """ + + +solid-mask yaw: {labeller} + +

Yaw labels: {labeller}

+

Label the face between the yellow ticks. Frontal: both eyes visible, face mostly towards the camera. +Three-quarter: both eyes visible, face clearly turned. Profile: one eye or none visible. +Unclear: you can't tell. Keys 1 to 4 choose; Backspace goes back one face.

+

Label alone, and don't open the other author's file until you have both finished.

+

+ +
{buttons}
+

eval/labels/yaw-{labeller}.csv

+

Everything in this box, the labels already in the file and the new ones, replaces the file.

+ + + + +""" + + +def sample(images, faces, n=SAMPLE): + """The yaw sample: the first n faces, in the order of their SHA-256 key, among the report split's faces that are + not depictions. images maps imageId to its images.csv row; faces are faces.csv rows.""" + pool = [f for f in faces if images[f['imageId']]['split'] == 'report' and f['isDepiction'] != '1'] + return sorted(pool, key=lambda f: build_manifest.digest('yaw', f'{f["imageId"]}:{f["faceId"]}'))[:n] + + +def read_labels(path): + """{(imageId, faceId): label} from a labels file, after checking every row.""" + labels = {} + rows = build_manifest.read_csv(path) if path.exists() else [] + if rows and list(rows[0]) != LABEL_COLUMNS: + sys.exit(f'{path}: the columns must be {",".join(LABEL_COLUMNS)}') + for line, row in enumerate(rows, start=2): + face = (row['imageId'], row['faceId']) + if row['yaw'] not in LABELS or face in labels: + sys.exit(f'{path}, line {line}: expected a face not labelled before and one of {", ".join(LABELS)}') + labels[face] = row['yaw'] + return labels + + +def crop(image, face, size=CROP): + """A size x size crop centred on the face's box, twice its longer side, with mid-grey beyond the edges of the + image and yellow ticks just outside the corners of the box.""" + x, y, w, h = (float(face[k]) for k in 'xywh') + side = 2 * max(w, h) + left, top = x + w / 2 - side / 2, y + h / 2 - side / 2 + scale = size / side + x0, y0 = max(0.0, left), max(0.0, top) + x1, y1 = min(float(image.width), left + side), min(float(image.height), top + side) + canvas = Image.new('RGB', (size, size), transforms.FILL) + part = image.resize((max(1, round((x1 - x0) * scale)), max(1, round((y1 - y0) * scale))), + Image.Resampling.LANCZOS, box=(x0, y0, x1, y1)) + canvas.paste(part, (round((x0 - left) * scale), round((y0 - top) * scale))) + pen = ImageDraw.Draw(canvas) + bx0, by0 = (x - left) * scale - 3, (y - top) * scale - 3 + bx1, by1 = (x + w - left) * scale + 3, (y + h - top) * scale + 3 + arm = 12 + for cx, cy, dx, dy in ((bx0, by0, 1, 1), (bx1, by0, -1, 1), (bx0, by1, 1, -1), (bx1, by1, -1, -1)): + pen.line([(cx, cy + dy * arm), (cx, cy), (cx + dx * arm, cy)], fill=TICK, width=2) + return canvas + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--labeller', required=True, help='a short name in lower case, such as shashwat') + ap.add_argument('--manifest', type=Path, default=ROOT / 'eval/manifest') + ap.add_argument('--labels', type=Path, default=ROOT / 'eval/labels') + ap.add_argument('--images', type=Path, default=ROOT / 'data/openimages/images') + ap.add_argument('--out', type=Path, default=ROOT / 'data/yaw') + args = ap.parse_args(argv) + if not re.fullmatch(r'[a-z0-9-]+', args.labeller): + ap.error('--labeller must be lower-case letters, digits and hyphens') + + images = {r['imageId']: r for r in build_manifest.read_csv(args.manifest / 'images.csv')} + chosen = sample(images, build_manifest.read_csv(args.manifest / 'faces.csv')) + existing = read_labels(args.labels / f'yaw-{args.labeller}.csv') + todo = [f for f in chosen if (f['imageId'], f['faceId']) not in existing] + out = args.out / args.labeller + out.mkdir(parents=True, exist_ok=True) + for i in dict.fromkeys(f['imageId'] for f in todo): + image = transforms.upright(download.image_path(args.images, i, images[i]['url']), int(images[i]['rotation'])) + for face in (f for f in todo if f['imageId'] == i): + crop(image, face).save(out / f'{i}_{face["faceId"]}.jpg', quality=92) + page_faces = [{'imageId': f['imageId'], 'faceId': f['faceId'], 'src': f'{f["imageId"]}_{f["faceId"]}.jpg'} + for f in todo] + buttons = ''.join(f'' + for n, label in enumerate(LABELS, start=1)) + existing_pairs = [[f'{i},{n}', label] for (i, n), label in sorted(existing.items())] + (out / 'index.html').write_text(PAGE.format( + labeller=html.escape(args.labeller), buttons=buttons, faces=json.dumps(page_faces), + existing=json.dumps(existing_pairs), names=json.dumps(LABELS)), encoding='utf-8') + outside = len(set(existing) - {(f['imageId'], f['faceId']) for f in chosen}) + print(f'{len(chosen)} faces in the sample; {len(chosen) - len(todo)} labelled by {args.labeller}, {len(todo)} to go' + + (f'; {outside} labelled faces are no longer in the sample' if outside else '') + + f'. Open {out / "index.html"} in a browser.') + + +if __name__ == '__main__': + main()