Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion backend/apps/benchmarks/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,12 @@ def get_benchmark_case(request: HttpRequest, suite: str, case_id: str, revision:
raise HttpError(404, f"Revision {target.revision} has no public case {case_id!r}.")
runs = leaderboard.latest_per_tool_and_case(leaderboard.approved_runs(target), case_id=case.pk)
with_public_screenshots(runs)
return {**_case_out(case), "revision": target.revision, "runs": [_run_out(r) for r in runs]}
return {
**_case_out(case),
"revision": target.revision,
"revision_is_current": target.is_current,
"runs": [_run_out(r) for r in runs],
}


@router.get(
Expand Down Expand Up @@ -298,6 +303,7 @@ def get_benchmark_tool_report(
"tool": leaderboard.tool_ref(tool),
"suite": suite,
"revision": target.revision,
"revision_is_current": target.is_current,
"scope": scope,
"case_count": target.cases.count(),
"surfaces": surfaces,
Expand Down Expand Up @@ -331,6 +337,7 @@ def get_benchmark_run(request: HttpRequest, run_id: str):
"tier": submission.tier,
"notes": submission.notes,
"revision": run.case.revision.revision,
"revision_is_current": run.case.revision.is_current,
}


Expand Down
6 changes: 6 additions & 0 deletions backend/apps/benchmarks/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,14 @@ class RunDetailOut(RunOut):
tier: str
notes: str
revision: str
#: False for a superseded revision - the site links only those with `?revision=`.
revision_is_current: bool


class CaseDetailOut(CaseOut):
revision: str
#: False for a superseded revision - the site links only those with `?revision=`.
revision_is_current: bool
runs: list[RunOut]


Expand All @@ -187,6 +191,8 @@ class ToolReportOut(Schema):
tool: ToolRefOut
suite: str
revision: str
#: False for a superseded revision - the site links only those with `?revision=`.
revision_is_current: bool
scope: Literal["all", "verified"]
case_count: int
surfaces: list[ToolSurfaceOut]
Expand Down
17 changes: 16 additions & 1 deletion backend/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -7198,6 +7198,10 @@
"title": "Revision",
"type": "string"
},
"revision_is_current": {
"title": "Revision Is Current",
"type": "boolean"
},
"runs": {
"items": {
"$ref": "#/components/schemas/RunOut"
Expand All @@ -7215,6 +7219,7 @@
"pdf_url",
"preview",
"revision",
"revision_is_current",
"runs"
],
"title": "CaseDetailOut",
Expand Down Expand Up @@ -7482,6 +7487,10 @@
"title": "Revision",
"type": "string"
},
"revision_is_current": {
"title": "Revision Is Current",
"type": "boolean"
},
"scope": {
"enum": [
"all",
Expand All @@ -7506,6 +7515,7 @@
"tool",
"suite",
"revision",
"revision_is_current",
"scope",
"case_count",
"surfaces"
Expand Down Expand Up @@ -7712,6 +7722,10 @@
"revision": {
"title": "Revision",
"type": "string"
},
"revision_is_current": {
"title": "Revision Is Current",
"type": "boolean"
}
},
"required": [
Expand Down Expand Up @@ -7741,7 +7755,8 @@
"tool_version",
"tier",
"notes",
"revision"
"revision",
"revision_is_current"
],
"title": "RunDetailOut",
"type": "object"
Expand Down
23 changes: 23 additions & 0 deletions backend/tests/test_benchmarks_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,29 @@ def test_a_run_page_carries_the_whole_report(client, published):
assert body["scored_by"] == "submitter"


def _detail_urls(run):
return [
f"{BASE}/suites/pdf/cases/extraction-conditions-1?revision=v0.1.1",
f"{BASE}/suites/pdf/tools/pdf-redaction?revision=v0.1.1",
f"{BASE}/runs/{run.run_id}",
]


def test_a_detail_page_says_its_revision_is_the_current_one(client, published):
"""So the site can link the clean URL rather than a `?revision=` copy of it."""
for url in _detail_urls(published):
assert client.get(url).json()["revision_is_current"] is True, url


def test_a_detail_page_says_when_its_revision_has_been_superseded(client, published):
revision = published.case.revision
revision.is_current = False
revision.save()

for url in _detail_urls(published):
assert client.get(url).json()["revision_is_current"] is False, url


def test_a_run_not_yet_approved_is_not_public(client, published):
published.submission.status = SubmissionStatus.PENDING_REVIEW
published.submission.save()
Expand Down
8 changes: 5 additions & 3 deletions frontend/src/app/(site)/benchmarks/[suite]/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { notFound } from 'next/navigation'

import { BenchmarkSuiteView } from '@/features/benchmarks/benchmark-suite-view'
import { getGetBenchmarkSuiteQueryKey } from '@/lib/api/generated/benchmarks/benchmarks'
import { parseBenchmarkParams } from '@/lib/benchmarks/params'
import { isAlternateView, parseBenchmarkParams } from '@/lib/benchmarks/params'
import { fetchBenchmarkSuite } from '@/lib/benchmarks/server'
import { clientEnv } from '@/lib/env'
import { getQueryClient } from '@/lib/query/client'
Expand All @@ -25,8 +25,10 @@ export async function generateMetadata({
title: `${data.name} benchmark ${data.revision.revision} — leak rates by tool`,
description: `${data.leaderboard.length} redaction tools scored on ${data.cases.length} synthetic cases: how many sensitive values each leaves recoverable, with 95% intervals.`,
...canonicalMetadata(`/benchmarks/${suite}`),
// Another revision or the verified-only view is the same page sliced differently.
robots: query.revision || query.scope !== 'all' ? { index: false, follow: true } : undefined,
// An older revision or the verified-only view is the same page sliced differently.
robots: isAlternateView(query, data.revision.is_current)
? { index: false, follow: true }
: undefined,
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { notFound } from 'next/navigation'
import { ToolReportView } from '@/features/benchmarks/tool-report-view'
import { getGetBenchmarkToolReportQueryKey } from '@/lib/api/generated/benchmarks/benchmarks'
import { formatRate } from '@/lib/benchmarks/format'
import { parseBenchmarkParams } from '@/lib/benchmarks/params'
import { isAlternateView, parseBenchmarkParams } from '@/lib/benchmarks/params'
import { fetchBenchmarkToolReport } from '@/lib/benchmarks/server'
import { getQueryClient } from '@/lib/query/client'
import { canonicalMetadata } from '@/lib/seo/canonical'
Expand All @@ -25,7 +25,9 @@ export async function generateMetadata({
title: `${report.tool.name} redaction benchmark — ${headline ? formatRate(headline.leak_rate) : ''} leak rate`,
description: `How many sensitive values ${report.tool.name} leaves recoverable in our ${suite.toUpperCase()} benchmark ${report.revision}, by layer, category and case, with overlays.`,
...canonicalMetadata(`/benchmarks/${suite}/tools/${slug}`),
robots: query.revision || query.scope !== 'all' ? { index: false, follow: true } : undefined,
robots: isAlternateView(query, report.revision_is_current)
? { index: false, follow: true }
: undefined,
}
}

Expand Down
23 changes: 22 additions & 1 deletion frontend/src/features/benchmarks/benchmark-suite-view.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,15 @@ describe('BenchmarkSuiteView', () => {
name: /first page of extraction-conditions-1/i,
})
expect(thumbnail.closest('a')).toHaveAttribute(
'href',
'/benchmarks/pdf/cases/extraction-conditions-1',
)
})

it('names a superseded revision in its case links', () => {
render(makeSuite({ revision: makeRevision({ is_current: false }) }))

expect(screen.getByRole('link', { name: 'extraction-conditions-1' })).toHaveAttribute(
'href',
'/benchmarks/pdf/cases/extraction-conditions-1?revision=v0.1.1',
)
Expand Down Expand Up @@ -65,12 +74,24 @@ describe('BenchmarkSuiteView', () => {
expect(screen.getByText('v0.1.1')).toHaveAttribute('aria-current', 'page')
})

// Viewing an old revision, the way back is the plain page, not a named copy of it.
it('links the current revision by its plain URL', () => {
render(
makeSuite({
revision: makeRevision({ revision: 'v0.1.0', is_current: false }),
revisions: [makeRevision(), makeRevision({ revision: 'v0.1.0', is_current: false })],
}),
)

expect(screen.getByRole('link', { name: 'v0.1.1' })).toHaveAttribute('href', '/benchmarks/pdf')
})

it('switches between every result and the ones we scored or verified', () => {
render()

expect(screen.getByRole('link', { name: /verified only/i })).toHaveAttribute(
'href',
'/benchmarks/pdf?revision=v0.1.1&scope=verified',
'/benchmarks/pdf?scope=verified',
)
})

Expand Down
27 changes: 18 additions & 9 deletions frontend/src/features/benchmarks/benchmark-suite-view.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { Card, CardDescription, CardTitle } from '@/components/ui/card'
import { Skeleton } from '@/components/ui/skeleton'
import { useGetBenchmarkSuite } from '@/lib/api/generated/benchmarks/benchmarks'
import type { GetBenchmarkSuiteParams } from '@/lib/api/generated/model'
import { benchmarkHref } from '@/lib/benchmarks/params'
import { cn } from '@/lib/utils'

import { previewAlt } from './case-preview'
Expand All @@ -30,7 +31,8 @@ export function BenchmarkSuiteView({
if (!data) return <Skeleton className="h-64 w-full" />

const revision = data.revision.revision
const base = `/benchmarks/${suite}?revision=${revision}`
const at = { revision, isCurrent: data.revision.is_current }
const caseHref = (caseId: string) => benchmarkHref(`/benchmarks/${suite}/cases/${caseId}`, at)

return (
<div className="space-y-10">
Expand Down Expand Up @@ -67,7 +69,10 @@ export function BenchmarkSuiteView({
) : (
<Link
key={r.revision}
href={`/benchmarks/${suite}?revision=${r.revision}`}
href={benchmarkHref(`/benchmarks/${suite}`, {
revision: r.revision,
isCurrent: r.is_current,
})}
className="text-muted-foreground hover:underline"
>
{r.revision}
Expand All @@ -77,17 +82,24 @@ export function BenchmarkSuiteView({
</nav>
</div>
<nav aria-label="Which results" className="flex gap-1 text-sm">
<ScopeLink href={base} active={data.scope === 'all'}>
<ScopeLink
href={benchmarkHref(`/benchmarks/${suite}`, at)}
active={data.scope === 'all'}
>
All results
</ScopeLink>
<ScopeLink href={`${base}&scope=verified`} active={data.scope === 'verified'}>
<ScopeLink
href={benchmarkHref(`/benchmarks/${suite}`, { ...at, scope: 'verified' })}
active={data.scope === 'verified'}
>
Verified only
</ScopeLink>
</nav>
</div>
<LeaderboardTable
suite={suite}
revision={revision}
isCurrent={data.revision.is_current}
rows={data.leaderboard}
caseCount={data.cases.length + data.holdout_case_count}
/>
Expand Down Expand Up @@ -117,7 +129,7 @@ export function BenchmarkSuiteView({
<Card className="space-y-3 p-4">
{item.preview ? (
<Link
href={`/benchmarks/${suite}/cases/${item.case_id}?revision=${revision}`}
href={caseHref(item.case_id)}
className="border-border block overflow-hidden rounded-md border"
>
{/* The page itself: which case is which is easier to see than to read. */}
Expand All @@ -136,10 +148,7 @@ export function BenchmarkSuiteView({
</Link>
) : null}
<CardTitle className="font-mono text-sm">
<Link
href={`/benchmarks/${suite}/cases/${item.case_id}?revision=${revision}`}
className="hover:underline"
>
<Link href={caseHref(item.case_id)} className="hover:underline">
{item.case_id}
</Link>
</CardTitle>
Expand Down
9 changes: 8 additions & 1 deletion frontend/src/features/benchmarks/case-view.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { Button } from '@/components/ui/button'
import { Skeleton } from '@/components/ui/skeleton'
import { useGetBenchmarkCase } from '@/lib/api/generated/benchmarks/benchmarks'
import type { GetBenchmarkCaseParams } from '@/lib/api/generated/model'
import { benchmarkHref } from '@/lib/benchmarks/params'

import { CasePreview } from './case-preview'
import { RunCards } from './run-cards'
Expand Down Expand Up @@ -37,7 +38,13 @@ export function CaseView({
<div className="space-y-10">
<header className="space-y-3">
<p className="text-muted-foreground text-sm">
<Link href={`/benchmarks/${suite}?revision=${data.revision}`} className="hover:underline">
<Link
href={benchmarkHref(`/benchmarks/${suite}`, {
revision: data.revision,
isCurrent: data.revision_is_current,
})}
className="hover:underline"
>
{suite.toUpperCase()} benchmark {data.revision}
</Link>
</p>
Expand Down
10 changes: 9 additions & 1 deletion frontend/src/features/benchmarks/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ export function makeToolReport(overrides: Partial<ToolReportOut> = {}): ToolRepo
tool: row.tool,
suite: 'pdf',
revision: 'v0.1.1',
revision_is_current: true,
scope: 'all',
case_count: 4,
surfaces: [
Expand All @@ -176,7 +177,13 @@ export function makeToolReport(overrides: Partial<ToolReportOut> = {}): ToolRepo
}

export function makeCaseDetail(overrides: Partial<CaseDetailOut> = {}): CaseDetailOut {
return { ...makeCase(), revision: 'v0.1.1', runs: [makeRun()], ...overrides }
return {
...makeCase(),
revision: 'v0.1.1',
revision_is_current: true,
runs: [makeRun()],
...overrides,
}
}

export function makeRunDetail(overrides: Partial<RunDetailOut> = {}): RunDetailOut {
Expand All @@ -201,6 +208,7 @@ export function makeRunDetail(overrides: Partial<RunDetailOut> = {}): RunDetailO
tier: 'Free',
notes: '',
revision: 'v0.1.1',
revision_is_current: true,
...overrides,
}
}
Expand Down
Loading
Loading