diff --git a/frontend/content/blog/redactable-pdf-redaction-benchmark.mdx b/frontend/content/blog/redactable-pdf-redaction-benchmark.mdx new file mode 100644 index 0000000..38135f2 --- /dev/null +++ b/frontend/content/blog/redactable-pdf-redaction-benchmark.mdx @@ -0,0 +1,161 @@ +--- +title: 'Redactable on the 54-name page: the best result yet, and too much removed' +description: Redactable removed 53 of 54 copies of a planted name, including every rotated and stacked copy. Its wizard also blacked out the byline, URLs and dates, so it fails our usability bar. +date: '2026-10-08' +tags: ['Benchmarks', 'PDF redaction', 'Reviews'] +authors: ['mykola-melnyk'] +image: '/images/blog/redactable-pdf-redaction-benchmark/overlay.png' +keywords: + [ + 'Redactable review', + 'Redactable redaction', + 'PDF redaction benchmark', + 'AI redaction accuracy', + 'over-redaction', + ] +--- + +**TL;DR:** We ran [Redactable](/tool/redactable) through the +[extraction-conditions page](/blog/pdf-redaction-extraction-conditions-benchmark), the PDF +that carries one invented name 54 times in 54 different forms. It removed **53 of 54**. That +is the lowest leak rate on the leaderboard so far, and it includes all four letter-by-letter +stacks that defeated every other tool running its own detection. It also removed things +nobody asked it to: the byline, both header URLs, the case number and the date. Only 81% of +the page's harmless text is still selectable, which fails our 90% usability bar. + +If you haven't read the original write-up, it explains how the page is built and how a leak +is decided. This post only covers what is new. + +## What Redactable is + +Redactable is a cloud service built for redaction alone, not a PDF editor with a redaction +button. You upload a document, and its **Auto** wizard lists suggested redactions by category +(person and company names, credentials and so on). You tick what applies and it removes the +content. It can also issue a redaction certificate that records what was removed. + +It runs in the cloud only, so documents leave your machine. Pricing is by documents per month, +with a free tier for evaluation. The plans and their sources are in +[the catalog](/tool/redactable). + +## The run + +We uploaded `extraction-conditions-1.pdf` through the web app, ran the Auto wizard with +default settings and applied its suggestions. The wizard reported **122 suggested +redactions** on a page that contains one name. + +![Redactable's editor with the test page loaded. The Redaction Wizard panel reports 122 suggested redactions; the first category, Company Names, lists "Redaction Tools" and "Freya".](/images/blog/redactable-pdf-redaction-benchmark/wizard.png) + +We scored the PDF it returned on our server, the same way as every other run: + +![The scored output. Every cell has a green box (removed) except t040, an upside-down handwritten form entry, boxed in red.](/images/blog/redactable-pdf-redaction-benchmark/overlay.png) + +| Where the name was | Removed | Note | +| -------------------------- | --------: | -------------------------------------------- | +| Text layer (real text) | 27/27 | upright, small, rotated, vertical stacks | +| Image only (no text layer) | 26/27 | scans, handwriting, inverse, watermarked | +| **Total** | **53/54** | leak rate 0.019, 95% interval [0.003, 0.098] | + +The [full run](/benchmarks/pdf/runs/20261005T140938-redactable-web-extraction-conditions-1-a1-b5eb0d) +has the output PDF, so you can check all of this yourself. + +## What it got right + +### Vertical stacks + +Four names on the page are stacked one letter per line. They are real, selectable text, but +a parser returns them one character at a time, so a name detector never sees `Freya Yamamoto`. +PDF Redaction, AI-Redact, SafeRedact and KeptPDF, which all rely on their own detection, +removed none of the four. Blinded removed all four, but only because it was told what to +search for. + +Redactable removed all four on its own detection. So it rebuilds words from where the letters +sit on the page, not only from the order the parser outputs them in. + +### Rotated and scanned text + +It removed 18 of the 19 rotated and skewed names, and every name in the turned-and-stacked +band. The one rotated name it missed is the leak below. AI-Redact and SafeRedact removed none +of the 19. + +### Names behind a watermark + +Three cells carry the name under a light "CONFIDENTIAL" watermark, with no text layer. This +was PDF Redaction's weakest condition: it missed all three. Redactable removed the name in all +three without drawing a box at all. The watermark is still there, and the name is gone. + +### Removal, not covering + +The name appears nowhere in the output's text layer. The page is still an A4 PDF with real, +selectable text, not a flattened image. The metadata contains only `Producer: Redacted by +Redactable`. + +## The one leak + +`t040` is a form line, `Name:`, with the name handwritten in cursive and upside down, as on a +form that went through the scanner the wrong way round. Redactable drew a box over the +"Freya" end and left "Yamamoto" readable: + +![The output at t040: "Name:" followed by the cursive word Yamamoto, upside down and fully legible, and a black box where Freya was.](/images/blog/redactable-pdf-redaction-benchmark/t040-leak.png) + +PDF Redaction leaked the same cell. A half-redacted name is still a leak, and a surname on its +own is often enough to identify someone. + +## What it removed that it shouldn't have + +![The header of the output page. The byline after "by", both URLs on the right, the case number and the generated timestamp are all blacked out.](/images/blog/redactable-pdf-redaction-benchmark/header-over-redaction.png) + +Applying the wizard's suggestions also removed: + +- the byline **"Redaction Tools"**, which the wizard listed as a company name; +- both URLs in the header; +- the case number at the end of `extraction-conditions-1`; +- the generated timestamp. + +None of these is personal data about Freya Yamamoto. Together they bring **text retention** to +0.807: about a fifth of the page's harmless text can no longer be selected. Our usability gate +is 0.9, so the run fails it. + +On this page that cost is small. On a contract or a court bundle, blacking out every company +name, date and URL can make the document useless to whoever receives it. The alternative is +to review 122 suggestions by hand. That is the trade-off of a wizard tuned to catch +everything: you trade leaks for review time. + +The output has one more oddity. In cell `t037`, the name was printed upside down in light grey +as real text. The output keeps a near-white, scrambled remnant of it, upright, and the text +layer reads `Frmmammm Yyyoto`. Our checker does not count that as the name, and we agree it +cannot be read as one. But it suggests Redactable rewrote the characters in that cell rather +than deleting them, and a remnant in the text layer is worth knowing about. + +## Where it sits + +| Tool | Names removed | Leak rate, 95% interval | Text retention | +| ---------------------------------------------------- | ------------: | ----------------------- | -------------: | +| [Redactable](/benchmarks/pdf/tools/redactable) | 53/54 | 0.02 [0.00, 0.10] | 0.807 | +| [Blinded](/benchmarks/pdf/tools/blinded) | 51/54 | 0.06 [0.02, 0.15] | 0.876 | +| [PDF Redaction](/benchmarks/pdf/tools/pdf-redaction) | 42/54 | 0.22 [0.13, 0.35] | 0.932 | +| [AI-Redact](/benchmarks/pdf/tools/ai-redact) | 30/54 | 0.44 [0.32, 0.58] | 0.951 | +| [KeptPDF](/benchmarks/pdf/tools/keptpdf) | 19/54 | 0.65 [0.51, 0.76] | 0.010 | +| [SafeRedact](/benchmarks/pdf/tools/saferedact-app) | 14/54 | 0.74 [0.61, 0.84] | 0.000 | + +The figures are from the [live leaderboard](/benchmarks/pdf), dataset revision v0.1.1. +Redactable's interval overlaps Blinded's, so this page alone cannot say which of the two is +ahead. Both fail the retention gate: so far, the two lowest leak rates both come with more +than a tenth of the page's harmless text removed. + +Disclosure: Redaction Tools is run by StabRise, which also makes PDF Redaction. The +[About page](/about) explains how we handle that. Redactable's run was scored by the same +server, with the same checker, as ours. + +## Caveats + +- **One page, one run, one category.** Every probe is a person's name, so this measures + reading, not how well Redactable picks categories. A wizard that suggests 122 redactions + for one name would likely look different on a page with many kinds of data. +- **Settings.** We used the Auto wizard with defaults and applied its suggestions. Redactable + supports excluded terms and category selection. With the byline excluded, retention would + be higher, and we have not measured whether the leak count would change. +- **Simulated handwriting.** The handwriting on the page is drawn from fonts, which makes it + tidier than real handwriting. + +Redactable can [submit its own runs](/docs/benchmarks/submitting-results), for example with +tuned settings. We rescore every one, and an editor reviews it before it is listed. diff --git a/frontend/public/images/blog/redactable-pdf-redaction-benchmark/header-over-redaction.png b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/header-over-redaction.png new file mode 100644 index 0000000..0f2b923 Binary files /dev/null and b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/header-over-redaction.png differ diff --git a/frontend/public/images/blog/redactable-pdf-redaction-benchmark/overlay.png b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/overlay.png new file mode 100644 index 0000000..74896f4 Binary files /dev/null and b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/overlay.png differ diff --git a/frontend/public/images/blog/redactable-pdf-redaction-benchmark/t037-remnant.png b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/t037-remnant.png new file mode 100644 index 0000000..782fa95 Binary files /dev/null and b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/t037-remnant.png differ diff --git a/frontend/public/images/blog/redactable-pdf-redaction-benchmark/t040-leak.png b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/t040-leak.png new file mode 100644 index 0000000..10e54e5 Binary files /dev/null and b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/t040-leak.png differ diff --git a/frontend/public/images/blog/redactable-pdf-redaction-benchmark/wizard.png b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/wizard.png new file mode 100644 index 0000000..038cbc9 Binary files /dev/null and b/frontend/public/images/blog/redactable-pdf-redaction-benchmark/wizard.png differ diff --git a/frontend/src/features/catalog/tool-profile.test.tsx b/frontend/src/features/catalog/tool-profile.test.tsx index 24a8ad5..6a44cd0 100644 --- a/frontend/src/features/catalog/tool-profile.test.tsx +++ b/frontend/src/features/catalog/tool-profile.test.tsx @@ -125,6 +125,20 @@ describe('ToolProfile', () => { expect(within(facts).getByText('No')).toBeInTheDocument() }) + // Staff add media in the admin; a list kept here would hide every new one. + it('lists every media facet the API sends, by its label', () => { + render( + makeToolDetail({ + facets: [ + { dimension: 'media', dimension_label: 'Media', slug: 'pdf', label: 'PDF' }, + { dimension: 'media', dimension_label: 'Media', slug: 'word', label: 'Word' }, + ], + }), + ) + + expect(within(screen.getByTestId('key-facts')).getByText('PDF, Word')).toBeInTheDocument() + }) + it('describes a trial as a trial in the key facts', () => { render() diff --git a/frontend/src/features/catalog/tool-profile.tsx b/frontend/src/features/catalog/tool-profile.tsx index 97e01cc..7a2ced6 100644 --- a/frontend/src/features/catalog/tool-profile.tsx +++ b/frontend/src/features/catalog/tool-profile.tsx @@ -38,10 +38,6 @@ import { ToolLogo } from './tool-logo' import { ToolScreenshots } from './tool-screenshots' import { VendorMaintainedBadge } from './vendor-maintained-badge' -const MEDIA = new Set(['pdf', 'image', 'video', 'audio', 'text']) -const DEPLOYMENT = new Set(['online', 'desktop', 'self-hosted', 'api-tools', 'browser-extension']) -const METHOD = new Set(['manual-redaction', 'ai', 'hybrid', 'rule-based']) - /** The plan's own published figure, shared with the cost calculator so the two * tables can never quote the same plan differently. */ export function planPrice(plan: PlanOut): string { @@ -240,16 +236,21 @@ function HeaderLinks({ tool }: { tool: ToolDetailOut }) { } function KeyFacts({ tool }: { tool: ToolDetailOut }) { - const facets = (allowed: Set) => - tool.facet_slugs.filter((slug) => allowed.has(slug)).join(', ') || 'Not recorded' + // From the API's facets rather than a slug list here, so a value staff add + // in the admin shows up without a deploy. + const facets = (dimension: string) => + tool.facets + .filter((facet) => facet.dimension === dimension) + .map((facet) => facet.label) + .join(', ') || 'Not recorded' const summary = tool.price_summary const facts: [string, string][] = [ ['Vendor', tool.vendor.name], ['Headquarters', tool.vendor.hq_country || 'Not recorded'], - ['Media', facets(MEDIA)], - ['Deployment', facets(DEPLOYMENT)], - ['Method', facets(METHOD)], + ['Media', facets('media')], + ['Deployment', facets('deployment')], + ['Method', facets('method')], ['Free tier', summary.has_free_tier ? 'Yes' : 'No'], ['Trial', summary.is_trial && summary.trial_days ? `${summary.trial_days}-day trial` : 'None'], [ diff --git a/frontend/src/features/catalog/tool-table.test.tsx b/frontend/src/features/catalog/tool-table.test.tsx index 6ce7971..0c2fd88 100644 --- a/frontend/src/features/catalog/tool-table.test.tsx +++ b/frontend/src/features/catalog/tool-table.test.tsx @@ -2,7 +2,8 @@ import { screen, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { describe, expect, it } from 'vitest' -import { getListToolsQueryKey } from '@/lib/api/generated/catalog/catalog' +import { getListFacetsQueryKey, getListToolsQueryKey } from '@/lib/api/generated/catalog/catalog' +import type { FacetDimensionOut } from '@/lib/api/generated/model' import type { CatalogFilters } from '@/lib/catalog/filters' import { makeQueryClient } from '@/lib/query/client' import { renderWithProviders } from '@/test/render' @@ -10,9 +11,26 @@ import { renderWithProviders } from '@/test/render' import { makePage, makePlan, makePrice, makeTool } from './fixtures' import { ToolTable } from './tool-table' +const medium = (slug: string, label: string) => ({ + code: slug, + slug, + label, + has_landing_page: false, + tool_count: 1, +}) + +const FACETS: FacetDimensionOut[] = [ + { + code: 'media', + label: 'Media', + values: [medium('pdf', 'PDF'), medium('word', 'Word'), medium('image', 'Image')], + }, +] + function render(tools = [makeTool()], filters: CatalogFilters = {}) { const queryClient = makeQueryClient() queryClient.setQueryData(getListToolsQueryKey(filters), makePage(tools)) + queryClient.setQueryData(getListFacetsQueryKey(), FACETS) return renderWithProviders(, { queryClient }) } @@ -82,6 +100,18 @@ describe('ToolTable', () => { ).toEqual(['PDF', 'Image']) }) + // Staff add media in the admin; a list kept here would hide every new one. + it('shows a medium the taxonomy gained after this was written', () => { + render([makeTool({ facet_slugs: ['word', 'pdf'] })]) + + const covers = screen.getByTestId('covers-adobe-acrobat') + expect( + within(covers) + .getAllByRole('listitem') + .map((chip) => chip.textContent), + ).toEqual(['PDF', 'Word']) + }) + // A "from" price answers what the cheapest plan costs, not what the reader's // month would - a per-page rate and a flat fee cannot be ranked by it. it('shows what 1,000 pages a month cost on the cheapest plan', () => { diff --git a/frontend/src/features/catalog/tool-table.tsx b/frontend/src/features/catalog/tool-table.tsx index cf194d1..5235000 100644 --- a/frontend/src/features/catalog/tool-table.tsx +++ b/frontend/src/features/catalog/tool-table.tsx @@ -14,7 +14,7 @@ import { TableRow, } from '@/components/ui/table' import { Tooltip } from '@/components/ui/tooltip' -import { useListTools } from '@/lib/api/generated/catalog/catalog' +import { useListFacets, useListTools } from '@/lib/api/generated/catalog/catalog' import type { PriceSummaryOut } from '@/lib/api/generated/model' import { cheapestCost, type DocumentInput } from '@/lib/catalog/document-cost' import type { CatalogFilters } from '@/lib/catalog/filters' @@ -165,9 +165,16 @@ export function ToolTable({ filters }: { filters: CatalogFilters }) { const HEADER = 'text-xs font-medium' -/** Media as chips, in the taxonomy's order rather than whatever order the row carries. */ +/** + * Media as chips, in the taxonomy's order rather than whatever order the row carries. + * + * Read from the facets endpoint, which the hub prefetches beside the list, so a + * medium staff add in the admin shows up without a deploy. + */ function MediaChips({ facetSlugs }: { facetSlugs: string[] }) { - const media = MEDIA.filter((medium) => facetSlugs.includes(medium.slug)) + const { data } = useListFacets() + const taxonomy = data?.find((dimension) => dimension.code === 'media')?.values ?? [] + const media = taxonomy.filter((medium) => facetSlugs.includes(medium.slug)) if (!media.length) return — return (