From 8b2223ba41829fceb54e4d25fdcc30683a15c58e Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sat, 25 Jul 2026 16:16:45 -0500 Subject: [PATCH] Fixture/snapshot test suite for split/merge/overlap/analyzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deferred since v3.1; now that the Chunkability score makes chunking behavior load-bearing, any change to it should fail a test. 30 tests, node:test only — no new dependencies. Run: npm test. - api/chunk.js gains named exports for the test surface; runtime is unchanged (Vercel only uses the default handler) - tests/chunking.test.js: recursiveSplit limits/ordering/paragraph preference/force-split, mergeSmallChunks, addOverlap (prefix content, recorded count, the exact 29-word clamp from the page that started this, no-op paths), initialism, Flesch-Kincaid ordering - tests/analyzer.test.js: one isolated scenario per flag, including the single-word-heading skip, UWM initialism anchoring, near-duplicates counted once per pair, and overlap-prefix exclusion; per-chunk-average score math; determinism; empty input - tests/fixture-snapshot.test.js: two committed HTML fixtures through the full offline pipeline, deep-equal against committed snapshots (UPDATE_SNAPSHOTS=1 npm test to regenerate intentionally). nested-sections.html is the regression fixture for the v3.3 parent-swallows-child extraction fix - Snapshots use the estimate tokenizer so they don't churn with gpt-tokenizer versions - .github/workflows/test.yml runs the suite on PRs and main pushes - npm test uses a glob: the directory form of node --test fails silently on Node 22.22 Co-Authored-By: Claude Fable 5 --- .github/workflows/test.yml | 18 ++ SESSION_LOG.md | 15 +- api/chunk.js | 16 + package.json | 1 + tasks/todo.md | 2 +- tests/analyzer.test.js | 226 ++++++++++++++ tests/chunking.test.js | 103 +++++++ tests/fixture-snapshot.test.js | 113 +++++++ tests/fixtures/marketing-page.html | 31 ++ tests/fixtures/marketing-page.snapshot.json | 300 +++++++++++++++++++ tests/fixtures/nested-sections.html | 21 ++ tests/fixtures/nested-sections.snapshot.json | 157 ++++++++++ 12 files changed, 999 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 tests/analyzer.test.js create mode 100644 tests/chunking.test.js create mode 100644 tests/fixture-snapshot.test.js create mode 100644 tests/fixtures/marketing-page.html create mode 100644 tests/fixtures/marketing-page.snapshot.json create mode 100644 tests/fixtures/nested-sections.html create mode 100644 tests/fixtures/nested-sections.snapshot.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f9d020c --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,18 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm test diff --git a/SESSION_LOG.md b/SESSION_LOG.md index 6b1156e..00ee9a6 100644 --- a/SESSION_LOG.md +++ b/SESSION_LOG.md @@ -25,10 +25,19 @@ **Backlog cleanup:** `feature/enhanced-chunking` turned out to be already deleted (backlog item was stale); pruned leftover local tracking refs for the merged `chore/remove-dead-css` + `feature/marketing-shell-parity` branches. Branch list is now just main + the three chunk-quality branches. +## 2026-07-25 (cont. 2) — Fixture/snapshot test suite (backlog item since v3.1) DONE + +Will approved the fresh-scope recommendation. `feature/fixture-tests` (stacked on v3.4), 30 tests green in ~0.6s via `npm test` (node:test, zero new deps): +- **Named exports added** at the bottom of `api/chunk.js` for the test surface (runtime unaffected — Vercel only uses the default handler; verified API still healthy afterward). +- `tests/chunking.test.js` — recursiveSplit (limit, word preservation, paragraph boundaries, force-split), mergeSmallChunks, addOverlap (prefix content, recorded count, the exact 29-word clamp case from Chuck's page, no-op paths), initialism, Flesch-Kincaid ordering. +- `tests/analyzer.test.js` — one isolated scenario per flag (incl. single-word-heading skip, UWM initialism anchoring, near-duplicate counted once per pair, overlap-prefix exclusion), per-chunk-average score math (85/B case), determinism, empty→null. Gotcha found: synthetic filler words are accidentally polysyllabic → FK grade >12, so the "simple prose" case is hand-written. +- `tests/fixture-snapshot.test.js` — two committed HTML fixtures run through the full offline pipeline (extractSourceMetadata → buildChunks → enhanceChunks → analyzeChunks, tokenizer 'estimate' so snapshots don't churn with gpt-tokenizer versions); full-result deep-equal vs committed `*.snapshot.json`; regenerate intentionally with `UPDATE_SNAPSHOTS=1 npm test`. `nested-sections.html` is the regression fixture for the v3.3 extraction fix (parent must not swallow child `
` content; explicit asserts + snapshot). +- `.github/workflows/test.yml` — npm ci + npm test on PRs and main pushes, Node 22. +- Note: `node --test tests/` (directory form) fails silently on Node 22.22 — script uses the `'tests/*.test.js'` glob. + **Open at day's end:** -1. **Merge chain awaiting Will:** #10 (v3.2, base main) → #11 (v3.3, retarget to main after #10) → #12 (v3.4, retarget after #11). Merging main auto-deploys. -2. Fixture/snapshot tests (split/merge/overlap/analyzer) — deferred since v3.1, still the one real backlog item. -3. Phase 4 optional: template query generator, `?queries=` Ontologizer deep-link, compare mode, README rubric docs. +1. **Merge chain awaiting Will:** #10 (v3.2, base main) → #11 (v3.3) → #12 (v3.4) → #13 (tests) — retarget each to main as its base lands. Merging to main auto-deploys. +2. Phase 4 optional: template query generator, `?queries=` Ontologizer deep-link, compare mode, README rubric docs. ## 2026-07-24 — Chunk Quality plan (from Chuck's feedback) + repo moved out of archive diff --git a/api/chunk.js b/api/chunk.js index c6da2e2..d5493cb 100644 --- a/api/chunk.js +++ b/api/chunk.js @@ -1095,3 +1095,19 @@ function slugify(text) { .replace(/\s+/g, '-') .slice(0, 64); } + +// Named exports for the test suite (tests/). Vercel only uses the default +// handler; these change nothing at runtime. +export { + CHUNK_SIZES, + countWords, + recursiveSplit, + mergeSmallChunks, + addOverlap, + buildChunks, + enhanceChunks, + analyzeChunks, + extractSourceMetadata, + initialism, + fleschKincaidGrade, +}; diff --git a/package.json b/package.json index 9d5d3e0..42bd1fe 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "type": "module", "main": "api/chunk.js", "scripts": { + "test": "node --test 'tests/*.test.js'", "serve": "python3 -m http.server 8080", "demo": "open public/index.html" }, diff --git a/tasks/todo.md b/tasks/todo.md index ea72483..ef7a197 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -157,4 +157,4 @@ Branch: `feature/chunk-quality-v3.4` **Deviations from plan, all reasoned:** score model became per-chunk-average instead of flat deductions (proportionality); thin-section got an absolute 100w floor; anchor terms gained initialisms; answer-buried requires ≥2 heading content words; the near-duplicate flag surfaced the deferred nested-content extraction bug, fixed at the root in `extractContentPieces`/`stopCondition`. -**Remaining backlog (not in this plan):** fixture/snapshot tests for split/merge/overlap/analyzer (deferred since v3.1); Phase 4 optional items (template query generator, `?queries=` deep-link from Ontologizer, compare mode, README rubric docs). +**Remaining backlog (not in this plan):** ~~fixture/snapshot tests for split/merge/overlap/analyzer~~ **DONE 2026-07-25** (PR #13: 30 node:test tests, 2 HTML fixtures + committed snapshots, nested-section regression coverage, CI workflow); Phase 4 optional items (template query generator, `?queries=` deep-link from Ontologizer, compare mode, README rubric docs). diff --git a/tests/analyzer.test.js b/tests/analyzer.test.js new file mode 100644 index 0000000..098cdc4 --- /dev/null +++ b/tests/analyzer.test.js @@ -0,0 +1,226 @@ +// analyzeChunks() scenario tests: one isolated case per flag, scoring math, +// and determinism. analyzeChunks mutates its input, so every scenario builds +// fresh chunk objects. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { analyzeChunks } from '../api/chunk.js'; + +const OPTIONS = { chunkSize: 'medium', strategy: 'heading', overlap: 35 }; + +const SOURCE = { + url: 'https://example.com/rockets', + title: 'Rocket Engine Maintenance Guide - Acme Rocket Supply', + site: 'Acme Rocket Supply', + jsonld: [{ '@type': 'Organization', name: 'Acme Rocket Supply' }], +}; + +// Unique-vocabulary filler: short, simple sentences whose words never repeat +// across seeds, so scenarios can't collide via the near-duplicate check. +function filler(seed, sentences) { + return Array.from( + { length: sentences }, + (_, i) => `${seed}${i}ax ${seed}${i}bx ${seed}${i}cx runs fine.` + ).join(' '); +} + +function mkChunk(index, title, text, extra = {}) { + return { + big_chunk_index: index, + title, + level: 2, + fragment: `#chunk-${index}`, + small_chunks: [{ text, chunk_index: 1 }], + ...extra, + }; +} + +// ~105 words, anchored, heading echoed in the first sentence, no pronoun +// opener, low reading grade — carries no non-info flags. +function goodChunk(index, seed = 'alpha') { + return mkChunk( + index, + 'Rocket Engine Maintenance', + `Rocket engine maintenance keeps every Acme Rocket Supply vehicle safe. ${filler(seed, 19)}` + ); +} + +function nonInfoCodes(chunk) { + return chunk.flags.filter((f) => f.severity !== 'info').map((f) => f.code).sort(); +} + +test('clean page scores 100/A with no fixes', () => { + const chunks = [goodChunk(1, 'alpha'), goodChunk(2, 'bravo')]; + const result = analyzeChunks(chunks, SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunks[0]), []); + assert.deepEqual(nonInfoCodes(chunks[1]), []); + assert.equal(result.score, 100); + assert.equal(result.grade, 'A'); + assert.deepEqual(result.top_fixes, []); +}); + +test('dangling-reference: pronoun opener', () => { + const chunk = mkChunk( + 1, + 'Rocket Engine Maintenance', + `Our rocket engine maintenance keeps every Acme Rocket Supply vehicle safe. ${filler('charlie', 19)}` + ); + analyzeChunks([chunk], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), ['dangling-reference']); + assert.match(chunk.flags.find((f) => f.code === 'dangling-reference').message, /"Our"/); +}); + +test('thin-section: under the 100-word floor', () => { + const chunk = mkChunk( + 1, + 'Rocket Engine Maintenance', + `Rocket engine maintenance keeps every Acme Rocket Supply vehicle safe. ${filler('delta', 6)}` + ); + analyzeChunks([chunk], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), ['thin-section']); +}); + +test('oversized-section: over the medium max with no subheading', () => { + const chunk = mkChunk( + 1, + 'Rocket Engine Maintenance', + `Rocket engine maintenance keeps every Acme Rocket Supply vehicle safe. ${filler('echo', 110)}` + ); + analyzeChunks([chunk], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), ['oversized-section']); +}); + +test('generic-heading flags the stoplist and skips answer-buried', () => { + const chunk = mkChunk( + 1, + 'Overview', + `Rocket engine maintenance keeps every Acme Rocket Supply vehicle safe. ${filler('foxtrot', 19)}` + ); + analyzeChunks([chunk], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), ['generic-heading']); +}); + +test('answer-buried: first sentence ignores a multiword heading', () => { + const chunk = mkChunk( + 1, + 'Pricing And Contracts', + `Acme Rocket Supply ships worldwide from three depots. ${filler('golf', 19)}` + ); + analyzeChunks([chunk], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), ['answer-buried']); +}); + +test('answer-buried skips single-word headings', () => { + const chunk = mkChunk( + 1, + 'History', + `Acme Rocket Supply ships worldwide from three depots. ${filler('hotel', 19)}` + ); + analyzeChunks([chunk], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), []); +}); + +test('no-entity-anchor: section never names any page entity', () => { + const chunk = mkChunk( + 1, + 'Shipping Options', + `Shipping options include ground and air delivery for every order. ${filler('india', 19)}` + ); + analyzeChunks([chunk], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), ['no-entity-anchor']); +}); + +test('no-entity-anchor accepts the initialism (UWM anchors the full name)', () => { + const source = { + url: 'https://example.edu/', + title: 'Admissions - University of Wisconsin-Milwaukee', + site: 'University of Wisconsin-Milwaukee', + jsonld: [], + }; + const chunk = mkChunk( + 1, + 'Application Deadlines', + `Application deadlines at UWM fall in January and August. ${filler('juliet', 19)}` + ); + analyzeChunks([chunk], source, OPTIONS); + assert.deepEqual(nonInfoCodes(chunk), []); +}); + +test('near-duplicate: identical chunks flag each other, counted once as a pair', () => { + const text = `Rocket engine maintenance keeps every Acme Rocket Supply vehicle safe. ${filler('kilo', 19)}`; + const a = mkChunk(1, 'Rocket Engine Maintenance', text); + const b = mkChunk(2, 'Rocket Engine Maintenance', text); + const result = analyzeChunks([a, b], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(a), ['near-duplicate']); + assert.deepEqual(nonInfoCodes(b), ['near-duplicate']); + const fix = result.top_fixes.find((f) => f.code === 'near-duplicate'); + assert.equal(fix.count, 1); // one pair, not two instances +}); + +test('overlap prefixes are excluded from duplicate and thin checks', () => { + const a = goodChunk(1, 'lima'); + const aWords = a.small_chunks[0].text.trim().split(/\s+/); + const prefix = aWords.slice(-60).join(' '); + const b = mkChunk(2, 'Launch Day Checklist', '', { + small_chunks: [{ + text: `${prefix} The launch day checklist covers Acme Rocket Supply steps. ${filler('mike', 20)}`, + chunk_index: 1, + overlap_word_count: 60, + }], + }); + analyzeChunks([a, b], SOURCE, OPTIONS); + assert.deepEqual(nonInfoCodes(a), []); + assert.deepEqual(nonInfoCodes(b), []); +}); + +test('readability: dense prose gets an info flag, simple prose does not', () => { + const dense = mkChunk( + 1, + 'Rocket Engine Maintenance', + 'Rocket engine maintenance necessitates extraordinarily comprehensive organizational preparation incorporating interdependent technological capabilities alongside institutional considerations regarding Acme Rocket Supply operational infrastructure modernization initiatives representing considerable administrative complexity throughout implementation phases requiring deliberate prioritization frameworks additionally encompassing evaluation methodologies ' + + Array.from({ length: 12 }, (_, i) => `furthermore incorporating supplementary considerations regarding notwithstanding organizational deliberations ${i}`).join(' ') + ); + // Genuinely plain prose — the synthetic filler() words are multi-syllable + // by accident, so a hand-written low-grade text is used here instead. + const simple = mkChunk( + 2, + 'Rocket Engine Maintenance', + 'Rocket engine maintenance keeps each Acme Rocket Supply craft safe. We check the pump. We check the seals. We log each part. The crew signs off. Then we test fire the stand. It takes one day.' + ); + analyzeChunks([dense, simple], SOURCE, OPTIONS); + assert.ok(dense.flags.some((f) => f.code === 'readability' && f.severity === 'info')); + assert.ok(!simple.flags.some((f) => f.code === 'readability')); +}); + +test('score is the per-chunk average and top fixes report point gains', () => { + const clean = goodChunk(1, 'oscar'); + const flagged = mkChunk( + 2, + 'Rocket Engine Maintenance', + `Our rocket engine maintenance keeps every Acme Rocket Supply vehicle safe. ${filler('papa', 19)}` + ); + const result = analyzeChunks([clean, flagged], SOURCE, OPTIONS); + assert.equal(clean.chunk_score, 100); + assert.equal(flagged.chunk_score, 70); // dangling-reference weighs 30 + assert.equal(result.score, 85); + assert.equal(result.grade, 'B'); + assert.deepEqual(result.top_fixes.map((f) => [f.code, f.count, f.points]), [ + ['dangling-reference', 1, 15], + ]); +}); + +test('analysis is deterministic: identical input, identical output', () => { + const build = () => [ + goodChunk(1, 'quebec'), + mkChunk(2, 'Overview', `Acme Rocket Supply ships worldwide from three depots. ${filler('romeo', 6)}`), + ]; + const first = build(); + const second = build(); + const r1 = analyzeChunks(first, SOURCE, OPTIONS); + const r2 = analyzeChunks(second, SOURCE, OPTIONS); + assert.deepEqual(r1, r2); + assert.deepEqual(first, second); // flags + chunk_score attached identically +}); + +test('empty chunk list returns null', () => { + assert.equal(analyzeChunks([], SOURCE, OPTIONS), null); +}); diff --git a/tests/chunking.test.js b/tests/chunking.test.js new file mode 100644 index 0000000..53ce499 --- /dev/null +++ b/tests/chunking.test.js @@ -0,0 +1,103 @@ +// Unit tests for the chunking primitives: split, merge, overlap, helpers. +// Run: npm test +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + countWords, + recursiveSplit, + mergeSmallChunks, + addOverlap, + initialism, + fleschKincaidGrade, +} from '../api/chunk.js'; + +const sentence = 'The quick brown fox jumps over the lazy dog again today.'; +const longText = Array.from({ length: 40 }, () => sentence).join(' '); + +test('recursiveSplit returns text unchanged when under the limit', () => { + assert.deepEqual(recursiveSplit('Short text here.', 100), ['Short text here.']); +}); + +test('recursiveSplit keeps every piece within maxWords', () => { + const pieces = recursiveSplit(longText, 50); + assert.ok(pieces.length > 1); + for (const piece of pieces) { + assert.ok(countWords(piece) <= 50, `piece has ${countWords(piece)} words`); + } +}); + +test('recursiveSplit preserves all words in order', () => { + const pieces = recursiveSplit(longText, 50); + assert.equal(pieces.join(' ').split(/\s+/).join(' '), longText.split(/\s+/).join(' ')); +}); + +test('recursiveSplit prefers paragraph boundaries', () => { + const para = Array.from({ length: 4 }, () => sentence).join(' '); // 44 words + const text = `${para}\n\n${para}\n\n${para}`; + const pieces = recursiveSplit(text, 50); + assert.equal(pieces.length, 3); + assert.deepEqual(pieces, [para, para, para]); +}); + +test('recursiveSplit force-splits a single giant sentence', () => { + const giant = Array.from({ length: 120 }, (_, i) => `word${i}`).join(' '); + const pieces = recursiveSplit(giant, 50); + assert.equal(pieces.length, 3); + assert.ok(pieces.every((p) => countWords(p) <= 50)); +}); + +test('mergeSmallChunks merges below-minimum pieces forward', () => { + const merged = mergeSmallChunks(['one two three', 'four five six', sentence], 5); + assert.equal(merged.length, 2); + assert.equal(merged[0], 'one two three\n\nfour five six'); + assert.equal(merged[1], sentence); +}); + +test('mergeSmallChunks leaves compliant chunks alone', () => { + const chunks = [sentence, sentence, sentence]; + assert.deepEqual(mergeSmallChunks(chunks, 5), chunks); +}); + +test('addOverlap leaves the first chunk untouched and records the count', () => { + const chunks = [{ text: longText }, { text: sentence }]; + const out = addOverlap(chunks, 10); + assert.equal(out[0].text, longText); + assert.equal(out[0].overlap_word_count, undefined); + assert.equal(out[1].overlap_word_count, 10); + const expectedPrefix = longText.trim().split(/\s+/).slice(-10).join(' '); + assert.ok(out[1].text.startsWith(`${expectedPrefix} `)); + assert.ok(out[1].text.endsWith(sentence)); +}); + +test('addOverlap clamps to the previous chunk length (the 29-word case)', () => { + // Chuck's page: a 29-word section with a 35-word overlap window — the + // whole previous section gets prepended and the count must say 29, not 35. + const short = Array.from({ length: 29 }, (_, i) => `w${i}`).join(' '); + const out = addOverlap([{ text: short }, { text: sentence }], 35); + assert.equal(out[1].overlap_word_count, 29); + assert.equal(out[1].text, `${short} ${sentence}`); +}); + +test('addOverlap is a no-op for zero overlap or a single chunk', () => { + const chunks = [{ text: sentence }, { text: sentence }]; + assert.deepEqual(addOverlap(chunks, 0), chunks); + assert.deepEqual(addOverlap([{ text: sentence }], 10), [{ text: sentence }]); +}); + +test('initialism builds acronyms from multiword names, skipping stopwords', () => { + assert.equal(initialism('University of Wisconsin-Milwaukee'), 'uwm'); + assert.equal(initialism('Retrieval-augmented generation'), 'rag'); + assert.equal(initialism('Seaside'), null); // single word + assert.equal(initialism('Acme Rockets'), null); // 2 letters — too short to match +}); + +test('fleschKincaidGrade is deterministic and orders simple < dense text', () => { + const simple = 'We fix cars. We do it fast. You can trust us. Call us today.'; + const dense = + 'Notwithstanding organizational considerations, the implementation of comprehensive infrastructural modernization necessitates extraordinarily deliberate prioritization of interdependent technological capabilities.'; + const simpleGrade = fleschKincaidGrade(simple); + const denseGrade = fleschKincaidGrade(dense); + assert.equal(simpleGrade, fleschKincaidGrade(simple)); + assert.ok(simpleGrade < 6, `simple text graded ${simpleGrade}`); + assert.ok(denseGrade > 15, `dense text graded ${denseGrade}`); +}); diff --git a/tests/fixture-snapshot.test.js b/tests/fixture-snapshot.test.js new file mode 100644 index 0000000..c61ea9d --- /dev/null +++ b/tests/fixture-snapshot.test.js @@ -0,0 +1,113 @@ +// Fixture snapshot tests: committed HTML pages run through the full offline +// pipeline (extract metadata → build chunks → enhance → analyze), and the +// entire result is compared against a committed snapshot. Any change to +// splitting, merging, overlap, extraction, or scoring shows up as a diff. +// +// Intentional behavior change? Regenerate with: UPDATE_SNAPSHOTS=1 npm test +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { load } from 'cheerio'; +import { + extractSourceMetadata, + buildChunks, + enhanceChunks, + analyzeChunks, +} from '../api/chunk.js'; + +const FIXTURES = new URL('./fixtures/', import.meta.url); + +// Mirrors chunkUrl() minus the network fetch and Defuddle (cheerio path only, +// fully deterministic). Tokenizer 'estimate' keeps snapshots independent of +// gpt-tokenizer versions. +function runPipeline(fixtureName, url, finalOptions) { + const html = readFileSync(new URL(fixtureName, FIXTURES), 'utf8'); + const $ = load(html); + const source = extractSourceMetadata($, url); + const warnings = []; + const bigChunks = buildChunks($, finalOptions, warnings, source.title); + const enhanced = enhanceChunks(bigChunks, 'estimate'); + const chunkability = analyzeChunks(enhanced, source, finalOptions); + return { source, warnings, big_chunks: enhanced, chunkability }; +} + +function checkSnapshot(name, actual) { + const file = new URL(`${name}.snapshot.json`, FIXTURES); + if (process.env.UPDATE_SNAPSHOTS) { + writeFileSync(file, `${JSON.stringify(actual, null, 2)}\n`); + return; + } + let expected; + try { + expected = JSON.parse(readFileSync(file, 'utf8')); + } catch { + assert.fail(`Missing snapshot ${name}.snapshot.json — run UPDATE_SNAPSHOTS=1 npm test`); + } + assert.deepEqual(actual, expected); +} + +const MARKETING_OPTIONS = { chunkSize: 'small', strategy: 'heading', overlap: 15 }; + +test('marketing page: full pipeline matches snapshot', () => { + const result = runPipeline( + 'marketing-page.html', + 'https://maplegrove.example/assisted-living', + MARKETING_OPTIONS + ); + checkSnapshot('marketing-page', result); +}); + +test('marketing page: expected structure and flags', () => { + const result = runPipeline( + 'marketing-page.html', + 'https://maplegrove.example/assisted-living', + MARKETING_OPTIONS + ); + const titles = result.big_chunks.map((c) => c.title); + assert.deepEqual(titles, [ + 'Assisted Living in Springfield', + 'Personalized Care Plans', + 'Overview', + 'Amenities at Maple Grove', + 'Frequently Asked Questions About Cost', + ]); + + const flagsFor = (title) => + result.big_chunks.find((c) => c.title === title).flags.map((f) => f.code); + assert.ok(flagsFor('Personalized Care Plans').includes('dangling-reference'), 'pronoun opener'); + assert.ok(flagsFor('Overview').includes('generic-heading'), 'stoplist heading'); + assert.ok(result.chunkability.score > 0 && result.chunkability.score < 100); + + // Overlap prefixes recorded wherever a chunk has multiple sections + for (const chunk of result.big_chunks) { + chunk.small_chunks.forEach((sc, i) => { + if (i === 0) assert.equal(sc.overlap_word_count, 0); + else assert.ok(sc.overlap_word_count > 0, `${chunk.title} section ${i + 1}`); + }); + } +}); + +test('nested sections: parent chunk must not swallow child content', () => { + const result = runPipeline( + 'nested-sections.html', + 'https://acme.example/engine-tuning', + { chunkSize: 'small', strategy: 'heading', overlap: 0 } + ); + const byTitle = Object.fromEntries(result.big_chunks.map((c) => [c.title, c])); + assert.deepEqual(Object.keys(byTitle), ['Improvements', 'Injector', 'Turbopump']); + + const improvementsText = byTitle.Improvements.small_chunks.map((sc) => sc.text).join(' '); + assert.ok(improvementsText.includes('three points in the assembly flow'), 'keeps its own intro'); + assert.ok(!improvementsText.includes('nozzle plate'), 'must not contain Injector content'); + assert.ok(!improvementsText.includes('balancing rig'), 'must not contain Turbopump content'); + + const injectorText = byTitle.Injector.small_chunks.map((sc) => sc.text).join(' '); + assert.ok(injectorText.includes('machined manifold'), 'child section extracted on its own'); + + // And no near-duplicate flags — the old extraction bug produced them + for (const chunk of result.big_chunks) { + assert.ok(!chunk.flags.some((f) => f.code === 'near-duplicate'), chunk.title); + } + + checkSnapshot('nested-sections', result); +}); diff --git a/tests/fixtures/marketing-page.html b/tests/fixtures/marketing-page.html new file mode 100644 index 0000000..c93ac1b --- /dev/null +++ b/tests/fixtures/marketing-page.html @@ -0,0 +1,31 @@ + + + + Assisted Living in Springfield - Maple Grove Senior Living + + + + + + + +
+

Assisted Living in Springfield

+

Assisted living at Maple Grove Senior Living gives Springfield seniors daily support without giving up independence. Residents keep their own apartments and routines. Licensed staff help with medication, bathing, and dressing whenever help is wanted. Families get a partner they can reach at any hour of the day.

+

Personalized Care Plans

+

Our nurses write a care plan with each resident before move-in day. The plan covers medication schedules, mobility support, and personal goals. It is reviewed every ninety days with the family.

+

Overview

+

Residents enjoy chef-prepared meals, weekly housekeeping, and a full calendar of outings around Springfield.

+

Amenities at Maple Grove

+
    +
  • Chef-prepared meals served three times daily in the dining room
    • +
  • Weekly housekeeping and linen service for every apartment
    • +
  • Scheduled transportation to Springfield appointments and shopping
    • +
  • Secure courtyard garden with raised planting beds
    • +
+

Frequently Asked Questions About Cost

+

Maple Grove Senior Living charges one monthly fee that covers rent, meals, housekeeping, and scheduled transportation. Care services are billed in four levels, and our team reassesses the level with families each quarter. Most residents pay between four and six thousand dollars per month depending on apartment size and care level. Veterans benefits and long-term care insurance can offset part of the fee, and the billing office helps families file both. A refundable deposit holds an apartment for up to thirty days while a family decides. Respite stays are offered by the week when a caregiver needs a break, and those stays include the same meals, housekeeping, and care services that long-term residents receive every day.

+
+

Privacy Policy

+ + diff --git a/tests/fixtures/marketing-page.snapshot.json b/tests/fixtures/marketing-page.snapshot.json new file mode 100644 index 0000000..8790f98 --- /dev/null +++ b/tests/fixtures/marketing-page.snapshot.json @@ -0,0 +1,300 @@ +{ + "source": { + "url": "https://maplegrove.example/assisted-living", + "title": "Assisted Living in Springfield - Maple Grove Senior Living", + "description": "Assisted living apartments, care plans, and amenities at Maple Grove Senior Living in Springfield.", + "canonical": "https://maplegrove.example/assisted-living", + "language": "en", + "opengraph": { + "site_name": "Maple Grove Senior Living" + }, + "jsonld": [ + { + "@context": "https://schema.org", + "@type": "Organization", + "name": "Maple Grove Senior Living", + "url": "https://maplegrove.example" + } + ], + "site": "Maple Grove Senior Living" + }, + "warnings": [], + "big_chunks": [ + { + "big_chunk_index": 1, + "title": "Assisted Living in Springfield", + "level": 1, + "heading_path": [ + "Assisted Living in Springfield - Maple Grove Senior Living", + "Assisted Living in Springfield" + ], + "fragment": "#assisted-living-in-springfield", + "small_chunks": [ + { + "text": "Assisted living at Maple Grove Senior Living gives Springfield seniors daily support without giving up independence. Residents keep their own apartments and routines. Licensed staff help with medication, bathing, and dressing whenever help is wanted. Families get a partner they can reach at any hour of the day.", + "content_type": "prose", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 48, + "char_count": 312, + "tokens": 36, + "token_estimate": 36, + "start_char": 0, + "end_char": 312, + "percent_through_doc": 0 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 48, + "total_characters": 312, + "total_tokens": 36, + "total_tokens_estimate": 36 + }, + "flags": [ + { + "code": "thin-section", + "label": "Thin section", + "severity": "warn", + "message": "48 words of content — under 100 words a section rarely carries enough context to rank or be quoted on its own.", + "fix": "Merge or expand short sections." + } + ], + "chunk_score": 80 + }, + { + "big_chunk_index": 2, + "title": "Personalized Care Plans", + "level": 2, + "heading_path": [ + "Assisted Living in Springfield - Maple Grove Senior Living", + "Assisted Living in Springfield", + "Personalized Care Plans" + ], + "fragment": "#personalized-care-plans", + "small_chunks": [ + { + "text": "Our nurses write a care plan with each resident before move-in day. The plan covers medication schedules, mobility support, and personal goals. It is reviewed every ninety days with the family.", + "content_type": "prose", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 31, + "char_count": 193, + "tokens": 24, + "token_estimate": 24, + "start_char": 314, + "end_char": 507, + "percent_through_doc": 0.197 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 31, + "total_characters": 193, + "total_tokens": 24, + "total_tokens_estimate": 24 + }, + "flags": [ + { + "code": "thin-section", + "label": "Thin section", + "severity": "warn", + "message": "31 words of content — under 100 words a section rarely carries enough context to rank or be quoted on its own.", + "fix": "Merge or expand short sections." + }, + { + "code": "dangling-reference", + "label": "Dangling reference", + "severity": "high", + "message": "Opens with \"Our\" — retrieved on its own, nothing in this section says who or what that refers to.", + "fix": "Open sections with the entity name instead of a pronoun." + }, + { + "code": "no-entity-anchor", + "label": "No entity anchor", + "severity": "warn", + "message": "Never mentions Maple Grove Senior Living (or any page-level entity term) — an AI quoting this section can't attribute it.", + "fix": "Name the brand or entity inside each section." + } + ], + "chunk_score": 25 + }, + { + "big_chunk_index": 3, + "title": "Overview", + "level": 2, + "heading_path": [ + "Assisted Living in Springfield - Maple Grove Senior Living", + "Assisted Living in Springfield", + "Overview" + ], + "fragment": "#overview", + "small_chunks": [ + { + "text": "Residents enjoy chef-prepared meals, weekly housekeeping, and a full calendar of outings around Springfield.", + "content_type": "prose", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 14, + "char_count": 108, + "tokens": 11, + "token_estimate": 11, + "start_char": 509, + "end_char": 617, + "percent_through_doc": 0.319 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 14, + "total_characters": 108, + "total_tokens": 11, + "total_tokens_estimate": 11 + }, + "flags": [ + { + "code": "thin-section", + "label": "Thin section", + "severity": "warn", + "message": "14 words of content — under 100 words a section rarely carries enough context to rank or be quoted on its own.", + "fix": "Merge or expand short sections." + }, + { + "code": "generic-heading", + "label": "Generic heading", + "severity": "warn", + "message": "\"Overview\" says nothing about the topic — as a retrieval label it matches almost any query weakly and none well.", + "fix": "Rename vague headings to name the topic." + } + ], + "chunk_score": 60 + }, + { + "big_chunk_index": 4, + "title": "Amenities at Maple Grove", + "level": 2, + "heading_path": [ + "Assisted Living in Springfield - Maple Grove Senior Living", + "Assisted Living in Springfield", + "Amenities at Maple Grove" + ], + "fragment": "#amenities-at-maple-grove", + "small_chunks": [ + { + "text": "- Chef-prepared meals served three times daily in the dining room\n- Weekly housekeeping and linen service for every apartment\n- Scheduled transportation to Springfield appointments and shopping\n- Secure courtyard garden with raised planting beds", + "content_type": "list", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 36, + "char_count": 245, + "tokens": 27, + "token_estimate": 27, + "start_char": 619, + "end_char": 864, + "percent_through_doc": 0.388 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 36, + "total_characters": 245, + "total_tokens": 27, + "total_tokens_estimate": 27 + }, + "flags": [ + { + "code": "thin-section", + "label": "Thin section", + "severity": "warn", + "message": "36 words of content — under 100 words a section rarely carries enough context to rank or be quoted on its own.", + "fix": "Merge or expand short sections." + }, + { + "code": "answer-buried", + "label": "Answer buried", + "severity": "warn", + "message": "The first sentence shares no content words with the heading \"Amenities at Maple Grove\" — the direct answer sits lower in the section, where ranking functions weight it less.", + "fix": "Answer the heading in the first sentence." + }, + { + "code": "readability", + "label": "Hard to read", + "severity": "info", + "message": "Reads at ~grade 18.3 (Flesch-Kincaid). Long sentences make clean quoting harder.", + "fix": "Shorten sentences; prefer plain words." + } + ], + "chunk_score": 65 + }, + { + "big_chunk_index": 5, + "title": "Frequently Asked Questions About Cost", + "level": 2, + "heading_path": [ + "Assisted Living in Springfield - Maple Grove Senior Living", + "Assisted Living in Springfield", + "Frequently Asked Questions About Cost" + ], + "fragment": "#frequently-asked-questions-about-cost", + "small_chunks": [ + { + "text": "Maple Grove Senior Living charges one monthly fee that covers rent, meals, housekeeping, and scheduled transportation. Care services are billed in four levels, and our team reassesses the level with families each quarter. Most residents pay between four and six thousand dollars per month depending on apartment size and care level. Veterans benefits and long-term care insurance can offset part of the fee, and the billing office helps families file both. A refundable deposit holds an apartment for up to thirty days while a family decides. Respite stays are offered by the week when a caregiver needs a break, and those stays include the same meals, housekeeping, and care services that long-term residents receive every day.", + "content_type": "prose", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 116, + "char_count": 728, + "tokens": 87, + "token_estimate": 87, + "start_char": 866, + "end_char": 1594, + "percent_through_doc": 0.543 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 116, + "total_characters": 728, + "total_tokens": 87, + "total_tokens_estimate": 87 + }, + "flags": [ + { + "code": "answer-buried", + "label": "Answer buried", + "severity": "warn", + "message": "The first sentence shares no content words with the heading \"Frequently Asked Questions About Cost\" — the direct answer sits lower in the section, where ranking functions weight it less.", + "fix": "Answer the heading in the first sentence." + } + ], + "chunk_score": 85 + } + ], + "chunkability": { + "score": 63, + "grade": "D", + "top_fixes": [ + { + "code": "thin-section", + "label": "Thin section", + "count": 4, + "points": 16, + "fix": "Merge or expand short sections." + }, + { + "code": "answer-buried", + "label": "Answer buried", + "count": 2, + "points": 6, + "fix": "Answer the heading in the first sentence." + }, + { + "code": "dangling-reference", + "label": "Dangling reference", + "count": 1, + "points": 6, + "fix": "Open sections with the entity name instead of a pronoun." + } + ] + } +} diff --git a/tests/fixtures/nested-sections.html b/tests/fixtures/nested-sections.html new file mode 100644 index 0000000..0a63580 --- /dev/null +++ b/tests/fixtures/nested-sections.html @@ -0,0 +1,21 @@ + + + + Engine Tuning Handbook - Acme Rocket Supply + + + +
+

Improvements

+

Improvements to the base engine can be applied at three points in the assembly flow at Acme Rocket Supply.

+
+

Injector

+

Injector upgrades swap the stock nozzle plate for a machined manifold. The machined manifold atomizes fuel into a finer spray, which burns more evenly across the chamber and lifts thrust by a measurable margin on the Acme Rocket Supply test stand.

+
+
+

Turbopump

+

Turbopump balancing removes vibration at high spool speeds. Technicians spin each rotor on the balancing rig, add trim weights, and log the result before the pump returns to the Acme Rocket Supply assembly line.

+
+
+ + diff --git a/tests/fixtures/nested-sections.snapshot.json b/tests/fixtures/nested-sections.snapshot.json new file mode 100644 index 0000000..5054201 --- /dev/null +++ b/tests/fixtures/nested-sections.snapshot.json @@ -0,0 +1,157 @@ +{ + "source": { + "url": "https://acme.example/engine-tuning", + "title": "Engine Tuning Handbook - Acme Rocket Supply", + "description": null, + "canonical": null, + "language": "en", + "opengraph": { + "site_name": "Acme Rocket Supply" + }, + "site": "Acme Rocket Supply" + }, + "warnings": [], + "big_chunks": [ + { + "big_chunk_index": 1, + "title": "Improvements", + "level": 2, + "heading_path": [ + "Engine Tuning Handbook - Acme Rocket Supply", + "Improvements" + ], + "fragment": "#improvements", + "small_chunks": [ + { + "text": "Improvements to the base engine can be applied at three points in the assembly flow at Acme Rocket Supply.", + "content_type": "prose", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 19, + "char_count": 106, + "tokens": 15, + "token_estimate": 15, + "start_char": 0, + "end_char": 106, + "percent_through_doc": 0 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 19, + "total_characters": 106, + "total_tokens": 15, + "total_tokens_estimate": 15 + }, + "flags": [ + { + "code": "thin-section", + "label": "Thin section", + "severity": "warn", + "message": "19 words of content — under 100 words a section rarely carries enough context to rank or be quoted on its own.", + "fix": "Merge or expand short sections." + } + ], + "chunk_score": 80 + }, + { + "big_chunk_index": 2, + "title": "Injector", + "level": 3, + "heading_path": [ + "Engine Tuning Handbook - Acme Rocket Supply", + "Improvements", + "Injector" + ], + "fragment": "#injector", + "small_chunks": [ + { + "text": "Injector upgrades swap the stock nozzle plate for a machined manifold. The machined manifold atomizes fuel into a finer spray, which burns more evenly across the chamber and lifts thrust by a measurable margin on the Acme Rocket Supply test stand.", + "content_type": "prose", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 41, + "char_count": 247, + "tokens": 31, + "token_estimate": 31, + "start_char": 108, + "end_char": 355, + "percent_through_doc": 0.19 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 41, + "total_characters": 247, + "total_tokens": 31, + "total_tokens_estimate": 31 + }, + "flags": [ + { + "code": "thin-section", + "label": "Thin section", + "severity": "warn", + "message": "41 words of content — under 100 words a section rarely carries enough context to rank or be quoted on its own.", + "fix": "Merge or expand short sections." + } + ], + "chunk_score": 80 + }, + { + "big_chunk_index": 3, + "title": "Turbopump", + "level": 3, + "heading_path": [ + "Engine Tuning Handbook - Acme Rocket Supply", + "Improvements", + "Turbopump" + ], + "fragment": "#turbopump", + "small_chunks": [ + { + "text": "Turbopump balancing removes vibration at high spool speeds. Technicians spin each rotor on the balancing rig, add trim weights, and log the result before the pump returns to the Acme Rocket Supply assembly line.", + "content_type": "prose", + "overlap_word_count": 0, + "chunk_index": 1, + "word_count": 34, + "char_count": 211, + "tokens": 26, + "token_estimate": 26, + "start_char": 357, + "end_char": 568, + "percent_through_doc": 0.629 + } + ], + "metadata": { + "total_small_chunks": 1, + "total_words": 34, + "total_characters": 211, + "total_tokens": 26, + "total_tokens_estimate": 26 + }, + "flags": [ + { + "code": "thin-section", + "label": "Thin section", + "severity": "warn", + "message": "34 words of content — under 100 words a section rarely carries enough context to rank or be quoted on its own.", + "fix": "Merge or expand short sections." + } + ], + "chunk_score": 80 + } + ], + "chunkability": { + "score": 80, + "grade": "B", + "top_fixes": [ + { + "code": "thin-section", + "label": "Thin section", + "count": 3, + "points": 20, + "fix": "Merge or expand short sections." + } + ] + } +}