From d2e0f86b8f4987f28cae33fc932bf92894d968d9 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Sat, 25 Jul 2026 15:53:37 -0500 Subject: [PATCH] Chunkability score + per-chunk quality flags (v3.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Chunks view showed structure but passed no judgment — "then what?" had no answer. This adds the diagnosis layer, all deterministic (regex, word counts, stoplists, Jaccard shingles, Flesch-Kincaid) — zero AI. API: analyzeChunks() attaches flags[] and chunk_score to each big chunk and returns chunkability {score, grade, top_fixes}. Flags: - dangling-reference: opens with a pronoun (Our/We/This/...) - near-duplicate: 5-gram Jaccard >= 0.6 between de-overlapped chunks - no-entity-anchor: no page-entity term (title/site/JSON-LD org/H1 stems + initialisms, so "RAG" anchors "Retrieval-augmented generation") - thin-section (< min(target.min, 100) words), oversized-section, generic-heading (~40 stoplist), answer-buried (heading vs first sentence, >= 2 heading content words), readability (FK > 12, info) Scoring: each chunk starts at 100 and loses its flags' weights; the page score is the average — proportional to page size by construction. Extraction fix the near-duplicate flag surfaced: containers wrapping nested sections were swallowed wholesale into the parent chunk (Wikipedia "Improvements" contained all of "Encoder"). stopCondition now treats a container holding the next heading as the boundary, and heading-bearing containers that never became chunks are skipped. UI: color-banded score card with top 3 fixes (+N pts each), red/amber/ info flag chips per chunk with fix tooltips, "show only flagged" toggle. Tuned on 5 real pages: Wikipedia RAG 79/C (highest, as the control should be) ... Hayes Barton Place 51/F (the page that started all this). Co-Authored-By: Claude Fable 5 --- SESSION_LOG.md | 14 +++ api/chunk.js | 258 ++++++++++++++++++++++++++++++++++++++++++++++ public/index.html | 157 +++++++++++++++++++++++++++- tasks/todo.md | 32 +++--- 4 files changed, 444 insertions(+), 17 deletions(-) diff --git a/SESSION_LOG.md b/SESSION_LOG.md index c7f78cf..370b8ae 100644 --- a/SESSION_LOG.md +++ b/SESSION_LOG.md @@ -1,5 +1,19 @@ # getchunks Session Log +## 2026-07-25 — Phase 2 (v3.3): flags + Chunkability score, on `feature/chunk-quality-v3.3` (stacked on v3.2) + +**Trigger:** Will asked (twice) for the "actionable layer — now what?" after approving v3.2. That's Phase 2 of the approved plan; executed it. + +**API (`api/chunk.js`):** `analyzeChunks()` — pure/deterministic, zero AI. Per-chunk `flags[]` (code/label/severity/message/fix) + `chunk_score`, top-level `chunkability {score, grade, top_fixes}`. Flags: dangling-reference, near-duplicate (5-gram Jaccard ≥ 0.6 on de-overlapped text), no-entity-anchor (title+site+JSON-LD org names+H1, plus initialisms so "RAG" anchors "Retrieval-augmented generation"), thin-section (floor = min(target.min, 100)), oversized-section, generic-heading (~40 stoplist), answer-buried (needs ≥2 heading content words), readability (FK > 12, info-only). **Scoring model:** per-chunk 100-minus-flag-weights, page = average (flat deductions punished many-chunk pages — Wikipedia scored F). Weights: dangling/dupe 30, no-anchor 25, thin/oversized/generic 20, answer-buried 15. + +**Bonus root-cause fix:** the near-duplicate flag exposed the long-deferred nested-content extraction bug — containers wrapping nested `
`s were swallowed wholesale into the parent chunk (Wikipedia "Improvements" contained all of "Encoder"). Fixed in `stopCondition` (boundary when a container holds the next heading) + `extractContentPieces` (skip containers holding uncollected headings). That open item can come off the backlog. + +**UI:** score card (color-banded score+grade, top 3 fixes with "up to +N pts"), red/amber/info flag chips per chunk card with fix tooltips, green "✓ No flags" chip, "show only flagged" toggle, legend bullets for score + chips. + +**Tuning (5 pages):** Wikipedia RAG 79/C (control, highest ✓) · Liberty 73/C · seasidefl 68/D (answers Will's "it seems to chunk well" — 9 sections, all under 100 words) · SI home 65/D · Hayes (Chuck's page) 51/F with fixes "merge thin sections, replace pronoun openers, answer the heading first." + +**Next:** Will reviews v3.3 → then Phase 3 (v3.4): BM25 retrieval simulator, query coverage matrix, markdown optimization-report export. + ## 2026-07-24 — Chunk Quality plan (from Chuck's feedback) + repo moved out of archive **Session ran from ontologizer-next; planning only, no code changes here yet.** diff --git a/api/chunk.js b/api/chunk.js index c496ba4..c6da2e2 100644 --- a/api/chunk.js +++ b/api/chunk.js @@ -247,6 +247,7 @@ async function chunkUrl(url, options) { const bigChunks = buildChunks($, finalOptions, warnings, source.title); const enhancedChunks = enhanceChunks(bigChunks, options.tokenizer); + const chunkability = analyzeChunks(enhancedChunks, source, finalOptions); return { big_chunks: enhancedChunks, @@ -266,6 +267,7 @@ async function chunkUrl(url, options) { total_words: enhancedChunks.reduce((s, c) => s + c.metadata.total_words, 0), total_tokens: enhancedChunks.reduce((s, c) => s + c.metadata.total_tokens, 0), }, + chunkability, warnings, }; } @@ -420,6 +422,10 @@ function buildChunks($, finalOptions, warnings, docTitle) { const stopCondition = (current) => { if (nextHeading && current[0] === nextHeading.element[0]) return true; if (current.is('h1, h2, h3, h4, h5, h6') && current[0] !== heading.element[0]) return true; + // A container wrapping the next heading (nested
markup) is + // the section boundary too — extracting its text wholesale would + // duplicate the child section's content into this chunk. + if (nextHeading && current.find('h1, h2, h3, h4, h5, h6').toArray().includes(nextHeading.element[0])) return true; return false; }; @@ -527,6 +533,12 @@ function extractContentPieces($, startElement, stopCondition) { while (current.length) { if (stopCondition(current)) break; + // Containers holding headings that never became chunks (nav/aside blocks, + // headings under 3 chars) would pollute this section's text — skip them. + if (current.find('h1, h2, h3, h4, h5, h6').length) { + current = current.next(); + continue; + } let text = ''; let type = 'prose'; @@ -725,6 +737,252 @@ function enhanceChunks(bigChunks, tokenizer) { }); } +// --- Chunk quality analysis (v3.3) ----------------------------------------- +// Deterministic diagnosis: same input, same score. No AI calls — regex, word +// counts, stoplists, Jaccard shingles, and Flesch-Kincaid only. + +// Weights are per-chunk points: every chunk starts at 100, loses the weight +// of each flag it carries, and the page score is the average across chunks — +// proportional by construction, so a 12-chunk page with 2 flagged chunks +// outscores a 4-chunk page with 2 flagged chunks. +const FLAG_META = { + 'dangling-reference': { label: 'Dangling reference', severity: 'high', weight: 30, fix: 'Open sections with the entity name instead of a pronoun.' }, + 'near-duplicate': { label: 'Near-duplicate', severity: 'high', weight: 30, fix: 'Consolidate near-identical sections or differentiate them.' }, + 'no-entity-anchor': { label: 'No entity anchor', severity: 'warn', weight: 25, fix: 'Name the brand or entity inside each section.' }, + 'generic-heading': { label: 'Generic heading', severity: 'warn', weight: 20, fix: 'Rename vague headings to name the topic.' }, + 'oversized-section': { label: 'Oversized section', severity: 'warn', weight: 20, fix: 'Break long sections up with subheadings.' }, + 'thin-section': { label: 'Thin section', severity: 'warn', weight: 20, fix: 'Merge or expand short sections.' }, + 'answer-buried': { label: 'Answer buried', severity: 'warn', weight: 15, fix: 'Answer the heading in the first sentence.' }, + readability: { label: 'Hard to read', severity: 'info', weight: 0, fix: 'Shorten sentences; prefer plain words.' }, +}; + +const GENERIC_HEADINGS = new Set([ + 'overview', 'introduction', 'intro', 'learn more', 'read more', 'more', 'more info', + 'more information', 'additional information', 'details', 'why choose us', 'why us', + 'our services', 'services', 'our approach', 'our story', 'our mission', 'about', + 'about us', 'welcome', 'get started', 'getting started', 'features', 'benefits', + 'faq', 'faqs', 'frequently asked questions', 'resources', 'conclusion', 'summary', + 'next steps', 'contact', 'contact us', 'get in touch', 'our team', 'testimonials', + 'gallery', 'miscellaneous', 'misc', 'other', +]); + +const ANALYSIS_STOPWORDS = new Set(('a an the and or but if then else for nor so yet at by in of on to up as is are was were be been ' + + 'being am do does did have has had will would can could may might must shall should with from into onto over under about after ' + + 'before between during through above below out off again further once here there all any both each few most other some such no ' + + 'not only own same than too very just more less much many what which who whom whose when where why how this that these those it ' + + 'its they their them we our us you your yours he she his her i me my mine thing things page home welcome official website site online').split(' ')); + +function tokenizeTerms(text) { + return (text || '') + .toLowerCase() + .replace(/[^a-z0-9\s'-]/g, ' ') + .split(/\s+/) + .map((w) => w.replace(/^['-]+|['-]+$/g, '')) + .filter((w) => w.length >= 3 && !ANALYSIS_STOPWORDS.has(w)); +} + +function lightStem(word) { + return word.length > 4 + ? word.replace(/(ings|ing|ers|er|ies|ed|es|s)$/, '') + : word.replace(/s$/, ''); +} + +function stemSet(text) { + return new Set(tokenizeTerms(text).map(lightStem)); +} + +function setsIntersect(a, b) { + for (const item of a) if (b.has(item)) return true; + return false; +} + +// Words prepended by addOverlap() are context carry-over, not page content — +// every quality check runs on the de-overlapped text. +function deOverlappedText(chunk) { + return chunk.small_chunks + .map((sc) => { + if (!sc.overlap_word_count) return sc.text; + return sc.text.trim().split(/\s+/).slice(sc.overlap_word_count).join(' '); + }) + .join('\n\n'); +} + +function firstSentenceOf(text) { + const plain = text.replace(/^[->\s]+/, '').replace(/\s+/g, ' ').trim(); + const match = plain.match(/^.*?[.!?](?=\s|$)/); + return (match ? match[0] : plain).trim(); +} + +// "University of Wisconsin-Milwaukee" → "uwm", "Retrieval-augmented +// generation" → "rag" — pages routinely fall back to the initialism after +// first mention, and that still anchors the entity. +function initialism(name) { + const words = (name || '') + .toLowerCase() + .replace(/[^a-z0-9\s-]/g, ' ') + .split(/[\s-]+/) + .filter((w) => w && !ANALYSIS_STOPWORDS.has(w)); + if (words.length < 2) return null; + const initials = words.map((w) => w[0]).join(''); + return initials.length >= 3 ? initials : null; +} + +function buildAnchorTerms(source, chunks) { + const titleMain = (source.title || '').split(/\s+[|–—-]\s+/)[0]; + const names = [titleMain, source.site]; + const jsonldNodes = (source.jsonld || []).flatMap((n) => (n && n['@graph'] ? n['@graph'] : [n])); + for (const node of jsonldNodes) { + if (!node || typeof node.name !== 'string') continue; + const types = [].concat(node['@type'] || []).join(' '); + if (/Organization|Business|WebSite|WebPage|Brand|Hotel|Store|Restaurant|Resort/i.test(types)) { + names.push(node.name); + } + } + const h1 = chunks.find((c) => c.level === 1); + if (h1) names.push(h1.title); + + const anchors = stemSet(`${source.title || ''} ${names.join(' ')}`); + for (const name of names) { + const init = initialism(name); + if (init) anchors.add(init); + } + return anchors; +} + +function countSyllables(word) { + const w = word.toLowerCase().replace(/[^a-z]/g, ''); + if (!w) return 0; + if (w.length <= 3) return 1; + const trimmed = w.replace(/(?:[^laeiouy]e|ed|es)$/, ''); + const groups = trimmed.match(/[aeiouy]{1,2}/g); + return Math.max(1, groups ? groups.length : 1); +} + +function fleschKincaidGrade(text) { + const words = text.split(/\s+/).filter((w) => /[a-zA-Z]/.test(w)); + if (!words.length) return 0; + const sentences = Math.max(1, (text.match(/[.!?]+(?=\s|$)/g) || []).length); + const syllables = words.reduce((sum, w) => sum + countSyllables(w), 0); + const grade = 0.39 * (words.length / sentences) + 11.8 * (syllables / words.length) - 15.59; + return Math.round(grade * 10) / 10; +} + +function wordShingles(text, size = 5) { + const words = tokenizeTerms(text).map(lightStem); + const set = new Set(); + for (let i = 0; i + size <= words.length; i++) { + set.add(words.slice(i, i + size).join(' ')); + } + return set; +} + +function jaccard(a, b) { + if (!a.size || !b.size) return 0; + let intersection = 0; + for (const item of a) if (b.has(item)) intersection++; + return intersection / (a.size + b.size - intersection); +} + +// Attaches flags[] to each enhanced big chunk (mutates) and returns the +// page-level chunkability verdict. +function analyzeChunks(chunks, source, finalOptions) { + if (!chunks.length) return null; + const target = CHUNK_SIZES[finalOptions.chunkSize]; + const anchorTerms = buildAnchorTerms(source, chunks); + const entityName = source.site || (chunks.find((c) => c.level === 1) || {}).title || 'the brand'; + const tally = {}; + const addFlag = (chunk, code, message) => { + const meta = FLAG_META[code]; + chunk.flags.push({ code, label: meta.label, severity: meta.severity, message, fix: meta.fix }); + tally[code] = (tally[code] || 0) + 1; + }; + + const texts = chunks.map((chunk) => deOverlappedText(chunk)); + + chunks.forEach((chunk, i) => { + chunk.flags = []; + const text = texts[i]; + const wordCount = countWords(text); + const syntheticHeading = !chunk.fragment; // fixed-strategy fallback ("Main Content") + + // Absolute floor, not the chunk-size minimum: a 300-word section on a + // "large"-sized page is still plenty to rank — under ~100 words it isn't. + const thinMin = Math.min(target.min, 100); + if (wordCount < thinMin) { + addFlag(chunk, 'thin-section', `${wordCount} words of content — under ${thinMin} words a section rarely carries enough context to rank or be quoted on its own.`); + } else if (wordCount > target.max) { + addFlag(chunk, 'oversized-section', `${wordCount} words under one heading (${finalOptions.chunkSize} max is ${target.max}). Retrieval had to split it mid-flow, so pieces lose their framing.`); + } + + const normalizedTitle = (chunk.title || '').toLowerCase().replace(/[^a-z0-9\s]/g, ' ').replace(/\s+/g, ' ').trim(); + const isGeneric = !syntheticHeading && GENERIC_HEADINGS.has(normalizedTitle); + if (isGeneric) { + addFlag(chunk, 'generic-heading', `"${chunk.title}" says nothing about the topic — as a retrieval label it matches almost any query weakly and none well.`); + } + + const opening = (chunk.small_chunks[0]?.text || '').replace(/^[->\s]+/, ''); + const pronounMatch = opening.match(/^(our|we|this|these|those|it|its|they|their|he|she)\b/i); + if (pronounMatch) { + addFlag(chunk, 'dangling-reference', `Opens with "${pronounMatch[1]}" — retrieved on its own, nothing in this section says who or what that refers to.`); + } + + if (anchorTerms.size && !setsIntersect(stemSet(`${chunk.title} ${text}`), anchorTerms)) { + addFlag(chunk, 'no-entity-anchor', `Never mentions ${entityName} (or any page-level entity term) — an AI quoting this section can't attribute it.`); + } + + if (!syntheticHeading && !isGeneric) { + const headingStems = stemSet(chunk.title); + const sentence = firstSentenceOf(text); + // Single-word headings ("History") make lexical-echo checks pure noise + if (headingStems.size >= 2 && sentence && !setsIntersect(headingStems, stemSet(sentence))) { + addFlag(chunk, 'answer-buried', `The first sentence shares no content words with the heading "${chunk.title}" — the direct answer sits lower in the section, where ranking functions weight it less.`); + } + } + + if (wordCount >= 30) { + const grade = fleschKincaidGrade(text); + if (grade > 12) { + addFlag(chunk, 'readability', `Reads at ~grade ${grade} (Flesch-Kincaid). Long sentences make clean quoting harder.`); + } + } + }); + + const shingleSets = texts.map((t) => (countWords(t) >= 20 ? wordShingles(t) : null)); + let duplicatePairs = 0; + for (let i = 0; i < chunks.length; i++) { + for (let j = i + 1; j < chunks.length; j++) { + if (!shingleSets[i] || !shingleSets[j]) continue; + const similarity = jaccard(shingleSets[i], shingleSets[j]); + if (similarity >= 0.6) { + duplicatePairs++; + const pct = `${Math.round(similarity * 100)}%`; + addFlag(chunks[i], 'near-duplicate', `${pct} similar to Chunk ${chunks[j].big_chunk_index} ("${chunks[j].title}") — retrieval returns them interchangeably, splitting relevance between them.`); + addFlag(chunks[j], 'near-duplicate', `${pct} similar to Chunk ${chunks[i].big_chunk_index} ("${chunks[i].title}") — retrieval returns them interchangeably, splitting relevance between them.`); + } + } + } + + chunks.forEach((chunk) => { + const lost = chunk.flags.reduce((sum, f) => sum + FLAG_META[f.code].weight, 0); + chunk.chunk_score = Math.max(0, 100 - lost); + }); + const score = Math.round(chunks.reduce((sum, c) => sum + c.chunk_score, 0) / chunks.length); + + // Page-level points recoverable per flag type: weight × occurrences, + // averaged over the chunk count — the same math the score is built from. + const fixGroups = []; + for (const [code, meta] of Object.entries(FLAG_META)) { + const count = code === 'near-duplicate' ? duplicatePairs : (tally[code] || 0); + if (!count || !meta.weight) continue; + const points = Math.round((meta.weight * (tally[code] || 0)) / chunks.length); + fixGroups.push({ code, label: meta.label, count, points, fix: meta.fix }); + } + fixGroups.sort((a, b) => b.points - a.points || b.count - a.count); + + const grade = score >= 90 ? 'A' : score >= 80 ? 'B' : score >= 70 ? 'C' : score >= 60 ? 'D' : 'F'; + return { score, grade, top_fixes: fixGroups.slice(0, 3) }; +} + // --- Format converters ----------------------------------------------------- function toJSONL(result) { diff --git a/public/index.html b/public/index.html index 00e73ab..12ec2b2 100644 --- a/public/index.html +++ b/public/index.html @@ -649,6 +649,107 @@ margin: 0; } + /* Chunkability score card + flag chips (v3.3) */ + .chunkability-body { + display: flex; + gap: 30px; + align-items: flex-start; + flex-wrap: wrap; + } + + .chunkability-score { + flex: 0 0 auto; + text-align: center; + padding: 18px 26px; + border-radius: 12px; + border: 2px solid; + } + + .chunkability-score .score-number { + display: block; + font-size: 2.6rem; + font-weight: 800; + line-height: 1.1; + } + + .chunkability-score .score-grade { + display: block; + font-size: 0.9rem; + font-weight: 700; + letter-spacing: 0.08em; + } + + .chunkability-score.band-good { border-color: var(--si-report-green); color: var(--si-report-green); background: rgba(36, 171, 89, 0.08); } + .chunkability-score.band-mid { border-color: #f59e0b; color: #b45309; background: rgba(245, 158, 11, 0.08); } + .chunkability-score.band-low { border-color: var(--error-red); color: var(--error-red); background: rgba(231, 76, 60, 0.08); } + + .chunkability-fixes { + flex: 1 1 260px; + } + + .chunkability-fixes h4 { + font-size: 0.85rem; + font-weight: 800; + text-transform: uppercase; + letter-spacing: 0.06em; + margin-bottom: 10px; + color: var(--secondary-content); + } + + .chunkability-fixes ol { + margin: 0 0 0 20px; + } + + .chunkability-fixes li { + margin-bottom: 8px; + } + + .flagged-toggle { + margin-top: 18px; + background: var(--content-bg); + border: 1px solid var(--border-gray); + color: var(--secondary-content); + padding: 8px 16px; + border-radius: 20px; + font-size: 0.85rem; + font-weight: 700; + font-family: inherit; + cursor: pointer; + } + + .flagged-toggle.active { + background: var(--si-navy); + border-color: var(--si-navy); + color: var(--white); + } + + .flags-only .big-chunk:not(.has-flags) { + display: none; + } + + .chunk-flags { + padding: 12px 30px; + background: var(--content-bg); + border-bottom: 1px solid var(--border-gray); + display: flex; + flex-wrap: wrap; + gap: 8px; + } + + .flag-chip { + display: inline-block; + font-size: 0.78rem; + font-weight: 700; + padding: 3px 10px; + border-radius: 12px; + border: 1px solid; + } + + .flag-chip.high { background: rgba(231, 76, 60, 0.1); border-color: rgba(231, 76, 60, 0.5); color: #c0392b; } + .flag-chip.warn { background: rgba(245, 158, 11, 0.1); border-color: rgba(245, 158, 11, 0.55); color: #b45309; } + .flag-chip.info { background: rgba(52, 144, 181, 0.1); border-color: rgba(52, 144, 181, 0.4); color: var(--info-blue); } + .flag-chip.clean { background: rgba(36, 171, 89, 0.08); border-color: rgba(36, 171, 89, 0.45); color: var(--si-report-green); cursor: default; } + /* Pure-CSS tooltips — native title="" is unreliable (long hover delay, never shown in some embedded browsers), so hints render as a styled ::after that appears immediately on hover. */ @@ -2072,6 +2173,16 @@

Frequently asked questions

URL.revokeObjectURL(url); }); + // "Show only flagged" toggle — delegated because the Chunks view + // re-renders its HTML on every extraction + chunksContent.addEventListener('click', function(e) { + const btn = e.target.closest('#flagged-filter'); + if (!btn) return; + const on = chunksContent.classList.toggle('flags-only'); + btn.classList.toggle('active', on); + btn.textContent = on ? 'Showing flagged chunks — show all' : 'Show only flagged chunks'; + }); + // New extraction document.getElementById('new-extraction').addEventListener('click', function() { hideAll(); @@ -2177,6 +2288,7 @@

Frequently asked questions

function displayChunksView(data) { let html = ''; + chunksContent.classList.remove('flags-only'); // fresh run, fresh filter // Collapsible legend (v3.2) — defines the vocabulary before the data const tw = data.settings?.target_words; @@ -2197,8 +2309,37 @@

Frequently asked questions

if (overlapSetting > 0) { html += `
  • ↔ overlap — the dimmed opening words repeat the end of the previous section on purpose (up to ${overlapSetting} words on this run). Standard RAG practice so a section still makes sense when retrieved alone — not duplicate content.
    • `; } + if (data.chunkability) { + html += `
  • Chunkability — 0–100: every chunk starts at 100, loses points per flag, and the page score is the average. Fully deterministic — the same page always scores the same, so re-run after editing the page to see the delta.
    • `; + html += `
  • Flag chips — red breaks standalone retrieval, amber weakens ranking or attribution, blue informational. Hover any chip for the specific issue and its fix.
    • `; + } html += ``; + // Chunkability score card (v3.3) + if (data.chunkability) { + const ca = data.chunkability; + const band = ca.score >= 80 ? 'good' : ca.score >= 60 ? 'mid' : 'low'; + const scoreTip = 'Deterministic 0–100 score: every chunk starts at 100, loses points for each flag it carries, and the page score is the average. No AI involved — edit the page, re-run, and the score moves for exactly the reasons shown.'; + html += `
    `; + html += `

    🎯 Chunkability

    `; + html += `
    `; + html += `
    ${ca.score}GRADE ${escapeHtml(ca.grade)}
    `; + if (ca.top_fixes && ca.top_fixes.length) { + html += `

    Top fixes

      `; + ca.top_fixes.forEach(f => { + const times = f.count > 1 ? ` ×${f.count}` : ''; + const pts = f.points ? ` (up to +${f.points} pts)` : ''; + html += `
    1. ${escapeHtml(f.label)}${times}: ${escapeHtml(f.fix)}${escapeHtml(pts)}
      2. `; + }); + html += `
    `; + } else { + html += `

    Top fixes

    No structural issues found — this page chunks cleanly.

    `; + } + html += `
    `; + html += ``; + html += `
    `; + } + // Source metadata block (new in v3) if (data.source) { const s = data.source; @@ -2251,7 +2392,8 @@

    Frequently asked questions

    if (data.big_chunks && data.big_chunks.length > 0) { data.big_chunks.forEach((chunk) => { - html += `
    `; + const flags = Array.isArray(chunk.flags) ? chunk.flags : []; + html += `
    `; // Big chunk header with metadata html += `
    `; @@ -2287,6 +2429,19 @@

    Frequently asked questions

    html += `
    ${metaText}
    `; html += `
    `; + // Flag chips (v3.3) — hover for the issue + fix + if (data.chunkability) { + html += `
    `; + if (flags.length) { + flags.forEach(f => { + html += `${escapeHtml(f.label)}`; + }); + } else { + html += `✓ No flags`; + } + html += `
    `; + } + // Small chunks container html += `
    `; chunk.small_chunks.forEach((smallChunk, smallIndex) => { diff --git a/tasks/todo.md b/tasks/todo.md index f249524..148ef7e 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -119,22 +119,22 @@ Branch: `feature/chunk-quality-v3.2` ### Phase 2 — Diagnosis (v3.3): flags + Chunkability score Branch: `feature/chunk-quality-v3.3` -- [ ] `api/chunk.js`: new pure function `analyzeChunks(bigChunks, source, settings)` → per-chunk `flags[]` (`{code, severity, message, fix}`) + top-level `chunkability: {score, grade, top_fixes[]}` -- [ ] Anchor-term set: tokenized page title + H1 + `og:site_name` + JSON-LD Organization/LocalBusiness names (all already captured in `source`) -- [ ] Flags (all deterministic): - - [ ] `no-entity-anchor` — no anchor term appears in chunk text - - [ ] `dangling-reference` — chunk opens `^(Our|This|These|It|They|He|She)\b` - - [ ] `thin-section` — big chunk word count under target min - - [ ] `oversized-section` — big chunk over target max with no subheading - - [ ] `generic-heading` — heading in stoplist (~30 entries: "Why Choose Us", "Learn More", "Overview", …) - - [ ] `answer-buried` — zero stemmed content-word overlap between heading and first sentence - - [ ] `near-duplicate` — word 5-gram Jaccard ≥ 0.6 between chunk pair, after subtracting the known overlap prefix - - [ ] `readability` — Flesch-Kincaid grade per chunk (info-level, no score deduction initially) -- [ ] Score: 100 minus weighted deductions; same input → same score (re-run after edits shows the delta) -- [ ] UI: score card in `chunks-summary` (score + grade + top 3 fixes) -- [ ] UI: color-coded flag chips on chunk cards (red/amber/info), each expanding to its one-line fix -- [ ] UI: "show only flagged" filter toggle -- [ ] Tune weights + stoplist against 5 real client pages before shipping +- [x] `api/chunk.js`: new pure function `analyzeChunks(bigChunks, source, settings)` → per-chunk `flags[]` (`{code, label, severity, message, fix}`) + top-level `chunkability: {score, grade, top_fixes[]}`; also attaches `chunk_score` per chunk +- [x] Anchor-term set: tokenized page title + H1 + `og:site_name` + JSON-LD Organization/LocalBusiness names — plus initialisms ("Retrieval-augmented generation" → "rag", "University of Wisconsin-Milwaukee" → "uwm") after tuning showed acronym-usage false positives +- [x] Flags (all deterministic): + - [x] `no-entity-anchor` — no anchor term appears in chunk title+text + - [x] `dangling-reference` — chunk opens with a pronoun (`^(our|we|this|these|those|it|its|they|their|he|she)\b`) + - [x] `thin-section` — under `min(target.min, 100)` de-overlapped words (absolute 100w floor added in tuning: 500w min on "large" pages flagged every Wikipedia section) + - [x] `oversized-section` — over target max with no subheading + - [x] `generic-heading` — heading in ~40-entry stoplist + - [x] `answer-buried` — zero stemmed content-word overlap between heading and first sentence; requires ≥2 heading content words (single-word headings like "History" were pure noise) + - [x] `near-duplicate` — word 5-gram Jaccard ≥ 0.6 between de-overlapped chunk pairs. Caught a REAL extraction bug: containers wrapping nested sections were swallowed wholesale into the parent chunk (the long-deferred nested-content fix) — fixed in `extractContentPieces`/`stopCondition`, not muted in the analyzer + - [x] `readability` — Flesch-Kincaid per chunk, info-only flag when grade > 12, no deduction +- [x] Score — reworked during tuning: flat per-instance deductions punished many-chunk pages, so each chunk starts at 100, loses its flags' weights (dangling/dupe 30, no-anchor 25, thin/oversized/generic 20, answer-buried 15), page score = average. Proportional by construction, per-chunk explainable, still deterministic +- [x] UI: score card at top of Chunks view (score + grade color-banded, top 3 fixes with "up to +N pts") +- [x] UI: color-coded flag chips on chunk cards (red/amber/info/green "No flags"), fix shown via the CSS tooltip +- [x] UI: "show only flagged" filter toggle (delegated listener, resets per run) +- [x] Tuned against 5 real pages: Wikipedia RAG 79/C (control, highest ✓), Liberty Senior Living 73/C, seasidefl.com 68/D (looks clean, but 9 sections all under 100w), SI home 65/D, Hayes Barton Place 51/F (Chuck's page, worst ✓) ### Phase 3 — Action (v3.4): retrieval simulator + report export Branch: `feature/chunk-quality-v3.4`