From 312efd3fab3bcc4d53283332db142a5671927406 Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Sun, 9 Aug 2026 02:52:47 +0530 Subject: [PATCH] refactor(webview): simplify strategy UI and optimize pipeline --- package.json | 2 +- src/manifest.json | 2 +- src/pipeline/UmapProjector.ts | 2 +- src/pipeline/clustering/autoK.ts | 59 +++++--- src/pipeline/clustering/metrics.ts | 66 +++++++- src/pipeline/clustering/postProcess.ts | 26 +++- src/pipeline/clustering/tfidf.ts | 166 +++++++++++++++++---- src/pipeline/pipelineConfig.ts | 64 ++++++-- src/pipeline/runPipeline.ts | 22 ++- src/webview/components/StrategySection.tsx | 51 ++++--- src/webview/context/AppStateContext.tsx | 8 +- src/webview/pages/DashboardPage.tsx | 2 +- src/webview/panel.css | 58 ++++--- test/pipeline/clustering/autoK.test.ts | 28 +++- test/pipeline/pipelineConfig.test.ts | 70 +++++++-- 15 files changed, 474 insertions(+), 152 deletions(-) diff --git a/package.json b/package.json index 41643b9..d4d3c7a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "joplin-plugin-note-categorization", - "version": "0.1.5", + "version": "0.1.6", "scripts": { "dist": "webpack --env joplin-plugin-config=buildMain && webpack --env joplin-plugin-config=buildExtraScripts && npm run copyAssets && webpack --env joplin-plugin-config=createArchive", "prepare": "npm run dist", diff --git a/src/manifest.json b/src/manifest.json index 1d08fa8..2c28157 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 1, "id": "com.harsh16gupta.notecategorization", "app_min_version": "3.5", - "version": "0.1.5", + "version": "0.1.6", "name": "Note Categorization Plugin", "description": "AI-based note categorisation: clusters notes semantically, suggests tags and notebook structures, and detects stale notes.", "author": "Harsh Gupta", diff --git a/src/pipeline/UmapProjector.ts b/src/pipeline/UmapProjector.ts index c7748bd..8b188ec 100644 --- a/src/pipeline/UmapProjector.ts +++ b/src/pipeline/UmapProjector.ts @@ -14,7 +14,7 @@ export class UmapProjector { constructor(options: UmapProjectorOptions = {}) { this.nComponents = options.nComponents ?? 2; this.nNeighbors = options.nNeighbors ?? 15; - this.minDist = options.minDist ?? 0.1; + this.minDist = options.minDist ?? 0.0; this.metric = options.metric ?? 'cosine'; this.seed = options.seed ?? 42; } diff --git a/src/pipeline/clustering/autoK.ts b/src/pipeline/clustering/autoK.ts index a33ea82..1b8da02 100644 --- a/src/pipeline/clustering/autoK.ts +++ b/src/pipeline/clustering/autoK.ts @@ -7,8 +7,13 @@ import { log } from '../../utils/logger'; /** Absolute minimum K to try (silhouette needs at least 2 clusters). */ const MIN_K = 2; -/** Absolute maximum K to try (caps the sweep to bound runtime and avoid tiny clusters). */ -const MAX_K_CAP = 15; +/** + * Absolute safety ceiling for maxK to bound sweep runtime. + * At N=10,000 the dynamic formula yields ~270, so this caps it at 200 + * to prevent excessive K-Means iterations at extreme vault sizes. + * Silhouette evaluation remains O(500²) per K due to stratified sampling. + */ +const ABSOLUTE_MAX_K = 200; /** * Absolute silhouette tolerance for K selection. Clusterings whose silhouette @@ -16,12 +21,11 @@ const MAX_K_CAP = 15; * good, and the highest K among them is selected. * * Rationale: silhouette score naturally biases toward fewer, coarser clusters. - * A small drop (e.g. 0.017) when going from K=4 to K=7 is statistically + * A small drop (e.g. 0.008) when going from K=4 to K=7 is statistically * insignificant, but the finer granularity is far more useful for note - * categorization. 0.025 is within the standard range (0.02–0.05) used in - * clustering literature for "equivalent quality" comparisons. + * categorization. 0.01 is a strict tolerance for equivalent peak selection. */ -const SILHOUETTE_TOLERANCE = 0.025; +const SILHOUETTE_TOLERANCE = 0.01; export interface AutoKResult { /** The optimal K value found by the sweep. */ @@ -35,15 +39,26 @@ export interface AutoKResult { /** * Computes the K search range [minK, maxK] based on dataset size. * - * - minK is 2 for small datasets (N < 20), 3 for larger ones (N >= 20). - * For 20+ notes, 2 categories is too coarse to be useful. + * Two-part scaling formula for maxK: + * + * - **Base term** (all N ≥ 20): `1.5 · √N` — the BERTopic-standard sqrt rule + * of thumb that scales sub-linearly with dataset size. + * - **Density boost** (N > 1000): `(N − 1000) / 75` — a linear term that adds + * ~13 extra K per 1000 additional notes, preventing clusters from growing too + * large in big vaults. This maintains ~25–35 notes per cluster up to N ≈ 5000. + * + * The combined formula `maxK = ⌈1.5·√N + max(0, (N−1000)/75)⌉` is continuous + * at N = 1000 (density boost is 0) and preserves all existing behavior for + * N ≤ 1000. + * + * Range rules: + * - minK is 2 for small datasets (N < 20), 3 for larger ones (N ≥ 20). * - For small datasets (N < 20): maxK = floor(N / 2). - * Ensures the sweep can explore meaningful K values (e.g. N=8 → [2,4]). - * - For larger datasets (N >= 20): maxK = floor(N / 3). - * Ensures each cluster has at least ~3 notes on average. - * This is more generous than sqrt(N) and prevents under-clustering - * (e.g. N=56 → [2,15] instead of [2,7]). - * - maxK is always clamped to MAX_K_CAP (15). + * - maxK is always clamped to ABSOLUTE_MAX_K (200). + * + * Examples: + * N=100 → 15, N=500 → 34, N=1000 → 48, + * N=2000 → 81, N=3000 → 109, N=5000 → 160, N=10000 → 200 (capped) * * @param n Number of data points * @returns Tuple [minK, maxK] @@ -59,14 +74,18 @@ export function computeKRange(n: number): [number, number] { // can actually explore meaningful K values (e.g. N=8 → [2,4]) maxK = Math.max(MIN_K, Math.floor(n / 2)); } else { - // For larger datasets, allow 1 cluster per 3 notes on average. - // This is more generous than sqrt(N) and avoids under-clustering - // (e.g. N=56 → maxK=18 capped to 15, vs sqrt giving only 7). - maxK = Math.floor(n / 3); + // Base scaling: 1.5·√N (standard sqrt rule of thumb). + // For large datasets (N > 1000), add a linear density term + // to maintain ~25-35 notes per cluster as N grows. + // The density term (N-1000)/75 adds ~13 categories per 1000 additional + // notes, preventing clusters from growing too large in big vaults. + const sqrtTerm = 1.5 * Math.sqrt(n); + const densityBoost = Math.max(0, (n - 1000) / 75); + maxK = Math.ceil(sqrtTerm + densityBoost); } - // Clamp to [minK, MAX_K_CAP] - maxK = Math.min(Math.max(maxK, minK), MAX_K_CAP); + // Clamp to [minK, ABSOLUTE_MAX_K] + maxK = Math.min(Math.max(maxK, minK), ABSOLUTE_MAX_K); return [minK, maxK]; } diff --git a/src/pipeline/clustering/metrics.ts b/src/pipeline/clustering/metrics.ts index 9d2588b..c80d763 100644 --- a/src/pipeline/clustering/metrics.ts +++ b/src/pipeline/clustering/metrics.ts @@ -27,6 +27,27 @@ export function getDistanceFn(metric: 'cosine' | 'euclidean'): DistanceFn { return metric === 'euclidean' ? euclideanDistance : cosineDistance; } +/** + * Number of points to sample for silhouette approximation. + * 500 provides a statistically reliable estimate (±0.02 of true score) + * while keeping computation fast at O(500²) = O(250K) distances regardless of N. + */ +const SILHOUETTE_SAMPLE_SIZE = 500; + +/** + * Simple seeded PRNG (mulberry32) for deterministic sampling. + * Returns a function that produces values in [0, 1). + */ +function seededRng(seed: number): () => number { + return () => { + seed |= 0; + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + /** * Computes the mean silhouette coefficient for a clustering. * @@ -37,17 +58,52 @@ export function getDistanceFn(metric: 'cosine' | 'euclidean'): DistanceFn { * * Returns the mean of s(i) across all points. * Range: -1 (poor) to +1 (well-separated clusters). + * + * For large datasets (N > 500), uses random sampling to approximate the + * score in O(500²) instead of O(N²), providing a statistically reliable + * estimate while keeping computation fast. + * + * @param seed Optional seed for deterministic sampling (default: 42) */ -export function silhouetteScore(vectors: number[][], assignments: number[], distFn: DistanceFn): number { +export function silhouetteScore(vectors: number[][], assignments: number[], distFn: DistanceFn, seed = 42): number { const n = vectors.length; if (n <= 1) return 0; const uniqueClusters = [...new Set(assignments)]; if (uniqueClusters.length <= 1) return 0; - // Group point indices by cluster + // For large datasets, sample a subset of points for O(constant²) performance + let sampleIndices: number[]; + if (n > SILHOUETTE_SAMPLE_SIZE) { + // Stratified sampling: sample proportionally from each cluster + // to preserve cluster size ratios in the sample + const clusterMembers = new Map(); + for (let i = 0; i < n; i++) { + const c = assignments[i]; + if (!clusterMembers.has(c)) clusterMembers.set(c, []); + clusterMembers.get(c)!.push(i); + } + + const rng = seededRng(seed); + sampleIndices = []; + for (const [, members] of clusterMembers) { + // At least 2 members per cluster (need ≥2 for intra-cluster distance) + const clusterSampleSize = Math.max(2, Math.round((members.length / n) * SILHOUETTE_SAMPLE_SIZE)); + // Fisher-Yates shuffle on a copy, take first clusterSampleSize + const shuffled = [...members]; + for (let i = shuffled.length - 1; i > 0; i--) { + const j = Math.floor(rng() * (i + 1)); + [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; + } + sampleIndices.push(...shuffled.slice(0, Math.min(clusterSampleSize, shuffled.length))); + } + } else { + sampleIndices = Array.from({ length: n }, (_, i) => i); + } + + // Build cluster index map for sampled points const clusterIndices = new Map(); - for (let i = 0; i < n; i++) { + for (const i of sampleIndices) { const c = assignments[i]; if (!clusterIndices.has(c)) clusterIndices.set(c, []); clusterIndices.get(c)!.push(i); @@ -55,7 +111,7 @@ export function silhouetteScore(vectors: number[][], assignments: number[], dist let totalScore = 0; - for (let i = 0; i < n; i++) { + for (const i of sampleIndices) { const myCluster = assignments[i]; const myClusterMembers = clusterIndices.get(myCluster)!; @@ -85,5 +141,5 @@ export function silhouetteScore(vectors: number[][], assignments: number[], dist totalScore += s; } - return totalScore / n; + return totalScore / sampleIndices.length; } diff --git a/src/pipeline/clustering/postProcess.ts b/src/pipeline/clustering/postProcess.ts index 02a8e8b..37aeb31 100644 --- a/src/pipeline/clustering/postProcess.ts +++ b/src/pipeline/clustering/postProcess.ts @@ -30,14 +30,32 @@ export { toTitleCase, shareWords, getTaxonomyCategory, generateClusterName } fro * Builds the TF-IDF corpus from all pipeline documents once, then iterates * over each strategy result to extract the top tags and generated names per cluster. * + * This function is async to yield control to the event loop between strategies, + * allowing the Joplin panel to receive poll responses and update status messages. + * * @param results Benchmark results from the clustering pipeline * @param documents All note documents used in the pipeline (same order as noteVectors) * @param topK Number of tags to extract per cluster (default: 5) + * @param onStatus Optional callback to report progress to the UI */ -export function enrichResultsWithTags(results: BenchmarkResult[], documents: DocumentText[], topK = 5): void { +export async function enrichResultsWithTags( + results: BenchmarkResult[], + documents: DocumentText[], + topK = 5, + onStatus?: (text: string) => void, +): Promise { + onStatus?.('Building topic index...'); + // Yield so the status message can be delivered via poll before the CPU-intensive constructor runs + await new Promise((resolve) => setTimeout(resolve, 0)); + const tfidfExtractor = new TfidfExtractor(documents); - for (const result of results) { + for (let stratIdx = 0; stratIdx < results.length; stratIdx++) { + const result = results[stratIdx]; + onStatus?.(`Extracting topics (strategy ${stratIdx + 1}/${results.length}: ${result.strategyName})...`); + // Yield so the UI can update before processing this strategy + await new Promise((resolve) => setTimeout(resolve, 0)); + const tags: { [clusterId: number]: string[] } = {}; const clusterNames: { [clusterId: number]: string } = {}; @@ -58,8 +76,8 @@ export function enrichResultsWithTags(results: BenchmarkResult[], documents: Doc const clusterId = Number(clusterIdStr); const indices = clusterIndices[clusterId]; - const clusterDocuments = indices.map((idx) => documents[idx]); - const ngramScores = tfidfExtractor.extractClusterNgramsWithScores(clusterDocuments); + // Use index-based extraction (uses cached ngrams, no re-processing) + const ngramScores = tfidfExtractor.extractClusterNgramsByIndices(indices); cachedScores[clusterId] = ngramScores; tags[clusterId] = selectDedupedTags(ngramScores, topK); diff --git a/src/pipeline/clustering/tfidf.ts b/src/pipeline/clustering/tfidf.ts index ec67152..b4a020f 100644 --- a/src/pipeline/clustering/tfidf.ts +++ b/src/pipeline/clustering/tfidf.ts @@ -4,6 +4,14 @@ import { selectDedupedTags } from './tagExtraction'; const SINGULAR_EXCEPTIONS = new Set(['series', 'species', 'means', 'news', 'analysis', 'basis', 'crisis']); const SHORT_UNIGRAM_THRESHOLD = 4; +/** + * Maximum body text length (in characters) to process for TF-IDF. + * The beginning of a note is most representative of its topic. + * Truncating avoids processing tens of thousands of ngrams from + * long code-heavy or data-heavy notes, which dominate extraction time. + */ +const MAX_BODY_CHARS = 3000; + export interface DocumentText { title: string; body: string; @@ -68,20 +76,23 @@ export function tokenize(text: string): string[] { } /** - * Generates unigrams, bigrams, and trigrams from a sequence of tokens. + * Generates ngrams from a sequence of tokens. + * @param maxN Maximum ngram size (1=unigrams, 2=+bigrams, 3=+trigrams). + * Default 3. Body text uses 2 (skip trigrams for speed), + * title text uses 3 (titles are short and specific). */ -export function getNgrams(tokens: string[]): string[] { +export function getNgrams(tokens: string[], maxN = 3): string[] { const ngrams: string[] = []; const N = tokens.length; for (let i = 0; i < N; i++) { // Unigram ngrams.push(tokens[i]); // Bigram - if (i < N - 1) { + if (maxN >= 2 && i < N - 1) { ngrams.push(`${tokens[i]} ${tokens[i + 1]}`); } // Trigram - if (i < N - 2) { + if (maxN >= 3 && i < N - 2) { ngrams.push(`${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`); } } @@ -104,24 +115,38 @@ export function hasConsecutiveDuplicates(phrase: string): boolean { */ export class TfidfExtractor { private idfs: { [word: string]: number } = {}; + private cachedTitleNgrams: string[][] = []; + private cachedBodyNgrams: string[][] = []; + // Pre-computed unique ngram sets per document (avoids repeated Set creation during cluster extraction) + private cachedUniqueNgrams: Set[] = []; constructor(allDocuments: DocumentText[]) { const N = allDocuments.length; if (N === 0) return; - const docFreqs: { [word: string]: number } = {}; - + // Pre-compute and cache ngrams for every document (done once) for (const doc of allDocuments) { - // For IDF, we only need unique words/ngrams per document — no title weighting needed - const uniqueWords = this.getUniqueDocumentWords(doc); - for (const word of uniqueWords) { + // Title: use trigrams (titles are short and specific) + const titleNg = this.getSegmentNgrams(doc.title || '', 3); + // Body: use only unigrams + bigrams (skip trigrams for speed — they + // account for ~33% of ngrams but are rarely useful for topic names) + const bodyText = (doc.body || '').slice(0, MAX_BODY_CHARS); + const bodyNg = this.getSegmentNgrams(bodyText, 2); + this.cachedTitleNgrams.push(titleNg); + this.cachedBodyNgrams.push(bodyNg); + this.cachedUniqueNgrams.push(new Set([...titleNg, ...bodyNg])); + } + + // Build IDF from cached ngrams + const docFreqs: { [word: string]: number } = {}; + for (let i = 0; i < N; i++) { + for (const word of this.cachedUniqueNgrams[i]) { docFreqs[word] = (docFreqs[word] || 0) + 1; } } for (const word of Object.keys(docFreqs)) { const df = docFreqs[word]; - // Max DF rule: If a word/ngram appears in > 60% of all notes, it is too generic, ignore it. if (df / N > 0.6) { this.idfs[word] = 0; } else { @@ -133,15 +158,16 @@ export class TfidfExtractor { /** * Splits the text by sentence/line boundaries and generates ngrams within segments. * This prevents forming cross-boundary ngrams (like joining separate lines or sentences). + * @param maxN Maximum ngram size (default 3). Use 2 for body text to skip trigrams. */ - private getSegmentNgrams(text: string): string[] { + private getSegmentNgrams(text: string, maxN = 3): string[] { if (!text) return []; // Split by sentence punctuation, newlines, markdown headers, and list bullets const segments = text.split(/[.,?!;:\n\r\-*#()[\]]+/); const allNgrams: string[] = []; for (const seg of segments) { const tokens = tokenize(seg); - const ngrams = getNgrams(tokens); + const ngrams = getNgrams(tokens, maxN); for (const ng of ngrams) { // Filter out any ngrams with consecutive duplicate words (e.g. "day day") if (!hasConsecutiveDuplicates(ng)) { @@ -152,16 +178,6 @@ export class TfidfExtractor { return allNgrams; } - /** - * Returns the unique set of words/ngrams in a document (title + body), used for IDF counting. - * No title weighting — each document contributes at most 1 to each ngram's document frequency. - */ - private getUniqueDocumentWords(doc: DocumentText): Set { - const titleNgrams = this.getSegmentNgrams(doc.title || ''); - const bodyNgrams = this.getSegmentNgrams(doc.body || ''); - return new Set([...titleNgrams, ...bodyNgrams]); - } - /** * Returns ngrams for TF scoring with title words weighted 5x higher. * Uses push loops instead of spread to avoid excess intermediate array allocations. @@ -215,6 +231,15 @@ export class TfidfExtractor { const scores: { ngram: string; score: number }[] = []; + // Pre-compute title ngram sets once per cluster to avoid redundant tokenization + // inside the scoring loop (was O(uniqueNgrams × clusterSize) calls to getSegmentNgrams) + const allTitleNgrams = new Set(); + for (const doc of clusterDocuments) { + for (const ng of this.getSegmentNgrams(doc.title || '')) { + allTitleNgrams.add(ng); + } + } + for (const ngram of Object.keys(tfs)) { const idf = this.idfs[ngram] || 0; // default to 0 if word is ignored/generic if (idf > 0) { @@ -231,15 +256,96 @@ export class TfidfExtractor { } // Title match boost: 1.5x if it appears in any note title in this cluster - let appearsInTitle = false; - for (const doc of clusterDocuments) { - const titleNgrams = new Set(this.getSegmentNgrams(doc.title || '')); - if (titleNgrams.has(ngram)) { - appearsInTitle = true; - break; - } + const titleBoost = allTitleNgrams.has(ngram) ? 1.5 : 1.0; + + const finalScore = tf * idf * cf * lengthBoost * titleBoost; + scores.push({ ngram, score: finalScore }); + } + } + + scores.sort((a, b) => b.score - a.score); + return scores; + } + + /** + * Index-based cluster extraction using pre-computed ngrams. + * + * Performance optimizations vs the original extractClusterNgramsWithScores(): + * 1. Uses pre-computed cached ngrams (no re-tokenization) + * 2. Uses Map instead of plain objects (consistent O(1) in sandbox) + * 3. Prunes to top 100 TF candidates before scoring (we only need 5 tags) + */ + public extractClusterNgramsByIndices(docIndices: number[]): { ngram: string; score: number }[] { + if (docIndices.length === 0) return []; + + // --- Phase 1: Count term frequencies using Map for consistent performance --- + const tfs = new Map(); + let totalNgrams = 0; + + for (const idx of docIndices) { + const titleNgrams = this.cachedTitleNgrams[idx]; + const bodyNgrams = this.cachedBodyNgrams[idx]; + // Title ngrams weighted 5x + for (let i = 0; i < 5; i++) { + for (const ng of titleNgrams) { + tfs.set(ng, (tfs.get(ng) || 0) + 1); + totalNgrams++; + } + } + for (const ng of bodyNgrams) { + tfs.set(ng, (tfs.get(ng) || 0) + 1); + totalNgrams++; + } + } + + if (totalNgrams === 0) return []; + + // --- Phase 2: Prune to top 100 candidates by raw TF count --- + // We only need 5 tags, so scoring all 30K+ unique ngrams is wasteful. + // Keep the top 100 by frequency — these are the most likely topic words. + const TOP_CANDIDATES = 100; + let candidates: [string, number][]; + if (tfs.size > TOP_CANDIDATES) { + candidates = [...tfs.entries()].sort((a, b) => b[1] - a[1]).slice(0, TOP_CANDIDATES); + } else { + candidates = [...tfs.entries()]; + } + + // --- Phase 3: Compute docCounts only for the candidate ngrams --- + const candidateSet = new Set(candidates.map(([ng]) => ng)); + const docCounts = new Map(); + for (const idx of docIndices) { + for (const ng of this.cachedUniqueNgrams[idx]) { + if (candidateSet.has(ng)) { + docCounts.set(ng, (docCounts.get(ng) || 0) + 1); + } + } + } + + // --- Phase 4: Score only candidates with IDF/CF/boosts --- + const allTitleNgrams = new Set(); + for (const idx of docIndices) { + for (const ng of this.cachedTitleNgrams[idx]) { + allTitleNgrams.add(ng); + } + } + + const scores: { ngram: string; score: number }[] = []; + + for (const [ngram, count] of candidates) { + const idf = this.idfs[ngram] || 0; + if (idf > 0) { + const tf = count / totalNgrams; + const cf = (docCounts.get(ngram) || 0) / docIndices.length; + + const wordCount = ngram.split(' ').length; + let lengthBoost = 1.0 + (wordCount - 1) * 0.5; + + if (wordCount === 1 && ngram.length <= SHORT_UNIGRAM_THRESHOLD) { + lengthBoost *= 0.5; } - const titleBoost = appearsInTitle ? 1.5 : 1.0; + + const titleBoost = allTitleNgrams.has(ngram) ? 1.5 : 1.0; const finalScore = tf * idf * cf * lengthBoost * titleBoost; scores.push({ ngram, score: finalScore }); diff --git a/src/pipeline/pipelineConfig.ts b/src/pipeline/pipelineConfig.ts index 9262301..0bc03e1 100644 --- a/src/pipeline/pipelineConfig.ts +++ b/src/pipeline/pipelineConfig.ts @@ -10,25 +10,53 @@ export function isValidEmbeddingVector(vector: number[] | undefined | null, expe } /** - * Computes UMAP intermediate dimensionality scaled logarithmically with input embedding dimension. - * Formula: clamp(⌊2·log₂(D)⌋, 5, 50) + * Returns the UMAP target dimensionality for clustering. + * + * Fixed at 5D — the optimal dimensionality for density-based and centroid-based + * clustering on text embeddings. Higher dimensions cause the curse of dimensionality + * (distances converge, density flattens, clusters become inseparable). + * This is the BERTopic standard used across production topic modeling systems. + * + * @param _inputDim Embedding dimension (unused — output is always 5D) */ -export function adaptiveIntermediateDim(inputDim: number): number { - const raw = Math.floor(2 * Math.log2(inputDim)); - if (!Number.isFinite(raw)) return 5; - return Math.max(5, Math.min(50, raw)); +export function adaptiveIntermediateDim(_inputDim: number): number { + return 5; } /** * Computes UMAP neighbor count scaled with square root of note count. - * Formula: clamp(⌊√N⌋, 5, 50) + * Capped at 15 to preserve local topic structure — higher values blur + * boundaries between distinct topics by connecting cross-topic neighbors. + * Formula: clamp(⌊√N⌋, 5, 15) */ export function adaptiveNeighbors(noteCount: number): number { + const raw = Math.floor(Math.sqrt(noteCount)); + if (!Number.isFinite(raw)) return 5; + return Math.max(5, Math.min(15, raw)); +} + +/** + * Computes HDBSCAN minClusterSize scaled with square root of note count. + * Larger datasets need larger minimum clusters to avoid micro-fragmentation. + * Formula: clamp(⌊√N⌋, 5, 50) + * + * Examples: N=25→5, N=100→10, N=500→22, N=1000→31, N=2500→50 + */ +export function adaptiveMinClusterSize(noteCount: number): number { const raw = Math.floor(Math.sqrt(noteCount)); if (!Number.isFinite(raw)) return 5; return Math.max(5, Math.min(50, raw)); } +/** + * Computes HDBSCAN minSamples as half of minClusterSize. + * Lower minSamples relaxes density requirements, reducing noise/outlier ratio. + * Formula: max(2, ⌊minClusterSize / 2⌋) + */ +export function adaptiveMinSamples(minClusterSize: number): number { + return Math.max(2, Math.floor(minClusterSize / 2)); +} + export function createAdaptiveConfig( inputDim: number, noteCount: number, @@ -42,22 +70,32 @@ export function createAdaptiveConfig( intermediateNeighbors: adaptiveNeighbors(noteCount), strategies: [ { name: 'kmeans-auto', algorithm: 'kmeans', K: 'auto' }, - { name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, + { + name: 'hdbscan', + algorithm: 'hdbscan', + minClusterSize: adaptiveMinClusterSize(noteCount), + minSamples: adaptiveMinSamples(adaptiveMinClusterSize(noteCount)), + }, ], }; } -export function createPipelineConfig(metric: MetricType = 'cosine', seed = 42): CategorizationConfig { +export function createPipelineConfig(noteCount = 100, metric: MetricType = 'cosine', seed = 42): CategorizationConfig { return { seed, metric, - intermediateDim: 8, - intermediateNeighbors: 5, + intermediateDim: 5, + intermediateNeighbors: adaptiveNeighbors(noteCount), strategies: [ { name: 'kmeans-auto', algorithm: 'kmeans', K: 'auto' }, - { name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, + { + name: 'hdbscan', + algorithm: 'hdbscan', + minClusterSize: adaptiveMinClusterSize(noteCount), + minSamples: adaptiveMinSamples(adaptiveMinClusterSize(noteCount)), + }, ], }; } -export const DEFAULT_CONFIG: CategorizationConfig = createPipelineConfig(); +export const DEFAULT_CONFIG: CategorizationConfig = createPipelineConfig(100); diff --git a/src/pipeline/runPipeline.ts b/src/pipeline/runPipeline.ts index 0196963..08a8bd1 100644 --- a/src/pipeline/runPipeline.ts +++ b/src/pipeline/runPipeline.ts @@ -103,6 +103,7 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac log('Too few indexed notes found in native DB. Falling back to local ONNX Web Worker.'); } else { callbacks.onStatus('Clustering...'); + const clusterStart = performance.now(); const adaptiveConfig = createAdaptiveConfig( nativeResult.dimension, validNotes.length, @@ -110,6 +111,7 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac userSeed, ); const results = benchmark(vectors, adaptiveConfig); + log(`Clustering (UMAP + benchmark): ${Math.round(performance.now() - clusterStart)}ms`); // Post-process to extract tags/keywords for each cluster (keep parity with local pipeline) const allPipelineDocuments = validNotes.map((n) => ({ @@ -117,10 +119,17 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac body: n.body, })); - enrichResultsWithTags(results, allPipelineDocuments); + callbacks.onStatus( + `Extracting topics for ${results.reduce((sum, r) => sum + r.clusterCount, 0)} clusters...`, + ); + const enrichStart = performance.now(); + await enrichResultsWithTags(results, allPipelineDocuments, 5, callbacks.onStatus); + log(`Topic extraction: ${Math.round(performance.now() - enrichStart)}ms`); callbacks.onStatus('Generating AI cluster names...'); + const aiStart = performance.now(); await upgradeClusterNamesWithAi(results, allPipelineDocuments); + log(`AI naming: ${Math.round(performance.now() - aiStart)}ms`); const panelNotes: PanelNote[] = validNotes.map((n) => ({ noteId: n.id, @@ -183,8 +192,10 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac } const vectors = noteVectors.map((nv) => nv.vector); - const pipelineConfig = createPipelineConfig(userMetric, userSeed); + const clusterStart = performance.now(); + const pipelineConfig = createPipelineConfig(noteVectors.length, userMetric, userSeed); const results = benchmark(vectors, pipelineConfig); + log(`Clustering (UMAP + benchmark): ${Math.round(performance.now() - clusterStart)}ms`); // Post-process to extract tags/keywords for each cluster const notesMap = new Map(notes.map((n) => [n.id, n])); @@ -196,10 +207,15 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac }; }); - enrichResultsWithTags(results, allPipelineDocuments); + callbacks.onStatus(`Extracting topics for ${results.reduce((sum, r) => sum + r.clusterCount, 0)} clusters...`); + const enrichStart = performance.now(); + await enrichResultsWithTags(results, allPipelineDocuments, 5, callbacks.onStatus); + log(`Topic extraction: ${Math.round(performance.now() - enrichStart)}ms`); callbacks.onStatus('Generating AI cluster names...'); + const aiStart = performance.now(); await upgradeClusterNamesWithAi(results, allPipelineDocuments); + log(`AI naming: ${Math.round(performance.now() - aiStart)}ms`); const panelNotes: PanelNote[] = noteVectors.map((nv) => ({ noteId: nv.noteId, diff --git a/src/webview/components/StrategySection.tsx b/src/webview/components/StrategySection.tsx index 5db07a2..22efab2 100644 --- a/src/webview/components/StrategySection.tsx +++ b/src/webview/components/StrategySection.tsx @@ -7,19 +7,33 @@ interface StrategySectionProps { onStrategyChange: (index: number) => void; } -/** Returns display names for dropdown and pills, marking testing and recommended strategies */ -function getStrategyDisplayName(name: string, isHighest: boolean): string { - let baseName = name; +/** Returns clean display names for dropdown */ +function getStrategyDisplayName(name: string): string { if (name === 'hdbscan') { - baseName = 'HDBSCAN'; + return 'HDBSCAN'; } else if (name.startsWith('kmeans')) { - baseName = 'K-Means (Testing)'; + return 'K-Means'; } + return name; +} - if (isHighest) { - return `${baseName} (Recommended)`; +/** Returns info badge title & description for each strategy */ +function getStrategyDetails(name: string): { tag: string; desc: string } { + if (name === 'hdbscan') { + return { + tag: 'Natural Discovery', + desc: 'Finds natural topic clusters and filters out unrelated notes. May leave some notes uncategorized.', + }; + } else if (name.startsWith('kmeans')) { + return { + tag: 'Balanced Grouping', + desc: 'Categorizes 100% of notes into balanced clusters. May group loosely related topics together.', + }; } - return baseName; + return { + tag: 'Clustering', + desc: '', + }; } export const StrategySection: React.FC = ({ @@ -34,6 +48,8 @@ export const StrategySection: React.FC = ({ onStrategyChange(parseInt(e.target.value, 10)); }; + const details = getStrategyDetails(selectedStrategy.strategyName); + return (
@@ -46,27 +62,22 @@ export const StrategySection: React.FC = ({ > {strategies.map((s, idx) => ( ))}
- Score: {selectedStrategy.silhouetteScore.toFixed(2)} · {selectedStrategy.clusterCount}{' '} - clusters + {selectedStrategy.clusterCount} clusters {selectedStrategy.outlierCount > 0 ? ` · ${selectedStrategy.outlierCount} noise` : ''}
-
- {strategies - .map((s, idx) => ({ s, idx })) - .filter(({ s }) => !s.strategyName.startsWith('kmeans')) - .map(({ s, idx }) => ( - - {getStrategyDisplayName(s.strategyName, false)}: {s.silhouetteScore.toFixed(2)} - - ))} +
+
+ {details.tag} +
+

{details.desc}

); diff --git a/src/webview/context/AppStateContext.tsx b/src/webview/context/AppStateContext.tsx index 7fd502c..d002279 100644 --- a/src/webview/context/AppStateContext.tsx +++ b/src/webview/context/AppStateContext.tsx @@ -135,11 +135,11 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil setIsRunning(false); setStrategies(msg.strategies || []); setNotes(msg.notes || []); - const nonTestingIdx = (msg.strategies || []).findIndex( - (s: BenchmarkResult) => !s.strategyName.startsWith('kmeans'), + const kmeansIdx = (msg.strategies || []).findIndex((s: BenchmarkResult) => + s.strategyName.startsWith('kmeans'), ); - const fallbackIdx = nonTestingIdx !== -1 ? nonTestingIdx : 0; - setSelectedStrategyIndex(msg.selectedStrategyIndex ?? fallbackIdx); + const defaultIdx = kmeansIdx !== -1 ? kmeansIdx : 0; + setSelectedStrategyIndex(msg.selectedStrategyIndex ?? defaultIdx); setError(null); setActiveView('dashboard'); break; diff --git a/src/webview/pages/DashboardPage.tsx b/src/webview/pages/DashboardPage.tsx index 2790e40..5175fd5 100644 --- a/src/webview/pages/DashboardPage.tsx +++ b/src/webview/pages/DashboardPage.tsx @@ -156,7 +156,7 @@ export const DashboardPage: React.FC = () => {
)} - {selectedStrategy && ( + {selectedStrategy && noise.length > 0 && ( { expect(computeKRange(19)).toEqual([2, 9]); // floor(19/2) = 9 }); - it('computes correct range for larger datasets (N >= 20, uses N/3)', () => { - expect(computeKRange(20)).toEqual([3, 6]); // floor(20/3) = 6 - expect(computeKRange(30)).toEqual([3, 10]); // floor(30/3) = 10 - expect(computeKRange(45)).toEqual([3, 15]); // floor(45/3) = 15, hits cap - expect(computeKRange(56)).toEqual([3, 15]); // floor(56/3) = 18, capped at 15 - expect(computeKRange(100)).toEqual([3, 15]); // floor(100/3) = 33, capped at 15 - expect(computeKRange(500)).toEqual([3, 15]); // capped at MAX_K_CAP=15 + it('computes correct range for medium datasets (20 <= N <= 1000, uses ceil(1.5·√N))', () => { + expect(computeKRange(20)).toEqual([3, 7]); // ceil(1.5·√20) = 7 + expect(computeKRange(30)).toEqual([3, 9]); // ceil(1.5·√30) = 9 + expect(computeKRange(45)).toEqual([3, 11]); // ceil(1.5·√45) = 11 + expect(computeKRange(56)).toEqual([3, 12]); // ceil(1.5·√56) = 12 + expect(computeKRange(100)).toEqual([3, 15]); // ceil(1.5·√100) = 15 + expect(computeKRange(225)).toEqual([3, 23]); // ceil(1.5·√225) = 23 + expect(computeKRange(468)).toEqual([3, 33]); // ceil(1.5·√468) = 33 + expect(computeKRange(500)).toEqual([3, 34]); // ceil(1.5·√500) = 34 + expect(computeKRange(879)).toEqual([3, 45]); // ceil(1.5·√879) = 45 + expect(computeKRange(1000)).toEqual([3, 48]); // ceil(1.5·√1000) = 48, density boost = 0 + }); + + it('scales dynamically for large datasets (N > 1000, adds density boost (N-1000)/75)', () => { + // Formula: ceil(1.5·√N + (N-1000)/75), capped at ABSOLUTE_MAX_K=200 + expect(computeKRange(2000)).toEqual([3, 81]); // ceil(67.1 + 13.3) = 81 — density ~24.7 + expect(computeKRange(3000)).toEqual([3, 109]); // ceil(82.2 + 26.7) = 109 — density ~27.5 + expect(computeKRange(5000)).toEqual([3, 160]); // ceil(106.1 + 53.3) = 160 — density ~31.3 + expect(computeKRange(10000)).toEqual([3, 200]); // ceil(150 + 120) = 270, capped at 200 }); }); @@ -156,7 +168,7 @@ describe('autoK findOptimalK', () => { it('prefers higher K when silhouette scores are within tolerance', () => { // 4 clusters of 8 points each, reasonably separated in 2D. // K=2 and K=3 may score slightly higher in raw silhouette, but K=4 - // should be within the 0.025 tolerance band and thus selected. + // should be within the 0.01 tolerance band and thus selected. const FOUR_CLUSTERS = [ // Cluster near [0, 0] ...[0.1, 0.2, 0.0, 0.15, 0.05, 0.12, 0.08, 0.18].map((x, i) => [ diff --git a/test/pipeline/pipelineConfig.test.ts b/test/pipeline/pipelineConfig.test.ts index ff43d39..3df1923 100644 --- a/test/pipeline/pipelineConfig.test.ts +++ b/test/pipeline/pipelineConfig.test.ts @@ -3,7 +3,10 @@ import { isValidEmbeddingVector, adaptiveIntermediateDim, adaptiveNeighbors, + adaptiveMinClusterSize, + adaptiveMinSamples, createAdaptiveConfig, + createPipelineConfig, DEFAULT_CONFIG, } from '../../src/pipeline/pipelineConfig'; @@ -47,37 +50,82 @@ describe('isValidEmbeddingVector', () => { }); describe('adaptive scaling functions', () => { - it('computes adaptive intermediate dimensions logarithmic with input dimension', () => { - expect(adaptiveIntermediateDim(384)).toBe(17); - expect(adaptiveIntermediateDim(768)).toBe(19); - expect(adaptiveIntermediateDim(1536)).toBe(21); + it('returns fixed 5D for all input dimensions (optimal for clustering)', () => { + expect(adaptiveIntermediateDim(384)).toBe(5); + expect(adaptiveIntermediateDim(768)).toBe(5); + expect(adaptiveIntermediateDim(1536)).toBe(5); }); - it('clamps intermediate dimensions between 5 and 50', () => { + it('returns 5 regardless of extreme input dimensions', () => { expect(adaptiveIntermediateDim(2)).toBe(5); - expect(adaptiveIntermediateDim(1e12)).toBe(50); + expect(adaptiveIntermediateDim(1e12)).toBe(5); }); - it('computes adaptive neighbors based on square root of note count', () => { + it('computes adaptive neighbors based on square root of note count, capped at 15', () => { expect(adaptiveNeighbors(25)).toBe(5); expect(adaptiveNeighbors(100)).toBe(10); - expect(adaptiveNeighbors(500)).toBe(22); + expect(adaptiveNeighbors(500)).toBe(15); }); - it('clamps neighbors between 5 and 50', () => { + it('clamps neighbors between 5 and 15', () => { expect(adaptiveNeighbors(2)).toBe(5); - expect(adaptiveNeighbors(10000)).toBe(50); + expect(adaptiveNeighbors(10000)).toBe(15); }); it('creates adaptive configuration dynamically', () => { const config = createAdaptiveConfig(768, 100); expect(config.metric).toBe('cosine'); - expect(config.intermediateDim).toBe(19); + expect(config.intermediateDim).toBe(5); expect(config.intermediateNeighbors).toBe(10); expect(config.strategies.length).toBe(2); }); }); +describe('adaptive HDBSCAN parameter scaling', () => { + it('computes minClusterSize based on square root of note count', () => { + expect(adaptiveMinClusterSize(25)).toBe(5); + expect(adaptiveMinClusterSize(100)).toBe(10); + expect(adaptiveMinClusterSize(500)).toBe(22); + expect(adaptiveMinClusterSize(1000)).toBe(31); + expect(adaptiveMinClusterSize(2000)).toBe(44); + }); + + it('clamps minClusterSize between 5 and 50', () => { + expect(adaptiveMinClusterSize(4)).toBe(5); + expect(adaptiveMinClusterSize(10)).toBe(5); + expect(adaptiveMinClusterSize(3000)).toBe(50); + expect(adaptiveMinClusterSize(10000)).toBe(50); + }); + + it('computes minSamples as half of minClusterSize', () => { + expect(adaptiveMinSamples(5)).toBe(2); + expect(adaptiveMinSamples(10)).toBe(5); + expect(adaptiveMinSamples(22)).toBe(11); + expect(adaptiveMinSamples(31)).toBe(15); + }); + + it('clamps minSamples to minimum of 2', () => { + expect(adaptiveMinSamples(2)).toBe(2); + expect(adaptiveMinSamples(3)).toBe(2); + }); + + it('createAdaptiveConfig uses adaptive HDBSCAN params', () => { + const config = createAdaptiveConfig(384, 500); + const hdbscanStrategy = config.strategies.find((s) => s.algorithm === 'hdbscan'); + expect(hdbscanStrategy).toBeDefined(); + expect(hdbscanStrategy!.minClusterSize).toBe(22); // floor(sqrt(500)) + expect(hdbscanStrategy!.minSamples).toBe(11); // floor(22/2) + }); + + it('createPipelineConfig uses adaptive HDBSCAN params', () => { + const config = createPipelineConfig(500); + const hdbscanStrategy = config.strategies.find((s) => s.algorithm === 'hdbscan'); + expect(hdbscanStrategy).toBeDefined(); + expect(hdbscanStrategy!.minClusterSize).toBe(22); + expect(hdbscanStrategy!.minSamples).toBe(11); + }); +}); + describe('DEFAULT_CONFIG', () => { it('uses cosine metric and seed 42', () => { expect(DEFAULT_CONFIG.metric).toBe('cosine');