refactor: simplify strategy UI and optimize Auto-K pipeline for 1,000–5,000+ notes

package.json

@@ -1,6 +1,6 @@
 {
   "name": "joplin-plugin-note-categorization",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "scripts": {
     "dist": "webpack --env joplin-plugin-config=buildMain && webpack --env joplin-plugin-config=buildExtraScripts && npm run copyAssets && webpack --env joplin-plugin-config=createArchive",
     "prepare": "npm run dist",

src/manifest.json

@@ -2,7 +2,7 @@
 	"manifest_version": 1,
 	"id": "com.harsh16gupta.notecategorization",
 	"app_min_version": "3.5",
-	"version": "0.1.5",
+	"version": "0.1.6",
 	"name": "Note Categorization Plugin",
 	"description": "AI-based note categorisation: clusters notes semantically, suggests tags and notebook structures, and detects stale notes.",
 	"author": "Harsh Gupta",

src/pipeline/UmapProjector.ts

@@ -14,7 +14,7 @@ export class UmapProjector {
 	constructor(options: UmapProjectorOptions = {}) {
 		this.nComponents = options.nComponents ?? 2;
 		this.nNeighbors = options.nNeighbors ?? 15;
-		this.minDist = options.minDist ?? 0.1;
+		this.minDist = options.minDist ?? 0.0;
 		this.metric = options.metric ?? 'cosine';
 		this.seed = options.seed ?? 42;
 	}

src/pipeline/clustering/autoK.ts

@@ -7,21 +7,25 @@ import { log } from '../../utils/logger';
 /** Absolute minimum K to try (silhouette needs at least 2 clusters). */
 const MIN_K = 2;
 
-/** Absolute maximum K to try (caps the sweep to bound runtime and avoid tiny clusters). */
-const MAX_K_CAP = 15;
+/**
+ * Absolute safety ceiling for maxK to bound sweep runtime.
+ * At N=10,000 the dynamic formula yields ~270, so this caps it at 200
+ * to prevent excessive K-Means iterations at extreme vault sizes.
+ * Silhouette evaluation remains O(500²) per K due to stratified sampling.
+ */
+const ABSOLUTE_MAX_K = 200;
 
 /**
  * Absolute silhouette tolerance for K selection. Clusterings whose silhouette
  * score is within this margin of the sweep's peak are considered equivalently
  * good, and the highest K among them is selected.
  *
  * Rationale: silhouette score naturally biases toward fewer, coarser clusters.
- * A small drop (e.g. 0.017) when going from K=4 to K=7 is statistically
+ * A small drop (e.g. 0.008) when going from K=4 to K=7 is statistically
  * insignificant, but the finer granularity is far more useful for note
- * categorization. 0.025 is within the standard range (0.02–0.05) used in
- * clustering literature for "equivalent quality" comparisons.
+ * categorization. 0.01 is a strict tolerance for equivalent peak selection.
  */
-const SILHOUETTE_TOLERANCE = 0.025;
+const SILHOUETTE_TOLERANCE = 0.01;
 
 export interface AutoKResult {
 	/** The optimal K value found by the sweep. */
@@ -35,15 +39,26 @@ export interface AutoKResult {
 /**
  * Computes the K search range [minK, maxK] based on dataset size.
  *
- * - minK is 2 for small datasets (N < 20), 3 for larger ones (N >= 20).
- *   For 20+ notes, 2 categories is too coarse to be useful.
+ * Two-part scaling formula for maxK:
+ *
+ * - **Base term** (all N ≥ 20): `1.5 · √N` — the BERTopic-standard sqrt rule
+ *   of thumb that scales sub-linearly with dataset size.
+ * - **Density boost** (N > 1000): `(N − 1000) / 75` — a linear term that adds
+ *   ~13 extra K per 1000 additional notes, preventing clusters from growing too
+ *   large in big vaults. This maintains ~25–35 notes per cluster up to N ≈ 5000.
+ *
+ * The combined formula `maxK = ⌈1.5·√N + max(0, (N−1000)/75)⌉` is continuous
+ * at N = 1000 (density boost is 0) and preserves all existing behavior for
+ * N ≤ 1000.
+ *
+ * Range rules:
+ * - minK is 2 for small datasets (N < 20), 3 for larger ones (N ≥ 20).
  * - For small datasets (N < 20): maxK = floor(N / 2).
- *   Ensures the sweep can explore meaningful K values (e.g. N=8 → [2,4]).
- * - For larger datasets (N >= 20): maxK = floor(N / 3).
- *   Ensures each cluster has at least ~3 notes on average.
- *   This is more generous than sqrt(N) and prevents under-clustering
- *   (e.g. N=56 → [2,15] instead of [2,7]).
- * - maxK is always clamped to MAX_K_CAP (15).
+ * - maxK is always clamped to ABSOLUTE_MAX_K (200).
+ *
+ * Examples:
+ *   N=100 → 15, N=500 → 34, N=1000 → 48,
+ *   N=2000 → 81, N=3000 → 109, N=5000 → 160, N=10000 → 200 (capped)
  *
  * @param n  Number of data points
  * @returns  Tuple [minK, maxK]
@@ -59,14 +74,18 @@ export function computeKRange(n: number): [number, number] {
 		// can actually explore meaningful K values (e.g. N=8 → [2,4])
 		maxK = Math.max(MIN_K, Math.floor(n / 2));
 	} else {
-		// For larger datasets, allow 1 cluster per 3 notes on average.
-		// This is more generous than sqrt(N) and avoids under-clustering
-		// (e.g. N=56 → maxK=18 capped to 15, vs sqrt giving only 7).
-		maxK = Math.floor(n / 3);
+		// Base scaling: 1.5·√N (standard sqrt rule of thumb).
+		// For large datasets (N > 1000), add a linear density term
+		// to maintain ~25-35 notes per cluster as N grows.
+		// The density term (N-1000)/75 adds ~13 categories per 1000 additional
+		// notes, preventing clusters from growing too large in big vaults.
+		const sqrtTerm = 1.5 * Math.sqrt(n);
+		const densityBoost = Math.max(0, (n - 1000) / 75);
+		maxK = Math.ceil(sqrtTerm + densityBoost);
 	}
 
-	// Clamp to [minK, MAX_K_CAP]
-	maxK = Math.min(Math.max(maxK, minK), MAX_K_CAP);
+	// Clamp to [minK, ABSOLUTE_MAX_K]
+	maxK = Math.min(Math.max(maxK, minK), ABSOLUTE_MAX_K);
 
 	return [minK, maxK];
 }

src/pipeline/clustering/metrics.ts

@@ -27,6 +27,27 @@ export function getDistanceFn(metric: 'cosine' | 'euclidean'): DistanceFn {
 	return metric === 'euclidean' ? euclideanDistance : cosineDistance;
 }
 
+/**
+ * Number of points to sample for silhouette approximation.
+ * 500 provides a statistically reliable estimate (±0.02 of true score)
+ * while keeping computation fast at O(500²) = O(250K) distances regardless of N.
+ */
+const SILHOUETTE_SAMPLE_SIZE = 500;
+
+/**
+ * Simple seeded PRNG (mulberry32) for deterministic sampling.
+ * Returns a function that produces values in [0, 1).
+ */
+function seededRng(seed: number): () => number {
+	return () => {
+		seed |= 0;
+		seed = (seed + 0x6d2b79f5) | 0;
+		let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
+		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+	};
+}
+
 /**
  * Computes the mean silhouette coefficient for a clustering.
  *
@@ -37,25 +58,60 @@ export function getDistanceFn(metric: 'cosine' | 'euclidean'): DistanceFn {
  *
  * Returns the mean of s(i) across all points.
  * Range: -1 (poor) to +1 (well-separated clusters).
+ *
+ * For large datasets (N > 500), uses random sampling to approximate the
+ * score in O(500²) instead of O(N²), providing a statistically reliable
+ * estimate while keeping computation fast.
+ *
+ * @param seed  Optional seed for deterministic sampling (default: 42)
  */
-export function silhouetteScore(vectors: number[][], assignments: number[], distFn: DistanceFn): number {
+export function silhouetteScore(vectors: number[][], assignments: number[], distFn: DistanceFn, seed = 42): number {
 	const n = vectors.length;
 	if (n <= 1) return 0;
 
 	const uniqueClusters = [...new Set(assignments)];
 	if (uniqueClusters.length <= 1) return 0;
 
-	// Group point indices by cluster
+	// For large datasets, sample a subset of points for O(constant²) performance
+	let sampleIndices: number[];
+	if (n > SILHOUETTE_SAMPLE_SIZE) {
+		// Stratified sampling: sample proportionally from each cluster
+		// to preserve cluster size ratios in the sample
+		const clusterMembers = new Map<number, number[]>();
+		for (let i = 0; i < n; i++) {
+			const c = assignments[i];
+			if (!clusterMembers.has(c)) clusterMembers.set(c, []);
+			clusterMembers.get(c)!.push(i);
+		}
+
+		const rng = seededRng(seed);
+		sampleIndices = [];
+		for (const [, members] of clusterMembers) {
+			// At least 2 members per cluster (need ≥2 for intra-cluster distance)
+			const clusterSampleSize = Math.max(2, Math.round((members.length / n) * SILHOUETTE_SAMPLE_SIZE));
+			// Fisher-Yates shuffle on a copy, take first clusterSampleSize
+			const shuffled = [...members];
+			for (let i = shuffled.length - 1; i > 0; i--) {
+				const j = Math.floor(rng() * (i + 1));
+				[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+			}
+			sampleIndices.push(...shuffled.slice(0, Math.min(clusterSampleSize, shuffled.length)));
+		}
+	} else {
+		sampleIndices = Array.from({ length: n }, (_, i) => i);
+	}
+
+	// Build cluster index map for sampled points
 	const clusterIndices = new Map<number, number[]>();
-	for (let i = 0; i < n; i++) {
+	for (const i of sampleIndices) {
 		const c = assignments[i];
 		if (!clusterIndices.has(c)) clusterIndices.set(c, []);
 		clusterIndices.get(c)!.push(i);
 	}
 
 	let totalScore = 0;
 
-	for (let i = 0; i < n; i++) {
+	for (const i of sampleIndices) {
 		const myCluster = assignments[i];
 		const myClusterMembers = clusterIndices.get(myCluster)!;
 
@@ -85,5 +141,5 @@ export function silhouetteScore(vectors: number[][], assignments: number[], dist
 		totalScore += s;
 	}
 
-	return totalScore / n;
+	return totalScore / sampleIndices.length;
 }

src/pipeline/clustering/postProcess.ts

@@ -30,14 +30,32 @@ export { toTitleCase, shareWords, getTaxonomyCategory, generateClusterName } fro
  * Builds the TF-IDF corpus from all pipeline documents once, then iterates
  * over each strategy result to extract the top tags and generated names per cluster.
  *
+ * This function is async to yield control to the event loop between strategies,
+ * allowing the Joplin panel to receive poll responses and update status messages.
+ *
  * @param results    Benchmark results from the clustering pipeline
  * @param documents  All note documents used in the pipeline (same order as noteVectors)
  * @param topK       Number of tags to extract per cluster (default: 5)
+ * @param onStatus   Optional callback to report progress to the UI
  */
-export function enrichResultsWithTags(results: BenchmarkResult[], documents: DocumentText[], topK = 5): void {
+export async function enrichResultsWithTags(
+	results: BenchmarkResult[],
+	documents: DocumentText[],
+	topK = 5,
+	onStatus?: (text: string) => void,
+): Promise<void> {
+	onStatus?.('Building topic index...');
+	// Yield so the status message can be delivered via poll before the CPU-intensive constructor runs
+	await new Promise((resolve) => setTimeout(resolve, 0));
+
 	const tfidfExtractor = new TfidfExtractor(documents);
 
-	for (const result of results) {
+	for (let stratIdx = 0; stratIdx < results.length; stratIdx++) {
+		const result = results[stratIdx];
+		onStatus?.(`Extracting topics (strategy ${stratIdx + 1}/${results.length}: ${result.strategyName})...`);
+		// Yield so the UI can update before processing this strategy
+		await new Promise((resolve) => setTimeout(resolve, 0));
+
 		const tags: { [clusterId: number]: string[] } = {};
 		const clusterNames: { [clusterId: number]: string } = {};
 
@@ -58,8 +76,8 @@ export function enrichResultsWithTags(results: BenchmarkResult[], documents: Doc
 			const clusterId = Number(clusterIdStr);
 			const indices = clusterIndices[clusterId];
 
-			const clusterDocuments = indices.map((idx) => documents[idx]);
-			const ngramScores = tfidfExtractor.extractClusterNgramsWithScores(clusterDocuments);
+			// Use index-based extraction (uses cached ngrams, no re-processing)
+			const ngramScores = tfidfExtractor.extractClusterNgramsByIndices(indices);
 			cachedScores[clusterId] = ngramScores;
 
 			tags[clusterId] = selectDedupedTags(ngramScores, topK);