From 34b2fb27a219c1471322e869f108152c2ff17c5a Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Wed, 7 Oct 2026 23:58:45 +0300 Subject: [PATCH 1/2] refactor(search): move the dense semantic search out of the open core The embedding model, its embedder and the hybrid engine now live in the private seforim-semantic package (Zayit's official builds). The public bundle no longer carries the model, and DownloadEmbedModel is gone. The generator still indexes dense vectors given -PvectorsBin. Signed-off-by: Elie Gambache --- generator/packaging/build.gradle.kts | 27 +- .../packaging/DownloadEmbedModel.kt | 84 ----- .../packaging/PackageArtifacts.kt | 21 -- search/build.gradle.kts | 5 - .../search/HebrewV5Normalizer.kt | 27 -- .../search/HybridSearchEngine.kt | 324 ------------------ .../seforimlibrary/search/SeforimEmbedder.kt | 146 -------- .../seforimlibrary/search/VectorSearcher.kt | 67 ---- .../search/SeforimEmbedderTest.kt | 39 --- 9 files changed, 1 insertion(+), 739 deletions(-) delete mode 100644 generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/DownloadEmbedModel.kt delete mode 100644 search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewV5Normalizer.kt delete mode 100644 search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HybridSearchEngine.kt delete mode 100644 search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedder.kt delete mode 100644 search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/VectorSearcher.kt delete mode 100644 search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedderTest.kt diff --git a/generator/packaging/build.gradle.kts b/generator/packaging/build.gradle.kts index e32d2481..698c0fe8 100644 --- a/generator/packaging/build.gradle.kts +++ b/generator/packaging/build.gradle.kts @@ -80,37 +80,12 @@ tasks.register("downloadLexicalDb") { jvmArgs = listOf("-Xmx512m") } -// Download the dense embedding model (int8 ONNX + tokenizer) from the private -// SeforimEmbedding v4-int8 release next to seforim.db, so it gets bundled. -// Needs GITHUB_TOKEN/GH_TOKEN; fails soft (bundle without model) if unavailable. -// Usage: -// GITHUB_TOKEN=… ./gradlew :packaging:downloadEmbedModel -tasks.register("downloadEmbedModel") { - group = "application" - description = "Download int8 embedding model + tokenizer from the private v4-int8 release next to seforim.db." - - dependsOn("jvmJar") - mainClass.set("io.github.kdroidfilter.seforimlibrary.packaging.DownloadEmbedModelKt") - classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath") - - if (project.hasProperty("seforimDb")) { - systemProperty("seforimDb", project.property("seforimDb") as String) - } else if (System.getenv("SEFORIM_DB") != null) { - systemProperty("seforimDb", System.getenv("SEFORIM_DB")) - } else { - val defaultDbPath = rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath - systemProperty("seforimDb", defaultDbPath) - } - - jvmArgs = listOf("-Xmx256m") -} - // Package DB + Lucene indexes into single tar.zst and split tasks.register("packageArtifacts") { group = "application" description = "Create seforim_bundle.tar.zst (DB + indexes + release info) with zstd and split into ~1.9GiB parts." - dependsOn("jvmJar", "writeReleaseInfo", "downloadLexicalDb", "downloadEmbedModel") + dependsOn("jvmJar", "writeReleaseInfo", "downloadLexicalDb") mainClass.set("io.github.kdroidfilter.seforimlibrary.packaging.PackageArtifactsKt") classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath") diff --git a/generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/DownloadEmbedModel.kt b/generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/DownloadEmbedModel.kt deleted file mode 100644 index 008a2d5b..00000000 --- a/generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/DownloadEmbedModel.kt +++ /dev/null @@ -1,84 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.packaging - -import co.touchlab.kermit.Logger -import co.touchlab.kermit.Severity -import io.github.kdroidfilter.seforimlibrary.common.OptimizedHttpClient -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - -private const val RELEASE_API = - "https://api.github.com/repos/kdroidFilter/SeforimEmbedding/releases/tags/v5-int8" -private const val USER_AGENT = "SeforimLibrary-DownloadEmbedModel/1.0" - -// Runtime dense-search artifacts pulled from the private SeforimEmbedding release. -private val ASSETS = listOf("seforim-embed-v5-int8.onnx", "tokenizer.json") - -/** - * Download the int8 embedding model + tokenizer from the private `v5-int8` release - * and place them next to `seforim.db` so [PackageArtifacts] bundles them. - * - * Requires a token with read access to the private repo via `GITHUB_TOKEN` / `GH_TOKEN` - * (consumed by [OptimizedHttpClient]). On any failure (no token, network, missing - * asset) it logs a warning and exits 0 so packaging proceeds WITHOUT the model - * (the app then degrades to lexical-only search). - * - * Usage: - * ./gradlew :packaging:downloadEmbedModel - * ./gradlew :packaging:downloadEmbedModel -PseforimDb=/path/to/seforim.db - */ -fun main(args: Array) { - Logger.setMinSeverity(Severity.Info) - val logger = Logger.withTag("DownloadEmbedModel") - - val dbPath = resolveDbPath(args) - - val present = ASSETS.all { name -> - val p = dbPath.resolveSibling(name) - Files.exists(p) && Files.isRegularFile(p) && Files.size(p) > 0 - } - if (present) { - logger.i { "Embedding model already present next to ${dbPath.fileName}; skipping download" } - return - } - - runCatching { - val json = OptimizedHttpClient.fetchJson(RELEASE_API, USER_AGENT, logger) - for (name in ASSETS) { - val out = dbPath.resolveSibling(name) - if (Files.exists(out) && Files.size(out) > 0) { - logger.i { "Using existing $name" } - continue - } - val url = assetApiUrl(json, name) - ?: throw IllegalStateException("Asset '$name' not found in v4-int8 release") - Files.createDirectories(out.parent) - val tmp = out.resolveSibling("${out.fileName}.part") - Files.deleteIfExists(tmp) - // Asset API url + Accept: octet-stream + token -> works for private repos. - OptimizedHttpClient.downloadFile(url, tmp, USER_AGENT, logger, "Downloading $name") - Files.deleteIfExists(out) - Files.move(tmp, out) - logger.i { "Downloaded $name -> ${out.toAbsolutePath()}" } - } - }.onFailure { - logger.w(it) { "Could not download the embedding model; bundle will omit it (dense search disabled)" } - } -} - -private fun resolveDbPath(args: Array): Path { - val dbPathStr = args.getOrNull(0) - ?: System.getProperty("seforimDb") - ?: System.getenv("SEFORIM_DB") - ?: Paths.get("build", "seforim.db").toString() - return Paths.get(dbPathStr) -} - -/** Extract the GitHub *asset API* url for a given asset name from the release JSON. */ -private fun assetApiUrl(json: String, name: String): String? { - val re = Regex( - "\\{\"url\":\"(https://api\\.github\\.com/[^\"]+?/assets/\\d+)\"[^{}]*?\"name\":\"" + - Regex.escape(name) + "\"", - ) - return re.find(json)?.groupValues?.get(1) -} diff --git a/generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/PackageArtifacts.kt b/generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/PackageArtifacts.kt index c553fe25..1fb4f34c 100644 --- a/generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/PackageArtifacts.kt +++ b/generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/PackageArtifacts.kt @@ -72,11 +72,6 @@ fun main(args: Array) { // Resolve lexical DB next to the DB val lexicalDbPath: Path = dbPath.resolveSibling("lexical.db") - // Resolve the dense embedding model (int8 ONNX) + tokenizer next to the DB. - // Bundled so the app gets dense search out of the box; absent -> lexical only. - val embedModelPath: Path = dbPath.resolveSibling("seforim-embed-v5-int8.onnx") - val embedTokenizerPath: Path = dbPath.resolveSibling("tokenizer.json") - if (!textIndexDir.toFile().isDirectory) { logger.w { "Lucene text index directory missing: $textIndexDir (will skip)" } } @@ -131,8 +126,6 @@ fun main(args: Array) { " - Catalog: $catalogPath\n" + " - Release info: $releaseInfoPath\n" + " - Lexical DB: $lexicalDbPath\n" + - " - Embed model: $embedModelPath\n" + - " - Embed tokenizer: $embedTokenizerPath\n" + " - Text index: $textIndexDir\n" + " - Lookup index: $lookupIndexDir\n" + " -> Bundle .tar.zst: $bundleOutputPath\n" + @@ -178,20 +171,6 @@ fun main(args: Array) { logger.w { "Lexical DB missing: $lexicalDbPath (skipped)" } } - // Add the dense embedding model + tokenizer if available - if (embedModelPath.exists()) { - addFileToTar(tar, embedModelPath, embedModelPath.fileName.toString(), logger) - logger.i { "Added embedding model to bundle" } - } else { - logger.w { "Embedding model missing: $embedModelPath (skipped, dense search disabled)" } - } - if (embedTokenizerPath.exists()) { - addFileToTar(tar, embedTokenizerPath, embedTokenizerPath.fileName.toString(), logger) - logger.i { "Added embedding tokenizer to bundle" } - } else { - logger.w { "Embedding tokenizer missing: $embedTokenizerPath (skipped)" } - } - // Add the precomputed catalog if available if (haveCatalog) { addFileToTar(tar, catalogPath, catalogPath.fileName.toString(), logger) diff --git a/search/build.gradle.kts b/search/build.gradle.kts index 4a2e4111..52c283fb 100644 --- a/search/build.gradle.kts +++ b/search/build.gradle.kts @@ -20,11 +20,6 @@ kotlin { implementation(libs.sqlDelight.driver.sqlite) implementation(libs.kermit) implementation(libs.jsoup) - // Dense semantic search: ONNX Runtime (query embedding) + HuggingFace tokenizer. - // Stock Maven `onnxruntime` is CPU-only on desktop JVM (no DirectML/CoreML/XNNPACK - // native — those need a custom build; CUDA needs the separate onnxruntime_gpu). - implementation(libs.onnxruntime) - implementation(libs.djl.huggingface.tokenizers) } jvmTest.dependencies { diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewV5Normalizer.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewV5Normalizer.kt deleted file mode 100644 index 304d725b..00000000 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HebrewV5Normalizer.kt +++ /dev/null @@ -1,27 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search - -import org.jsoup.parser.Parser - -/** - * Exact JVM port of `normalize_he_v5.py`: strip HTML/nikud/teamim and non-Hebrew - * characters, then fold final (sofit) letters (ך→כ ם→מ ן→נ ף→פ ץ→צ). It MUST be - * applied to any text before embedding it with a v5 model (both indexed lines and - * query), so vectors stay comparable. Pair it with a v5 model + a v5-built index. - */ -object HebrewV5Normalizer { - private val TAG = Regex("<[^>]+>") - private val MARKS = Regex("[֑-ׇ]") - private val DROP = Regex("[^א-ת0-9\\s.,:;!?()\\[\\]\"'\\-/׳״]") - private val WS = Regex("\\s+") - // Final (sofit) -> base letter folding. - private val FINALS = mapOf('ך' to 'כ', 'ם' to 'מ', 'ן' to 'נ', 'ף' to 'פ', 'ץ' to 'צ') - - fun clean(text: String): String { - var s = TAG.replace(text, " ") - s = Parser.unescapeEntities(s, false) - s = MARKS.replace(s, "") - s = DROP.replace(s, " ") - s = WS.replace(s, " ").trim() - return buildString(s.length) { for (c in s) append(FINALS[c] ?: c) } - } -} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HybridSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HybridSearchEngine.kt deleted file mode 100644 index e1b61805..00000000 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/HybridSearchEngine.kt +++ /dev/null @@ -1,324 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search - -import co.touchlab.kermit.Logger -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withContext -import org.jsoup.Jsoup -import org.jsoup.safety.Safelist -import java.nio.file.Files -import java.nio.file.Path - -/** - * Hybrid search = lexical (BM25 + MagicDictionary, [LuceneSearchEngine]) fused with - * dense semantic search (v4 embedding + [VectorSearcher]) via Reciprocal Rank Fusion. - * - * RRF(doc) = Σ 1/(K + rank_in_list) - * - * Implements [SearchEngine] so the app uses it transparently: [openSession] runs the - * fused search (the session pages over the in-memory fused list); all other methods - * (facets, snippets, title prefix) delegate to the lexical engine. - * - * Falls back to pure lexical when: the model/vector index is absent, OR the filters - * are not supported by the dense index (categoryFilter / lineIds). Book / base-book - * filters ARE supported by both paths. - * - * Dense-only hits (lines the lexical path missed) are turned into full [LineHit]s via - * [resolveLine], supplied by the caller (fetches title + text + snippet by line id). - */ -class HybridSearchEngine( - private val lexical: LuceneSearchEngine, - private val modelDir: Path?, - private val indexDir: Path, - private val rrfK: Int = 60, - private val candidates: Int = 150, - private val resolveLine: suspend (lineId: Long, bookId: Long, query: String) -> LineHit?, -) : SearchEngine { - - private val logger = Logger.withTag("HybridSearch") - - // The embedder (heavy OrtSession) + vector searcher are loaded LAZILY on the first - // dense search, off the main thread (see [ensureDense]) — so the first query just - // shows the normal search spinner instead of freezing the UI while the model loads. - @Volatile private var embedder: SeforimEmbedder? = null - @Volatile private var vectorSearcher: VectorSearcher? = null - @Volatile private var denseTried = false - private val denseMutex = Mutex() - - // Cheap, no-load check: are the model + vector index even present? Decides whether to - // take the dense path (then load lazily); the actual OrtSession is built in fuse(). - private val denseConfigured: Boolean = - Files.isDirectory(indexDir) && SeforimEmbedder.isAvailable(modelDir) - - val denseEnabled: Boolean get() = embedder != null && vectorSearcher != null - - /** Load the embedder + vector searcher once, on a background thread. Idempotent. */ - private suspend fun ensureDense() { - if (denseTried) return - denseMutex.withLock { - if (denseTried) return - withContext(Dispatchers.IO) { - val emb = SeforimEmbedder.tryLoad(modelDir) - val vs = if (emb != null && Files.isDirectory(indexDir)) { - runCatching { VectorSearcher(indexDir) }.getOrNull() - } else null - if (vs != null) { - embedder = emb; vectorSearcher = vs - logger.i { "dense ready (vector index $indexDir)" } - } else { - runCatching { emb?.close() } - logger.i { "dense unavailable -> lexical only" } - } - } - denseTried = true - } - } - - override fun openSession( - query: String, - near: Int, - bookFilter: Long?, - categoryFilter: Long?, - bookIds: Collection?, - lineIds: Collection?, - baseBookOnly: Boolean, - ): SearchSession? { - if (query.isBlank()) return null - // Dense index supports book / base-book filters only. For category/line filters - // (or when dense isn't configured), use pure lexical to stay correct. The model - // itself is loaded lazily in fuse() — denseConfigured is a cheap no-load check. - val denseOk = denseConfigured && categoryFilter == null && lineIds == null - if (!denseOk) { - return lexical.openSession(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) - } - val effBookIds = bookIds ?: bookFilter?.let { listOf(it) } - return HybridSession(query, near, effBookIds, baseBookOnly) - } - - override fun searchBooksByTitlePrefix(query: String, limit: Int): List = - lexical.searchBooksByTitlePrefix(query, limit) - - override fun buildSnippet(rawText: String, query: String, near: Int): String = - lexical.buildSnippet(rawText, query, near) - - /** - * Picks the passage of [text] closest in meaning to [query], using the SAME dense - * encoder as semantic search — so the highlight reflects meaning instead of scattering - * dictionary word matches. Returns the winning passage verbatim, or null when dense is - * unavailable or the text isn't worth localizing (a single short clause). - */ - override suspend fun semanticSpan(query: String, text: String): String? { - if (query.isBlank() || text.isBlank()) return null - ensureDense() - val emb = embedder ?: return null - val clauses = splitClauses(text) - if (clauses.size < 2) return null - return withContext(Dispatchers.Default) { - val qVec = emb.embed(query) - clauses - .map { it to cosine(qVec, emb.embed(it)) } - .maxByOrNull { it.second } - ?.first - } - } - - /** - * Embedding-based find-in-page in one book: dense KNN over the index scoped to [bookId]. - * Reuses the line vectors already in the index, so this is just the query embedding + a - * filtered KNN — the per-line passage is computed by the caller on the displayed text. - */ - override suspend fun denseReady(): Boolean { - ensureDense() - return denseEnabled - } - - override suspend fun semanticFind(query: String, bookId: Long, limit: Int): List { - if (query.isBlank()) return emptyList() - ensureDense() - val emb = embedder ?: return emptyList() - val vs = vectorSearcher ?: return emptyList() - return withContext(Dispatchers.Default) { - vs.search(emb.embed(query), limit, baseBookOnly = false, bookIds = listOf(bookId)).map { it.lineId } - } - } - - override fun computeFacets( - query: String, near: Int, bookFilter: Long?, categoryFilter: Long?, - bookIds: Collection?, lineIds: Collection?, baseBookOnly: Boolean, - ): SearchFacets? = lexical.computeFacets(query, near, bookFilter, categoryFilter, bookIds, lineIds, baseBookOnly) - - override fun close() { - runCatching { vectorSearcher?.close() } - runCatching { embedder?.close() } - runCatching { lexical.close() } - } - - /** The fused, RRF-ordered hits plus the ids that the lexical path matched (so the dense-only - * ones can get a meaning-based snippet lazily — their lexical snippet is meaningless). */ - private class FusedResult(val hits: List, val lexicalIds: Set) - - /** Lexical page + dense KNN, fused by RRF, resolved to full LineHits. */ - private suspend fun fuse(query: String, near: Int, bookIds: Collection?, baseOnly: Boolean): FusedResult { - ensureDense() // first call loads the model off-main (covered by the search spinner) - val lexHits = lexical.openSession(query, near = near, bookIds = bookIds, baseBookOnly = baseOnly) - ?.use { it.nextPage(candidates)?.hits ?: emptyList() } ?: emptyList() - - // Dense failed to load -> lexical-only result (still correct, just no semantic recall). - val emb = embedder - val vs = vectorSearcher - if (emb == null || vs == null) return FusedResult(lexHits, lexHits.map { it.lineId }.toSet()) - - // Don't let a dense failure sink the whole search: degrade to lexical and log. This - // also catches native-image gaps (e.g. a missing Lucene KNN vectors-format SPI entry). - val denseHits = try { - withContext(Dispatchers.Default) { - val qVec = emb.embed(query) - vs.search(qVec, candidates, baseOnly, bookIds) - } - } catch (e: Exception) { - logger.w(e) { "dense KNN failed; falling back to lexical-only" } - return FusedResult(lexHits, lexHits.map { it.lineId }.toSet()) - } - - val rrf = HashMap() - val bookOf = HashMap() - lexHits.forEachIndexed { rank, h -> - rrf.merge(h.lineId, 1.0 / (rrfK + rank + 1), Double::plus); bookOf[h.lineId] = h.bookId - } - denseHits.forEachIndexed { rank, h -> - rrf.merge(h.lineId, 1.0 / (rrfK + rank + 1), Double::plus); bookOf.putIfAbsent(h.lineId, h.bookId) - } - val lexById = lexHits.associateBy { it.lineId } - val ordered = rrf.entries.sortedByDescending { it.value } - val out = ArrayList(ordered.size) - for ((lineId, score) in ordered) { - val hit = lexById[lineId] ?: resolveLine(lineId, bookOf[lineId] ?: -1L, query) - if (hit != null) out += hit.copy(score = score.toFloat()) - } - return FusedResult(out, lexById.keys) - } - - /** - * Snippet for a dense-only hit: the lexical builder can't anchor (the query words aren't in - * the text) and, fed the passage, it tokenizes and bolds each word separately — so short - * function words (ואם, או…) get highlighted on their own. Instead, locate the passage closest - * in meaning and bold it as ONE contiguous span inside a context window. Null -> keep lexical. - */ - private suspend fun semanticSnippet(query: String, rawText: String, near: Int): String? { - // Jsoup.clean returns HTML-escaped text; substrings stay escaped, so we splice in - // directly (same convention as the lexical snippet builder). - val clean = Jsoup.clean(rawText, Safelist.none()) - val passage = semanticSpan(query, clean) ?: return null - val idx = clean.indexOf(passage) - if (idx < 0) return lexical.buildSnippet(rawText, passage, near) - val from = (idx - SNIPPET_CONTEXT).coerceAtLeast(0) - val to = (idx + passage.length + SNIPPET_CONTEXT).coerceAtMost(clean.length) - return buildString { - if (from > 0) append("…") - append(clean, from, idx) - append("").append(passage).append("") - append(clean, idx + passage.length, to) - if (to < clean.length) append("…") - } - } - - private inner class HybridSession( - private val query: String, - private val near: Int, - private val bookIds: Collection?, - private val baseOnly: Boolean, - ) : SearchSession { - private var fused: FusedResult? = null - private var offset = 0 - - override suspend fun nextPage(limit: Int): SearchPage? { - val all = fused ?: fuse(query, near, bookIds, baseOnly).also { fused = it } - if (offset >= all.hits.size) return null - val end = minOf(offset + limit, all.hits.size) - // Re-snippet only the dense-only hits on THIS page (bounded work) so semantic - // results show the passage that actually matched, not the line's opening words. - val slice = all.hits.subList(offset, end).map { hit -> - if (hit.lineId in all.lexicalIds) { - hit - } else { - semanticSnippet(query, hit.rawText, near)?.let { hit.copy(snippet = it) } ?: hit - } - } - offset = end - return SearchPage(hits = slice, totalHits = all.hits.size.toLong(), isLastPage = offset >= all.hits.size) - } - - override fun close() {} - } - - /** - * Splits [text] into passage candidates: first on strong clause delimiters (incl. the - * Hebrew sof-pasuk ׃), then sliding word windows over any segment too long to localize. - * Substrings are verbatim slices of [text] (original spacing preserved) so the caller - * can match them back diacritic-insensitively. Tiny fragments are dropped to keep a - * highlight that is a *passage*, not a stray word. - */ - private fun splitClauses(text: String): List { - val delimiters = charArrayOf('.', '!', '?', ':', ';', '׃', '\n') - val segments = ArrayList() - var start = 0 - for (i in text.indices) { - if (text[i] in delimiters) { - if (i > start) segments += start..i - start = i + 1 - } - } - if (start < text.length) segments += start until text.length - - val out = ArrayList() - for (seg in segments) { - val sub = text.substring(seg.first, seg.last + 1) - val wordRanges = WORD_RE.findAll(sub).map { it.range }.toList() - if (wordRanges.size < MIN_PASSAGE_WORDS) continue - if (wordRanges.size <= MAX_PASSAGE_WORDS) { - out += sub.substring(wordRanges.first().first, wordRanges.last().last + 1) - } else { - // Segment too long to localize: slide word windows over the original slice. - var w = 0 - while (w < wordRanges.size) { - val from = wordRanges[w].first - val toIdx = minOf(w + MAX_PASSAGE_WORDS - 1, wordRanges.size - 1) - out += sub.substring(from, wordRanges[toIdx].last + 1) - if (toIdx == wordRanges.size - 1) break - w += WINDOW_STRIDE_WORDS - } - } - } - return if (out.isEmpty()) listOf(text.trim()) else out - } - - private fun cosine(a: FloatArray, b: FloatArray): Float { - var s = 0f - val n = minOf(a.size, b.size) - for (i in 0 until n) s += a[i] * b[i] - return s - } - - companion object { - private val logger = Logger.withTag("HybridSearch") - - private val WORD_RE = Regex("\\S+") - private const val MIN_PASSAGE_WORDS = 3 - private const val MAX_PASSAGE_WORDS = 14 - private const val WINDOW_STRIDE_WORDS = 8 - private const val SNIPPET_CONTEXT = 90 // chars of context kept on each side of the passage - - fun create( - lexical: LuceneSearchEngine, - indexDir: Path, - modelDir: Path? = null, - resolveLine: suspend (Long, Long, String) -> LineHit?, - ): HybridSearchEngine { - // ONE index: dense vectors live in the SAME Lucene index as the text - // (seforim.db.lucene) — no separate vector index. The embedder + searcher - // load lazily on the first dense search (no UI freeze). - return HybridSearchEngine(lexical, modelDir, indexDir, resolveLine = resolveLine) - } - } -} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedder.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedder.kt deleted file mode 100644 index a72738ad..00000000 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedder.kt +++ /dev/null @@ -1,146 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search - -import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer -import ai.onnxruntime.OnnxTensor -import ai.onnxruntime.OrtEnvironment -import ai.onnxruntime.OrtSession -import co.touchlab.kermit.Logger -import java.io.Closeable -import java.nio.file.Files -import java.nio.file.Path - -/** - * Produces L2-normalized 384-d sentence embeddings for Hebrew/Aramaic text on the JVM, - * using the v5 model trained in the SeforimEmbedding project (ONNX export). - * - * The ONNX graph bakes in pooling + projection + L2 normalization, so [embed] returns - * a vector ready for a Lucene `KnnFloatVectorField` (cosine). Query text is normalized - * with [HebrewV5Normalizer] to match the training distribution. - * - * Model artifacts (from the SeforimEmbedding release): - * - `seforim-embed-v5-int8.onnx` (the model) - * - `tokenizer.json` (the matching tokenizer) - * Place both in a directory and point to it via `-DseforimEmbedModelDir=…`, - * the `SEFORIM_EMBED_MODEL` env var, or one of the default candidate locations. - * If not found, [tryLoad] returns null and dense search is simply disabled. - */ -class SeforimEmbedder private constructor( - onnxModel: Path, - tokenizerJson: Path, - private val maxLen: Int = 128, -) : Closeable { - - val dim: Int = 384 - - private val tokenizer: HuggingFaceTokenizer = HuggingFaceTokenizer.newInstance(tokenizerJson) - private val env: OrtEnvironment = OrtEnvironment.getEnvironment() - private val session: OrtSession = openSession(env, onnxModel) - - // The stock Maven `onnxruntime` artifact is CPU-only on desktop JVM. Hardware EPs only - // work if a capable native is bundled: CUDA (swap to the onnxruntime_gpu artifact + - // system CUDA/cuDNN, NVIDIA), or DirectML/CoreML (custom ORT build). They're opt-in via - // -DseforimEmbedGpu=true / SEFORIM_EMBED_GPU=1; otherwise an optimized CPU session - // (full graph opt + all cores). int8 is already CPU-friendly. - private fun openSession(env: OrtEnvironment, model: Path): OrtSession { - val wantGpu = System.getProperty("seforimEmbedGpu")?.toBoolean() == true || - System.getenv("SEFORIM_EMBED_GPU")?.toBoolean() == true - if (wantGpu) { - val eps = listOf Unit>>( - "CUDA" to { it.addCUDA(0) }, - "DirectML" to { it.addDirectML(0) }, - "CoreML" to { it.addCoreML() }, - ) - for ((name, add) in eps) { - runCatching { - val o = OrtSession.SessionOptions().apply { - setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT); add(this) - } - return env.createSession(model.toString(), o).also { logger.i { "ONNX EP: $name (GPU)" } } - } - } - logger.i { "No GPU EP available in this build; using CPU" } - } - val o = OrtSession.SessionOptions().apply { - setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT) - runCatching { setIntraOpNumThreads(Runtime.getRuntime().availableProcessors()) } - } - return env.createSession(model.toString(), o) - } - - // v5 models were trained on final-folded text; queries must be normalized - // the same way so query and indexed vectors stay comparable. - private val normalize: (String) -> String = HebrewV5Normalizer::clean - - /** Embed a single text (normalized like the corpus) into a normalized float[dim]. */ - fun embed(text: String): FloatArray { - val enc = tokenizer.encode(normalize(text)) - var ids = enc.ids - var mask = enc.attentionMask - if (ids.size > maxLen) { - ids = ids.copyOf(maxLen) - mask = mask.copyOf(maxLen) - } - OnnxTensor.createTensor(env, arrayOf(ids)).use { idsT -> - OnnxTensor.createTensor(env, arrayOf(mask)).use { maskT -> - session.run(mapOf("input_ids" to idsT, "attention_mask" to maskT)).use { res -> - @Suppress("UNCHECKED_CAST") - return (res[0].value as Array)[0] - } - } - } - } - - override fun close() { - session.close() - env.close() - tokenizer.close() - } - - companion object { - private val logger = Logger.withTag("SeforimEmbedder") - - /** - * Locate the model and load an embedder, or return null if unavailable - * (dense search then degrades gracefully to lexical-only). - */ - private fun candidateDirs(explicitDir: Path?): List = listOfNotNull( - explicitDir, - System.getProperty("seforimEmbedModelDir")?.let { Path.of(it) }, - System.getenv("SEFORIM_EMBED_MODEL")?.let { Path.of(it) }, - Path.of(System.getProperty("user.home"), "IdeaProjects/SeforimEmbedding/artifacts"), - ).distinct() - - /** Cheap presence check (model + tokenizer files) WITHOUT creating the heavy - * OrtSession — lets callers decide to take the dense path then load lazily. */ - fun isAvailable(explicitDir: Path? = null): Boolean = - candidateDirs(explicitDir).any { findOnnx(it) != null && findTokenizer(it) != null } - - fun tryLoad(explicitDir: Path? = null): SeforimEmbedder? { - val dirs = candidateDirs(explicitDir) - for (dir in dirs) { - val onnx = findOnnx(dir) ?: continue - val tok = findTokenizer(dir) ?: continue - return runCatching { - logger.i { "Loading dense embedder: onnx=$onnx tokenizer=$tok" } - SeforimEmbedder(onnx, tok) - }.onFailure { logger.w(it) { "Failed to load embedder from $dir" } }.getOrNull() - } - logger.i { "No embedding model found; dense search disabled. Checked: $dirs" } - return null - } - - // Prefer the newest model and the int8-quantized variant (4x smaller, ~3x - // faster CPU embedding) when present. v5 models fold final letters; the - // matching normalization is selected automatically from the filename. - private fun findOnnx(dir: Path): Path? = listOf( - "seforim-embed-v5-int8.onnx", "seforim-embed-v5.onnx", - "seforim-embed-v4-int8.onnx", "model.onnx", "seforim-embed-v4.onnx", - ).map { dir.resolve(it) }.firstOrNull { Files.isRegularFile(it) } - - private fun findTokenizer(dir: Path): Path? = listOf( - dir.resolve("tokenizer.json"), - dir.resolve("tokenizer_v4/tokenizer.json"), - dir.resolve("model_v4_phase2a/tokenizer.json"), - ).firstOrNull { Files.isRegularFile(it) } - } -} diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/VectorSearcher.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/VectorSearcher.kt deleted file mode 100644 index c62f4bd3..00000000 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/VectorSearcher.kt +++ /dev/null @@ -1,67 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search - -import org.apache.lucene.document.IntPoint -import org.apache.lucene.index.DirectoryReader -import org.apache.lucene.search.BooleanClause -import org.apache.lucene.search.BooleanQuery -import org.apache.lucene.search.IndexSearcher -import org.apache.lucene.search.KnnFloatVectorQuery -import org.apache.lucene.search.Query -import org.apache.lucene.store.FSDirectory -import org.apache.lucene.store.NIOFSDirectory -import java.io.Closeable -import java.nio.file.Path - -/** A dense (semantic) hit from the fused Lucene index. */ -data class DenseHit(val lineId: Long, val bookId: Long, val score: Float) - -/** - * Runs a filtered KNN query over the SINGLE fused Lucene index (text fields + - * dense `KnnFloatVectorField` per line, built by the generator's text index - * writer). Filters (base-books / specific books) are applied as a pre-filter so - * the KNN only considers eligible documents. - * - * Returns line ids that are joined back to the DB by the caller. - */ -class VectorSearcher(indexDir: Path) : Closeable { - // GraalVM native image can't instantiate MMapDirectory's MemorySegmentIndexInputProvider - // (Panama foreign downcalls) — use NIOFSDirectory there, like LuceneSearchEngine does. - private val dir = - if (System.getProperty("org.graalvm.nativeimage.imagecode") != null) { - NIOFSDirectory(indexDir) - } else { - FSDirectory.open(indexDir) - } - - private fun filterQuery(baseBookOnly: Boolean, bookIds: Collection?): Query? { - val b = BooleanQuery.Builder() - var any = false - if (baseBookOnly) { - // Field name matches the fused text index (LuceneTextIndexWriter.FIELD_IS_BASE_BOOK). - b.add(IntPoint.newExactQuery("is_base_book", 1), BooleanClause.Occur.FILTER); any = true - } - if (!bookIds.isNullOrEmpty()) { - b.add(IntPoint.newSetQuery("book_id", *bookIds.map { it.toInt() }.toIntArray()), BooleanClause.Occur.FILTER); any = true - } - return if (any) b.build() else null - } - - fun search(query: FloatArray, k: Int, baseBookOnly: Boolean = false, bookIds: Collection? = null): List { - DirectoryReader.open(dir).use { reader -> - val searcher = IndexSearcher(reader) - val knn = KnnFloatVectorQuery("vec", query, k, filterQuery(baseBookOnly, bookIds)) - val top = searcher.search(knn, k) - val stored = searcher.storedFields() - return top.scoreDocs.map { sd -> - val d = stored.document(sd.doc) - DenseHit( - lineId = d.getField("line_id").numericValue().toLong(), - bookId = d.getField("book_id").numericValue().toLong(), - score = sd.score, - ) - } - } - } - - override fun close() = dir.close() -} diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedderTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedderTest.kt deleted file mode 100644 index 5aa05a60..00000000 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SeforimEmbedderTest.kt +++ /dev/null @@ -1,39 +0,0 @@ -package io.github.kdroidfilter.seforimlibrary.search - -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertTrue - -class SeforimEmbedderTest { - - private fun cos(a: FloatArray, b: FloatArray): Float { - var s = 0f - for (i in a.indices) s += a[i] * b[i] - return s - } - - @Test - fun embedderLoadsAndProducesUsableVectors() { - val embedder = SeforimEmbedder.tryLoad() - if (embedder == null) { - println("[skip] no embedding model found (set -DseforimEmbedModelDir) — dense search disabled") - return - } - embedder.use { e -> - val q = e.embed("מה מברכים על אוכל") - assertEquals(384, q.size, "embedding dim") - - // deterministic + normalized: same text twice -> cosine ~1 - val q2 = e.embed("מה מברכים על אוכל") - assertTrue(cos(q, q2) > 0.999f, "self-cosine should be ~1.0") - - // sanity: a topically related text should be closer than an unrelated one - val related = e.embed("ברכת הנהנין על פירות וירקות") - val unrelated = e.embed("הלכות טומאה וטהרה של כלים") - val cr = cos(q, related) - val cu = cos(q, unrelated) - println("[embedder] cos(related)=$cr cos(unrelated)=$cu") - assertTrue(cr > cu, "related ($cr) should be closer than unrelated ($cu)") - } - } -} From 2a8c980154fcc6a1b367df6c8ee36b2c95927f7b Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Wed, 7 Oct 2026 23:58:46 +0300 Subject: [PATCH 2/2] feat(searchindex): embed the corpus with a Gradle task when the semantic package is readable With the token that reads seforim-semantic (the siddur's, as in Zayit, or gh), buildLuceneIndexDefault depends on embedCorpus, which runs the package's EmbedCorpus: fp32 model on an NVIDIA GPU with the CUDA 12 / cuDNN 9 runtime fetched from NVIDIA's redistributables (-PcudaArchivesDir for archives at hand), else int8 on the CPU. No Python. Community builds stay lexical only. Signed-off-by: Elie Gambache --- build.gradle.kts | 2 +- generator/searchindex/build.gradle.kts | 135 ++++++++++++++++++++++--- gradle/libs.versions.toml | 9 +- settings.gradle.kts | 38 +++++++ 4 files changed, 166 insertions(+), 18 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b01c0b05..07a80c98 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -56,7 +56,7 @@ project(":otzariasqlite").tasks.matching { it.name == "generateHavroutaLinks" }. project(":catalog").tasks.matching { it.name == "buildCatalog" }.configureEach { mustRunAfter(":otzariasqlite:generateHavroutaLinks") } -project(":searchindex").tasks.matching { it.name == "buildLuceneIndexDefault" }.configureEach { +project(":searchindex").tasks.matching { it.name in setOf("embedCorpus", "buildLuceneIndexDefault") }.configureEach { mustRunAfter(":catalog:buildCatalog") } project(":packaging").tasks.matching { it.name == "writeReleaseInfo" }.configureEach { diff --git a/generator/searchindex/build.gradle.kts b/generator/searchindex/build.gradle.kts index f5284199..877a4dfd 100644 --- a/generator/searchindex/build.gradle.kts +++ b/generator/searchindex/build.gradle.kts @@ -1,12 +1,35 @@ +import java.net.URI +import java.nio.file.Files +import java.nio.file.StandardCopyOption +import java.security.MessageDigest + plugins { alias(libs.plugins.multiplatform) } // Generator forked-JVM heap. Honors -PgeneratorHeap=… (CI lowers it on 16 GB runners). // Default 10g matches local workstation use; CI sets 5g via the workflow. +val seforimDb: String = (project.findProperty("seforimDb") as String?) + ?: System.getenv("SEFORIM_DB") + ?: rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath + +// The semantic search (open core): with the token of its private package (settings.gradle.kts), the corpus is embedded +// into the index's dense vectors. -PvectorsBin= brings precomputed ones instead; -PnoVectors=true leaves them out. +val withSemantic = gradle.extensions.extraProperties.run { has("semanticEnabled") && get("semanticEnabled") == true } +val explicitVectors = project.findProperty("vectorsBin") as String? +val embedVectors = withSemantic && explicitVectors == null && project.findProperty("noVectors") != "true" +val vectorsDir: File = rootProject.layout.buildDirectory.dir("vectors").get().asFile + +// Embedding runs on an NVIDIA GPU (Linux x64), with the CUDA runtime fetched below; elsewhere on the CPU, for hours +val cuda = embedVectors && + System.getProperty("os.name").startsWith("Linux") && + System.getProperty("os.arch") in setOf("amd64", "x86_64") && + File("/proc/driver/nvidia/version").exists() + +// The vectors (~5.5 GB) are loaded in the indexer's heap val generatorHeap: String = (project.findProperty("generatorHeap") as String?) ?: System.getenv("SEFORIM_GENERATOR_HEAP") - ?: "10g" + ?: if (embedVectors || explicitVectors != null) "20g" else "10g" kotlin { @@ -48,15 +71,7 @@ tasks.register("buildLuceneIndexDefault") { classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath") // Pass DB path as system property recognized by the Kotlin entrypoint - if (project.hasProperty("seforimDb")) { - systemProperty("seforimDb", project.property("seforimDb") as String) - } else if (System.getenv("SEFORIM_DB") != null) { - systemProperty("seforimDb", System.getenv("SEFORIM_DB")) - } else { - // Default to DB under root build/ - val defaultDbPath = rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath - systemProperty("seforimDb", defaultDbPath) - } + systemProperty("seforimDb", seforimDb) // Prefer in-memory DB for faster reads (override with -PinMemoryDb=false) val inMemory = project.findProperty("inMemoryDb") != "false" @@ -64,9 +79,13 @@ tasks.register("buildLuceneIndexDefault") { systemProperty("inMemoryDb", "true") } - // Optional: -PvectorsBin=/path (dir with ids.i64+vecs.f32+meta.txt) → SINGLE - // fused index (text + dense KnnFloatVectorField per line). - (project.findProperty("vectorsBin") as String?)?.let { systemProperty("vectorsBin", it) } + // Dense vectors (dir with ids.i64+vecs.f32+meta.txt) → SINGLE fused index (text + KnnFloatVectorField per line): + // -PvectorsBin=/path, or the corpus embedded by embedCorpus + explicitVectors?.let { systemProperty("vectorsBin", it) } + if (embedVectors) { + dependsOn("embedCorpus") + systemProperty("vectorsBin", vectorsDir.absolutePath) + } // Optional: -PindexThreads=N to cap concurrent indexing threads (lower = less RAM). (project.findProperty("indexThreads") as String?)?.let { systemProperty("indexThreads", it) } @@ -78,3 +97,93 @@ tasks.register("buildLuceneIndexDefault") { "--add-modules=jdk.incubator.vector" ) } + +if (embedVectors) { + val semanticEmbedder = configurations.create("semanticEmbedder") { + isCanBeConsumed = false + attributes { attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.JAVA_RUNTIME)) } + // onnxruntime_gpu: the same API, plus the CUDA execution provider + if (cuda) { + resolutionStrategy.dependencySubstitution { + substitute(module("com.microsoft.onnxruntime:onnxruntime")).using(module(libs.onnxruntime.gpu.get().toString())) + } + } + } + dependencies { + add(semanticEmbedder.name, libs.seforim.semantic) + // The fp32 model, for the GPU (the int8 one, in seforim-semantic, has no CUDA kernels for its quantized ops) + if (cuda) add(semanticEmbedder.name, libs.seforim.semantic.fp32) + } + + // CUDA 12 + cuDNN 9 for onnxruntime_gpu, from NVIDIA's redistributables (no CUDA install): only their shared + // libraries, in the Gradle user home, fetched once (~2 GB). -PcudaArchivesDir= takes the archives already + // there instead of downloading them (still checked against their sha256) + val localArchives = (project.findProperty("cudaArchivesDir") as String?)?.let(::File) + val cudaLibDir = gradle.gradleUserHomeDir.resolve("caches/seforim-cuda/cuda-12.8-cudnn-9.10/lib") + val prepareCudaRuntime = tasks.register("prepareCudaRuntime") { + val archives = mapOf( + "cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-12.8.90-archive.tar.xz" to + "8d566b5fe745c46842dc16945cf36686227536decd2302c372be86da37faca68", + "cuda/redist/libcublas/linux-x86_64/libcublas-linux-x86_64-12.8.4.1-archive.tar.xz" to + "21718957c2cf000bacd69d36c95708a2319199e39e056f8b4f0f68e3b9f323bb", + "cuda/redist/libcurand/linux-x86_64/libcurand-linux-x86_64-10.3.9.90-archive.tar.xz" to + "32a5ec30be446c1b7228d1bc502b2f029cc8b59a5e362c70d960754fa646778b", + "cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-9.10.2.21_cuda12-archive.tar.xz" to + "d0defcbc4c6dad711ff4cb66d254036a300c9071b07c7b64199aacab534313c1", + ) + val complete = cudaLibDir.resolve(".complete") + onlyIf { !complete.exists() } + doLast { + cudaLibDir.mkdirs() + archives.forEach { (path, sha256) -> + val name = path.substringAfterLast('/') + val local = localArchives?.resolve(name)?.takeIf { it.isFile } + val archive = local ?: temporaryDir.resolve(name) + if (local == null) { + logger.lifecycle("Downloading $name") + URI("https://developer.download.nvidia.com/compute/$path").toURL().openStream().use { + Files.copy(it, archive.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + } + val digest = MessageDigest.getInstance("SHA-256") + archive.inputStream().use { input -> + val buffer = ByteArray(1 shl 20) + while (true) { + val read = input.read(buffer) + if (read < 0) break + digest.update(buffer, 0, read) + } + } + val actual = digest.digest().joinToString("") { "%02x".format(it) } + check(actual == sha256) { "${archive.name}: sha256 $actual, expected $sha256" } + val tar = ProcessBuilder( + "tar", "-xJf", archive.path, "-C", cudaLibDir.path, + "--strip-components=2", "--wildcards", "*/lib/*.so*", + ).inheritIO().start() + check(tar.waitFor() == 0) { "Could not extract ${archive.name}" } + if (local == null) archive.delete() + } + complete.writeText("") + } + } + + // Embeds every indexable line of seforim.db into build/vectors (the private package's EmbedCorpus) + tasks.register("embedCorpus") { + group = "application" + description = "Embed the corpus of seforim.db into dense vectors for the fused Lucene index (semantic search)." + + classpath = semanticEmbedder + mainClass.set("io.github.kdroidfilter.seforim.semantic.EmbedCorpusKt") + args( + "--db", seforimDb, + "--out", vectorsDir.path, + "--gpu", cuda.toString(), + ) + if (cuda) { + dependsOn(prepareCudaRuntime) + val inherited = System.getenv("LD_LIBRARY_PATH") + environment("LD_LIBRARY_PATH", listOfNotNull(cudaLibDir.path, inherited).joinToString(File.pathSeparator)) + } + jvmArgs("-Xmx8g", "--enable-native-access=ALL-UNNAMED") + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e5a2909e..cce998fb 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,8 +5,8 @@ kotlin = "2.3.20" agp = "9.1.1" jvmToolchain = "25" lucene = "10.4.0" -onnxruntime = "1.20.0" -djlHuggingface = "0.30.0" +seforimSemantic = "0.3.0" +onnxruntime = "1.30.0" maven-publish = "0.36.0" kotlinx-coroutines = "1.10.2" kotlinx-serialization = "1.10.0" @@ -36,8 +36,9 @@ lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version. lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "lucene" } lucene-analysis-common = { module = "org.apache.lucene:lucene-analysis-common", version.ref = "lucene" } lucene-core = { module = "org.apache.lucene:lucene-core", version.ref = "lucene" } -onnxruntime = { module = "com.microsoft.onnxruntime:onnxruntime", version.ref = "onnxruntime" } -djl-huggingface-tokenizers = { module = "ai.djl.huggingface:tokenizers", version.ref = "djlHuggingface" } +seforim-semantic = { module = "io.github.kdroidfilter:seforim-semantic", version.ref = "seforimSemantic" } +seforim-semantic-fp32 = { module = "io.github.kdroidfilter:seforim-semantic-fp32", version.ref = "seforimSemantic" } +onnxruntime-gpu = { module = "com.microsoft.onnxruntime:onnxruntime_gpu", version.ref = "onnxruntime" } sqlDelight-driver-sqlite = { module = "app.cash.sqldelight:sqlite-driver", version.ref = "sqlDelight" } sqlDelight-driver-android = { module = "app.cash.sqldelight:android-driver", version.ref = "sqlDelight" } sqlDelight-driver-native = { module = "app.cash.sqldelight:native-driver", version.ref = "sqlDelight" } diff --git a/settings.gradle.kts b/settings.gradle.kts index ddb2c2f5..ca729eb9 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,8 +15,46 @@ pluginManagement { } } +/** + * The token that reads the semantic search's private package (open core): with it, the generator embeds the corpus into + * the index's dense vectors. semantic.token, SEFORIM_SEMANTIC_TOKEN, the siddur's token (siddur.token, + * SEFORIM_SIDDUR_TOKEN) as in Zayit, or the gh CLI's when it can read the package; none in a community build. + */ +val semanticToken: String? = + providers.gradleProperty("semantic.token").orNull + ?: providers.environmentVariable("SEFORIM_SEMANTIC_TOKEN").orNull + ?: providers.gradleProperty("siddur.token").orNull + ?: providers.environmentVariable("SEFORIM_SIDDUR_TOKEN").orNull + ?: run { + val gh = providers.environmentVariable("PATH").getOrElse("").split(File.pathSeparator) + .flatMap { listOf(File(it, "gh"), File(it, "gh.exe")) } + .firstOrNull { it.canExecute() } + ?.path + fun ghOutput(vararg args: String): String? = + gh?.let { + val exec = providers.exec { + commandLine(it, *args) + isIgnoreExitValue = true + } + exec.standardOutput.asText.get().trim().takeIf { exec.result.get().exitValue == 0 } + } + ghOutput("api", "users/kdroidFilter/packages/maven/io.github.kdroidfilter.seforim-semantic", "--jq", ".name") + ?.let { ghOutput("auth", "token") } + } +gradle.extensions.extraProperties["semanticEnabled"] = semanticToken != null + dependencyResolutionManagement { repositories { + // The semantic search, for the official builds (open core): a private package, read with a token + semanticToken?.let { token -> + maven("https://maven.pkg.github.com/kdroidFilter/SeforimEmbedding") { + credentials { + username = "token" + password = token + } + content { includeModuleByRegex("io\\.github\\.kdroidfilter", "seforim-semantic.*") } + } + } google { content { includeGroupByRegex("com\\.android.*")