Move the semantic search out of the open core; embed the corpus with Gradle

build.gradle.kts

@@ -56,7 +56,7 @@ project(":otzariasqlite").tasks.matching { it.name == "generateHavroutaLinks" }.
 project(":catalog").tasks.matching { it.name == "buildCatalog" }.configureEach {
     mustRunAfter(":otzariasqlite:generateHavroutaLinks")
 }
-project(":searchindex").tasks.matching { it.name == "buildLuceneIndexDefault" }.configureEach {
+project(":searchindex").tasks.matching { it.name in setOf("embedCorpus", "buildLuceneIndexDefault") }.configureEach {
     mustRunAfter(":catalog:buildCatalog")
 }
 project(":packaging").tasks.matching { it.name == "writeReleaseInfo" }.configureEach {

generator/packaging/build.gradle.kts

@@ -80,37 +80,12 @@ tasks.register<JavaExec>("downloadLexicalDb") {
     jvmArgs = listOf("-Xmx512m")
 }
 
-// Download the dense embedding model (int8 ONNX + tokenizer) from the private
-// SeforimEmbedding v4-int8 release next to seforim.db, so it gets bundled.
-// Needs GITHUB_TOKEN/GH_TOKEN; fails soft (bundle without model) if unavailable.
-// Usage:
-//   GITHUB_TOKEN=… ./gradlew :packaging:downloadEmbedModel
-tasks.register<JavaExec>("downloadEmbedModel") {
-    group = "application"
-    description = "Download int8 embedding model + tokenizer from the private v4-int8 release next to seforim.db."
-
-    dependsOn("jvmJar")
-    mainClass.set("io.github.kdroidfilter.seforimlibrary.packaging.DownloadEmbedModelKt")
-    classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath")
-
-    if (project.hasProperty("seforimDb")) {
-        systemProperty("seforimDb", project.property("seforimDb") as String)
-    } else if (System.getenv("SEFORIM_DB") != null) {
-        systemProperty("seforimDb", System.getenv("SEFORIM_DB"))
-    } else {
-        val defaultDbPath = rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath
-        systemProperty("seforimDb", defaultDbPath)
-    }
-
-    jvmArgs = listOf("-Xmx256m")
-}
-
 // Package DB + Lucene indexes into single tar.zst and split
 tasks.register<JavaExec>("packageArtifacts") {
     group = "application"
     description = "Create seforim_bundle.tar.zst (DB + indexes + release info) with zstd and split into ~1.9GiB parts."
 
-    dependsOn("jvmJar", "writeReleaseInfo", "downloadLexicalDb", "downloadEmbedModel")
+    dependsOn("jvmJar", "writeReleaseInfo", "downloadLexicalDb")
     mainClass.set("io.github.kdroidfilter.seforimlibrary.packaging.PackageArtifactsKt")
     classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath")
 

generator/packaging/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/packaging/PackageArtifacts.kt

@@ -72,11 +72,6 @@ fun main(args: Array<String>) {
     // Resolve lexical DB next to the DB
     val lexicalDbPath: Path = dbPath.resolveSibling("lexical.db")
 
-    // Resolve the dense embedding model (int8 ONNX) + tokenizer next to the DB.
-    // Bundled so the app gets dense search out of the box; absent -> lexical only.
-    val embedModelPath: Path = dbPath.resolveSibling("seforim-embed-v5-int8.onnx")
-    val embedTokenizerPath: Path = dbPath.resolveSibling("tokenizer.json")
-
     if (!textIndexDir.toFile().isDirectory) {
         logger.w { "Lucene text index directory missing: $textIndexDir (will skip)" }
     }
@@ -131,8 +126,6 @@ fun main(args: Array<String>) {
             " - Catalog: $catalogPath\n" +
             " - Release info: $releaseInfoPath\n" +
             " - Lexical DB: $lexicalDbPath\n" +
-            " - Embed model: $embedModelPath\n" +
-            " - Embed tokenizer: $embedTokenizerPath\n" +
             " - Text index: $textIndexDir\n" +
             " - Lookup index: $lookupIndexDir\n" +
             " -> Bundle .tar.zst: $bundleOutputPath\n" +
@@ -178,20 +171,6 @@ fun main(args: Array<String>) {
                             logger.w { "Lexical DB missing: $lexicalDbPath (skipped)" }
                         }
 
-                        // Add the dense embedding model + tokenizer if available
-                        if (embedModelPath.exists()) {
-                            addFileToTar(tar, embedModelPath, embedModelPath.fileName.toString(), logger)
-                            logger.i { "Added embedding model to bundle" }
-                        } else {
-                            logger.w { "Embedding model missing: $embedModelPath (skipped, dense search disabled)" }
-                        }
-                        if (embedTokenizerPath.exists()) {
-                            addFileToTar(tar, embedTokenizerPath, embedTokenizerPath.fileName.toString(), logger)
-                            logger.i { "Added embedding tokenizer to bundle" }
-                        } else {
-                            logger.w { "Embedding tokenizer missing: $embedTokenizerPath (skipped)" }
-                        }
-
                         // Add the precomputed catalog if available
                         if (haveCatalog) {
                             addFileToTar(tar, catalogPath, catalogPath.fileName.toString(), logger)

generator/searchindex/build.gradle.kts

@@ -1,12 +1,35 @@
+import java.net.URI
+import java.nio.file.Files
+import java.nio.file.StandardCopyOption
+import java.security.MessageDigest
+
 plugins {
     alias(libs.plugins.multiplatform)
 }
 
 // Generator forked-JVM heap. Honors -PgeneratorHeap=… (CI lowers it on 16 GB runners).
 // Default 10g matches local workstation use; CI sets 5g via the workflow.
+val seforimDb: String = (project.findProperty("seforimDb") as String?)
+    ?: System.getenv("SEFORIM_DB")
+    ?: rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath
+
+// The semantic search (open core): with the token of its private package (settings.gradle.kts), the corpus is embedded
+// into the index's dense vectors. -PvectorsBin=<dir> brings precomputed ones instead; -PnoVectors=true leaves them out.
+val withSemantic = gradle.extensions.extraProperties.run { has("semanticEnabled") && get("semanticEnabled") == true }
+val explicitVectors = project.findProperty("vectorsBin") as String?
+val embedVectors = withSemantic && explicitVectors == null && project.findProperty("noVectors") != "true"
+val vectorsDir: File = rootProject.layout.buildDirectory.dir("vectors").get().asFile
+
+// Embedding runs on an NVIDIA GPU (Linux x64), with the CUDA runtime fetched below; elsewhere on the CPU, for hours
+val cuda = embedVectors &&
+    System.getProperty("os.name").startsWith("Linux") &&
+    System.getProperty("os.arch") in setOf("amd64", "x86_64") &&
+    File("/proc/driver/nvidia/version").exists()
+
+// The vectors (~5.5 GB) are loaded in the indexer's heap
 val generatorHeap: String = (project.findProperty("generatorHeap") as String?)
     ?: System.getenv("SEFORIM_GENERATOR_HEAP")
-    ?: "10g"
+    ?: if (embedVectors || explicitVectors != null) "20g" else "10g"
 
 
 kotlin {
@@ -48,25 +71,21 @@ tasks.register<JavaExec>("buildLuceneIndexDefault") {
     classpath = files(tasks.named("jvmJar")) + configurations.getByName("jvmRuntimeClasspath")
 
     // Pass DB path as system property recognized by the Kotlin entrypoint
-    if (project.hasProperty("seforimDb")) {
-        systemProperty("seforimDb", project.property("seforimDb") as String)
-    } else if (System.getenv("SEFORIM_DB") != null) {
-        systemProperty("seforimDb", System.getenv("SEFORIM_DB"))
-    } else {
-        // Default to DB under root build/
-        val defaultDbPath = rootProject.layout.buildDirectory.file("seforim.db").get().asFile.absolutePath
-        systemProperty("seforimDb", defaultDbPath)
-    }
+    systemProperty("seforimDb", seforimDb)
 
     // Prefer in-memory DB for faster reads (override with -PinMemoryDb=false)
     val inMemory = project.findProperty("inMemoryDb") != "false"
     if (inMemory) {
         systemProperty("inMemoryDb", "true")
     }
 
-    // Optional: -PvectorsBin=/path (dir with ids.i64+vecs.f32+meta.txt) → SINGLE
-    // fused index (text + dense KnnFloatVectorField per line).
-    (project.findProperty("vectorsBin") as String?)?.let { systemProperty("vectorsBin", it) }
+    // Dense vectors (dir with ids.i64+vecs.f32+meta.txt) → SINGLE fused index (text + KnnFloatVectorField per line):
+    // -PvectorsBin=/path, or the corpus embedded by embedCorpus
+    explicitVectors?.let { systemProperty("vectorsBin", it) }
+    if (embedVectors) {
+        dependsOn("embedCorpus")
+        systemProperty("vectorsBin", vectorsDir.absolutePath)
+    }
     // Optional: -PindexThreads=N to cap concurrent indexing threads (lower = less RAM).
     (project.findProperty("indexThreads") as String?)?.let { systemProperty("indexThreads", it) }
 
@@ -78,3 +97,93 @@ tasks.register<JavaExec>("buildLuceneIndexDefault") {
         "--add-modules=jdk.incubator.vector"
     )
 }
+
+if (embedVectors) {
+    val semanticEmbedder = configurations.create("semanticEmbedder") {
+        isCanBeConsumed = false
+        attributes { attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage.JAVA_RUNTIME)) }
+        // onnxruntime_gpu: the same API, plus the CUDA execution provider
+        if (cuda) {
+            resolutionStrategy.dependencySubstitution {
+                substitute(module("com.microsoft.onnxruntime:onnxruntime")).using(module(libs.onnxruntime.gpu.get().toString()))
+            }
+        }
+    }
+    dependencies {
+        add(semanticEmbedder.name, libs.seforim.semantic)
+        // The fp32 model, for the GPU (the int8 one, in seforim-semantic, has no CUDA kernels for its quantized ops)
+        if (cuda) add(semanticEmbedder.name, libs.seforim.semantic.fp32)
+    }
+
+    // CUDA 12 + cuDNN 9 for onnxruntime_gpu, from NVIDIA's redistributables (no CUDA install): only their shared
+    // libraries, in the Gradle user home, fetched once (~2 GB). -PcudaArchivesDir=<dir> takes the archives already
+    // there instead of downloading them (still checked against their sha256)
+    val localArchives = (project.findProperty("cudaArchivesDir") as String?)?.let(::File)
+    val cudaLibDir = gradle.gradleUserHomeDir.resolve("caches/seforim-cuda/cuda-12.8-cudnn-9.10/lib")
+    val prepareCudaRuntime = tasks.register("prepareCudaRuntime") {
+        val archives = mapOf(
+            "cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-12.8.90-archive.tar.xz" to
+                "8d566b5fe745c46842dc16945cf36686227536decd2302c372be86da37faca68",
+            "cuda/redist/libcublas/linux-x86_64/libcublas-linux-x86_64-12.8.4.1-archive.tar.xz" to
+                "21718957c2cf000bacd69d36c95708a2319199e39e056f8b4f0f68e3b9f323bb",
+            "cuda/redist/libcurand/linux-x86_64/libcurand-linux-x86_64-10.3.9.90-archive.tar.xz" to
+                "32a5ec30be446c1b7228d1bc502b2f029cc8b59a5e362c70d960754fa646778b",
+            "cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-9.10.2.21_cuda12-archive.tar.xz" to
+                "d0defcbc4c6dad711ff4cb66d254036a300c9071b07c7b64199aacab534313c1",
+        )
+        val complete = cudaLibDir.resolve(".complete")
+        onlyIf { !complete.exists() }
+        doLast {
+            cudaLibDir.mkdirs()
+            archives.forEach { (path, sha256) ->
+                val name = path.substringAfterLast('/')
+                val local = localArchives?.resolve(name)?.takeIf { it.isFile }
+                val archive = local ?: temporaryDir.resolve(name)
+                if (local == null) {
+                    logger.lifecycle("Downloading $name")
+                    URI("https://developer.download.nvidia.com/compute/$path").toURL().openStream().use {
+                        Files.copy(it, archive.toPath(), StandardCopyOption.REPLACE_EXISTING)
+                    }
+                }
+                val digest = MessageDigest.getInstance("SHA-256")
+                archive.inputStream().use { input ->
+                    val buffer = ByteArray(1 shl 20)
+                    while (true) {
+                        val read = input.read(buffer)
+                        if (read < 0) break
+                        digest.update(buffer, 0, read)
+                    }
+                }
+                val actual = digest.digest().joinToString("") { "%02x".format(it) }
+                check(actual == sha256) { "${archive.name}: sha256 $actual, expected $sha256" }
+                val tar = ProcessBuilder(
+                    "tar", "-xJf", archive.path, "-C", cudaLibDir.path,
+                    "--strip-components=2", "--wildcards", "*/lib/*.so*",
+                ).inheritIO().start()
+                check(tar.waitFor() == 0) { "Could not extract ${archive.name}" }
+                if (local == null) archive.delete()
+            }
+            complete.writeText("")
+        }
+    }
+
+    // Embeds every indexable line of seforim.db into build/vectors (the private package's EmbedCorpus)
+    tasks.register<JavaExec>("embedCorpus") {
+        group = "application"
+        description = "Embed the corpus of seforim.db into dense vectors for the fused Lucene index (semantic search)."
+
+        classpath = semanticEmbedder
+        mainClass.set("io.github.kdroidfilter.seforim.semantic.EmbedCorpusKt")
+        args(
+            "--db", seforimDb,
+            "--out", vectorsDir.path,
+            "--gpu", cuda.toString(),
+        )
+        if (cuda) {
+            dependsOn(prepareCudaRuntime)
+            val inherited = System.getenv("LD_LIBRARY_PATH")
+            environment("LD_LIBRARY_PATH", listOfNotNull(cudaLibDir.path, inherited).joinToString(File.pathSeparator))
+        }
+        jvmArgs("-Xmx8g", "--enable-native-access=ALL-UNNAMED")
+    }
+}

gradle/libs.versions.toml

@@ -5,8 +5,8 @@ kotlin = "2.3.20"
 agp = "9.1.1"
 jvmToolchain = "25"
 lucene = "10.4.0"
-onnxruntime = "1.20.0"
-djlHuggingface = "0.30.0"
+seforimSemantic = "0.3.0"
+onnxruntime = "1.30.0"
 maven-publish = "0.36.0"
 kotlinx-coroutines = "1.10.2"
 kotlinx-serialization = "1.10.0"
@@ -36,8 +36,9 @@ lucene-highlighter = { module = "org.apache.lucene:lucene-highlighter", version.
 lucene-queryparser = { module = "org.apache.lucene:lucene-queryparser", version.ref = "lucene" }
 lucene-analysis-common = { module = "org.apache.lucene:lucene-analysis-common", version.ref = "lucene" }
 lucene-core = { module = "org.apache.lucene:lucene-core", version.ref = "lucene" }
-onnxruntime = { module = "com.microsoft.onnxruntime:onnxruntime", version.ref = "onnxruntime" }
-djl-huggingface-tokenizers = { module = "ai.djl.huggingface:tokenizers", version.ref = "djlHuggingface" }
+seforim-semantic = { module = "io.github.kdroidfilter:seforim-semantic", version.ref = "seforimSemantic" }
+seforim-semantic-fp32 = { module = "io.github.kdroidfilter:seforim-semantic-fp32", version.ref = "seforimSemantic" }
+onnxruntime-gpu = { module = "com.microsoft.onnxruntime:onnxruntime_gpu", version.ref = "onnxruntime" }
 sqlDelight-driver-sqlite = { module = "app.cash.sqldelight:sqlite-driver", version.ref = "sqlDelight" }
 sqlDelight-driver-android = { module = "app.cash.sqldelight:android-driver", version.ref = "sqlDelight" }
 sqlDelight-driver-native = { module = "app.cash.sqldelight:native-driver", version.ref = "sqlDelight" }

search/build.gradle.kts

@@ -20,11 +20,6 @@ kotlin {
             implementation(libs.sqlDelight.driver.sqlite)
             implementation(libs.kermit)
             implementation(libs.jsoup)
-            // Dense semantic search: ONNX Runtime (query embedding) + HuggingFace tokenizer.
-            // Stock Maven `onnxruntime` is CPU-only on desktop JVM (no DirectML/CoreML/XNNPACK
-            // native — those need a custom build; CUDA needs the separate onnxruntime_gpu).
-            implementation(libs.onnxruntime)
-            implementation(libs.djl.huggingface.tokenizers)
         }
 
         jvmTest.dependencies {