diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index 457070f8..3e7c4d78 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -468,7 +468,7 @@ internal class SefariaBookPayloadReader( } private fun recursiveSections( - sectionNames: List, + sectionNames: List, text: JsonElement?, depth: Int, level: Int, @@ -519,7 +519,8 @@ internal class SefariaBookPayloadReader( } if (text !is JsonArray) return val index = (sectionNames.size - depth).coerceAtLeast(0) - val sectionName = sectionNames.getOrNull(index) ?: "" + // null marks an unnamed level (Sefaria "Integer"): its heading is the bare number. + val sectionName = sectionNames.getOrElse(index) { "" } val nonEmptyCount = if (depth == 1) { text.count { !it.isTriviallyEmpty() } @@ -545,16 +546,17 @@ internal class SefariaBookPayloadReader( "" } - if (depth > 1 && sectionName.isNotBlank() && isReferenceable) { + if (depth > 1 && sectionName?.isBlank() != true && isReferenceable) { val tag = when (level) { 1 -> "

" to "

" 2 -> "

" to "

" 3 -> "

" to "

" else -> "
" to "
" } - output += "${tag.first}$sectionName $letter${tag.second}" + val title = sectionName?.let { "$it $letter" } ?: letter + output += "${tag.first}$title${tag.second}" headings += Heading( - title = "$sectionName $letter", + title = title, level = level, lineIndex = output.size - 1 ) @@ -631,8 +633,9 @@ internal class SefariaBookPayloadReader( * top-level Hebrew label (e.g. "תשובה", "סימן") was never filled in; the * English `sectionNames` still has it. Without this fallback the importer * skips the matching heading and responsa-style books end up with no TOC. + * A null entry marks an unnamed level (see [mapSectionNameToHebrew]). */ - private fun readHeSectionNames(obj: JsonObject): List { + private fun readHeSectionNames(obj: JsonObject): List { val he = obj["heSectionNames"]?.jsonArray?.map { it.jsonPrimitive.contentOrNull.orEmpty() } ?: emptyList() val en = obj["sectionNames"]?.jsonArray?.map { it.jsonPrimitive.contentOrNull.orEmpty() } ?: emptyList() val size = maxOf(he.size, en.size) @@ -643,7 +646,7 @@ internal class SefariaBookPayloadReader( heVal } else { val enVal = en.getOrNull(idx)?.trim().orEmpty() - mapSectionNameToHebrew(enVal.ifBlank { null }).orEmpty() + if (enVal.isEmpty()) "" else mapSectionNameToHebrew(enVal) } } } diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index 69d54f52..0ab0bdb6 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -40,12 +40,19 @@ internal fun cleanSefariaLine(raw: String): String { * matching `heSectionNames`, so the generated TOC labels fall back gracefully * instead of showing English strings mid-Hebrew UI. * - * Returns the original string if no mapping applies, or `null` for blank input. + * Returns the original string if no mapping applies, or `null` for blank input + * and for unnamed levels: Sefaria uses "Integer" as a placeholder name and shows + * those sections by their number alone (its Hebrew refs read "א׳:ב׳", and + * "Integer" has no Hebrew term in its dictionary). */ internal fun mapSectionNameToHebrew(base: String?): String? { if (base.isNullOrBlank()) return null - val norm = base.lowercase() + val norm = base.lowercase().trim('\'', '"', ' ') return when { + // Short tokens are matched exactly: as substrings they would hit unrelated names. + norm == "integer" -> null + norm == "ot" -> "אות" + norm == "dh" -> "דיבור המתחיל" "aliyah" in norm || "aliya" in norm -> "עליה" "daf" in norm -> "דף" "chapter" in norm -> "פרק" @@ -69,6 +76,37 @@ internal fun mapSectionNameToHebrew(base: String?): String? { "column" in norm -> "טור" "folio" in norm -> "דף" "segment" in norm -> "קטע" + "question" in norm -> "שאלה" + "comment" in norm -> "פירוש" + "footnote" in norm -> "הערה" + "pararaph" in norm -> "פסקה" + "se'if" in norm -> "סעיף" + "passuk" in norm -> "פסוק" + "sheilta" in norm -> "שאילתא" + "tosefta" in norm -> "תוספתא" + "midrash" in norm -> "מדרש" + "drush" in norm -> "דרוש" + "remez" in norm -> "רמז" + "inyan" in norm -> "ענין" + "mitzvah" in norm -> "מצוה" + "shorash" in norm -> "שורש" + "piyyut" in norm -> "פיוט" + "hadran" in norm -> "הדרן" + "kovetz" in norm -> "קובץ" + "gate" in norm || "shaar" in norm || "sha'ar" in norm -> "שער" + "essay" in norm || "treatise" in norm -> "מאמר" + "statement" in norm -> "סימן" + "principle" in norm -> "עיקר" + "epistle" in norm -> "אגרת" + "letter" in norm -> "אות" + "page" in norm -> "עמוד" + "word" in norm -> "ערך" + "room" in norm || "chamber" in norm -> "חדר" + "window" in norm -> "חלון" + "book" in norm -> "ספר" + "nahar" in norm -> "נהר" + "maayan" in norm -> "מעין" + "shoket" in norm -> "שוקת" else -> base } } diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSectionNameMappingTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSectionNameMappingTest.kt new file mode 100644 index 00000000..1e6aa979 --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSectionNameMappingTest.kt @@ -0,0 +1,94 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import co.touchlab.kermit.Logger +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Regression for https://github.com/kdroidFilter/Zayit/issues/501: the Epistle of + * Rav Sherira Gaon ships `heSectionNames=["", "פסקה"]` with `sectionNames=["Question", ...]`, + * and the unmapped English "Question" leaked into the Hebrew headings. + */ +class SefariaSectionNameMappingTest { + @Test + fun mapsEnglishSectionNamesFoundInSefariaSchemas() { + val expected = mapOf( + "Question" to "שאלה", + "Comment" to "פירוש", + "'Comment'" to "פירוש", + "DH" to "דיבור המתחיל", + "Ot" to "אות", + "Sheilta" to "שאילתא", + "Gate" to "שער", + "Sha'ar Ha'Gemul" to "שער", + "Se'if" to "סעיף", + "Passuk" to "פסוק", + "Room" to "חדר", + "Letter" to "אות", + "Principle" to "עיקר", + "Statement" to "סימן", + ) + expected.forEach { (en, he) -> assertEquals(he, mapSectionNameToHebrew(en), "mapping for '$en'") } + } + + @Test + fun shortTokensDoNotMatchAsSubstrings() { + // "ot" must not hijack names that merely contain it + assertEquals("הערה", mapSectionNameToHebrew("Footnote")) + assertEquals("Manuscript", mapSectionNameToHebrew("Manuscript")) + } + + @Test + fun integerLevelIsUnnamed() { + assertNull(mapSectionNameToHebrew("Integer")) + } + + /** + * Sefaria shows "Integer" levels by their bare number (its Hebrew refs read + * "א׳:א׳:א׳"), as in Sha'arei Torat Eretz Yisrael (Chapter/Halakhah/Integer/Integer). + */ + @Test + fun integerLevelHeadingIsTheBareNumber() = runBlocking { + val tempDir = Files.createTempDirectory("seforim-integer") + val schemaDir = Files.createDirectories(tempDir.resolve("schemas")) + val bookDir = Files.createDirectories(tempDir.resolve("json").resolve("Sample")) + Files.writeString( + schemaDir.resolve("Sample.json"), + """ + { + "title": "Sample", + "heTitle": "דוגמה", + "schema": { + "nodeType": "JaggedArrayNode", + "depth": 4, + "addressTypes": ["Perek", "Halakhah", "Integer", "Integer"], + "sectionNames": ["Chapter", "Halakhah", "Integer", "Integer"], + "title": "Sample", + "heTitle": "דוגמה" + } + } + """.trimIndent() + ) + Files.writeString( + bookDir.resolve("merged.json"), + """{"title": "Sample", "heTitle": "דוגמה", "text": [[[["a1", "a2"], ["b1"]]]]}""" + ) + + val reader = SefariaBookPayloadReader( + Json { ignoreUnknownKeys = true; coerceInputValues = true }, + Logger.withTag("SefariaSectionNameMappingTest") + ) + val payload = reader.readBooksInParallel( + tempDir.resolve("json"), schemaDir, reader.buildSchemaLookup(schemaDir) + ).single() + val titles = payload.headings.map { it.title } + + assertEquals(listOf("דוגמה", "פרק א", "הלכה א", "א", "ב"), titles) + assertTrue(payload.lines.none { it.contains("Integer") }, "no English label in ${payload.lines}") + } +}