Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ internal class SefariaBookPayloadReader(
}

private fun recursiveSections(
sectionNames: List<String>,
sectionNames: List<String?>,
text: JsonElement?,
depth: Int,
level: Int,
Expand Down Expand Up @@ -519,7 +519,8 @@ internal class SefariaBookPayloadReader(
}
if (text !is JsonArray) return
val index = (sectionNames.size - depth).coerceAtLeast(0)
val sectionName = sectionNames.getOrNull(index) ?: ""
// null marks an unnamed level (Sefaria "Integer"): its heading is the bare number.
val sectionName = sectionNames.getOrElse(index) { "" }

val nonEmptyCount = if (depth == 1) {
text.count { !it.isTriviallyEmpty() }
Expand All @@ -545,16 +546,17 @@ internal class SefariaBookPayloadReader(
""
}

if (depth > 1 && sectionName.isNotBlank() && isReferenceable) {
if (depth > 1 && sectionName?.isBlank() != true && isReferenceable) {
val tag = when (level) {
1 -> "<h2>" to "</h2>"
2 -> "<h3>" to "</h3>"
3 -> "<h4>" to "</h4>"
else -> "<h5>" to "</h5>"
}
output += "${tag.first}$sectionName $letter${tag.second}"
val title = sectionName?.let { "$it $letter" } ?: letter
output += "${tag.first}$title${tag.second}"
headings += Heading(
title = "$sectionName $letter",
title = title,
level = level,
lineIndex = output.size - 1
)
Expand Down Expand Up @@ -631,8 +633,9 @@ internal class SefariaBookPayloadReader(
* top-level Hebrew label (e.g. "תשובה", "סימן") was never filled in; the
* English `sectionNames` still has it. Without this fallback the importer
* skips the matching heading and responsa-style books end up with no TOC.
* A null entry marks an unnamed level (see [mapSectionNameToHebrew]).
*/
private fun readHeSectionNames(obj: JsonObject): List<String> {
private fun readHeSectionNames(obj: JsonObject): List<String?> {
val he = obj["heSectionNames"]?.jsonArray?.map { it.jsonPrimitive.contentOrNull.orEmpty() } ?: emptyList()
val en = obj["sectionNames"]?.jsonArray?.map { it.jsonPrimitive.contentOrNull.orEmpty() } ?: emptyList()
val size = maxOf(he.size, en.size)
Expand All @@ -643,7 +646,7 @@ internal class SefariaBookPayloadReader(
heVal
} else {
val enVal = en.getOrNull(idx)?.trim().orEmpty()
mapSectionNameToHebrew(enVal.ifBlank { null }).orEmpty()
if (enVal.isEmpty()) "" else mapSectionNameToHebrew(enVal)
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,19 @@ internal fun cleanSefariaLine(raw: String): String {
* matching `heSectionNames`, so the generated TOC labels fall back gracefully
* instead of showing English strings mid-Hebrew UI.
*
* Returns the original string if no mapping applies, or `null` for blank input.
* Returns the original string if no mapping applies, or `null` for blank input
* and for unnamed levels: Sefaria uses "Integer" as a placeholder name and shows
* those sections by their number alone (its Hebrew refs read "א׳:ב׳", and
* "Integer" has no Hebrew term in its dictionary).
*/
internal fun mapSectionNameToHebrew(base: String?): String? {
if (base.isNullOrBlank()) return null
val norm = base.lowercase()
val norm = base.lowercase().trim('\'', '"', ' ')
return when {
// Short tokens are matched exactly: as substrings they would hit unrelated names.
norm == "integer" -> null
norm == "ot" -> "אות"
norm == "dh" -> "דיבור המתחיל"
"aliyah" in norm || "aliya" in norm -> "עליה"
"daf" in norm -> "דף"
"chapter" in norm -> "פרק"
Expand All @@ -69,6 +76,37 @@ internal fun mapSectionNameToHebrew(base: String?): String? {
"column" in norm -> "טור"
"folio" in norm -> "דף"
"segment" in norm -> "קטע"
"question" in norm -> "שאלה"
"comment" in norm -> "פירוש"
"footnote" in norm -> "הערה"
"pararaph" in norm -> "פסקה"
"se'if" in norm -> "סעיף"
"passuk" in norm -> "פסוק"
"sheilta" in norm -> "שאילתא"
"tosefta" in norm -> "תוספתא"
"midrash" in norm -> "מדרש"
"drush" in norm -> "דרוש"
"remez" in norm -> "רמז"
"inyan" in norm -> "ענין"
"mitzvah" in norm -> "מצוה"
"shorash" in norm -> "שורש"
"piyyut" in norm -> "פיוט"
"hadran" in norm -> "הדרן"
"kovetz" in norm -> "קובץ"
"gate" in norm || "shaar" in norm || "sha'ar" in norm -> "שער"
"essay" in norm || "treatise" in norm -> "מאמר"
"statement" in norm -> "סימן"
"principle" in norm -> "עיקר"
"epistle" in norm -> "אגרת"
"letter" in norm -> "אות"
"page" in norm -> "עמוד"
"word" in norm -> "ערך"
"room" in norm || "chamber" in norm -> "חדר"
"window" in norm -> "חלון"
"book" in norm -> "ספר"
"nahar" in norm -> "נהר"
"maayan" in norm -> "מעין"
"shoket" in norm -> "שוקת"
else -> base
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package io.github.kdroidfilter.seforimlibrary.sefariasqlite

import co.touchlab.kermit.Logger
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.json.Json
import java.nio.file.Files
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertTrue

/**
* Regression for https://github.com/kdroidFilter/Zayit/issues/501: the Epistle of
* Rav Sherira Gaon ships `heSectionNames=["", "פסקה"]` with `sectionNames=["Question", ...]`,
* and the unmapped English "Question" leaked into the Hebrew headings.
*/
class SefariaSectionNameMappingTest {
@Test
fun mapsEnglishSectionNamesFoundInSefariaSchemas() {
val expected = mapOf(
"Question" to "שאלה",
"Comment" to "פירוש",
"'Comment'" to "פירוש",
"DH" to "דיבור המתחיל",
"Ot" to "אות",
"Sheilta" to "שאילתא",
"Gate" to "שער",
"Sha'ar Ha'Gemul" to "שער",
"Se'if" to "סעיף",
"Passuk" to "פסוק",
"Room" to "חדר",
"Letter" to "אות",
"Principle" to "עיקר",
"Statement" to "סימן",
)
expected.forEach { (en, he) -> assertEquals(he, mapSectionNameToHebrew(en), "mapping for '$en'") }
}

@Test
fun shortTokensDoNotMatchAsSubstrings() {
// "ot" must not hijack names that merely contain it
assertEquals("הערה", mapSectionNameToHebrew("Footnote"))
assertEquals("Manuscript", mapSectionNameToHebrew("Manuscript"))
}

@Test
fun integerLevelIsUnnamed() {
assertNull(mapSectionNameToHebrew("Integer"))
}

/**
* Sefaria shows "Integer" levels by their bare number (its Hebrew refs read
* "א׳:א׳:א׳"), as in Sha'arei Torat Eretz Yisrael (Chapter/Halakhah/Integer/Integer).
*/
@Test
fun integerLevelHeadingIsTheBareNumber() = runBlocking {
val tempDir = Files.createTempDirectory("seforim-integer")
val schemaDir = Files.createDirectories(tempDir.resolve("schemas"))
val bookDir = Files.createDirectories(tempDir.resolve("json").resolve("Sample"))
Files.writeString(
schemaDir.resolve("Sample.json"),
"""
{
"title": "Sample",
"heTitle": "דוגמה",
"schema": {
"nodeType": "JaggedArrayNode",
"depth": 4,
"addressTypes": ["Perek", "Halakhah", "Integer", "Integer"],
"sectionNames": ["Chapter", "Halakhah", "Integer", "Integer"],
"title": "Sample",
"heTitle": "דוגמה"
}
}
""".trimIndent()
)
Files.writeString(
bookDir.resolve("merged.json"),
"""{"title": "Sample", "heTitle": "דוגמה", "text": [[[["a1", "a2"], ["b1"]]]]}"""
)

val reader = SefariaBookPayloadReader(
Json { ignoreUnknownKeys = true; coerceInputValues = true },
Logger.withTag("SefariaSectionNameMappingTest")
)
val payload = reader.readBooksInParallel(
tempDir.resolve("json"), schemaDir, reader.buildSchemaLookup(schemaDir)
).single()
val titles = payload.headings.map { it.title }

assertEquals(listOf("דוגמה", "פרק א", "הלכה א", "א", "ב"), titles)
assertTrue(payload.lines.none { it.contains("Integer") }, "no English label in ${payload.lines}")
}
}
Loading