diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index 00e1abbd..fa6e89de 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -501,7 +501,7 @@ internal class SefariaBookPayloadReader( if (depth == 0 || (leafPrimitive != null && leafPrimitive.isString)) { val content = leafPrimitive?.takeIf { it.isString }?.content if (!content.isNullOrEmpty()) { - val cleaned = cleanSefariaLine(content) + val cleaned = cleanSefariaLine(content, stripFootnotes = hasTranslatorFootnotes(refPrefix)) if (cleaned.isNotEmpty()) { output += linePrefix + cleaned val cleanRef = trimTrailingSeparators(refPrefix) diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index 0ab0bdb6..6020d4d1 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -18,8 +18,49 @@ private val OTZAR_MARKUP_REGEX = Regex("""@\d{2}([^}]*)\}""") // continuous. Collapse them to a single space so paragraphs read as prose. private val HTML_LINE_BREAK_REGEX = Regex("""""", RegexOption.IGNORE_CASE) -internal fun cleanSefariaLine(raw: String): String { +// Ref prefixes of sections whose footnotes are a translator's apparatus rather +// than part of the printed sefer. Noda BiYehudah I's Orach Chaim comes from +// Harold Landa's edition, whose footnotes mix English notes with Hebrew +// sources followed by Sefaria's English translations (kdroidFilter/Zayit#439). +private val TRANSLATOR_FOOTNOTE_REF_PREFIXES = listOf("Noda BiYehudah I, Orach Chaim,") + +internal fun hasTranslatorFootnotes(ref: String): Boolean = + TRANSLATOR_FOOTNOTE_REF_PREFIXES.any { ref.startsWith(it) } + +private val FOOTNOTE_MARKER_REGEX = Regex(""".*?""") +private const val FOOTNOTE_OPEN = """""" +private val ITALIC_TAG_REGEX = Regex("""]*>|""", RegexOption.IGNORE_CASE) + +/** Drops footnote markers and bodies; bodies may nest `` tags, so closing tags are matched by depth. */ +internal fun stripSefariaFootnotes(line: String): String { + if (!line.contains(FOOTNOTE_OPEN)) return line + val withoutMarkers = FOOTNOTE_MARKER_REGEX.replace(line, "") + val out = StringBuilder(withoutMarkers.length) + var cursor = 0 + while (true) { + val start = withoutMarkers.indexOf(FOOTNOTE_OPEN, cursor) + if (start < 0) break + out.append(withoutMarkers, cursor, start) + var depth = 0 + var end = withoutMarkers.length + for (tag in ITALIC_TAG_REGEX.findAll(withoutMarkers, start)) { + depth += if (tag.value.startsWith("b
c")) } + + @Test + fun keepsFootnotesByDefault() { + val line = """א*הגה״ה. ב""" + assertEquals(line, cleanSefariaLine(line)) + } + + @Test + fun stripsFootnotesWithNestedItalics() { + // Shape of Noda BiYehudah I, Orach Chaim 1:20 (Landa's translator footnotes) + val line = "ע\"ש38הלכות תפילין ג:א
" + + "There are eight requirements in the making of tefillin. See Sukkah 8a.
" + + "והנה39See note 47, שכן נראה" + assertEquals("ע\"ש והנה, שכן נראה", cleanSefariaLine(line, stripFootnotes = true)) + } + + @Test + fun translatorFootnotesAreScopedToNodaBiYehudahOrachChaim() { + assertTrue(hasTranslatorFootnotes("Noda BiYehudah I, Orach Chaim, 1 20 ")) + assertFalse(hasTranslatorFootnotes("Noda BiYehudah I, Author's Introduction, 1 ")) + assertFalse(hasTranslatorFootnotes("Noda BiYehudah II, Orach Chaim, 1 ")) + } }