fix(sefariasqlite): drop translator footnotes from Noda BiYehudah I, Orach Chaim

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt

@@ -501,7 +501,7 @@ internal class SefariaBookPayloadReader(
         if (depth == 0 || (leafPrimitive != null && leafPrimitive.isString)) {
             val content = leafPrimitive?.takeIf { it.isString }?.content
             if (!content.isNullOrEmpty()) {
-                val cleaned = cleanSefariaLine(content)
+                val cleaned = cleanSefariaLine(content, stripFootnotes = hasTranslatorFootnotes(refPrefix))
                 if (cleaned.isNotEmpty()) {
                     output += linePrefix + cleaned
                     val cleanRef = trimTrailingSeparators(refPrefix)

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt

@@ -18,8 +18,49 @@ private val OTZAR_MARKUP_REGEX = Regex("""@\d{2}([^}]*)\}""")
 // continuous. Collapse them to a single space so paragraphs read as prose.
 private val HTML_LINE_BREAK_REGEX = Regex("""<br\s*/?>""", RegexOption.IGNORE_CASE)
 
-internal fun cleanSefariaLine(raw: String): String {
+// Ref prefixes of sections whose footnotes are a translator's apparatus rather
+// than part of the printed sefer. Noda BiYehudah I's Orach Chaim comes from
+// Harold Landa's edition, whose footnotes mix English notes with Hebrew
+// sources followed by Sefaria's English translations (kdroidFilter/Zayit#439).
+private val TRANSLATOR_FOOTNOTE_REF_PREFIXES = listOf("Noda BiYehudah I, Orach Chaim,")
+
+internal fun hasTranslatorFootnotes(ref: String): Boolean =
+    TRANSLATOR_FOOTNOTE_REF_PREFIXES.any { ref.startsWith(it) }
+
+private val FOOTNOTE_MARKER_REGEX = Regex("""<sup class="footnote-marker">.*?</sup>""")
+private const val FOOTNOTE_OPEN = """<i class="footnote">"""
+private val ITALIC_TAG_REGEX = Regex("""<i\b[^>]*>|</i>""", RegexOption.IGNORE_CASE)
+
+/** Drops footnote markers and bodies; bodies may nest `<i>` tags, so closing tags are matched by depth. */
+internal fun stripSefariaFootnotes(line: String): String {
+    if (!line.contains(FOOTNOTE_OPEN)) return line
+    val withoutMarkers = FOOTNOTE_MARKER_REGEX.replace(line, "")
+    val out = StringBuilder(withoutMarkers.length)
+    var cursor = 0
+    while (true) {
+        val start = withoutMarkers.indexOf(FOOTNOTE_OPEN, cursor)
+        if (start < 0) break
+        out.append(withoutMarkers, cursor, start)
+        var depth = 0
+        var end = withoutMarkers.length
+        for (tag in ITALIC_TAG_REGEX.findAll(withoutMarkers, start)) {
+            depth += if (tag.value.startsWith("</")) -1 else 1
+            if (depth == 0) {
+                end = tag.range.last + 1
+                break
+            }
+        }
+        cursor = end
+    }
+    out.append(withoutMarkers, cursor, withoutMarkers.length)
+    return out.toString()
+}
+
+internal fun cleanSefariaLine(raw: String, stripFootnotes: Boolean = false): String {
     var s = if (raw.contains('\n')) raw.replace("\n", "") else raw
+    if (stripFootnotes) {
+        s = stripSefariaFootnotes(s)
+    }
     if (OTZAR_MARKUP_REGEX.containsMatchIn(s)) {
         s = OTZAR_MARKUP_REGEX.replace(s, "$1")
     }

generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt

@@ -2,6 +2,8 @@ package io.github.kdroidfilter.seforimlibrary.sefariasqlite
 
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
 
 class SefariaCleanSefariaLineTest {
     @Test
@@ -58,4 +60,26 @@ class SefariaCleanSefariaLineTest {
     fun handlesSelfClosingAndUppercaseBr() {
         assertEquals("a b c", cleanSefariaLine("a<br/>b<BR />c"))
     }
+
+    @Test
+    fun keepsFootnotesByDefault() {
+        val line = """א<sup class="footnote-marker">*</sup><i class="footnote">הגה״ה</i>. ב"""
+        assertEquals(line, cleanSefariaLine(line))
+    }
+
+    @Test
+    fun stripsFootnotesWithNestedItalics() {
+        // Shape of Noda BiYehudah I, Orach Chaim 1:20 (Landa's translator footnotes)
+        val line = "ע\"ש<sup class=\"footnote-marker\">38</sup><i class=\"footnote\">הלכות תפילין ג:א<br>" +
+            "<b>There are eight requirements in the making of <i>tefillin</i>.</b> See <i>Sukkah</i> 8a.</i> " +
+            "והנה<sup class=\"footnote-marker\">39</sup><i class=\"footnote\">See note 47</i>, שכן נראה"
+        assertEquals("ע\"ש והנה, שכן נראה", cleanSefariaLine(line, stripFootnotes = true))
+    }
+
+    @Test
+    fun translatorFootnotesAreScopedToNodaBiYehudahOrachChaim() {
+        assertTrue(hasTranslatorFootnotes("Noda BiYehudah I, Orach Chaim,  1 20 "))
+        assertFalse(hasTranslatorFootnotes("Noda BiYehudah I, Author's Introduction,  1 "))
+        assertFalse(hasTranslatorFootnotes("Noda BiYehudah II, Orach Chaim,  1 "))
+    }
 }