From 23ddc2963650ba34e469f68e964afd407585d033 Mon Sep 17 00:00:00 2001
From: Elie Gambache <elyahou.hadass@gmail.com>
Date: Thu, 8 Oct 2026 13:07:11 +0300
Subject: [PATCH] fix(sefariasqlite): drop translator footnotes from Noda
 BiYehudah I, Orach Chaim
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Sefaria takes this section from Harold Landa's edition, whose 60 footnotes
are a translator's apparatus: English notes ("See note 47") and Hebrew
sources followed by Sefaria's English translations. They were inlined as
text, so the reader saw long English passages inside the sefer.

Strip footnote markers and bodies (nested <i> tags handled) for refs under
"Noda BiYehudah I, Orach Chaim". Footnotes elsewhere are kept: the author's
own הגה״ה in the introduction, and the English bibliography of other books.

Fixes kdroidFilter/Zayit#439

Signed-off-by: Elie Gambache <elyahou.hadass@gmail.com>
---
 .../sefariasqlite/SefariaBookPayloadReader.kt |  2 +-
 .../sefariasqlite/SefariaImportText.kt        | 43 ++++++++++++++++++-
 .../SefariaCleanSefariaLineTest.kt            | 24 +++++++++++
 3 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt
index 00e1abbd..fa6e89de 100644
--- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt
+++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt
@@ -501,7 +501,7 @@ internal class SefariaBookPayloadReader(
         if (depth == 0 || (leafPrimitive != null && leafPrimitive.isString)) {
             val content = leafPrimitive?.takeIf { it.isString }?.content
             if (!content.isNullOrEmpty()) {
-                val cleaned = cleanSefariaLine(content)
+                val cleaned = cleanSefariaLine(content, stripFootnotes = hasTranslatorFootnotes(refPrefix))
                 if (cleaned.isNotEmpty()) {
                     output += linePrefix + cleaned
                     val cleanRef = trimTrailingSeparators(refPrefix)
diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt
index 0ab0bdb6..6020d4d1 100644
--- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt
+++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt
@@ -18,8 +18,49 @@ private val OTZAR_MARKUP_REGEX = Regex("""@\d{2}([^}]*)\}""")
 // continuous. Collapse them to a single space so paragraphs read as prose.
 private val HTML_LINE_BREAK_REGEX = Regex("""<br\s*/?>""", RegexOption.IGNORE_CASE)
 
-internal fun cleanSefariaLine(raw: String): String {
+// Ref prefixes of sections whose footnotes are a translator's apparatus rather
+// than part of the printed sefer. Noda BiYehudah I's Orach Chaim comes from
+// Harold Landa's edition, whose footnotes mix English notes with Hebrew
+// sources followed by Sefaria's English translations (kdroidFilter/Zayit#439).
+private val TRANSLATOR_FOOTNOTE_REF_PREFIXES = listOf("Noda BiYehudah I, Orach Chaim,")
+
+internal fun hasTranslatorFootnotes(ref: String): Boolean =
+    TRANSLATOR_FOOTNOTE_REF_PREFIXES.any { ref.startsWith(it) }
+
+private val FOOTNOTE_MARKER_REGEX = Regex("""<sup class="footnote-marker">.*?</sup>""")
+private const val FOOTNOTE_OPEN = """<i class="footnote">"""
+private val ITALIC_TAG_REGEX = Regex("""<i\b[^>]*>|</i>""", RegexOption.IGNORE_CASE)
+
+/** Drops footnote markers and bodies; bodies may nest `<i>` tags, so closing tags are matched by depth. */
+internal fun stripSefariaFootnotes(line: String): String {
+    if (!line.contains(FOOTNOTE_OPEN)) return line
+    val withoutMarkers = FOOTNOTE_MARKER_REGEX.replace(line, "")
+    val out = StringBuilder(withoutMarkers.length)
+    var cursor = 0
+    while (true) {
+        val start = withoutMarkers.indexOf(FOOTNOTE_OPEN, cursor)
+        if (start < 0) break
+        out.append(withoutMarkers, cursor, start)
+        var depth = 0
+        var end = withoutMarkers.length
+        for (tag in ITALIC_TAG_REGEX.findAll(withoutMarkers, start)) {
+            depth += if (tag.value.startsWith("</")) -1 else 1
+            if (depth == 0) {
+                end = tag.range.last + 1
+                break
+            }
+        }
+        cursor = end
+    }
+    out.append(withoutMarkers, cursor, withoutMarkers.length)
+    return out.toString()
+}
+
+internal fun cleanSefariaLine(raw: String, stripFootnotes: Boolean = false): String {
     var s = if (raw.contains('\n')) raw.replace("\n", "") else raw
+    if (stripFootnotes) {
+        s = stripSefariaFootnotes(s)
+    }
     if (OTZAR_MARKUP_REGEX.containsMatchIn(s)) {
         s = OTZAR_MARKUP_REGEX.replace(s, "$1")
     }
diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt
index 41843c10..9da06b5f 100644
--- a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt
+++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaCleanSefariaLineTest.kt
@@ -2,6 +2,8 @@ package io.github.kdroidfilter.seforimlibrary.sefariasqlite
 
 import kotlin.test.Test
 import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
 
 class SefariaCleanSefariaLineTest {
     @Test
@@ -58,4 +60,26 @@ class SefariaCleanSefariaLineTest {
     fun handlesSelfClosingAndUppercaseBr() {
         assertEquals("a b c", cleanSefariaLine("a<br/>b<BR />c"))
     }
+
+    @Test
+    fun keepsFootnotesByDefault() {
+        val line = """א<sup class="footnote-marker">*</sup><i class="footnote">הגה״ה</i>. ב"""
+        assertEquals(line, cleanSefariaLine(line))
+    }
+
+    @Test
+    fun stripsFootnotesWithNestedItalics() {
+        // Shape of Noda BiYehudah I, Orach Chaim 1:20 (Landa's translator footnotes)
+        val line = "ע\"ש<sup class=\"footnote-marker\">38</sup><i class=\"footnote\">הלכות תפילין ג:א<br>" +
+            "<b>There are eight requirements in the making of <i>tefillin</i>.</b> See <i>Sukkah</i> 8a.</i> " +
+            "והנה<sup class=\"footnote-marker\">39</sup><i class=\"footnote\">See note 47</i>, שכן נראה"
+        assertEquals("ע\"ש והנה, שכן נראה", cleanSefariaLine(line, stripFootnotes = true))
+    }
+
+    @Test
+    fun translatorFootnotesAreScopedToNodaBiYehudahOrachChaim() {
+        assertTrue(hasTranslatorFootnotes("Noda BiYehudah I, Orach Chaim,  1 20 "))
+        assertFalse(hasTranslatorFootnotes("Noda BiYehudah I, Author's Introduction,  1 "))
+        assertFalse(hasTranslatorFootnotes("Noda BiYehudah II, Orach Chaim,  1 "))
+    }
 }