fix(sefariasqlite): link the whole Darkhei Moshe to the Tur through the Beit Yosef

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt

@@ -11,6 +11,7 @@ import io.github.kdroidfilter.seforimlibrary.common.ids.InMemoryIdAllocator
 import io.github.kdroidfilter.seforimlibrary.core.models.Author
 import io.github.kdroidfilter.seforimlibrary.core.models.Book
 import io.github.kdroidfilter.seforimlibrary.core.models.Category
+import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType
 import io.github.kdroidfilter.seforimlibrary.core.models.Line
 import io.github.kdroidfilter.seforimlibrary.core.text.HebrewTextUtils
 import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository
@@ -444,15 +445,29 @@ class SefariaDirectImporter(
         val linksDir = dbRoot.resolve("links")
         if (linksDir.exists()) {
             logger.i { "Processing links (${headingLineIds.size} heading lines will be excluded from link targets)..." }
+            // Commentary links between the books of sparsely linked commentaries (see
+            // SefariaSparseCommentaryLinks.kt). Filled by the single inserter coroutine.
+            val sparseCommentaries = collectSparseCommentaries(allRefsWithPath, lineKeyToId, lineIdToBookId)
+            val sparseLinks = sparseCommentaries.associateWith { SparseCommentaryLinks() }
             linksImporter.processLinksInParallel(
                 linksDir = linksDir,
                 refsByCanonical = refsByCanonical,
                 refsByBase = refsByBase,
                 lineKeyToId = lineKeyToId,
                 lineIdToBookId = lineIdToBookId,
                 bookMetaById = bookMetaById,
-                headingLineIds = headingLineIds
+                headingLineIds = headingLineIds,
+                onLinkStored = { link ->
+                    if (link.connectionType == ConnectionType.COMMENTARY) {
+                        sparseLinks.forEach { (commentary, links) -> links.record(commentary, link) }
+                    }
+                },
             )
+            sparseLinks.forEach { (commentary, links) ->
+                val anchors = anchorUnlinkedCommentaryLines(commentary, links)
+                linksImporter.insertCommentaryAnchors(commentary, anchors, bookMetaById)
+                logger.i { "Anchored ${anchors.size} lines of book ${commentary.commentaryBookId} to its base text" }
+            }
             logger.i { "Links processed" }
         }
 

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt

@@ -25,7 +25,8 @@ internal class SefariaLinksImporter(
         lineKeyToId: Map<Pair<String, Int>, Long>,
         lineIdToBookId: Map<Long, Long>,
         bookMetaById: Map<Long, BookMeta>,
-        headingLineIds: Set<Long> = emptySet()
+        headingLineIds: Set<Long> = emptySet(),
+        onLinkStored: (Link) -> Unit = {},
     ) = coroutineScope {
         // Pre-register all connection types we'll use so their ids are stable
         // (so `link.connectionTypeId` is reproducible across builds).
@@ -60,6 +61,7 @@ internal class SefariaLinksImporter(
         val inserter = launch {
             val batch = mutableListOf<Link>()
             for (link in linkChannel) {
+                onLinkStored(link)
                 batch += link
                 if (batch.size >= SefariaImportTuning.LINK_BATCH_SIZE) {
                     repository.insertLinksBatch(batch)
@@ -232,6 +234,36 @@ internal class SefariaLinksImporter(
         }
     }
 
+    /**
+     * Stores the [anchors] of a sparsely linked commentary as COMMENTARY links from its base
+     * text. See [anchorUnlinkedCommentaryLines].
+     */
+    suspend fun insertCommentaryAnchors(
+        commentary: SparseCommentary,
+        anchors: List<CommentaryAnchor>,
+        bookMetaById: Map<Long, BookMeta>,
+    ) {
+        val typeId = bindings.upsertConnectionType(ConnectionType.COMMENTARY.name)
+        val isDeclaredBase = bookMetaById[commentary.commentaryBookId]
+            ?.sefariaDeclaredBaseTextBookIds
+            ?.contains(commentary.baseBookId) == true
+        anchors
+            .map { anchor ->
+                Link(
+                    id = bindings.allocator.linkId(anchor.baseLineId, anchor.commentaryLine.lineId, typeId),
+                    sourceBookId = commentary.baseBookId,
+                    targetBookId = commentary.commentaryBookId,
+                    sourceLineId = anchor.baseLineId,
+                    targetLineId = anchor.commentaryLine.lineId,
+                    targetLineIndex = anchor.commentaryLine.lineIndex,
+                    connectionType = ConnectionType.COMMENTARY,
+                    isDeclaredBase = isDeclaredBase,
+                )
+            }
+            .chunked(SefariaImportTuning.LINK_BATCH_SIZE)
+            .forEach { repository.insertLinksBatch(it) }
+    }
+
     private data class StoredLink(
         val srcBookId: Long,
         val tgtBookId: Long,

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt

@@ -0,0 +1,126 @@
+package io.github.kdroidfilter.seforimlibrary.sefariasqlite
+
+import io.github.kdroidfilter.seforimlibrary.core.models.Link
+
+/**
+ * A commentary that Sefaria links to its base text only partly, the rest of it being linked
+ * to an intermediate commentary on that base. English titles.
+ */
+private data class SparseCommentaryTitles(val commentary: String, val intermediate: String, val base: String)
+
+/**
+ * The Darkhei Moshe glosses the Beit Yosef and follows its seif numbering. Sefaria links a
+ * Darkhei Moshe seif to the Tur only when the Tur carries its own marker for it (about 2,000
+ * of 6,000 seifim); the others are linked to the Beit Yosef seif they gloss, so they never
+ * showed next to the Tur (Zayit issue #462).
+ */
+private val sparselyLinkedCommentaries = listOf(
+    SparseCommentaryTitles(commentary = "Darkhei Moshe", intermediate = "Beit Yosef", base = "Tur"),
+)
+
+/**
+ * A content line of a book with its top-level section (e.g. "orach chayim 5") and its
+ * segment address (e.g. "orach chayim 5:2" for "5:2:1").
+ */
+internal data class SectionLine(val lineId: Long, val lineIndex: Int, val section: String, val segment: String)
+
+/** A sparsely linked commentary, the commentary it is linked through, and their base text. */
+internal data class SparseCommentary(
+    val commentaryBookId: Long,
+    val intermediateBookId: Long,
+    val baseBookId: Long,
+    val commentaryLines: List<SectionLine>,
+    val baseLines: List<SectionLine>,
+)
+
+/** A missing link from a base line to a commentary line. */
+internal data class CommentaryAnchor(val baseLineId: Long, val commentaryLine: SectionLine)
+
+/** Collects the books of every [sparselyLinkedCommentaries] entry present in the export. */
+internal fun collectSparseCommentaries(
+    refs: List<RefEntry>,
+    lineKeyToId: Map<Pair<String, Int>, Long>,
+    lineIdToBookId: Map<Long, Long>,
+): List<SparseCommentary> {
+    fun linesOf(title: String): Pair<Long, List<SectionLine>>? {
+        val lines = refs
+            .filter { it.ref.startsWith("$title, ") }
+            .mapNotNull { entry ->
+                val lineId = lineKeyToId[entry.path to entry.lineIndex - 1] ?: return@mapNotNull null
+                val address = canonicalCitation(entry.ref.removePrefix(title))
+                SectionLine(
+                    lineId = lineId,
+                    lineIndex = entry.lineIndex - 1,
+                    section = address.substringBefore(':').trim(),
+                    segment = address.substringBeforeLast(':').trim(),
+                )
+            }
+            .sortedBy { it.lineIndex }
+        val bookId = lines.firstOrNull()?.let { lineIdToBookId[it.lineId] } ?: return null
+        return bookId to lines
+    }
+
+    return sparselyLinkedCommentaries.mapNotNull { titles ->
+        val (commentaryBookId, commentaryLines) = linesOf(titles.commentary) ?: return@mapNotNull null
+        val (intermediateBookId, _) = linesOf(titles.intermediate) ?: return@mapNotNull null
+        val (baseBookId, baseLines) = linesOf(titles.base) ?: return@mapNotNull null
+        SparseCommentary(commentaryBookId, intermediateBookId, baseBookId, commentaryLines, baseLines)
+    }
+}
+
+/**
+ * Commentary links stored between the books of a [SparseCommentary], each keyed by the
+ * dependant line id with the ids of the lines it depends on.
+ */
+internal class SparseCommentaryLinks {
+    val baseByCommentaryLine = HashMap<Long, MutableSet<Long>>()
+    val intermediateByCommentaryLine = HashMap<Long, MutableSet<Long>>()
+    val baseByIntermediateLine = HashMap<Long, MutableSet<Long>>()
+
+    /** Records a stored COMMENTARY [link] (base → dependant) if it joins two books of [commentary]. */
+    fun record(commentary: SparseCommentary, link: Link) {
+        val map = when (link.sourceBookId to link.targetBookId) {
+            commentary.baseBookId to commentary.commentaryBookId -> baseByCommentaryLine
+            commentary.intermediateBookId to commentary.commentaryBookId -> intermediateByCommentaryLine
+            commentary.baseBookId to commentary.intermediateBookId -> baseByIntermediateLine
+            else -> return
+        }
+        map.getOrPut(link.targetLineId) { HashSet() } += link.sourceLineId
+    }
+}
+
+/**
+ * Anchors the commentary lines without a link to their base text through the intermediate
+ * commentary: a line linked to an intermediate segment gets that segment's base lines. Lines
+ * of a segment share its anchors, so a segment's later paragraphs follow its first one, and a
+ * segment linked to the base keeps those links. Only base lines of the commentary line's own
+ * section are kept, so lateral citations of other sections are not followed. Lines linked to
+ * neither book stay unlinked.
+ */
+internal fun anchorUnlinkedCommentaryLines(
+    commentary: SparseCommentary,
+    links: SparseCommentaryLinks,
+): List<CommentaryAnchor> {
+    val baseSectionByLineId = commentary.baseLines.associate { it.lineId to it.section }
+
+    fun SectionLine.inOwnSection(baseLineIds: Iterable<Long>): Set<Long> =
+        baseLineIds.filterTo(LinkedHashSet()) { baseSectionByLineId[it] == section }
+
+    fun SectionLine.directBase(): Set<Long> = inOwnSection(links.baseByCommentaryLine[lineId].orEmpty())
+
+    fun SectionLine.baseThroughIntermediate(): Set<Long> = inOwnSection(
+        links.intermediateByCommentaryLine[lineId].orEmpty()
+            .flatMap { links.baseByIntermediateLine[it].orEmpty() }
+    )
+
+    return commentary.commentaryLines
+        .groupBy { it.segment }
+        .values
+        .flatMap { segment ->
+            val baseLineIds = segment.flatMapTo(LinkedHashSet()) { it.directBase() }
+                .ifEmpty { segment.flatMapTo(LinkedHashSet()) { it.baseThroughIntermediate() } }
+            segment
+                .filter { it.directBase().isEmpty() }
+                .flatMap { line -> baseLineIds.map { CommentaryAnchor(it, line) } }
+        }
+}

generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt

@@ -0,0 +1,87 @@
+package io.github.kdroidfilter.seforimlibrary.sefariasqlite
+
+import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType
+import io.github.kdroidfilter.seforimlibrary.core.models.Link
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+/**
+ * Regression for Zayit issue #462: Sefaria links most Darkhei Moshe seifim to the Beit Yosef
+ * seif they gloss instead of the Tur, so they never showed next to the Tur.
+ */
+class SefariaSparseCommentaryLinksTest {
+    private val tur = 1L
+    private val beitYosef = 2L
+    private val darkheiMoshe = 3L
+
+    @Test
+    fun collectsTheConfiguredBooksWithTheirSections() {
+        val refs = listOf(
+            RefEntry("Tur, Orach Chayim, 1:1", "", "tur", 1),
+            RefEntry("Beit Yosef, Orach Chayim 1:1:1", "", "by", 1),
+            RefEntry("Darkhei Moshe, Orach Chayim 1:2:1", "", "dm", 2),
+            RefEntry("Darkhei Moshe (Amiel) 1:1", "", "amiel", 1),
+        )
+        val lineKeyToId = mapOf(
+            ("tur" to 0) to 10L,
+            ("by" to 0) to 20L,
+            ("dm" to 1) to 30L,
+            ("amiel" to 0) to 40L,
+        )
+        val lineIdToBookId = mapOf(10L to tur, 20L to beitYosef, 30L to darkheiMoshe, 40L to 9L)
+
+        val commentary = collectSparseCommentaries(refs, lineKeyToId, lineIdToBookId).single()
+
+        assertEquals(Triple(darkheiMoshe, beitYosef, tur), commentary.let {
+            Triple(it.commentaryBookId, it.intermediateBookId, it.baseBookId)
+        })
+        assertEquals(listOf(SectionLine(30L, 1, "orach chayim 1", "orach chayim 1:2")), commentary.commentaryLines)
+        assertEquals(listOf(SectionLine(10L, 0, "orach chayim 1", "orach chayim 1")), commentary.baseLines)
+    }
+
+    @Test
+    fun unlinkedSeifimAreAnchoredThroughTheBeitYosef() {
+        val turLines = listOf(
+            SectionLine(100, 0, "orach chayim 1", "orach chayim 1"),
+            SectionLine(101, 1, "orach chayim 1", "orach chayim 1"),
+            SectionLine(200, 2, "orach chayim 2", "orach chayim 2"),
+        )
+        val dmLines = listOf(
+            dmLine(1, "1:1"), // linked to the Tur
+            dmLine(2, "1:1"), // second paragraph of 1:1: follows the seif's Tur link
+            dmLine(3, "1:2"), // linked to Beit Yosef 1:2, itself on Tur line 101
+            dmLine(4, "1:3"), // Beit Yosef seif in another siman: not followed
+            dmLine(5, "1:4"), // linked to neither: stays unlinked
+        )
+        val commentary = SparseCommentary(darkheiMoshe, beitYosef, tur, dmLines, turLines)
+        val links = SparseCommentaryLinks()
+        listOf(
+            link(tur, darkheiMoshe, 100, 1),
+            link(beitYosef, darkheiMoshe, 50, 3),
+            link(tur, beitYosef, 101, 50),
+            link(beitYosef, darkheiMoshe, 60, 4),
+            link(tur, beitYosef, 200, 60),
+            link(tur, 9L, 100, 5), // unrelated book: ignored
+        ).forEach { links.record(commentary, it) }
+
+        val anchors = anchorUnlinkedCommentaryLines(commentary, links)
+
+        assertEquals(
+            listOf(100L to 2L, 101L to 3L),
+            anchors.map { it.baseLineId to it.commentaryLine.lineId },
+        )
+    }
+
+    private fun dmLine(id: Long, seif: String) =
+        SectionLine(id, id.toInt(), "orach chayim ${seif.substringBefore(':')}", "orach chayim $seif")
+
+    private fun link(sourceBook: Long, targetBook: Long, sourceLine: Long, targetLine: Long) = Link(
+        id = 0,
+        sourceBookId = sourceBook,
+        targetBookId = targetBook,
+        sourceLineId = sourceLine,
+        targetLineId = targetLine,
+        targetLineIndex = 0,
+        connectionType = ConnectionType.COMMENTARY,
+    )
+}