From ab49ff306fadbfb98787ea1358e2a96a2c568209 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Thu, 8 Oct 2026 13:25:03 +0300 Subject: [PATCH] fix(sefariasqlite): link the whole Darkhei Moshe to the Tur through the Beit Yosef The Darkhei Moshe glosses the Beit Yosef and follows its seif numbering. Sefaria links a Darkhei Moshe seif to the Tur only when the Tur carries its own marker for it (about 2,000 of 6,000 seifim); the others are linked to the Beit Yosef seif they gloss. So studying the Tur with its commentaries showed only part of the Darkhei Moshe. After the links import, anchor each Darkhei Moshe line without a Tur link to the Tur lines its Beit Yosef seif is linked to, keeping only Tur lines of its own siman so lateral citations are not followed. Lines linked to neither book stay unlinked. On the current corpus: 3,889 lines anchored, 177 left unlinked. Fixes kdroidFilter/Zayit#462 Signed-off-by: Elie Gambache --- .../sefariasqlite/SefariaDirectImporter.kt | 17 ++- .../sefariasqlite/SefariaLinksImporter.kt | 34 ++++- .../SefariaSparseCommentaryLinks.kt | 126 ++++++++++++++++++ .../SefariaSparseCommentaryLinksTest.kt | 87 ++++++++++++ 4 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt create mode 100644 generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt index 55772ae8..b1d11b29 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt @@ -11,6 +11,7 @@ import io.github.kdroidfilter.seforimlibrary.common.ids.InMemoryIdAllocator import io.github.kdroidfilter.seforimlibrary.core.models.Author import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.Category +import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType import io.github.kdroidfilter.seforimlibrary.core.models.Line import io.github.kdroidfilter.seforimlibrary.core.text.HebrewTextUtils import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository @@ -444,6 +445,10 @@ class SefariaDirectImporter( val linksDir = dbRoot.resolve("links") if (linksDir.exists()) { logger.i { "Processing links (${headingLineIds.size} heading lines will be excluded from link targets)..." } + // Commentary links between the books of sparsely linked commentaries (see + // SefariaSparseCommentaryLinks.kt). Filled by the single inserter coroutine. + val sparseCommentaries = collectSparseCommentaries(allRefsWithPath, lineKeyToId, lineIdToBookId) + val sparseLinks = sparseCommentaries.associateWith { SparseCommentaryLinks() } linksImporter.processLinksInParallel( linksDir = linksDir, refsByCanonical = refsByCanonical, @@ -451,8 +456,18 @@ class SefariaDirectImporter( lineKeyToId = lineKeyToId, lineIdToBookId = lineIdToBookId, bookMetaById = bookMetaById, - headingLineIds = headingLineIds + headingLineIds = headingLineIds, + onLinkStored = { link -> + if (link.connectionType == ConnectionType.COMMENTARY) { + sparseLinks.forEach { (commentary, links) -> links.record(commentary, link) } + } + }, ) + sparseLinks.forEach { (commentary, links) -> + val anchors = anchorUnlinkedCommentaryLines(commentary, links) + linksImporter.insertCommentaryAnchors(commentary, anchors, bookMetaById) + logger.i { "Anchored ${anchors.size} lines of book ${commentary.commentaryBookId} to its base text" } + } logger.i { "Links processed" } } diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt index 6af82aeb..a03dce9c 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt @@ -25,7 +25,8 @@ internal class SefariaLinksImporter( lineKeyToId: Map, Long>, lineIdToBookId: Map, bookMetaById: Map, - headingLineIds: Set = emptySet() + headingLineIds: Set = emptySet(), + onLinkStored: (Link) -> Unit = {}, ) = coroutineScope { // Pre-register all connection types we'll use so their ids are stable // (so `link.connectionTypeId` is reproducible across builds). @@ -60,6 +61,7 @@ internal class SefariaLinksImporter( val inserter = launch { val batch = mutableListOf() for (link in linkChannel) { + onLinkStored(link) batch += link if (batch.size >= SefariaImportTuning.LINK_BATCH_SIZE) { repository.insertLinksBatch(batch) @@ -232,6 +234,36 @@ internal class SefariaLinksImporter( } } + /** + * Stores the [anchors] of a sparsely linked commentary as COMMENTARY links from its base + * text. See [anchorUnlinkedCommentaryLines]. + */ + suspend fun insertCommentaryAnchors( + commentary: SparseCommentary, + anchors: List, + bookMetaById: Map, + ) { + val typeId = bindings.upsertConnectionType(ConnectionType.COMMENTARY.name) + val isDeclaredBase = bookMetaById[commentary.commentaryBookId] + ?.sefariaDeclaredBaseTextBookIds + ?.contains(commentary.baseBookId) == true + anchors + .map { anchor -> + Link( + id = bindings.allocator.linkId(anchor.baseLineId, anchor.commentaryLine.lineId, typeId), + sourceBookId = commentary.baseBookId, + targetBookId = commentary.commentaryBookId, + sourceLineId = anchor.baseLineId, + targetLineId = anchor.commentaryLine.lineId, + targetLineIndex = anchor.commentaryLine.lineIndex, + connectionType = ConnectionType.COMMENTARY, + isDeclaredBase = isDeclaredBase, + ) + } + .chunked(SefariaImportTuning.LINK_BATCH_SIZE) + .forEach { repository.insertLinksBatch(it) } + } + private data class StoredLink( val srcBookId: Long, val tgtBookId: Long, diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt new file mode 100644 index 00000000..4dbec3c7 --- /dev/null +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt @@ -0,0 +1,126 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import io.github.kdroidfilter.seforimlibrary.core.models.Link + +/** + * A commentary that Sefaria links to its base text only partly, the rest of it being linked + * to an intermediate commentary on that base. English titles. + */ +private data class SparseCommentaryTitles(val commentary: String, val intermediate: String, val base: String) + +/** + * The Darkhei Moshe glosses the Beit Yosef and follows its seif numbering. Sefaria links a + * Darkhei Moshe seif to the Tur only when the Tur carries its own marker for it (about 2,000 + * of 6,000 seifim); the others are linked to the Beit Yosef seif they gloss, so they never + * showed next to the Tur (Zayit issue #462). + */ +private val sparselyLinkedCommentaries = listOf( + SparseCommentaryTitles(commentary = "Darkhei Moshe", intermediate = "Beit Yosef", base = "Tur"), +) + +/** + * A content line of a book with its top-level section (e.g. "orach chayim 5") and its + * segment address (e.g. "orach chayim 5:2" for "5:2:1"). + */ +internal data class SectionLine(val lineId: Long, val lineIndex: Int, val section: String, val segment: String) + +/** A sparsely linked commentary, the commentary it is linked through, and their base text. */ +internal data class SparseCommentary( + val commentaryBookId: Long, + val intermediateBookId: Long, + val baseBookId: Long, + val commentaryLines: List, + val baseLines: List, +) + +/** A missing link from a base line to a commentary line. */ +internal data class CommentaryAnchor(val baseLineId: Long, val commentaryLine: SectionLine) + +/** Collects the books of every [sparselyLinkedCommentaries] entry present in the export. */ +internal fun collectSparseCommentaries( + refs: List, + lineKeyToId: Map, Long>, + lineIdToBookId: Map, +): List { + fun linesOf(title: String): Pair>? { + val lines = refs + .filter { it.ref.startsWith("$title, ") } + .mapNotNull { entry -> + val lineId = lineKeyToId[entry.path to entry.lineIndex - 1] ?: return@mapNotNull null + val address = canonicalCitation(entry.ref.removePrefix(title)) + SectionLine( + lineId = lineId, + lineIndex = entry.lineIndex - 1, + section = address.substringBefore(':').trim(), + segment = address.substringBeforeLast(':').trim(), + ) + } + .sortedBy { it.lineIndex } + val bookId = lines.firstOrNull()?.let { lineIdToBookId[it.lineId] } ?: return null + return bookId to lines + } + + return sparselyLinkedCommentaries.mapNotNull { titles -> + val (commentaryBookId, commentaryLines) = linesOf(titles.commentary) ?: return@mapNotNull null + val (intermediateBookId, _) = linesOf(titles.intermediate) ?: return@mapNotNull null + val (baseBookId, baseLines) = linesOf(titles.base) ?: return@mapNotNull null + SparseCommentary(commentaryBookId, intermediateBookId, baseBookId, commentaryLines, baseLines) + } +} + +/** + * Commentary links stored between the books of a [SparseCommentary], each keyed by the + * dependant line id with the ids of the lines it depends on. + */ +internal class SparseCommentaryLinks { + val baseByCommentaryLine = HashMap>() + val intermediateByCommentaryLine = HashMap>() + val baseByIntermediateLine = HashMap>() + + /** Records a stored COMMENTARY [link] (base → dependant) if it joins two books of [commentary]. */ + fun record(commentary: SparseCommentary, link: Link) { + val map = when (link.sourceBookId to link.targetBookId) { + commentary.baseBookId to commentary.commentaryBookId -> baseByCommentaryLine + commentary.intermediateBookId to commentary.commentaryBookId -> intermediateByCommentaryLine + commentary.baseBookId to commentary.intermediateBookId -> baseByIntermediateLine + else -> return + } + map.getOrPut(link.targetLineId) { HashSet() } += link.sourceLineId + } +} + +/** + * Anchors the commentary lines without a link to their base text through the intermediate + * commentary: a line linked to an intermediate segment gets that segment's base lines. Lines + * of a segment share its anchors, so a segment's later paragraphs follow its first one, and a + * segment linked to the base keeps those links. Only base lines of the commentary line's own + * section are kept, so lateral citations of other sections are not followed. Lines linked to + * neither book stay unlinked. + */ +internal fun anchorUnlinkedCommentaryLines( + commentary: SparseCommentary, + links: SparseCommentaryLinks, +): List { + val baseSectionByLineId = commentary.baseLines.associate { it.lineId to it.section } + + fun SectionLine.inOwnSection(baseLineIds: Iterable): Set = + baseLineIds.filterTo(LinkedHashSet()) { baseSectionByLineId[it] == section } + + fun SectionLine.directBase(): Set = inOwnSection(links.baseByCommentaryLine[lineId].orEmpty()) + + fun SectionLine.baseThroughIntermediate(): Set = inOwnSection( + links.intermediateByCommentaryLine[lineId].orEmpty() + .flatMap { links.baseByIntermediateLine[it].orEmpty() } + ) + + return commentary.commentaryLines + .groupBy { it.segment } + .values + .flatMap { segment -> + val baseLineIds = segment.flatMapTo(LinkedHashSet()) { it.directBase() } + .ifEmpty { segment.flatMapTo(LinkedHashSet()) { it.baseThroughIntermediate() } } + segment + .filter { it.directBase().isEmpty() } + .flatMap { line -> baseLineIds.map { CommentaryAnchor(it, line) } } + } +} diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt new file mode 100644 index 00000000..f8ee5c25 --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt @@ -0,0 +1,87 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType +import io.github.kdroidfilter.seforimlibrary.core.models.Link +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Regression for Zayit issue #462: Sefaria links most Darkhei Moshe seifim to the Beit Yosef + * seif they gloss instead of the Tur, so they never showed next to the Tur. + */ +class SefariaSparseCommentaryLinksTest { + private val tur = 1L + private val beitYosef = 2L + private val darkheiMoshe = 3L + + @Test + fun collectsTheConfiguredBooksWithTheirSections() { + val refs = listOf( + RefEntry("Tur, Orach Chayim, 1:1", "", "tur", 1), + RefEntry("Beit Yosef, Orach Chayim 1:1:1", "", "by", 1), + RefEntry("Darkhei Moshe, Orach Chayim 1:2:1", "", "dm", 2), + RefEntry("Darkhei Moshe (Amiel) 1:1", "", "amiel", 1), + ) + val lineKeyToId = mapOf( + ("tur" to 0) to 10L, + ("by" to 0) to 20L, + ("dm" to 1) to 30L, + ("amiel" to 0) to 40L, + ) + val lineIdToBookId = mapOf(10L to tur, 20L to beitYosef, 30L to darkheiMoshe, 40L to 9L) + + val commentary = collectSparseCommentaries(refs, lineKeyToId, lineIdToBookId).single() + + assertEquals(Triple(darkheiMoshe, beitYosef, tur), commentary.let { + Triple(it.commentaryBookId, it.intermediateBookId, it.baseBookId) + }) + assertEquals(listOf(SectionLine(30L, 1, "orach chayim 1", "orach chayim 1:2")), commentary.commentaryLines) + assertEquals(listOf(SectionLine(10L, 0, "orach chayim 1", "orach chayim 1")), commentary.baseLines) + } + + @Test + fun unlinkedSeifimAreAnchoredThroughTheBeitYosef() { + val turLines = listOf( + SectionLine(100, 0, "orach chayim 1", "orach chayim 1"), + SectionLine(101, 1, "orach chayim 1", "orach chayim 1"), + SectionLine(200, 2, "orach chayim 2", "orach chayim 2"), + ) + val dmLines = listOf( + dmLine(1, "1:1"), // linked to the Tur + dmLine(2, "1:1"), // second paragraph of 1:1: follows the seif's Tur link + dmLine(3, "1:2"), // linked to Beit Yosef 1:2, itself on Tur line 101 + dmLine(4, "1:3"), // Beit Yosef seif in another siman: not followed + dmLine(5, "1:4"), // linked to neither: stays unlinked + ) + val commentary = SparseCommentary(darkheiMoshe, beitYosef, tur, dmLines, turLines) + val links = SparseCommentaryLinks() + listOf( + link(tur, darkheiMoshe, 100, 1), + link(beitYosef, darkheiMoshe, 50, 3), + link(tur, beitYosef, 101, 50), + link(beitYosef, darkheiMoshe, 60, 4), + link(tur, beitYosef, 200, 60), + link(tur, 9L, 100, 5), // unrelated book: ignored + ).forEach { links.record(commentary, it) } + + val anchors = anchorUnlinkedCommentaryLines(commentary, links) + + assertEquals( + listOf(100L to 2L, 101L to 3L), + anchors.map { it.baseLineId to it.commentaryLine.lineId }, + ) + } + + private fun dmLine(id: Long, seif: String) = + SectionLine(id, id.toInt(), "orach chayim ${seif.substringBefore(':')}", "orach chayim $seif") + + private fun link(sourceBook: Long, targetBook: Long, sourceLine: Long, targetLine: Long) = Link( + id = 0, + sourceBookId = sourceBook, + targetBookId = targetBook, + sourceLineId = sourceLine, + targetLineId = targetLine, + targetLineIndex = 0, + connectionType = ConnectionType.COMMENTARY, + ) +}