diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt index 55772ae8..b1d11b29 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt @@ -11,6 +11,7 @@ import io.github.kdroidfilter.seforimlibrary.common.ids.InMemoryIdAllocator import io.github.kdroidfilter.seforimlibrary.core.models.Author import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.Category +import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType import io.github.kdroidfilter.seforimlibrary.core.models.Line import io.github.kdroidfilter.seforimlibrary.core.text.HebrewTextUtils import io.github.kdroidfilter.seforimlibrary.dao.repository.SeforimRepository @@ -444,6 +445,10 @@ class SefariaDirectImporter( val linksDir = dbRoot.resolve("links") if (linksDir.exists()) { logger.i { "Processing links (${headingLineIds.size} heading lines will be excluded from link targets)..." } + // Commentary links between the books of sparsely linked commentaries (see + // SefariaSparseCommentaryLinks.kt). Filled by the single inserter coroutine. + val sparseCommentaries = collectSparseCommentaries(allRefsWithPath, lineKeyToId, lineIdToBookId) + val sparseLinks = sparseCommentaries.associateWith { SparseCommentaryLinks() } linksImporter.processLinksInParallel( linksDir = linksDir, refsByCanonical = refsByCanonical, @@ -451,8 +456,18 @@ class SefariaDirectImporter( lineKeyToId = lineKeyToId, lineIdToBookId = lineIdToBookId, bookMetaById = bookMetaById, - headingLineIds = headingLineIds + headingLineIds = headingLineIds, + onLinkStored = { link -> + if (link.connectionType == ConnectionType.COMMENTARY) { + sparseLinks.forEach { (commentary, links) -> links.record(commentary, link) } + } + }, ) + sparseLinks.forEach { (commentary, links) -> + val anchors = anchorUnlinkedCommentaryLines(commentary, links) + linksImporter.insertCommentaryAnchors(commentary, anchors, bookMetaById) + logger.i { "Anchored ${anchors.size} lines of book ${commentary.commentaryBookId} to its base text" } + } logger.i { "Links processed" } } diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt index 6af82aeb..a03dce9c 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaLinksImporter.kt @@ -25,7 +25,8 @@ internal class SefariaLinksImporter( lineKeyToId: Map, Long>, lineIdToBookId: Map, bookMetaById: Map, - headingLineIds: Set = emptySet() + headingLineIds: Set = emptySet(), + onLinkStored: (Link) -> Unit = {}, ) = coroutineScope { // Pre-register all connection types we'll use so their ids are stable // (so `link.connectionTypeId` is reproducible across builds). @@ -60,6 +61,7 @@ internal class SefariaLinksImporter( val inserter = launch { val batch = mutableListOf() for (link in linkChannel) { + onLinkStored(link) batch += link if (batch.size >= SefariaImportTuning.LINK_BATCH_SIZE) { repository.insertLinksBatch(batch) @@ -232,6 +234,36 @@ internal class SefariaLinksImporter( } } + /** + * Stores the [anchors] of a sparsely linked commentary as COMMENTARY links from its base + * text. See [anchorUnlinkedCommentaryLines]. + */ + suspend fun insertCommentaryAnchors( + commentary: SparseCommentary, + anchors: List, + bookMetaById: Map, + ) { + val typeId = bindings.upsertConnectionType(ConnectionType.COMMENTARY.name) + val isDeclaredBase = bookMetaById[commentary.commentaryBookId] + ?.sefariaDeclaredBaseTextBookIds + ?.contains(commentary.baseBookId) == true + anchors + .map { anchor -> + Link( + id = bindings.allocator.linkId(anchor.baseLineId, anchor.commentaryLine.lineId, typeId), + sourceBookId = commentary.baseBookId, + targetBookId = commentary.commentaryBookId, + sourceLineId = anchor.baseLineId, + targetLineId = anchor.commentaryLine.lineId, + targetLineIndex = anchor.commentaryLine.lineIndex, + connectionType = ConnectionType.COMMENTARY, + isDeclaredBase = isDeclaredBase, + ) + } + .chunked(SefariaImportTuning.LINK_BATCH_SIZE) + .forEach { repository.insertLinksBatch(it) } + } + private data class StoredLink( val srcBookId: Long, val tgtBookId: Long, diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt new file mode 100644 index 00000000..4dbec3c7 --- /dev/null +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinks.kt @@ -0,0 +1,126 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import io.github.kdroidfilter.seforimlibrary.core.models.Link + +/** + * A commentary that Sefaria links to its base text only partly, the rest of it being linked + * to an intermediate commentary on that base. English titles. + */ +private data class SparseCommentaryTitles(val commentary: String, val intermediate: String, val base: String) + +/** + * The Darkhei Moshe glosses the Beit Yosef and follows its seif numbering. Sefaria links a + * Darkhei Moshe seif to the Tur only when the Tur carries its own marker for it (about 2,000 + * of 6,000 seifim); the others are linked to the Beit Yosef seif they gloss, so they never + * showed next to the Tur (Zayit issue #462). + */ +private val sparselyLinkedCommentaries = listOf( + SparseCommentaryTitles(commentary = "Darkhei Moshe", intermediate = "Beit Yosef", base = "Tur"), +) + +/** + * A content line of a book with its top-level section (e.g. "orach chayim 5") and its + * segment address (e.g. "orach chayim 5:2" for "5:2:1"). + */ +internal data class SectionLine(val lineId: Long, val lineIndex: Int, val section: String, val segment: String) + +/** A sparsely linked commentary, the commentary it is linked through, and their base text. */ +internal data class SparseCommentary( + val commentaryBookId: Long, + val intermediateBookId: Long, + val baseBookId: Long, + val commentaryLines: List, + val baseLines: List, +) + +/** A missing link from a base line to a commentary line. */ +internal data class CommentaryAnchor(val baseLineId: Long, val commentaryLine: SectionLine) + +/** Collects the books of every [sparselyLinkedCommentaries] entry present in the export. */ +internal fun collectSparseCommentaries( + refs: List, + lineKeyToId: Map, Long>, + lineIdToBookId: Map, +): List { + fun linesOf(title: String): Pair>? { + val lines = refs + .filter { it.ref.startsWith("$title, ") } + .mapNotNull { entry -> + val lineId = lineKeyToId[entry.path to entry.lineIndex - 1] ?: return@mapNotNull null + val address = canonicalCitation(entry.ref.removePrefix(title)) + SectionLine( + lineId = lineId, + lineIndex = entry.lineIndex - 1, + section = address.substringBefore(':').trim(), + segment = address.substringBeforeLast(':').trim(), + ) + } + .sortedBy { it.lineIndex } + val bookId = lines.firstOrNull()?.let { lineIdToBookId[it.lineId] } ?: return null + return bookId to lines + } + + return sparselyLinkedCommentaries.mapNotNull { titles -> + val (commentaryBookId, commentaryLines) = linesOf(titles.commentary) ?: return@mapNotNull null + val (intermediateBookId, _) = linesOf(titles.intermediate) ?: return@mapNotNull null + val (baseBookId, baseLines) = linesOf(titles.base) ?: return@mapNotNull null + SparseCommentary(commentaryBookId, intermediateBookId, baseBookId, commentaryLines, baseLines) + } +} + +/** + * Commentary links stored between the books of a [SparseCommentary], each keyed by the + * dependant line id with the ids of the lines it depends on. + */ +internal class SparseCommentaryLinks { + val baseByCommentaryLine = HashMap>() + val intermediateByCommentaryLine = HashMap>() + val baseByIntermediateLine = HashMap>() + + /** Records a stored COMMENTARY [link] (base → dependant) if it joins two books of [commentary]. */ + fun record(commentary: SparseCommentary, link: Link) { + val map = when (link.sourceBookId to link.targetBookId) { + commentary.baseBookId to commentary.commentaryBookId -> baseByCommentaryLine + commentary.intermediateBookId to commentary.commentaryBookId -> intermediateByCommentaryLine + commentary.baseBookId to commentary.intermediateBookId -> baseByIntermediateLine + else -> return + } + map.getOrPut(link.targetLineId) { HashSet() } += link.sourceLineId + } +} + +/** + * Anchors the commentary lines without a link to their base text through the intermediate + * commentary: a line linked to an intermediate segment gets that segment's base lines. Lines + * of a segment share its anchors, so a segment's later paragraphs follow its first one, and a + * segment linked to the base keeps those links. Only base lines of the commentary line's own + * section are kept, so lateral citations of other sections are not followed. Lines linked to + * neither book stay unlinked. + */ +internal fun anchorUnlinkedCommentaryLines( + commentary: SparseCommentary, + links: SparseCommentaryLinks, +): List { + val baseSectionByLineId = commentary.baseLines.associate { it.lineId to it.section } + + fun SectionLine.inOwnSection(baseLineIds: Iterable): Set = + baseLineIds.filterTo(LinkedHashSet()) { baseSectionByLineId[it] == section } + + fun SectionLine.directBase(): Set = inOwnSection(links.baseByCommentaryLine[lineId].orEmpty()) + + fun SectionLine.baseThroughIntermediate(): Set = inOwnSection( + links.intermediateByCommentaryLine[lineId].orEmpty() + .flatMap { links.baseByIntermediateLine[it].orEmpty() } + ) + + return commentary.commentaryLines + .groupBy { it.segment } + .values + .flatMap { segment -> + val baseLineIds = segment.flatMapTo(LinkedHashSet()) { it.directBase() } + .ifEmpty { segment.flatMapTo(LinkedHashSet()) { it.baseThroughIntermediate() } } + segment + .filter { it.directBase().isEmpty() } + .flatMap { line -> baseLineIds.map { CommentaryAnchor(it, line) } } + } +} diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt new file mode 100644 index 00000000..f8ee5c25 --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSparseCommentaryLinksTest.kt @@ -0,0 +1,87 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import io.github.kdroidfilter.seforimlibrary.core.models.ConnectionType +import io.github.kdroidfilter.seforimlibrary.core.models.Link +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Regression for Zayit issue #462: Sefaria links most Darkhei Moshe seifim to the Beit Yosef + * seif they gloss instead of the Tur, so they never showed next to the Tur. + */ +class SefariaSparseCommentaryLinksTest { + private val tur = 1L + private val beitYosef = 2L + private val darkheiMoshe = 3L + + @Test + fun collectsTheConfiguredBooksWithTheirSections() { + val refs = listOf( + RefEntry("Tur, Orach Chayim, 1:1", "", "tur", 1), + RefEntry("Beit Yosef, Orach Chayim 1:1:1", "", "by", 1), + RefEntry("Darkhei Moshe, Orach Chayim 1:2:1", "", "dm", 2), + RefEntry("Darkhei Moshe (Amiel) 1:1", "", "amiel", 1), + ) + val lineKeyToId = mapOf( + ("tur" to 0) to 10L, + ("by" to 0) to 20L, + ("dm" to 1) to 30L, + ("amiel" to 0) to 40L, + ) + val lineIdToBookId = mapOf(10L to tur, 20L to beitYosef, 30L to darkheiMoshe, 40L to 9L) + + val commentary = collectSparseCommentaries(refs, lineKeyToId, lineIdToBookId).single() + + assertEquals(Triple(darkheiMoshe, beitYosef, tur), commentary.let { + Triple(it.commentaryBookId, it.intermediateBookId, it.baseBookId) + }) + assertEquals(listOf(SectionLine(30L, 1, "orach chayim 1", "orach chayim 1:2")), commentary.commentaryLines) + assertEquals(listOf(SectionLine(10L, 0, "orach chayim 1", "orach chayim 1")), commentary.baseLines) + } + + @Test + fun unlinkedSeifimAreAnchoredThroughTheBeitYosef() { + val turLines = listOf( + SectionLine(100, 0, "orach chayim 1", "orach chayim 1"), + SectionLine(101, 1, "orach chayim 1", "orach chayim 1"), + SectionLine(200, 2, "orach chayim 2", "orach chayim 2"), + ) + val dmLines = listOf( + dmLine(1, "1:1"), // linked to the Tur + dmLine(2, "1:1"), // second paragraph of 1:1: follows the seif's Tur link + dmLine(3, "1:2"), // linked to Beit Yosef 1:2, itself on Tur line 101 + dmLine(4, "1:3"), // Beit Yosef seif in another siman: not followed + dmLine(5, "1:4"), // linked to neither: stays unlinked + ) + val commentary = SparseCommentary(darkheiMoshe, beitYosef, tur, dmLines, turLines) + val links = SparseCommentaryLinks() + listOf( + link(tur, darkheiMoshe, 100, 1), + link(beitYosef, darkheiMoshe, 50, 3), + link(tur, beitYosef, 101, 50), + link(beitYosef, darkheiMoshe, 60, 4), + link(tur, beitYosef, 200, 60), + link(tur, 9L, 100, 5), // unrelated book: ignored + ).forEach { links.record(commentary, it) } + + val anchors = anchorUnlinkedCommentaryLines(commentary, links) + + assertEquals( + listOf(100L to 2L, 101L to 3L), + anchors.map { it.baseLineId to it.commentaryLine.lineId }, + ) + } + + private fun dmLine(id: Long, seif: String) = + SectionLine(id, id.toInt(), "orach chayim ${seif.substringBefore(':')}", "orach chayim $seif") + + private fun link(sourceBook: Long, targetBook: Long, sourceLine: Long, targetLine: Long) = Link( + id = 0, + sourceBookId = sourceBook, + targetBookId = targetBook, + sourceLineId = sourceLine, + targetLineId = targetLine, + targetLineIndex = 0, + connectionType = ConnectionType.COMMENTARY, + ) +}