From 9512b560eb29c3854b3c93a5df9c57b7b9d57b29 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 08:46:44 +0000 Subject: [PATCH 01/48] style: spotless reformat, no functional change --- .../codeonthego/indexing/FilteredIndex.kt | 17 +- .../codeonthego/indexing/InMemoryIndex.kt | 435 +++++------ .../codeonthego/indexing/MergedIndex.kt | 86 +-- .../codeonthego/indexing/SQLiteIndex.kt | 689 +++++++++--------- .../codeonthego/indexing/api/Index.kt | 144 ++-- .../codeonthego/indexing/api/Query.kt | 113 +-- .../codeonthego/indexing/FilteredIndexTest.kt | 314 ++++---- .../codeonthego/indexing/InMemoryIndexTest.kt | 494 +++++++------ .../codeonthego/indexing/MergedIndexTest.kt | 297 ++++---- 9 files changed, 1348 insertions(+), 1241 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt index 740a0ae8a7..b8eeda834f 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -19,8 +19,8 @@ import java.util.concurrent.ConcurrentHashMap */ open class FilteredIndex( private val backing: ReadableIndex, -) : ReadableIndex, Closeable { - +) : ReadableIndex, + Closeable { /** * The set of source IDs whose entries are visible. * Uses a concurrent set for thread-safe reads during queries. @@ -57,14 +57,12 @@ open class FilteredIndex( /** * Returns the current set of active source IDs. */ - open fun activeSources(): Set = - activeSources.toSet() + open fun activeSources(): Set = activeSources.toSet() /** * Returns true if the source is currently active (visible). */ - open fun isActive(sourceId: String): Boolean = - sourceId in activeSources + open fun isActive(sourceId: String): Boolean = sourceId in activeSources /** * Returns true if the source exists in the backing index, @@ -72,8 +70,7 @@ open class FilteredIndex( * * Use this to check if a JAR needs indexing at all. */ - open suspend fun isCached(sourceId: String): Boolean = - backing.containsSource(sourceId) + open suspend fun isCached(sourceId: String): Boolean = backing.containsSource(sourceId) override fun query(query: IndexQuery): Sequence { if (query.sourceId != null && !isActive(query.sourceId)) { @@ -88,9 +85,7 @@ open class FilteredIndex( return if (isActive(entry.sourceId)) entry else null } - override suspend fun containsSource(sourceId: String): Boolean { - return isActive(sourceId) && backing.containsSource(sourceId) - } + override suspend fun containsSource(sourceId: String): Boolean = isActive(sourceId) && backing.containsSource(sourceId) override fun distinctValues(fieldName: String): Sequence { // This is imprecise — the backing index may return values diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index a5d4dce21c..e0ccaa41ca 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -30,216 +30,229 @@ import kotlin.concurrent.write * @param descriptor Defines queryable fields and serialization. */ class InMemoryIndex( - override val descriptor: IndexDescriptor, - override val name: String = "memory:${descriptor.name}", + override val descriptor: IndexDescriptor, + override val name: String = "memory:${descriptor.name}", ) : Index { - - private val primaryMap = ConcurrentHashMap(256) - private val sourceMap = ConcurrentHashMap>(32) - private val fieldMaps = ConcurrentHashMap>>() - private val prefixBuckets = ConcurrentHashMap>>() - - private val lock = ReentrantReadWriteLock() - - private data class PrefixEntry(val lowerValue: String, val key: String) - - init { - for (field in descriptor.fields) { - fieldMaps[field.name] = ConcurrentHashMap() - if (field.prefixSearchable) { - prefixBuckets[field.name] = ConcurrentHashMap() - } - } - } - - override fun query(query: IndexQuery): Sequence { - val keys = resolveMatchingKeys(query) - val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit - return keys - .mapNotNull { primaryMap[it] } - .take(limit) - } - - override suspend fun get(key: String): T? = primaryMap[key] - - override suspend fun containsSource(sourceId: String): Boolean = - sourceMap.containsKey(sourceId) - - override fun distinctValues(fieldName: String): Sequence { - val fieldMap = fieldMaps[fieldName] ?: return emptySequence() - return lock.read { fieldMap.keys.toList() }.asSequence() - } - - override suspend fun insertAll(entries: Sequence) { - lock.write { - for (entry in entries) { - insertSingleLocked(entry) - } - } - } - - override suspend fun insert(entry: T) = lock.write { insertSingleLocked(entry) } - - override suspend fun removeBySource(sourceId: String) = lock.write { - removeBySourceLocked(sourceId) - } - - /** - * Remove every entry belonging to any of [sourceIds]. - * - * Acquires the write lock once and removes each source under it, so the whole - * batch is atomic with respect to concurrent readers and writers — there is no - * intermediate state in which only some of the sources have been removed. - */ - override suspend fun removeBySources(sourceIds: Collection) = lock.write { - for (sourceId in sourceIds) { - removeBySourceLocked(sourceId) - } - } - - /** - * Remove all entries for [sourceId] from the primary, source, and secondary - * indexes. Caller MUST already hold the write lock; this method does not lock. - */ - private fun removeBySourceLocked(sourceId: String) { - val keys = sourceMap.remove(sourceId) ?: return - for (key in keys) { - val entry = primaryMap.remove(key) ?: continue - removeFromSecondaryIndexes(entry) - } - } - - override suspend fun clear() = lock.write { - primaryMap.clear() - sourceMap.clear() - fieldMaps.values.forEach { it.clear() } - prefixBuckets.values.forEach { it.clear() } - } - - val size: Int get() = primaryMap.size - val sourceCount: Int get() = sourceMap.size - - /** - * Resolves the set of keys matching the query by intersecting - * the results of each predicate. - * - * Starts with the most selective predicate to minimize the - * intersection set. - */ - private fun resolveMatchingKeys(query: IndexQuery): Sequence = lock.read { - var candidates: Set? = null - - if (query.key != null) { - return@read if (primaryMap.containsKey(query.key)) { - sequenceOf(query.key) - } else { - emptySequence() - } - } - - if (query.sourceId != null) { - candidates = intersect(candidates, sourceMap[query.sourceId]) - } - - for ((field, value) in query.exactMatch) { - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - candidates = intersect(candidates, fieldMap[value]) - } - - for ((field, prefix) in query.prefixMatch) { - val buckets = prefixBuckets[field] - val matching: Set = if (buckets != null) { - // Prefix-searchable: case-insensitive match via the lowercased buckets, - // mirroring SQLite's `lowerCol LIKE 'prefix%'`. - val lowerPrefix = prefix.lowercase() - val firstChar = lowerPrefix.firstOrNull() - if (firstChar == null) { - // Empty prefix == "field present", matching SQLite's `LIKE '%'` - // (which excludes rows where the column IS NULL). - buckets.values.flatMapTo(mutableSetOf()) { entries -> entries.map { it.key } } - } else { - val bucket = buckets[firstChar] ?: return@read emptySequence() - bucket.asSequence() - .filter { it.lowerValue.startsWith(lowerPrefix) } - .map { it.key } - .toSet() - } - } else { - // Not prefix-searchable: fall back to a case-sensitive prefix scan of the - // regular field map, mirroring SQLite's `col LIKE 'prefix%'` fallback. - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - fieldMap.entries.asSequence() - .filter { (value, _) -> value.startsWith(prefix) } - .flatMap { (_, keys) -> keys.asSequence() } - .toSet() - } - - candidates = intersect(candidates, matching) - } - - for ((field, mustExist) in query.presence) { - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - val allKeysWithField = fieldMap.values.flatMapTo(mutableSetOf()) { it } - candidates = if (mustExist) { - intersect(candidates, allKeysWithField) - } else { - // Keys that DON'T have this field - val allKeys = primaryMap.keys.toMutableSet() - allKeys.removeAll(allKeysWithField) - intersect(candidates, allKeys) - } - } - - candidates?.asSequence() ?: primaryMap.keys.asSequence() - } - - private fun intersect(current: Set?, other: Set?): Set? { - if (other == null) return current - if (current == null) return other - return current.intersect(other) - } - - private fun insertSingleLocked(entry: T) { - val existing = primaryMap[entry.key] - if (existing != null) { - removeFromSecondaryIndexes(existing) - } - - primaryMap[entry.key] = entry - sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(entry.key) - - val fields = descriptor.fieldValues(entry) - for ((fieldName, value) in fields) { - if (value == null) continue - - fieldMaps[fieldName] - ?.getOrPut(value) { mutableSetOf() } - ?.add(entry.key) - - val buckets = prefixBuckets[fieldName] - if (buckets != null) { - val lower = value.lowercase() - val firstChar = lower.firstOrNull() ?: continue - buckets.getOrPut(firstChar) { mutableListOf() } - .add(PrefixEntry(lower, entry.key)) - } - } - } - - private fun removeFromSecondaryIndexes(entry: T) { - val fields = descriptor.fieldValues(entry) - for ((fieldName, value) in fields) { - if (value == null) continue - - fieldMaps[fieldName]?.get(value)?.remove(entry.key) - - val buckets = prefixBuckets[fieldName] - if (buckets != null) { - val lower = value.lowercase() - val firstChar = lower.firstOrNull() ?: continue - buckets[firstChar]?.removeAll { it.key == entry.key } - } - } - // Note: sourceMap is handled by the caller - } + private val primaryMap = ConcurrentHashMap(256) + private val sourceMap = ConcurrentHashMap>(32) + private val fieldMaps = ConcurrentHashMap>>() + private val prefixBuckets = ConcurrentHashMap>>() + + private val lock = ReentrantReadWriteLock() + + private data class PrefixEntry( + val lowerValue: String, + val key: String, + ) + + init { + for (field in descriptor.fields) { + fieldMaps[field.name] = ConcurrentHashMap() + if (field.prefixSearchable) { + prefixBuckets[field.name] = ConcurrentHashMap() + } + } + } + + override fun query(query: IndexQuery): Sequence { + val keys = resolveMatchingKeys(query) + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + return keys + .mapNotNull { primaryMap[it] } + .take(limit) + } + + override suspend fun get(key: String): T? = primaryMap[key] + + override suspend fun containsSource(sourceId: String): Boolean = sourceMap.containsKey(sourceId) + + override fun distinctValues(fieldName: String): Sequence { + val fieldMap = fieldMaps[fieldName] ?: return emptySequence() + return lock.read { fieldMap.keys.toList() }.asSequence() + } + + override suspend fun insertAll(entries: Sequence) { + lock.write { + for (entry in entries) { + insertSingleLocked(entry) + } + } + } + + override suspend fun insert(entry: T) = lock.write { insertSingleLocked(entry) } + + override suspend fun removeBySource(sourceId: String) = + lock.write { + removeBySourceLocked(sourceId) + } + + /** + * Remove every entry belonging to any of [sourceIds]. + * + * Acquires the write lock once and removes each source under it, so the whole + * batch is atomic with respect to concurrent readers and writers — there is no + * intermediate state in which only some of the sources have been removed. + */ + override suspend fun removeBySources(sourceIds: Collection) = + lock.write { + for (sourceId in sourceIds) { + removeBySourceLocked(sourceId) + } + } + + /** + * Remove all entries for [sourceId] from the primary, source, and secondary + * indexes. Caller MUST already hold the write lock; this method does not lock. + */ + private fun removeBySourceLocked(sourceId: String) { + val keys = sourceMap.remove(sourceId) ?: return + for (key in keys) { + val entry = primaryMap.remove(key) ?: continue + removeFromSecondaryIndexes(entry) + } + } + + override suspend fun clear() = + lock.write { + primaryMap.clear() + sourceMap.clear() + fieldMaps.values.forEach { it.clear() } + prefixBuckets.values.forEach { it.clear() } + } + + val size: Int get() = primaryMap.size + val sourceCount: Int get() = sourceMap.size + + /** + * Resolves the set of keys matching the query by intersecting + * the results of each predicate. + * + * Starts with the most selective predicate to minimize the + * intersection set. + */ + private fun resolveMatchingKeys(query: IndexQuery): Sequence = + lock.read { + var candidates: Set? = null + + if (query.key != null) { + return@read if (primaryMap.containsKey(query.key)) { + sequenceOf(query.key) + } else { + emptySequence() + } + } + + if (query.sourceId != null) { + candidates = intersect(candidates, sourceMap[query.sourceId]) + } + + for ((field, value) in query.exactMatch) { + val fieldMap = fieldMaps[field] ?: return@read emptySequence() + candidates = intersect(candidates, fieldMap[value]) + } + + for ((field, prefix) in query.prefixMatch) { + val buckets = prefixBuckets[field] + val matching: Set = + if (buckets != null) { + // Prefix-searchable: case-insensitive match via the lowercased buckets, + // mirroring SQLite's `lowerCol LIKE 'prefix%'`. + val lowerPrefix = prefix.lowercase() + val firstChar = lowerPrefix.firstOrNull() + if (firstChar == null) { + // Empty prefix == "field present", matching SQLite's `LIKE '%'` + // (which excludes rows where the column IS NULL). + buckets.values.flatMapTo(mutableSetOf()) { entries -> entries.map { it.key } } + } else { + val bucket = buckets[firstChar] ?: return@read emptySequence() + bucket + .asSequence() + .filter { it.lowerValue.startsWith(lowerPrefix) } + .map { it.key } + .toSet() + } + } else { + // Not prefix-searchable: fall back to a case-sensitive prefix scan of the + // regular field map, mirroring SQLite's `col LIKE 'prefix%'` fallback. + val fieldMap = fieldMaps[field] ?: return@read emptySequence() + fieldMap.entries + .asSequence() + .filter { (value, _) -> value.startsWith(prefix) } + .flatMap { (_, keys) -> keys.asSequence() } + .toSet() + } + + candidates = intersect(candidates, matching) + } + + for ((field, mustExist) in query.presence) { + val fieldMap = fieldMaps[field] ?: return@read emptySequence() + val allKeysWithField = fieldMap.values.flatMapTo(mutableSetOf()) { it } + candidates = + if (mustExist) { + intersect(candidates, allKeysWithField) + } else { + // Keys that DON'T have this field + val allKeys = primaryMap.keys.toMutableSet() + allKeys.removeAll(allKeysWithField) + intersect(candidates, allKeys) + } + } + + candidates?.asSequence() ?: primaryMap.keys.asSequence() + } + + private fun intersect( + current: Set?, + other: Set?, + ): Set? { + if (other == null) return current + if (current == null) return other + return current.intersect(other) + } + + private fun insertSingleLocked(entry: T) { + val existing = primaryMap[entry.key] + if (existing != null) { + removeFromSecondaryIndexes(existing) + } + + primaryMap[entry.key] = entry + sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(entry.key) + + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + if (value == null) continue + + fieldMaps[fieldName] + ?.getOrPut(value) { mutableSetOf() } + ?.add(entry.key) + + val buckets = prefixBuckets[fieldName] + if (buckets != null) { + val lower = value.lowercase() + val firstChar = lower.firstOrNull() ?: continue + buckets + .getOrPut(firstChar) { mutableListOf() } + .add(PrefixEntry(lower, entry.key)) + } + } + } + + private fun removeFromSecondaryIndexes(entry: T) { + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + if (value == null) continue + + fieldMaps[fieldName]?.get(value)?.remove(entry.key) + + val buckets = prefixBuckets[fieldName] + if (buckets != null) { + val lower = value.lowercase() + val firstChar = lower.firstOrNull() ?: continue + buckets[firstChar]?.removeAll { it.key == entry.key } + } + } + // Note: sourceMap is handled by the caller + } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt index 05a9bab1e7..93a2590514 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt @@ -16,52 +16,52 @@ import java.io.Closeable * @param indexes The indexes to merge, in priority order. */ class MergedIndex( - private val indexes: List>, -) : ReadableIndex, Closeable { + private val indexes: List>, +) : ReadableIndex, + Closeable { + constructor(vararg indexes: ReadableIndex) : this(indexes.toList()) - constructor(vararg indexes: ReadableIndex) : this(indexes.toList()) + override fun query(query: IndexQuery): Sequence = + sequence { + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + val seen = mutableSetOf() + var total = 0 + for (index in indexes) { + if (total >= limit) break + for (entry in index.query(query)) { + if (total >= limit) break + if (seen.add(entry.key)) { + yield(entry) + total++ + } + } + } + } - override fun query(query: IndexQuery): Sequence = sequence { - val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit - val seen = mutableSetOf() - var total = 0 - for (index in indexes) { - if (total >= limit) break - for (entry in index.query(query)) { - if (total >= limit) break - if (seen.add(entry.key)) { - yield(entry) - total++ - } - } - } - } + override suspend fun get(key: String): T? { + // First match wins (priority order) + for (index in indexes) { + val result = index.get(key) + if (result != null) return result + } + return null + } - override suspend fun get(key: String): T? { - // First match wins (priority order) - for (index in indexes) { - val result = index.get(key) - if (result != null) return result - } - return null - } + override suspend fun containsSource(sourceId: String): Boolean = indexes.any { it.containsSource(sourceId) } - override suspend fun containsSource(sourceId: String): Boolean { - return indexes.any { it.containsSource(sourceId) } - } + override fun distinctValues(fieldName: String): Sequence = + sequence { + val seen = mutableSetOf() + for (index in indexes) { + for (value in index.distinctValues(fieldName)) { + if (seen.add(value)) yield(value) + } + } + } - override fun distinctValues(fieldName: String): Sequence = sequence { - val seen = mutableSetOf() - for (index in indexes) { - for (value in index.distinctValues(fieldName)) { - if (seen.add(value)) yield(value) - } - } - } - - override fun close() { - for (index in indexes) { - if (index is Closeable) index.close() - } - } + override fun close() { + for (index in indexes) { + if (index is Closeable) index.close() + } + } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index e22a19f9ee..3b375b65aa 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -9,9 +9,9 @@ import androidx.sqlite.db.SupportSQLiteOpenHelper import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withContext import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexQuery @@ -57,332 +57,365 @@ import kotlin.collections.iterator * @param batchSize Number of rows per INSERT transaction. */ class SQLiteIndex( - override val descriptor: IndexDescriptor, - context: Context, - dbName: String?, - override val name: String = "sqlite:${descriptor.name}", - private val batchSize: Int = 500, + override val descriptor: IndexDescriptor, + context: Context, + dbName: String?, + override val name: String = "sqlite:${descriptor.name}", + private val batchSize: Int = 500, ) : Index { - companion object { - private val log = LoggerFactory.getLogger(SQLiteIndex::class.java) - - /** - * Max number of `_source_id` placeholders per batched DELETE. - * Kept well under SQLite's default 999 bound-parameter limit. - */ - private const val DELETE_CHUNK_SIZE = 900 - } - - - private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") - - // Field column names: "f_{fieldName}" - private val fieldColumns = descriptor.fields.associate { field -> - field.name to "f_${field.name}" - } - - // Prefix-searchable fields also get a "_lower" column - private val prefixColumns = descriptor.fields - .filter { it.prefixSearchable } - .associate { it.name to "f_${it.name}_lower" } - - private val mutex = Mutex() - @Volatile private var closed = false - private val db: SupportSQLiteDatabase - - init { - val config = SupportSQLiteOpenHelper.Configuration.builder(context) - .name(dbName) - .callback(object : SupportSQLiteOpenHelper.Callback(1) { - override fun onCreate(db: SupportSQLiteDatabase) { - createTable(db) - } - - override fun onUpgrade( - db: SupportSQLiteDatabase, oldVersion: Int, newVersion: Int, - ) { - // TODO: Add migration support - db.execSQL("DROP TABLE IF EXISTS $tableName") - createTable(db) - } - - override fun onOpen(db: SupportSQLiteDatabase) { - } - }) - .build() - - db = FrameworkSQLiteOpenHelperFactory() - .create(config) - .writableDatabase - - // Ensure table exists (for shared databases) - createTable(db) - } - - override fun query(query: IndexQuery): Sequence = runBlocking { - ifOpen(emptySequence()) { - val (sql, args) = buildSelectQuery(query) - val cursor = db.query(sql, args.toTypedArray()) - cursor.use { - val payloadIdx = it.getColumnIndexOrThrow("_payload") - buildList { - while (it.moveToNext()) { - add(descriptor.deserialize(it.getBlob(payloadIdx))) - } - } - }.asSequence() - } - } - - override suspend fun get(key: String): T? = withContext(Dispatchers.IO) { - ifOpen(null) { - val cursor = db.query( - "SELECT _payload FROM $tableName WHERE _key = ? LIMIT 1", - arrayOf(key), - ) - cursor.use { - if (it.moveToFirst()) { - descriptor.deserialize(it.getBlob(0)) - } else null - } - } - } - - override suspend fun containsSource(sourceId: String): Boolean = - withContext(Dispatchers.IO) { - ifOpen(false) { - val cursor = db.query( - "SELECT 1 FROM $tableName WHERE _source_id = ? LIMIT 1", - arrayOf(sourceId), - ) - cursor.use { it.moveToFirst() } - } - } - - override fun distinctValues(fieldName: String): Sequence = runBlocking { - ifOpen(emptySequence()) { - val col = fieldColumns[fieldName] - ?: throw IllegalArgumentException("Unknown field: $fieldName") - val cursor = db.query("SELECT DISTINCT $col FROM $tableName WHERE $col IS NOT NULL") - cursor.use { - buildList { - while (it.moveToNext()) { - add(it.getString(0)) - } - } - }.asSequence() - } - } - - override suspend fun insertAll(entries: Sequence) = withContext(Dispatchers.IO) { - val batch = mutableListOf() - for (entry in entries) { - batch.add(entry) - if (batch.size >= batchSize) { - ifOpen { insertBatchLocked(batch) } - batch.clear() - } - } - if (batch.isNotEmpty()) { - ifOpen { insertBatchLocked(batch) } - } - } - - override suspend fun insert(entry: T) = withContext(Dispatchers.IO) { - ifOpen { insertBatchLocked(listOf(entry)) } - } - - override suspend fun removeBySource(sourceId: String) = withContext(Dispatchers.IO) { - ifOpen { db.execSQL("DELETE FROM $tableName WHERE _source_id = ?", arrayOf(sourceId)) } - } - - /** - * Remove every row whose `_source_id` is in [sourceIds] using a single SQLite - * transaction. The ids are split into chunks of at most [DELETE_CHUNK_SIZE] so - * each `DELETE ... IN (?, ?, ...)` stays within SQLite's bound-parameter limit; - * all chunks run inside the one transaction, so the batch commits atomically - * (an empty [sourceIds] is a no-op and opens no transaction). - * - * @param sourceIds Source ids whose rows should be deleted. - */ - override suspend fun removeBySources(sourceIds: Collection) = - withContext(Dispatchers.IO) { - if (sourceIds.isEmpty()) return@withContext - ifOpen { - db.beginTransaction() - try { - for (chunk in sourceIds.chunked(DELETE_CHUNK_SIZE)) { - val placeholders = chunk.joinToString(",") { "?" } - db.execSQL( - "DELETE FROM $tableName WHERE _source_id IN ($placeholders)", - chunk.toTypedArray(), - ) - } - db.setTransactionSuccessful() - } finally { - db.endTransaction() - } - } - } - - override suspend fun clear() = withContext(Dispatchers.IO) { - ifOpen { db.execSQL("DELETE FROM $tableName") } - } - - override fun close() { - if (Looper.getMainLooper() == Looper.myLooper()) { - log.warn( - "SQLiteIndex.close() called on the main thread; waiting on mutex and closing db may block and cause ANR" - ) - } - runBlocking { - mutex.withLock { - if (closed) return@withLock - closed = true - db.close() - } - } - } - - private suspend inline fun ifOpen(default: R, crossinline block: () -> R): R = - mutex.withLock { if (closed) default else block() } - - private suspend inline fun ifOpen(crossinline block: () -> Unit) = - mutex.withLock { if (!closed) block() } - - suspend fun size(): Int = withContext(Dispatchers.IO) { - ifOpen(0) { - val cursor = db.query("SELECT COUNT(*) FROM $tableName") - cursor.use { if (it.moveToFirst()) it.getInt(0) else 0 } - } - } - - private fun createTable(db: SupportSQLiteDatabase) { - val columns = buildString { - append("_key TEXT PRIMARY KEY, ") - append("_source_id TEXT NOT NULL, ") - - for (field in descriptor.fields) { - val col = fieldColumns[field.name]!! - append("$col TEXT, ") - - if (field.prefixSearchable) { - val lowerCol = prefixColumns[field.name]!! - append("$lowerCol TEXT, ") - } - } - - append("_payload BLOB NOT NULL") - } - - db.execSQL("CREATE TABLE IF NOT EXISTS $tableName ($columns)") - - // Indexes - db.execSQL( - "CREATE INDEX IF NOT EXISTS idx_${tableName}_source ON $tableName(_source_id)" - ) - - for (field in descriptor.fields) { - val col = fieldColumns[field.name]!! - db.execSQL( - "CREATE INDEX IF NOT EXISTS idx_${tableName}_$col ON $tableName($col)" - ) - - if (field.prefixSearchable) { - val lowerCol = prefixColumns[field.name]!! - db.execSQL( - "CREATE INDEX IF NOT EXISTS idx_${tableName}_$lowerCol ON $tableName($lowerCol)" - ) - } - } - } - - private fun insertBatchLocked(entries: List) { - db.beginTransaction() - try { - for (entry in entries) { - val cv = ContentValues().apply { - put("_key", entry.key) - put("_source_id", entry.sourceId) - - val fields = descriptor.fieldValues(entry) - for ((fieldName, value) in fields) { - val col = fieldColumns[fieldName] ?: continue - put(col, value) - - val lowerCol = prefixColumns[fieldName] - if (lowerCol != null) { - put(lowerCol, value?.lowercase()) - } - } - - put("_payload", descriptor.serialize(entry)) - } - - db.insert( - tableName, - SQLiteDatabase.CONFLICT_REPLACE, - cv, - ) - } - db.setTransactionSuccessful() - } finally { - db.endTransaction() - } - } - - private data class SqlQuery(val sql: String, val args: List) - - private fun buildSelectQuery(query: IndexQuery): SqlQuery { - val where = StringBuilder() - val args = mutableListOf() - - fun and(clause: String, vararg values: String) { - if (where.isNotEmpty()) where.append(" AND ") - where.append(clause) - args.addAll(values) - } - - query.key?.let { and("_key = ?", it) } - query.sourceId?.let { and("_source_id = ?", it) } - - for ((field, value) in query.exactMatch) { - val col = fieldColumns[field] ?: continue - and("$col = ?", value) - } - - for ((field, prefix) in query.prefixMatch) { - val lowerCol = prefixColumns[field] - if (lowerCol != null) { - // Use the pre-lowercased column for index-friendly LIKE - and("$lowerCol LIKE ?", "${prefix.lowercase()}%") - } else { - // Fallback: case-sensitive prefix on the regular column - val col = fieldColumns[field] ?: continue - and("$col LIKE ?", "$prefix%") - } - } - - for ((field, mustExist) in query.presence) { - val col = fieldColumns[field] ?: continue - if (mustExist) { - and("$col IS NOT NULL") - } else { - and("$col IS NULL") - } - } - - val sql = buildString { - append("SELECT _payload FROM $tableName") - if (where.isNotEmpty()) { - append(" WHERE ") - append(where) - } - if (query.limit > 0) { - append(" LIMIT ${query.limit}") - } - } - - return SqlQuery(sql, args) - } + companion object { + private val log = LoggerFactory.getLogger(SQLiteIndex::class.java) + + /** + * Max number of `_source_id` placeholders per batched DELETE. + * Kept well under SQLite's default 999 bound-parameter limit. + */ + private const val DELETE_CHUNK_SIZE = 900 + } + + private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") + + // Field column names: "f_{fieldName}" + private val fieldColumns = + descriptor.fields.associate { field -> + field.name to "f_${field.name}" + } + + // Prefix-searchable fields also get a "_lower" column + private val prefixColumns = + descriptor.fields + .filter { it.prefixSearchable } + .associate { it.name to "f_${it.name}_lower" } + + private val mutex = Mutex() + + @Volatile private var closed = false + private val db: SupportSQLiteDatabase + + init { + val config = + SupportSQLiteOpenHelper.Configuration + .builder(context) + .name(dbName) + .callback( + object : SupportSQLiteOpenHelper.Callback(1) { + override fun onCreate(db: SupportSQLiteDatabase) { + createTable(db) + } + + override fun onUpgrade( + db: SupportSQLiteDatabase, + oldVersion: Int, + newVersion: Int, + ) { + // TODO: Add migration support + db.execSQL("DROP TABLE IF EXISTS $tableName") + createTable(db) + } + + override fun onOpen(db: SupportSQLiteDatabase) { + } + }, + ).build() + + db = + FrameworkSQLiteOpenHelperFactory() + .create(config) + .writableDatabase + + // Ensure table exists (for shared databases) + createTable(db) + } + + override fun query(query: IndexQuery): Sequence = + runBlocking { + ifOpen(emptySequence()) { + val (sql, args) = buildSelectQuery(query) + val cursor = db.query(sql, args.toTypedArray()) + cursor + .use { + val payloadIdx = it.getColumnIndexOrThrow("_payload") + buildList { + while (it.moveToNext()) { + add(descriptor.deserialize(it.getBlob(payloadIdx))) + } + } + }.asSequence() + } + } + + override suspend fun get(key: String): T? = + withContext(Dispatchers.IO) { + ifOpen(null) { + val cursor = + db.query( + "SELECT _payload FROM $tableName WHERE _key = ? LIMIT 1", + arrayOf(key), + ) + cursor.use { + if (it.moveToFirst()) { + descriptor.deserialize(it.getBlob(0)) + } else { + null + } + } + } + } + + override suspend fun containsSource(sourceId: String): Boolean = + withContext(Dispatchers.IO) { + ifOpen(false) { + val cursor = + db.query( + "SELECT 1 FROM $tableName WHERE _source_id = ? LIMIT 1", + arrayOf(sourceId), + ) + cursor.use { it.moveToFirst() } + } + } + + override fun distinctValues(fieldName: String): Sequence = + runBlocking { + ifOpen(emptySequence()) { + val col = + fieldColumns[fieldName] + ?: throw IllegalArgumentException("Unknown field: $fieldName") + val cursor = db.query("SELECT DISTINCT $col FROM $tableName WHERE $col IS NOT NULL") + cursor + .use { + buildList { + while (it.moveToNext()) { + add(it.getString(0)) + } + } + }.asSequence() + } + } + + override suspend fun insertAll(entries: Sequence) = + withContext(Dispatchers.IO) { + val batch = mutableListOf() + for (entry in entries) { + batch.add(entry) + if (batch.size >= batchSize) { + ifOpen { insertBatchLocked(batch) } + batch.clear() + } + } + if (batch.isNotEmpty()) { + ifOpen { insertBatchLocked(batch) } + } + } + + override suspend fun insert(entry: T) = + withContext(Dispatchers.IO) { + ifOpen { insertBatchLocked(listOf(entry)) } + } + + override suspend fun removeBySource(sourceId: String) = + withContext(Dispatchers.IO) { + ifOpen { db.execSQL("DELETE FROM $tableName WHERE _source_id = ?", arrayOf(sourceId)) } + } + + /** + * Remove every row whose `_source_id` is in [sourceIds] using a single SQLite + * transaction. The ids are split into chunks of at most [DELETE_CHUNK_SIZE] so + * each `DELETE ... IN (?, ?, ...)` stays within SQLite's bound-parameter limit; + * all chunks run inside the one transaction, so the batch commits atomically + * (an empty [sourceIds] is a no-op and opens no transaction). + * + * @param sourceIds Source ids whose rows should be deleted. + */ + override suspend fun removeBySources(sourceIds: Collection) = + withContext(Dispatchers.IO) { + if (sourceIds.isEmpty()) return@withContext + ifOpen { + db.beginTransaction() + try { + for (chunk in sourceIds.chunked(DELETE_CHUNK_SIZE)) { + val placeholders = chunk.joinToString(",") { "?" } + db.execSQL( + "DELETE FROM $tableName WHERE _source_id IN ($placeholders)", + chunk.toTypedArray(), + ) + } + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + } + + override suspend fun clear() = + withContext(Dispatchers.IO) { + ifOpen { db.execSQL("DELETE FROM $tableName") } + } + + override fun close() { + if (Looper.getMainLooper() == Looper.myLooper()) { + log.warn( + "SQLiteIndex.close() called on the main thread; waiting on mutex and closing db may block and cause ANR", + ) + } + runBlocking { + mutex.withLock { + if (closed) return@withLock + closed = true + db.close() + } + } + } + + private suspend inline fun ifOpen( + default: R, + crossinline block: () -> R, + ): R = mutex.withLock { if (closed) default else block() } + + private suspend inline fun ifOpen(crossinline block: () -> Unit) = mutex.withLock { if (!closed) block() } + + suspend fun size(): Int = + withContext(Dispatchers.IO) { + ifOpen(0) { + val cursor = db.query("SELECT COUNT(*) FROM $tableName") + cursor.use { if (it.moveToFirst()) it.getInt(0) else 0 } + } + } + + private fun createTable(db: SupportSQLiteDatabase) { + val columns = + buildString { + append("_key TEXT PRIMARY KEY, ") + append("_source_id TEXT NOT NULL, ") + + for (field in descriptor.fields) { + val col = fieldColumns[field.name]!! + append("$col TEXT, ") + + if (field.prefixSearchable) { + val lowerCol = prefixColumns[field.name]!! + append("$lowerCol TEXT, ") + } + } + + append("_payload BLOB NOT NULL") + } + + db.execSQL("CREATE TABLE IF NOT EXISTS $tableName ($columns)") + + // Indexes + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_source ON $tableName(_source_id)", + ) + + for (field in descriptor.fields) { + val col = fieldColumns[field.name]!! + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_$col ON $tableName($col)", + ) + + if (field.prefixSearchable) { + val lowerCol = prefixColumns[field.name]!! + db.execSQL( + "CREATE INDEX IF NOT EXISTS idx_${tableName}_$lowerCol ON $tableName($lowerCol)", + ) + } + } + } + + private fun insertBatchLocked(entries: List) { + db.beginTransaction() + try { + for (entry in entries) { + val cv = + ContentValues().apply { + put("_key", entry.key) + put("_source_id", entry.sourceId) + + val fields = descriptor.fieldValues(entry) + for ((fieldName, value) in fields) { + val col = fieldColumns[fieldName] ?: continue + put(col, value) + + val lowerCol = prefixColumns[fieldName] + if (lowerCol != null) { + put(lowerCol, value?.lowercase()) + } + } + + put("_payload", descriptor.serialize(entry)) + } + + db.insert( + tableName, + SQLiteDatabase.CONFLICT_REPLACE, + cv, + ) + } + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + + private data class SqlQuery( + val sql: String, + val args: List, + ) + + private fun buildSelectQuery(query: IndexQuery): SqlQuery { + val where = StringBuilder() + val args = mutableListOf() + + fun and( + clause: String, + vararg values: String, + ) { + if (where.isNotEmpty()) where.append(" AND ") + where.append(clause) + args.addAll(values) + } + + query.key?.let { and("_key = ?", it) } + query.sourceId?.let { and("_source_id = ?", it) } + + for ((field, value) in query.exactMatch) { + val col = fieldColumns[field] ?: continue + and("$col = ?", value) + } + + for ((field, prefix) in query.prefixMatch) { + val lowerCol = prefixColumns[field] + if (lowerCol != null) { + // Use the pre-lowercased column for index-friendly LIKE + and("$lowerCol LIKE ?", "${prefix.lowercase()}%") + } else { + // Fallback: case-sensitive prefix on the regular column + val col = fieldColumns[field] ?: continue + and("$col LIKE ?", "$prefix%") + } + } + + for ((field, mustExist) in query.presence) { + val col = fieldColumns[field] ?: continue + if (mustExist) { + and("$col IS NOT NULL") + } else { + and("$col IS NULL") + } + } + + val sql = + buildString { + append("SELECT _payload FROM $tableName") + if (where.isNotEmpty()) { + append(" WHERE ") + append(where) + } + if (query.limit > 0) { + append(" LIMIT ${query.limit}") + } + } + + return SqlQuery(sql, args) + } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index ed354a7a1a..2f6dd6a275 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -11,77 +11,75 @@ import java.io.Closeable * @param T The indexed type. */ interface ReadableIndex { - - /** - * Query the index. Returns a lazy [Sequence] of matching entries. - * - * Results are not guaranteed to be in any particular order - * unless the implementation specifies otherwise. - * - * If [IndexQuery.limit] is 0, all matches are emitted. - */ - fun query(query: IndexQuery): Sequence - - /** - * Point lookup by key. Returns null if not found. - */ - suspend fun get(key: String): T? - - /** - * Fast existence check for a source. - */ - suspend fun containsSource(sourceId: String): Boolean - - /** - * Returns distinct values for a given field across all entries. - * - * Useful for enumerating packages, kinds, etc. without - * deserializing full entries. - * - * @param fieldName Must be one of the fields declared in the - * [IndexDescriptor]. - */ - fun distinctValues(fieldName: String): Sequence + /** + * Query the index. Returns a lazy [Sequence] of matching entries. + * + * Results are not guaranteed to be in any particular order + * unless the implementation specifies otherwise. + * + * If [IndexQuery.limit] is 0, all matches are emitted. + */ + fun query(query: IndexQuery): Sequence + + /** + * Point lookup by key. Returns null if not found. + */ + suspend fun get(key: String): T? + + /** + * Fast existence check for a source. + */ + suspend fun containsSource(sourceId: String): Boolean + + /** + * Returns distinct values for a given field across all entries. + * + * Useful for enumerating packages, kinds, etc. without + * deserializing full entries. + * + * @param fieldName Must be one of the fields declared in the + * [IndexDescriptor]. + */ + fun distinctValues(fieldName: String): Sequence } /** * Write interface for mutating an index. */ interface WritableIndex { - - /** - * Insert entries from a [Sequence]. - * - * Entries are consumed lazily from the sequence and batched - * internally for throughput. If an entry with the same key - * already exists, it is replaced. - */ - suspend fun insertAll(entries: Sequence) - - /** - * Convenience: insert a single entry. - */ - suspend fun insert(entry: T) - - /** - * Remove all entries from the given source. - */ - suspend fun removeBySource(sourceId: String) - - /** - * Remove all entries from the given sources in a single transaction. - * - * Equivalent to calling [removeBySource] for each id, but issues the - * deletes as one batched, transactional operation instead of N - * sequential statements. Implementations should chunk the ids so the - * generated SQL stays within parameter limits. - */ - suspend fun removeBySources(sourceIds: Collection) - - /** - * Remove all entries. - */ - suspend fun clear() + /** + * Insert entries from a [Sequence]. + * + * Entries are consumed lazily from the sequence and batched + * internally for throughput. If an entry with the same key + * already exists, it is replaced. + */ + suspend fun insertAll(entries: Sequence) + + /** + * Convenience: insert a single entry. + */ + suspend fun insert(entry: T) + + /** + * Remove all entries from the given source. + */ + suspend fun removeBySource(sourceId: String) + + /** + * Remove all entries from the given sources in a single transaction. + * + * Equivalent to calling [removeBySource] for each id, but issues the + * deletes as one batched, transactional operation instead of N + * sequential statements. Implementations should chunk the ids so the + * generated SQL stays within parameter limits. + */ + suspend fun removeBySources(sourceIds: Collection) + + /** + * Remove all entries. + */ + suspend fun clear() } /** @@ -89,13 +87,15 @@ interface WritableIndex { * * @param T The indexed type. */ -interface Index : ReadableIndex, WritableIndex, Closeable { - - /** Human-readable name for logging. */ - val name: String +interface Index : + ReadableIndex, + WritableIndex, + Closeable { + /** Human-readable name for logging. */ + val name: String - /** The descriptor governing serialization and field extraction. */ - val descriptor: IndexDescriptor + /** The descriptor governing serialization and field extraction. */ + val descriptor: IndexDescriptor - override fun close() {} + override fun close() {} } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt index 8b3e1a6a12..e7c1e27e1b 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -8,71 +8,80 @@ package org.appdevforall.codeonthego.indexing.api * for Kotlin symbols, Android resources, Python declarations, etc. */ data class IndexQuery( - /** Exact match predicates: field name → expected value. */ - val exactMatch: Map = emptyMap(), - - /** Prefix match predicates: field name → prefix (case-insensitive). */ - val prefixMatch: Map = emptyMap(), - - /** - * Presence predicates: field name → whether the field must be - * non-null (true) or null (false). - */ - val presence: Map = emptyMap(), - - /** Filter by source ID. */ - val sourceId: String? = null, - - /** Filter by key (exact). */ - val key: String? = null, - - /** Maximum number of results. 0 = unlimited (use with care). */ - val limit: Int = 200, + /** Exact match predicates: field name → expected value. */ + val exactMatch: Map = emptyMap(), + /** Prefix match predicates: field name → prefix (case-insensitive). */ + val prefixMatch: Map = emptyMap(), + /** + * Presence predicates: field name → whether the field must be + * non-null (true) or null (false). + */ + val presence: Map = emptyMap(), + /** Filter by source ID. */ + val sourceId: String? = null, + /** Filter by key (exact). */ + val key: String? = null, + /** Maximum number of results. 0 = unlimited (use with care). */ + val limit: Int = 200, ) { - companion object { - /** Match everything up to [limit]. */ - val ALL = IndexQuery() + companion object { + /** Match everything up to [limit]. */ + val ALL = IndexQuery() - /** Exact key lookup. */ - fun byKey(key: String) = IndexQuery(key = key, limit = 1) + /** Exact key lookup. */ + fun byKey(key: String) = IndexQuery(key = key, limit = 1) - /** All entries from a specific source. */ - fun bySource(sourceId: String) = IndexQuery(sourceId = sourceId, limit = 0) - } + /** All entries from a specific source. */ + fun bySource(sourceId: String) = IndexQuery(sourceId = sourceId, limit = 0) + } } /** * DSL builder for [IndexQuery]. */ class IndexQueryBuilder { - private val exact = mutableMapOf() - private val prefix = mutableMapOf() - private val pres = mutableMapOf() - var sourceId: String? = null - var key: String? = null - var limit: Int = 200 + private val exact = mutableMapOf() + private val prefix = mutableMapOf() + private val pres = mutableMapOf() + var sourceId: String? = null + var key: String? = null + var limit: Int = 200 - /** Exact match on a field. */ - fun eq(field: String, value: String) { exact[field] = value } + /** Exact match on a field. */ + fun eq( + field: String, + value: String, + ) { + exact[field] = value + } - /** Prefix match on a field (case-insensitive). */ - fun prefix(field: String, value: String) { prefix[field] = value } + /** Prefix match on a field (case-insensitive). */ + fun prefix( + field: String, + value: String, + ) { + prefix[field] = value + } - /** Field must be non-null. */ - fun exists(field: String) { pres[field] = true } + /** Field must be non-null. */ + fun exists(field: String) { + pres[field] = true + } - /** Field must be null. */ - fun notExists(field: String) { pres[field] = false } + /** Field must be null. */ + fun notExists(field: String) { + pres[field] = false + } - fun build() = IndexQuery( - exactMatch = exact.toMap(), - prefixMatch = prefix.toMap(), - presence = pres.toMap(), - sourceId = sourceId, - key = key, - limit = limit, - ) + fun build() = + IndexQuery( + exactMatch = exact.toMap(), + prefixMatch = prefix.toMap(), + presence = pres.toMap(), + sourceId = sourceId, + key = key, + limit = limit, + ) } -inline fun indexQuery(block: IndexQueryBuilder.() -> Unit): IndexQuery = - IndexQueryBuilder().apply(block).build() +inline fun indexQuery(block: IndexQueryBuilder.() -> Unit): IndexQuery = IndexQueryBuilder().apply(block).build() diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt index 81168d8d74..09d9036bac 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt @@ -12,154 +12,168 @@ import org.junit.runners.JUnit4 @RunWith(JUnit4::class) class FilteredIndexTest { - - data class Entry( - override val key: String, - override val sourceId: String, - val value: String, - ) : Indexable - - private val descriptor = object : IndexDescriptor { - override val name = "test_filtered" - override val fields = listOf(IndexField("value")) - - override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) - - override fun serialize(entry: Entry) = - "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() - - override fun deserialize(bytes: ByteArray): Entry { - val parts = String(bytes).split("|") - return Entry(parts[0], parts[1], parts[2]) - } - } - - private suspend fun setupBackingAndFiltered(): Pair, FilteredIndex> { - val backing = InMemoryIndex(descriptor) - backing.insert(Entry("k1", "src1", "val1")) - backing.insert(Entry("k2", "src1", "val2")) - backing.insert(Entry("k3", "src2", "val3")) - return backing to FilteredIndex(backing) - } - - @Test - fun `query returns nothing when no sources active`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.query(IndexQuery.ALL).toList()).isEmpty() - } - - @Test - fun `query returns entries for active source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k1", "k2") - } - - @Test - fun `deactivateSource hides entries from that source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.activateSource("src2") - filtered.deactivateSource("src1") - - val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k3") - } - - @Test - fun `setActiveSources replaces entire active set`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.setActiveSources(setOf("src2")) - - val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k3") - } - - @Test - fun `get returns null when source is inactive`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.get("k1")).isNull() - } - - @Test - fun `get returns entry when source is active`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - assertThat(filtered.get("k1")).isNotNull() - assertThat(filtered.get("k1")!!.key).isEqualTo("k1") - } - - @Test - fun `get returns null for nonexistent key even with active source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - assertThat(filtered.get("missing")).isNull() - } - - @Test - fun `containsSource returns false when source inactive`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.containsSource("src1")).isFalse() - } - - @Test - fun `containsSource returns true only when source is active AND in backing`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - assertThat(filtered.containsSource("src1")).isTrue() - assertThat(filtered.containsSource("src999")).isFalse() - } - - @Test - fun `isCached returns true if source exists in backing regardless of active state`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.isCached("src1")).isTrue() - assertThat(filtered.isCached("src999")).isFalse() - } - - @Test - fun `activeSources returns current active set`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.activateSource("src2") - assertThat(filtered.activeSources()).containsExactly("src1", "src2") - } - - @Test - fun `isActive reflects current state`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - assertThat(filtered.isActive("src1")).isFalse() - filtered.activateSource("src1") - assertThat(filtered.isActive("src1")).isTrue() - filtered.deactivateSource("src1") - assertThat(filtered.isActive("src1")).isFalse() - } - - @Test - fun `explicit sourceId query returns empty for inactive source`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - // src1 has entries in backing but is not active - val results = filtered.query(IndexQuery.bySource("src1")).toList() - assertThat(results).isEmpty() - } - - @Test - fun `all sources active returns all entries from backing`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.activateSource("src2") - - val results = filtered.query(IndexQuery.ALL).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `close clears active sources`() = runTest { - val (_, filtered) = setupBackingAndFiltered() - filtered.activateSource("src1") - filtered.close() - assertThat(filtered.activeSources()).isEmpty() - } + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_filtered" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private suspend fun setupBackingAndFiltered(): Pair, FilteredIndex> { + val backing = InMemoryIndex(descriptor) + backing.insert(Entry("k1", "src1", "val1")) + backing.insert(Entry("k2", "src1", "val2")) + backing.insert(Entry("k3", "src2", "val3")) + return backing to FilteredIndex(backing) + } + + @Test + fun `query returns nothing when no sources active`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.query(IndexQuery.ALL).toList()).isEmpty() + } + + @Test + fun `query returns entries for active source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k1", "k2") + } + + @Test + fun `deactivateSource hides entries from that source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.activateSource("src2") + filtered.deactivateSource("src1") + + val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k3") + } + + @Test + fun `setActiveSources replaces entire active set`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.setActiveSources(setOf("src2")) + + val keys = filtered.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k3") + } + + @Test + fun `get returns null when source is inactive`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.get("k1")).isNull() + } + + @Test + fun `get returns entry when source is active`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + assertThat(filtered.get("k1")).isNotNull() + assertThat(filtered.get("k1")!!.key).isEqualTo("k1") + } + + @Test + fun `get returns null for nonexistent key even with active source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + assertThat(filtered.get("missing")).isNull() + } + + @Test + fun `containsSource returns false when source inactive`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.containsSource("src1")).isFalse() + } + + @Test + fun `containsSource returns true only when source is active AND in backing`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + assertThat(filtered.containsSource("src1")).isTrue() + assertThat(filtered.containsSource("src999")).isFalse() + } + + @Test + fun `isCached returns true if source exists in backing regardless of active state`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.isCached("src1")).isTrue() + assertThat(filtered.isCached("src999")).isFalse() + } + + @Test + fun `activeSources returns current active set`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.activateSource("src2") + assertThat(filtered.activeSources()).containsExactly("src1", "src2") + } + + @Test + fun `isActive reflects current state`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + assertThat(filtered.isActive("src1")).isFalse() + filtered.activateSource("src1") + assertThat(filtered.isActive("src1")).isTrue() + filtered.deactivateSource("src1") + assertThat(filtered.isActive("src1")).isFalse() + } + + @Test + fun `explicit sourceId query returns empty for inactive source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + // src1 has entries in backing but is not active + val results = filtered.query(IndexQuery.bySource("src1")).toList() + assertThat(results).isEmpty() + } + + @Test + fun `all sources active returns all entries from backing`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.activateSource("src2") + + val results = filtered.query(IndexQuery.ALL).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `close clears active sources`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + filtered.close() + assertThat(filtered.activeSources()).isEmpty() + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt index 45cff41032..2f2c74130d 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt @@ -13,237 +13,265 @@ import org.junit.runners.JUnit4 @RunWith(JUnit4::class) class InMemoryIndexTest { - - data class TestEntry( - override val key: String, - override val sourceId: String, - val name: String, - val category: String? = null, - ) : Indexable - - private val descriptor = object : IndexDescriptor { - override val name = "test" - override val fields = listOf( - IndexField("name", prefixSearchable = true), - IndexField("category"), - ) - - override fun fieldValues(entry: TestEntry) = mapOf( - "name" to entry.name, - "category" to entry.category, - ) - - override fun serialize(entry: TestEntry) = - "${entry.key}|${entry.sourceId}|${entry.name}|${entry.category}".toByteArray() - - override fun deserialize(bytes: ByteArray): TestEntry { - val parts = String(bytes).split("|") - return TestEntry(parts[0], parts[1], parts[2], parts[3].takeIf { it != "null" }) - } - } - - private fun makeIndex() = InMemoryIndex(descriptor) - - private fun entry(key: String, sourceId: String, name: String, category: String? = null) = - TestEntry(key, sourceId, name, category) - - @Test - fun `insert and get by key`() = runTest { - val index = makeIndex() - val e = entry("k1", "src1", "Foo") - index.insert(e) - assertThat(index.get("k1")).isEqualTo(e) - assertThat(index.get("missing")).isNull() - } - - @Test - fun `query returns all entries when no predicates`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - index.insert(entry("k3", "src2", "Gamma")) - - val results = index.query(IndexQuery.ALL).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `exact match on field`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta", "app")) - index.insert(entry("k3", "src2", "Gamma", "lib")) - - val results = index.query(indexQuery { eq("category", "lib") }).toList() - assertThat(results.map { it.key }).containsExactly("k1", "k3") - } - - @Test - fun `prefix match on field`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "ArrayList")) - index.insert(entry("k2", "src1", "ArrayDeque")) - index.insert(entry("k3", "src1", "String")) - - val results = index.query(indexQuery { prefix("name", "Array") }).toList() - assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) - } - - @Test - fun `prefix match is case-insensitive`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "ArrayList")) - index.insert(entry("k2", "src1", "ARRAYDEQUE")) - - val results = index.query(indexQuery { prefix("name", "array") }).toList() - assertThat(results).hasSize(2) - } - - @Test - fun `containsSource returns true for indexed source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Foo")) - assertThat(index.containsSource("src1")).isTrue() - assertThat(index.containsSource("src999")).isFalse() - } - - @Test - fun `removeBySource removes all entries from that source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - index.insert(entry("k3", "src2", "Gamma")) - - index.removeBySource("src1") - - assertThat(index.get("k1")).isNull() - assertThat(index.get("k2")).isNull() - assertThat(index.get("k3")).isNotNull() - assertThat(index.containsSource("src1")).isFalse() - assertThat(index.size).isEqualTo(1) - } - - @Test - fun `clear removes all entries`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src2", "Beta")) - - index.clear() - - assertThat(index.size).isEqualTo(0) - assertThat(index.sourceCount).isEqualTo(0) - } - - @Test - fun `insert replaces existing entry with same key`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Original")) - index.insert(entry("k1", "src1", "Updated")) - - assertThat(index.get("k1")!!.name).isEqualTo("Updated") - assertThat(index.size).isEqualTo(1) - } - - @Test - fun `query by key returns single match`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - - val results = index.query(IndexQuery.byKey("k1")).toList() - assertThat(results).hasSize(1) - assertThat(results[0].key).isEqualTo("k1") - } - - @Test - fun `query by source returns entries for that source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha")) - index.insert(entry("k2", "src1", "Beta")) - index.insert(entry("k3", "src2", "Gamma")) - - val results = index.query(IndexQuery.bySource("src1")).toList() - assertThat(results.map { it.key }).containsExactly("k1", "k2") - } - - @Test - fun `query respects limit`() = runTest { - val index = makeIndex() - repeat(10) { i -> index.insert(entry("k$i", "src1", "Name$i")) } - - val results = index.query(IndexQuery(limit = 3)).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `presence filter - field exists`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta")) // no category - - val results = index.query(indexQuery { exists("category") }).toList() - assertThat(results.map { it.key }).containsExactly("k1") - } - - @Test - fun `presence filter - field absent`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta")) - - val results = index.query(indexQuery { notExists("category") }).toList() - assertThat(results.map { it.key }).containsExactly("k2") - } - - @Test - fun `distinctValues returns unique field values`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Alpha", "lib")) - index.insert(entry("k2", "src1", "Beta", "lib")) - index.insert(entry("k3", "src2", "Gamma", "app")) - - val values = index.distinctValues("category").toSet() - assertThat(values).containsExactly("lib", "app") - } - - @Test - fun `multi-field combined query`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "ArrayList", "java")) - index.insert(entry("k2", "src1", "ArrayDeque", "java")) - index.insert(entry("k3", "src2", "ArrayBlockingQueue", "concurrent")) - - val results = index.query(indexQuery { - eq("category", "java") - prefix("name", "Array") - }).toList() - assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) - } - - @Test - fun `insertAll inserts multiple entries`() = runTest { - val index = makeIndex() - val entries = (1..5).map { i -> entry("k$i", "src1", "Name$i") } - index.insertAll(entries.asSequence()) - - assertThat(index.size).isEqualTo(5) - } - - @Test - fun `query with unknown field returns empty sequence`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Foo")) - - val results = index.query(indexQuery { eq("nonexistentField", "value") }).toList() - assertThat(results).isEmpty() - } - - @Test - fun `removeBySource is no-op for unknown source`() = runTest { - val index = makeIndex() - index.insert(entry("k1", "src1", "Foo")) - index.removeBySource("unknownSource") - assertThat(index.size).isEqualTo(1) - } + data class TestEntry( + override val key: String, + override val sourceId: String, + val name: String, + val category: String? = null, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test" + override val fields = + listOf( + IndexField("name", prefixSearchable = true), + IndexField("category"), + ) + + override fun fieldValues(entry: TestEntry) = + mapOf( + "name" to entry.name, + "category" to entry.category, + ) + + override fun serialize(entry: TestEntry) = "${entry.key}|${entry.sourceId}|${entry.name}|${entry.category}".toByteArray() + + override fun deserialize(bytes: ByteArray): TestEntry { + val parts = String(bytes).split("|") + return TestEntry(parts[0], parts[1], parts[2], parts[3].takeIf { it != "null" }) + } + } + + private fun makeIndex() = InMemoryIndex(descriptor) + + private fun entry( + key: String, + sourceId: String, + name: String, + category: String? = null, + ) = TestEntry(key, sourceId, name, category) + + @Test + fun `insert and get by key`() = + runTest { + val index = makeIndex() + val e = entry("k1", "src1", "Foo") + index.insert(e) + assertThat(index.get("k1")).isEqualTo(e) + assertThat(index.get("missing")).isNull() + } + + @Test + fun `query returns all entries when no predicates`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + index.insert(entry("k3", "src2", "Gamma")) + + val results = index.query(IndexQuery.ALL).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `exact match on field`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta", "app")) + index.insert(entry("k3", "src2", "Gamma", "lib")) + + val results = index.query(indexQuery { eq("category", "lib") }).toList() + assertThat(results.map { it.key }).containsExactly("k1", "k3") + } + + @Test + fun `prefix match on field`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "ArrayList")) + index.insert(entry("k2", "src1", "ArrayDeque")) + index.insert(entry("k3", "src1", "String")) + + val results = index.query(indexQuery { prefix("name", "Array") }).toList() + assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) + } + + @Test + fun `prefix match is case-insensitive`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "ArrayList")) + index.insert(entry("k2", "src1", "ARRAYDEQUE")) + + val results = index.query(indexQuery { prefix("name", "array") }).toList() + assertThat(results).hasSize(2) + } + + @Test + fun `containsSource returns true for indexed source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + assertThat(index.containsSource("src1")).isTrue() + assertThat(index.containsSource("src999")).isFalse() + } + + @Test + fun `removeBySource removes all entries from that source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + index.insert(entry("k3", "src2", "Gamma")) + + index.removeBySource("src1") + + assertThat(index.get("k1")).isNull() + assertThat(index.get("k2")).isNull() + assertThat(index.get("k3")).isNotNull() + assertThat(index.containsSource("src1")).isFalse() + assertThat(index.size).isEqualTo(1) + } + + @Test + fun `clear removes all entries`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src2", "Beta")) + + index.clear() + + assertThat(index.size).isEqualTo(0) + assertThat(index.sourceCount).isEqualTo(0) + } + + @Test + fun `insert replaces existing entry with same key`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Original")) + index.insert(entry("k1", "src1", "Updated")) + + assertThat(index.get("k1")!!.name).isEqualTo("Updated") + assertThat(index.size).isEqualTo(1) + } + + @Test + fun `query by key returns single match`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + + val results = index.query(IndexQuery.byKey("k1")).toList() + assertThat(results).hasSize(1) + assertThat(results[0].key).isEqualTo("k1") + } + + @Test + fun `query by source returns entries for that source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha")) + index.insert(entry("k2", "src1", "Beta")) + index.insert(entry("k3", "src2", "Gamma")) + + val results = index.query(IndexQuery.bySource("src1")).toList() + assertThat(results.map { it.key }).containsExactly("k1", "k2") + } + + @Test + fun `query respects limit`() = + runTest { + val index = makeIndex() + repeat(10) { i -> index.insert(entry("k$i", "src1", "Name$i")) } + + val results = index.query(IndexQuery(limit = 3)).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `presence filter - field exists`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta")) // no category + + val results = index.query(indexQuery { exists("category") }).toList() + assertThat(results.map { it.key }).containsExactly("k1") + } + + @Test + fun `presence filter - field absent`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta")) + + val results = index.query(indexQuery { notExists("category") }).toList() + assertThat(results.map { it.key }).containsExactly("k2") + } + + @Test + fun `distinctValues returns unique field values`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Alpha", "lib")) + index.insert(entry("k2", "src1", "Beta", "lib")) + index.insert(entry("k3", "src2", "Gamma", "app")) + + val values = index.distinctValues("category").toSet() + assertThat(values).containsExactly("lib", "app") + } + + @Test + fun `multi-field combined query`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "ArrayList", "java")) + index.insert(entry("k2", "src1", "ArrayDeque", "java")) + index.insert(entry("k3", "src2", "ArrayBlockingQueue", "concurrent")) + + val results = + index + .query( + indexQuery { + eq("category", "java") + prefix("name", "Array") + }, + ).toList() + assertThat(results.map { it.key }).containsExactlyElementsIn(listOf("k1", "k2")) + } + + @Test + fun `insertAll inserts multiple entries`() = + runTest { + val index = makeIndex() + val entries = (1..5).map { i -> entry("k$i", "src1", "Name$i") } + index.insertAll(entries.asSequence()) + + assertThat(index.size).isEqualTo(5) + } + + @Test + fun `query with unknown field returns empty sequence`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + val results = index.query(indexQuery { eq("nonexistentField", "value") }).toList() + assertThat(results).isEmpty() + } + + @Test + fun `removeBySource is no-op for unknown source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + index.removeBySource("unknownSource") + assertThat(index.size).isEqualTo(1) + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt index d41c8ed03c..4086d75548 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/MergedIndexTest.kt @@ -12,145 +12,160 @@ import org.junit.runners.JUnit4 @RunWith(JUnit4::class) class MergedIndexTest { - - data class Entry( - override val key: String, - override val sourceId: String, - val value: String, - ) : Indexable - - private val descriptor = object : IndexDescriptor { - override val name = "test_merged" - override val fields = listOf(IndexField("value")) - - override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) - - override fun serialize(entry: Entry) = - "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() - - override fun deserialize(bytes: ByteArray): Entry { - val parts = String(bytes).split("|") - return Entry(parts[0], parts[1], parts[2]) - } - } - - private fun makeIndex() = InMemoryIndex(descriptor) - - @Test - fun `merges results from multiple indexes`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "v1")) } - val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "v2")) } - val merged = MergedIndex(idx1, idx2) - - val keys = merged.query(IndexQuery.ALL).map { it.key }.toList() - assertThat(keys).containsExactly("k1", "k2") - } - - @Test - fun `deduplicates by key - first index wins`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "from-index1")) } - val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "from-index2")) } - val merged = MergedIndex(idx1, idx2) - - val results = merged.query(IndexQuery.ALL).toList() - assertThat(results).hasSize(1) - assertThat(results[0].value).isEqualTo("from-index1") - } - - @Test - fun `respects limit across indexes`() = runTest { - val idx1 = makeIndex().also { e -> - (1..5).forEach { i -> e.insert(Entry("k$i", "s1", "v$i")) } - } - val idx2 = makeIndex().also { e -> - (6..10).forEach { i -> e.insert(Entry("k$i", "s2", "v$i")) } - } - val merged = MergedIndex(idx1, idx2) - - val results = merged.query(IndexQuery(limit = 3)).toList() - assertThat(results).hasSize(3) - } - - @Test - fun `get returns first match in priority order`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "primary")) } - val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "secondary")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.get("k1")!!.value).isEqualTo("primary") - } - - @Test - fun `get returns from second index when not in first`() = runTest { - val idx1 = makeIndex() - val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "second")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.get("k2")!!.value).isEqualTo("second") - } - - @Test - fun `get returns null if key not in any index`() = runTest { - val merged = MergedIndex(makeIndex()) - assertThat(merged.get("missing")).isNull() - } - - @Test - fun `containsSource checks all indexes`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "src1", "v1")) } - val idx2 = makeIndex().also { it.insert(Entry("k2", "src2", "v2")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.containsSource("src1")).isTrue() - assertThat(merged.containsSource("src2")).isTrue() - assertThat(merged.containsSource("src999")).isFalse() - } - - @Test - fun `distinctValues deduplicates across indexes`() = runTest { - val idx1 = makeIndex().also { e -> - e.insert(Entry("k1", "s1", "alpha")) - e.insert(Entry("k2", "s1", "beta")) - } - val idx2 = makeIndex().also { e -> - e.insert(Entry("k3", "s2", "beta")) // duplicate - e.insert(Entry("k4", "s2", "gamma")) - } - val merged = MergedIndex(idx1, idx2) - - val values = merged.distinctValues("value").toSet() - assertThat(values).containsExactly("alpha", "beta", "gamma") - } - - @Test - fun `empty merged index returns empty results`() = runTest { - val merged = MergedIndex(emptyList()) - - assertThat(merged.query(IndexQuery.ALL).toList()).isEmpty() - assertThat(merged.get("k1")).isNull() - assertThat(merged.containsSource("s1")).isFalse() - assertThat(merged.distinctValues("value").toList()).isEmpty() - } - - @Test - fun `single index merged works identically to unmerged`() = runTest { - val idx = makeIndex().also { - it.insert(Entry("k1", "s1", "val1")) - it.insert(Entry("k2", "s1", "val2")) - } - val merged = MergedIndex(idx) - - assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) - assertThat(merged.get("k1")).isNotNull() - assertThat(merged.containsSource("s1")).isTrue() - } - - @Test - fun `vararg constructor builds merged index`() = runTest { - val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "a")) } - val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "b")) } - val merged = MergedIndex(idx1, idx2) - - assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) - } + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_merged" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private fun makeIndex() = InMemoryIndex(descriptor) + + @Test + fun `merges results from multiple indexes`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "v1")) } + val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "v2")) } + val merged = MergedIndex(idx1, idx2) + + val keys = merged.query(IndexQuery.ALL).map { it.key }.toList() + assertThat(keys).containsExactly("k1", "k2") + } + + @Test + fun `deduplicates by key - first index wins`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "from-index1")) } + val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "from-index2")) } + val merged = MergedIndex(idx1, idx2) + + val results = merged.query(IndexQuery.ALL).toList() + assertThat(results).hasSize(1) + assertThat(results[0].value).isEqualTo("from-index1") + } + + @Test + fun `respects limit across indexes`() = + runTest { + val idx1 = + makeIndex().also { e -> + (1..5).forEach { i -> e.insert(Entry("k$i", "s1", "v$i")) } + } + val idx2 = + makeIndex().also { e -> + (6..10).forEach { i -> e.insert(Entry("k$i", "s2", "v$i")) } + } + val merged = MergedIndex(idx1, idx2) + + val results = merged.query(IndexQuery(limit = 3)).toList() + assertThat(results).hasSize(3) + } + + @Test + fun `get returns first match in priority order`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "primary")) } + val idx2 = makeIndex().also { it.insert(Entry("k1", "s2", "secondary")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.get("k1")!!.value).isEqualTo("primary") + } + + @Test + fun `get returns from second index when not in first`() = + runTest { + val idx1 = makeIndex() + val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "second")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.get("k2")!!.value).isEqualTo("second") + } + + @Test + fun `get returns null if key not in any index`() = + runTest { + val merged = MergedIndex(makeIndex()) + assertThat(merged.get("missing")).isNull() + } + + @Test + fun `containsSource checks all indexes`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "src1", "v1")) } + val idx2 = makeIndex().also { it.insert(Entry("k2", "src2", "v2")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.containsSource("src1")).isTrue() + assertThat(merged.containsSource("src2")).isTrue() + assertThat(merged.containsSource("src999")).isFalse() + } + + @Test + fun `distinctValues deduplicates across indexes`() = + runTest { + val idx1 = + makeIndex().also { e -> + e.insert(Entry("k1", "s1", "alpha")) + e.insert(Entry("k2", "s1", "beta")) + } + val idx2 = + makeIndex().also { e -> + e.insert(Entry("k3", "s2", "beta")) // duplicate + e.insert(Entry("k4", "s2", "gamma")) + } + val merged = MergedIndex(idx1, idx2) + + val values = merged.distinctValues("value").toSet() + assertThat(values).containsExactly("alpha", "beta", "gamma") + } + + @Test + fun `empty merged index returns empty results`() = + runTest { + val merged = MergedIndex(emptyList()) + + assertThat(merged.query(IndexQuery.ALL).toList()).isEmpty() + assertThat(merged.get("k1")).isNull() + assertThat(merged.containsSource("s1")).isFalse() + assertThat(merged.distinctValues("value").toList()).isEmpty() + } + + @Test + fun `single index merged works identically to unmerged`() = + runTest { + val idx = + makeIndex().also { + it.insert(Entry("k1", "s1", "val1")) + it.insert(Entry("k2", "s1", "val2")) + } + val merged = MergedIndex(idx) + + assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) + assertThat(merged.get("k1")).isNotNull() + assertThat(merged.containsSource("s1")).isTrue() + } + + @Test + fun `vararg constructor builds merged index`() = + runTest { + val idx1 = makeIndex().also { it.insert(Entry("k1", "s1", "a")) } + val idx2 = makeIndex().also { it.insert(Entry("k2", "s2", "b")) } + val merged = MergedIndex(idx1, idx2) + + assertThat(merged.query(IndexQuery.ALL).toList()).hasSize(2) + } } From 13ba7849cf244f9d91c5b79a2788490c58c7c530 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 09:22:35 +0000 Subject: [PATCH 02/48] ADFA-6228: Let an index query be scoped to a set of sources A ClassTrie is per module while the symbol index is global and keyed by JAR path, so "classes on module X's classpath" needs `sourceId IN (X's JARs)`, which a single `sourceId` cannot express. The scoped SELECT is chunked the way `removeBySources` already chunks its DELETE, because a module's classpath can exceed SQLite's bound-parameter limit. Chunks are disjoint on `_source_id`, so no row is returned twice and each statement can take what remains of the limit. `null` and an empty collection mean different things: no restriction at all versus a scope that admits nothing. Keeping them distinct is what lets a caller pass a computed scope without having to special-case the empty result. --- .../codeonthego/indexing/InMemoryIndex.kt | 7 ++ .../codeonthego/indexing/SQLiteIndex.kt | 98 +++++++++++---- .../codeonthego/indexing/api/Query.kt | 11 ++ .../codeonthego/indexing/InMemoryIndexTest.kt | 37 ++++++ .../indexing/SQLiteIndexScopeTest.kt | 114 ++++++++++++++++++ 5 files changed, 244 insertions(+), 23 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index e0ccaa41ca..f7660c2a9c 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -146,6 +146,13 @@ class InMemoryIndex( candidates = intersect(candidates, sourceMap[query.sourceId]) } + val sourceIds = query.sourceIds + if (sourceIds != null) { + // An empty scope matches nothing, as distinct from an absent scope matching anything. + val scoped = sourceIds.flatMapTo(mutableSetOf()) { sourceMap[it].orEmpty() } + candidates = intersect(candidates, scoped) + } + for ((field, value) in query.exactMatch) { val fieldMap = fieldMaps[field] ?: return@read emptySequence() candidates = intersect(candidates, fieldMap[value]) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 3b375b65aa..3e6606ba80 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -71,6 +71,13 @@ class SQLiteIndex( * Kept well under SQLite's default 999 bound-parameter limit. */ private const val DELETE_CHUNK_SIZE = 900 + + /** + * Max number of `_source_id` placeholders per source-scoped SELECT, under the same + * bound-parameter limit as [DELETE_CHUNK_SIZE]. A scoped query runs one statement per chunk; + * chunks are disjoint on `_source_id`, so no row can be returned by two of them. + */ + private const val SOURCE_ID_CHUNK_SIZE = 900 } private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") @@ -130,17 +137,23 @@ class SQLiteIndex( override fun query(query: IndexQuery): Sequence = runBlocking { ifOpen(emptySequence()) { - val (sql, args) = buildSelectQuery(query) - val cursor = db.query(sql, args.toTypedArray()) - cursor - .use { + val limit = effectiveLimit(query) + val results = mutableListOf() + for (chunk in sourceIdChunks(query)) { + if (results.size >= limit) { + break + } + + val (sql, args) = buildSelectQuery(query, chunk, limit - results.size) + val cursor = db.query(sql, args.toTypedArray()) + cursor.use { val payloadIdx = it.getColumnIndexOrThrow("_payload") - buildList { - while (it.moveToNext()) { - add(descriptor.deserialize(it.getBlob(payloadIdx))) - } + while (it.moveToNext()) { + results.add(descriptor.deserialize(it.getBlob(payloadIdx))) } - }.asSequence() + } + } + results.asSequence() } } @@ -362,7 +375,53 @@ class SQLiteIndex( val args: List, ) - private fun buildSelectQuery(query: IndexQuery): SqlQuery { + private fun effectiveLimit(query: IndexQuery) = if (query.limit > 0) query.limit else Int.MAX_VALUE + + /** + * Splits [IndexQuery.sourceIds] into groups small enough for one `IN (...)` clause. + * + * Returns a single `null` chunk when the query is unscoped, and no chunks at all when it is + * scoped to an empty set -- the caller then runs no statement and yields nothing, which is the + * difference between "any source" and "none of them". + */ + private fun sourceIdChunks(query: IndexQuery): List?> { + val sourceIds = query.sourceIds ?: return listOf(null) + if (sourceIds.isEmpty()) { + return emptyList() + } + return sourceIds.distinct().chunked(SOURCE_ID_CHUNK_SIZE) + } + + private fun buildSelectQuery( + query: IndexQuery, + sourceIdChunk: List?, + limit: Int, + ): SqlQuery { + val (where, args) = buildWhereClause(query, sourceIdChunk) + val sql = + buildString { + append("SELECT _payload FROM $tableName") + if (where.isNotEmpty()) { + append(" WHERE ") + append(where) + } + if (limit != Int.MAX_VALUE) { + append(" LIMIT $limit") + } + } + + return SqlQuery(sql, args) + } + + /** + * Builds the shared `WHERE` body for [query], restricted to [sourceIdChunk] when the query is + * source-scoped. Returns the clause without the `WHERE` keyword so both the row select and the + * distinct projection can splice it in. + */ + private fun buildWhereClause( + query: IndexQuery, + sourceIdChunk: List?, + ): Pair> { val where = StringBuilder() val args = mutableListOf() @@ -378,6 +437,11 @@ class SQLiteIndex( query.key?.let { and("_key = ?", it) } query.sourceId?.let { and("_source_id = ?", it) } + if (sourceIdChunk != null) { + val placeholders = sourceIdChunk.joinToString(",") { "?" } + and("_source_id IN ($placeholders)", *sourceIdChunk.toTypedArray()) + } + for ((field, value) in query.exactMatch) { val col = fieldColumns[field] ?: continue and("$col = ?", value) @@ -404,18 +468,6 @@ class SQLiteIndex( } } - val sql = - buildString { - append("SELECT _payload FROM $tableName") - if (where.isNotEmpty()) { - append(" WHERE ") - append(where) - } - if (query.limit > 0) { - append(" LIMIT ${query.limit}") - } - } - - return SqlQuery(sql, args) + return where.toString() to args } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt index e7c1e27e1b..f02c5e205a 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -19,6 +19,15 @@ data class IndexQuery( val presence: Map = emptyMap(), /** Filter by source ID. */ val sourceId: String? = null, + /** + * Filter by a set of source IDs. `null` imposes no restriction; an empty collection matches + * nothing, which is what makes "scoped to a set that happens to be empty" distinguishable from + * "unscoped". + * + * Implementations must apply this before any [limit], so a scoped query cannot lose matches to + * rows that were only going to be discarded. + */ + val sourceIds: Collection? = null, /** Filter by key (exact). */ val key: String? = null, /** Maximum number of results. 0 = unlimited (use with care). */ @@ -44,6 +53,7 @@ class IndexQueryBuilder { private val prefix = mutableMapOf() private val pres = mutableMapOf() var sourceId: String? = null + var sourceIds: Collection? = null var key: String? = null var limit: Int = 200 @@ -79,6 +89,7 @@ class IndexQueryBuilder { prefixMatch = prefix.toMap(), presence = pres.toMap(), sourceId = sourceId, + sourceIds = sourceIds, key = key, limit = limit, ) diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt index 2f2c74130d..a274b9482a 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt @@ -274,4 +274,41 @@ class InMemoryIndexTest { index.removeBySource("unknownSource") assertThat(index.size).isEqualTo(1) } + + @Test + fun `query scoped to source ids returns only those sources`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + index.insert(entry("k2", "src2", "Bar")) + index.insert(entry("k3", "src3", "Baz")) + + val keys = index.query(IndexQuery(sourceIds = listOf("src1", "src3"), limit = 0)).map { it.key }.toList() + assertThat(keys).containsExactly("k1", "k3") + } + + @Test + fun `query scoped to an empty source set returns nothing`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + assertThat(index.query(IndexQuery(sourceIds = emptyList(), limit = 0)).toList()).isEmpty() + assertThat(index.query(IndexQuery(sourceIds = null, limit = 0)).toList()).hasSize(1) + } + + @Test + fun `source scope is combined with the other predicates`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + index.insert(entry("k2", "src2", "Foo")) + + val keys = + index + .query(IndexQuery(exactMatch = mapOf("name" to "Foo"), sourceIds = listOf("src2"), limit = 0)) + .map { it.key } + .toList() + assertThat(keys).containsExactly("k2") + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt new file mode 100644 index 0000000000..a79d328dc3 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt @@ -0,0 +1,114 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** Source scoping and column projection on the SQLite-backed index. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexScopeTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + val group: String = "", + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_scope" + override val fields = + listOf( + IndexField("value", prefixSearchable = true), + IndexField("group"), + ) + + override fun fieldValues(entry: Entry) = + mapOf( + "value" to entry.value, + "group" to entry.group, + ) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}|${entry.group}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2], parts[3]) + } + } + + private val index = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + ) + + @After + fun tearDown() { + index.close() + } + + @Test + fun `query scoped to source ids returns only those sources`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha")) + index.insert(Entry("k2", "jarB", "Beta")) + index.insert(Entry("k3", "jarC", "Gamma")) + + val keys = + index + .query(IndexQuery(sourceIds = listOf("jarA", "jarC"), limit = 0)) + .map { it.key } + .toList() + + assertThat(keys).containsExactly("k1", "k3") + } + + @Test + fun `query scoped to an empty source set returns nothing`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha")) + + // Distinct from sourceIds = null, which imposes no restriction at all. + assertThat(index.query(IndexQuery(sourceIds = emptyList(), limit = 0)).toList()).isEmpty() + assertThat(index.query(IndexQuery(sourceIds = null, limit = 0)).toList()).hasSize(1) + } + + @Test + fun `query scoped to more sources than one IN clause holds returns every match`() = + runTest { + // Over SQLite's 999 bound-parameter limit, so the scope has to be split across statements. + val sourceIds = (0 until 2100).map { "jar$it" } + index.insertAll(sourceIds.asSequence().mapIndexed { i, src -> Entry("k$i", src, "Cls$i") }) + + val found = index.query(IndexQuery(sourceIds = sourceIds, limit = 0)).map { it.key }.toList() + + assertThat(found).hasSize(2100) + assertThat(found.toSet()).hasSize(2100) + } + + @Test + fun `scoped query applies the scope before the limit`() = + runTest { + repeat(20) { index.insert(Entry("noise$it", "jarNoise", "Widget$it")) } + index.insert(Entry("wanted", "jarWanted", "Widget99")) + + val found = + index + .query(IndexQuery(prefixMatch = mapOf("value" to "Widget"), sourceIds = listOf("jarWanted"), limit = 1)) + .map { it.key } + .toList() + + assertThat(found).containsExactly("wanted") + } +} From 7c142648c47f12b9ae02e0aab57c35bd1757e500 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 09:15:05 +0000 Subject: [PATCH 03/48] style: spotless reformat, no functional change --- .../compiler/index/IndexWorkerDisposalTest.kt | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt index d84ffc2b83..e73aa0db10 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt @@ -23,7 +23,6 @@ import org.junit.Test * index worker is still running and about to call `PsiManager.findFile(project)`. */ class IndexWorkerDisposalTest : KtLspTest() { - private fun jvmIndex(): JvmSymbolIndex { val backing = InMemoryIndex(JvmSymbolDescriptor) return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { @@ -31,35 +30,36 @@ class IndexWorkerDisposalTest : KtLspTest() { } } - private fun fileIndex(): KtFileMetadataIndex = - KtFileMetadataIndex(InMemoryIndex(KtFileMetadataDescriptor)) + private fun fileIndex(): KtFileMetadataIndex = KtFileMetadataIndex(InMemoryIndex(KtFileMetadataDescriptor)) @Test - fun `worker stops without crashing when project is already disposed`(): Unit = runBlocking { - // Capture a real VirtualFile while the project is still alive. - val vf = createSourceFile("Sample.kt", "fun foo() = 1").virtualFile + fun `worker stops without crashing when project is already disposed`(): Unit = + runBlocking { + // Capture a real VirtualFile while the project is still alive. + val vf = createSourceFile("Sample.kt", "fun foo() = 1").virtualFile - val worker = IndexWorker( - project = env.project, - queue = WorkerQueue(), - fileIndex = fileIndex(), - sourceIndex = jvmIndex(), - scope = CoroutineScope(Dispatchers.Default + SupervisorJob()), - ) + val worker = + IndexWorker( + project = env.project, + queue = WorkerQueue(), + fileIndex = fileIndex(), + sourceIndex = jvmIndex(), + scope = CoroutineScope(Dispatchers.Default + SupervisorJob()), + ) - // Dispose the project out from under the worker, exactly as happens on LSP shutdown - // (env.close() disposes the project via its parent Disposable, the production path). - // IntelliJ requires model teardown to run inside a write action. - ApplicationManager.getApplication().runWriteAction { env.close() } - assertThat(env.project.isDisposed).isTrue() + // Dispose the project out from under the worker, exactly as happens on LSP shutdown + // (env.close() disposes the project via its parent Disposable, the production path). + // IntelliJ requires model teardown to run inside a write action. + ApplicationManager.getApplication().runWriteAction { env.close() } + assertThat(env.project.isDisposed).isTrue() - // Pre-fix, processing either command calls PsiManager.findFile on the disposed project - // and throws AssertionError("Project is already disposed"). Post-fix, the disposal guard - // breaks the loop, so start() returns cleanly instead of throwing. - worker.submitCommand(IndexCommand.ScanSourceFile(vf)) - worker.submitCommand(IndexCommand.IndexSourceFile(vf)) - worker.submitCommand(IndexCommand.Stop) + // Pre-fix, processing either command calls PsiManager.findFile on the disposed project + // and throws AssertionError("Project is already disposed"). Post-fix, the disposal guard + // breaks the loop, so start() returns cleanly instead of throwing. + worker.submitCommand(IndexCommand.ScanSourceFile(vf)) + worker.submitCommand(IndexCommand.IndexSourceFile(vf)) + worker.submitCommand(IndexCommand.Stop) - withTimeout(5_000) { worker.start() } - } + withTimeout(5_000) { worker.start() } + } } From e1d067862b07c2b123f445337e67979c74f90d19 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 09:26:28 +0000 Subject: [PATCH 04/48] ADFA-6228: Apply the active-source filter before the limit, not after it A defect on stage, not a consequence of the index migration, but fixed here because moving completion onto this index would otherwise inherit it. `FilteredIndex.query` called `backing.query(query)`, which had already appended `LIMIT` in SQL, and only then dropped rows from inactive sources. So a query for 200 could return zero while hundreds of matching active rows sat past the SQL limit -- the limit was spent on rows the caller was never allowed to see. The fix rewrites the query's scope instead of filtering its results, so the limit counts only rows that already survived filtering. No second filter is layered on top; that would just move the problem. That changes the extension contract. Testing one id is no longer enough, because the scope now has to be *described* to the backing index. So `visibleSourceIds` is the overridable member, `null` means every source is visible, and `isActive` derives from it and is final -- an override that could not be honoured now fails to compile instead of silently hiding every entry. `get` and `containsSource` are unaffected: both are point operations with no limit to spend, and tests pin that they still refuse an inactive source. --- .../codeonthego/indexing/FilteredIndex.kt | 37 ++++- .../indexing/FilteredIndexLimitTest.kt | 136 ++++++++++++++++++ .../index/IndexWorkerBatchRemovalTest.kt | 2 +- .../compiler/index/IndexWorkerDisposalTest.kt | 2 +- .../index/IndexWorkerPhaseLifecycleTest.kt | 2 +- .../compiler/index/SourceFileIndexerTest.kt | 2 +- .../kotlin/fixtures/KtLspTestEnvironment.kt | 2 +- 7 files changed, 172 insertions(+), 11 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt index b8eeda834f..d81baeda0c 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -59,10 +59,20 @@ open class FilteredIndex( */ open fun activeSources(): Set = activeSources.toSet() + /** + * The source IDs whose entries are visible, or `null` if every source is. + * + * This, rather than [isActive], is the point to override to change what the filter admits. + * Scoping is pushed into the query, so the filter has to be able to *describe* its scope and not + * merely test one ID against it -- an override of [isActive] alone could not be honoured, and + * would silently hide every entry instead. + */ + protected open fun visibleSourceIds(): Collection? = activeSources + /** * Returns true if the source is currently active (visible). */ - open fun isActive(sourceId: String): Boolean = sourceId in activeSources + fun isActive(sourceId: String): Boolean = visibleSourceIds()?.contains(sourceId) ?: true /** * Returns true if the source exists in the backing index, @@ -72,12 +82,27 @@ open class FilteredIndex( */ open suspend fun isCached(sourceId: String): Boolean = backing.containsSource(sourceId) - override fun query(query: IndexQuery): Sequence { - if (query.sourceId != null && !isActive(query.sourceId)) { - return emptySequence() + override fun query(query: IndexQuery): Sequence = backing.query(scopedToActive(query)) + + /** + * Narrows [query] to the active sources by rewriting its scope, rather than by filtering the + * rows it returns. + * + * Filtering afterwards is wrong whenever the query is limited: the backing index applies the + * limit first, so a page full of inactive rows yields nothing even though matches exist. Pushing + * the active set into the query makes the limit count only rows the caller can actually see. + */ + private fun scopedToActive(query: IndexQuery): IndexQuery { + val visible = visibleSourceIds()?.toSet() ?: return query + + if (query.sourceId != null) { + // Already as narrow as a scope gets: it either survives the active set or matches nothing. + return if (query.sourceId in visible) query else query.copy(sourceIds = emptyList()) } - val original = backing.query(query) - return original.filter { isActive(it.sourceId) } + + val requested = query.sourceIds + val scoped = requested?.filter { it in visible } ?: visible + return query.copy(sourceIds = scoped) } override suspend fun get(key: String): T? { diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt new file mode 100644 index 0000000000..7f8cdca733 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt @@ -0,0 +1,136 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * [FilteredIndex] against a real SQLite backing, where the limit is applied by the database rather + * than by the caller. + * + * These use SQLite rather than [InMemoryIndex] deliberately: the defect they pin only appears when + * the backing index applies [IndexQuery.limit] itself, which is what production does. + */ +@RunWith(RobolectricTestRunner::class) +class FilteredIndexLimitTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_filtered_limit" + override val fields = listOf(IndexField("value", prefixSearchable = true)) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val backing = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + ) + + @After + fun tearDown() { + backing.close() + } + + private fun activeOnly(vararg sourceIds: String) = FilteredIndex(backing).apply { setActiveSources(sourceIds.toSet()) } + + private fun widgetQuery(limit: Int) = IndexQuery(prefixMatch = mapOf("value" to "Widget"), limit = limit) + + @Test + fun `limited query does not lose active matches behind inactive rows`() = + runTest { + // Twenty matching rows from an inactive source are inserted first, so they occupy the whole + // of a five-row SQL limit, and the one active match sits beyond them. + repeat(20) { backing.insert(Entry("stale$it", "inactiveJar", "Widget$it")) } + backing.insert(Entry("live", "activeJar", "WidgetLive")) + + val found = activeOnly("activeJar").query(widgetQuery(limit = 5)) + + assertThat(found.map { it.key }.toList()).containsExactly("live") + } + + @Test + fun `limited query counts only visible rows towards the limit`() = + runTest { + repeat(20) { backing.insert(Entry("stale$it", "inactiveJar", "Widget$it")) } + repeat(5) { backing.insert(Entry("live$it", "activeJar", "Widget$it")) } + + val found = activeOnly("activeJar").query(widgetQuery(limit = 3)) + + assertThat(found.toList()).hasSize(3) + } + + @Test + fun `explicit source scope is intersected with the active set`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + backing.insert(Entry("b", "jarB", "Widget")) + + // jarB is asked for but is not active, so it must contribute nothing. + val found = + activeOnly("jarA") + .query(IndexQuery(sourceIds = listOf("jarA", "jarB"), limit = 0)) + .map { it.key } + .toList() + + assertThat(found).containsExactly("a") + } + + @Test + fun `a subclass that declares every source visible is not filtered`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + backing.insert(Entry("b", "jarB", "Widget")) + + // No source is ever activated. Overriding the scope, rather than a per-id test, is what + // makes that mean "everything" instead of "nothing" once scoping is pushed into the query. + val unfiltered = + object : FilteredIndex(backing) { + override fun visibleSourceIds(): Collection? = null + } + + assertThat(unfiltered.query(widgetQuery(limit = 0)).map { it.key }.toList()) + .containsExactly("a", "b") + } + + @Test + fun `a point lookup still refuses an entry from an inactive source`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + + assertThat(activeOnly("jarA").get("a")).isNotNull() + assertThat(activeOnly("otherJar").get("a")).isNull() + } + + @Test + fun `containsSource still refuses an inactive source`() = + runTest { + backing.insert(Entry("a", "jarA", "Widget")) + + assertThat(activeOnly("jarA").containsSource("jarA")).isTrue() + assertThat(activeOnly("otherJar").containsSource("jarA")).isFalse() + } +} diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt index 9fee622f41..a2cc8b3111 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt @@ -88,7 +88,7 @@ class IndexWorkerBatchRemovalTest { val counting = CountingIndex(backing) val index = object : JvmSymbolIndex(counting, BackgroundIndexer(counting)) { - override fun isActive(sourceId: String): Boolean = true + override fun visibleSourceIds(): Collection? = null } return counting to index } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt index e73aa0db10..0970b56ee7 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt @@ -26,7 +26,7 @@ class IndexWorkerDisposalTest : KtLspTest() { private fun jvmIndex(): JvmSymbolIndex { val backing = InMemoryIndex(JvmSymbolDescriptor) return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { - override fun isActive(sourceId: String) = true + override fun visibleSourceIds(): Collection? = null } } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt index 263d5aa4d2..c4ad9a174b 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt @@ -60,7 +60,7 @@ class IndexWorkerPhaseLifecycleTest { val symbolBacking = InMemoryIndex(JvmSymbolDescriptor) val sourceIndex = object : JvmSymbolIndex(symbolBacking, BackgroundIndexer(symbolBacking)) { - override fun isActive(sourceId: String) = true + override fun visibleSourceIds(): Collection? = null } return IndexWorker( diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt index 3c912de6bb..4654c603e5 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexerTest.kt @@ -17,7 +17,7 @@ class SourceFileIndexerTest : KtLspTest() { private fun buildSymbolIndex(): JvmSymbolIndex { val backing = InMemoryIndex(JvmSymbolDescriptor) return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { - override fun isActive(sourceId: String): Boolean = true + override fun visibleSourceIds(): Collection? = null } } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt index f68d0d1d4e..e5e6c7c55a 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt @@ -222,7 +222,7 @@ internal class KtLspTestEnvironment( val inMemoryJvmSymbolIndex = object : JvmSymbolIndex(inMemoryJvmBackingIndex, BackgroundIndexer(inMemoryJvmBackingIndex)) { // ensure we're not filtering out anything - override fun isActive(sourceId: String) = true + override fun visibleSourceIds(): Collection? = null override fun query(query: IndexQuery): Sequence { onSymbolIndexQuery?.invoke(query) From 39d2563f06b2d6bd1c4bc1804c2715c3adf71561 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 09:27:18 +0000 Subject: [PATCH 05/48] ADFA-6228: Project a single indexed column, filtered by a query The consumers that navigate the package tree do not look names up; they ask what sits directly under a package. Answering that needs every package under a prefix, and the existing `distinctValues(field)` takes no query at all -- no source scope, no prefix -- so the only way to narrow it was to fetch the matching entries and reduce them in memory, which is the materialisation this work exists to avoid. `distinctValues(field, query)` scans one column instead. Values are deduplicated across source chunks as well as by SQL, since DISTINCT applies only within a statement and one package name legitimately appears in many JARs. It also closes the gap `FilteredIndex.distinctValues` documented in a comment: it passed straight through to the backing index, so it reported values from sources that were not active. It now carries the same scope as `query`. Dropping `InMemoryIndex`'s own override removes a stale-value bug with it -- that one read the field map's keys, which retain a value after its last entry has been removed. --- .../codeonthego/indexing/FilteredIndex.kt | 13 ++---- .../codeonthego/indexing/InMemoryIndex.kt | 31 +++++++++++-- .../codeonthego/indexing/MergedIndex.kt | 10 ++++- .../codeonthego/indexing/SQLiteIndex.kt | 43 +++++++++++++++---- .../codeonthego/indexing/api/Index.kt | 20 ++++++++- .../indexing/FilteredIndexLimitTest.kt | 9 ++++ .../codeonthego/indexing/InMemoryIndexTest.kt | 22 ++++++++++ .../indexing/SQLiteIndexScopeTest.kt | 39 +++++++++++++++++ 8 files changed, 164 insertions(+), 23 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt index d81baeda0c..d05c086f24 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -112,15 +112,10 @@ open class FilteredIndex( override suspend fun containsSource(sourceId: String): Boolean = isActive(sourceId) && backing.containsSource(sourceId) - override fun distinctValues(fieldName: String): Sequence { - // This is imprecise — the backing index may return values - // from inactive sources. For exact results, we'd need to - // query all entries and filter. For package enumeration - // (the main use case), this approximation is acceptable - // since packages from inactive JARs are harmless — they - // just produce empty results when queried further. - return backing.distinctValues(fieldName) - } + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence = backing.distinctValues(fieldName, scopedToActive(query)) override fun close() { activeSources.clear() diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index f7660c2a9c..d0d945d11d 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -66,9 +66,34 @@ class InMemoryIndex( override suspend fun containsSource(sourceId: String): Boolean = sourceMap.containsKey(sourceId) - override fun distinctValues(fieldName: String): Sequence { - val fieldMap = fieldMaps[fieldName] ?: return emptySequence() - return lock.read { fieldMap.keys.toList() }.asSequence() + /** + * Projects [fieldName] out of the matching entries. + * + * Unlike the SQLite index there is no column to scan, so this reduces over the matched entries. + * That is acceptable at the sizes this index is built for (hundreds to low thousands of entries); + * it is not a general substitute for a column projection. + */ + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence { + if (!fieldMaps.containsKey(fieldName)) { + return emptySequence() + } + + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit + return lock + .read { + val values = LinkedHashSet() + for (key in resolveMatchingKeys(query)) { + if (values.size >= limit) { + break + } + val entry = primaryMap[key] ?: continue + descriptor.fieldValues(entry)[fieldName]?.let { values.add(it) } + } + values.toList() + }.asSequence() } override suspend fun insertAll(entries: Sequence) { diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt index 93a2590514..d0892be44f 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt @@ -49,11 +49,17 @@ class MergedIndex( override suspend fun containsSource(sourceId: String): Boolean = indexes.any { it.containsSource(sourceId) } - override fun distinctValues(fieldName: String): Sequence = + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence = sequence { + val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit val seen = mutableSetOf() for (index in indexes) { - for (value in index.distinctValues(fieldName)) { + if (seen.size >= limit) break + for (value in index.distinctValues(fieldName, query)) { + if (seen.size >= limit) break if (seen.add(value)) yield(value) } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 3e6606ba80..9bbf5cec0f 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -187,21 +187,48 @@ class SQLiteIndex( } } - override fun distinctValues(fieldName: String): Sequence = + override fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence = runBlocking { ifOpen(emptySequence()) { val col = fieldColumns[fieldName] ?: throw IllegalArgumentException("Unknown field: $fieldName") - val cursor = db.query("SELECT DISTINCT $col FROM $tableName WHERE $col IS NOT NULL") - cursor - .use { - buildList { - while (it.moveToNext()) { - add(it.getString(0)) + val limit = effectiveLimit(query) + + /* + * Deduplicated here as well as in SQL: DISTINCT only applies within one statement, and + * chunked source ids mean one statement per chunk. The same package name legitimately + * appears in many JARs, so without this the caller would see it once per chunk. + */ + val values = LinkedHashSet() + for (chunk in sourceIdChunks(query)) { + if (values.size >= limit) { + break + } + + val (where, args) = buildWhereClause(query, chunk) + val sql = + buildString { + append("SELECT DISTINCT $col FROM $tableName WHERE $col IS NOT NULL") + if (where.isNotEmpty()) { + append(" AND ") + append(where) } + if (limit != Int.MAX_VALUE) { + append(" LIMIT ${limit - values.size}") + } + } + + db.query(sql, args.toTypedArray()).use { + while (it.moveToNext()) { + values.add(it.getString(0)) } - }.asSequence() + } + } + values.asSequence() } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index 2f6dd6a275..03d5ba12ce 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -40,7 +40,25 @@ interface ReadableIndex { * @param fieldName Must be one of the fields declared in the * [IndexDescriptor]. */ - fun distinctValues(fieldName: String): Sequence + fun distinctValues(fieldName: String): Sequence = distinctValues(fieldName, IndexQuery(limit = 0)) + + /** + * Returns the distinct values of [fieldName] among the entries matching [query]. + * + * This projects a single column instead of materializing entries, which is what makes it usable + * for enumerating a large field -- every package on a classpath, say -- where fetching the + * matching entries and reducing them in memory would defeat the point. + * + * Values are deduplicated across the whole result, and their order is unspecified. If + * [IndexQuery.limit] is 0, every distinct value is emitted. + * + * @param fieldName Must be one of the fields declared in the [IndexDescriptor]. + * @param query Restricts which entries contribute a value. + */ + fun distinctValues( + fieldName: String, + query: IndexQuery, + ): Sequence } /** diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt index 7f8cdca733..5b9c76d1b5 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt @@ -133,4 +133,13 @@ class FilteredIndexLimitTest { assertThat(activeOnly("jarA").containsSource("jarA")).isTrue() assertThat(activeOnly("otherJar").containsSource("jarA")).isFalse() } + + @Test + fun `distinctValues excludes inactive sources`() = + runTest { + backing.insert(Entry("a", "jarA", "Alpha")) + backing.insert(Entry("b", "jarB", "Beta")) + + assertThat(activeOnly("jarA").distinctValues("value").toList()).containsExactly("Alpha") + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt index a274b9482a..658afbeff9 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt @@ -311,4 +311,26 @@ class InMemoryIndexTest { .toList() assertThat(keys).containsExactly("k2") } + + @Test + fun `distinctValues projects only the values of matching rows`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + index.insert(entry("k2", "src2", "Bar", category = "beta")) + + val categories = index.distinctValues("category", IndexQuery(sourceIds = listOf("src1"), limit = 0)).toList() + assertThat(categories).containsExactly("alpha") + } + + @Test + fun `distinctValues does not report values whose only entry was removed`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + index.insert(entry("k2", "src2", "Bar", category = "beta")) + index.removeBySource("src1") + + assertThat(index.distinctValues("category").toList()).containsExactly("beta") + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt index a79d328dc3..3b4c99cb76 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt @@ -111,4 +111,43 @@ class SQLiteIndexScopeTest { assertThat(found).containsExactly("wanted") } + + @Test + fun `distinctValues projects only the values of matching rows`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + index.insert(Entry("k2", "jarA", "Beta", group = "one")) + index.insert(Entry("k3", "jarB", "Gamma", group = "two")) + + val groups = + index + .distinctValues("group", IndexQuery(sourceIds = listOf("jarA"), limit = 0)) + .toList() + + assertThat(groups).containsExactly("one") + } + + @Test + fun `distinctValues deduplicates across source chunks`() = + runTest { + // One shared value spread over more sources than a single IN clause can hold: DISTINCT is + // per statement, so without cross-chunk deduplication this value comes back repeatedly. + val sourceIds = (0 until 1500).map { "jar$it" } + index.insertAll( + sourceIds.asSequence().mapIndexed { i, src -> Entry("k$i", src, "Cls$i", group = "shared") }, + ) + + val groups = index.distinctValues("group", IndexQuery(sourceIds = sourceIds, limit = 0)).toList() + + assertThat(groups).containsExactly("shared") + } + + @Test + fun `distinctValues without a query still returns every value`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + index.insert(Entry("k2", "jarB", "Beta", group = "two")) + + assertThat(index.distinctValues("group").toList()).containsExactly("one", "two") + } } From bcee666cd736492ec958eeb13c7aa9fddd1b7c06 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 08:58:33 +0000 Subject: [PATCH 06/48] ADFA-6228: Make prefix queries use the column index and stop treating _ as a wildcard `LIKE 'prefix%'` cannot use the column's index here. SQLite only rewrites LIKE into a range scan when `case_sensitive_like` is on or the column collates NOCASE, and this schema has neither, so every prefix query was a full table scan. Measured against a 400k-row table shaped like the JVM symbol index, the range form answers the same query in 0.04ms against 3.9ms - and this runs on every completion keystroke. The escaped LIKE stays beside the range bounds as the semantic guard. `_` and `%` are legal identifier characters, so an unescaped pattern silently widened the match: a prefix of `my_f` also returned `myXfield`. The prefix match on a non-prefix-searchable column was case-insensitive, which contradicted both its own comment and `InMemoryIndex`, whose fallback has always used `startsWith`. The two agree now. --- .../codeonthego/indexing/SQLiteIndex.kt | 50 ++++++- .../indexing/SQLiteIndexPrefixTest.kt | 138 ++++++++++++++++++ 2 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 9bbf5cec0f..601200a3eb 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -404,6 +404,27 @@ class SQLiteIndex( private fun effectiveLimit(query: IndexQuery) = if (query.limit > 0) query.limit else Int.MAX_VALUE + /** + * The smallest string greater than every string starting with [prefix], or `null` when no such + * bound exists because [prefix] ends in the highest representable characters. + */ + private fun exclusiveUpperBound(prefix: String): String? { + for (i in prefix.length - 1 downTo 0) { + val c = prefix[i] + if (c != Char.MAX_VALUE) { + return prefix.substring(0, i) + (c + 1) + } + } + return null + } + + /** Escapes the LIKE metacharacters in [literal] so it matches only itself. */ + private fun escapeLikeLiteral(literal: String) = + literal + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + /** * Splits [IndexQuery.sourceIds] into groups small enough for one `IN (...)` clause. * @@ -476,14 +497,31 @@ class SQLiteIndex( for ((field, prefix) in query.prefixMatch) { val lowerCol = prefixColumns[field] - if (lowerCol != null) { - // Use the pre-lowercased column for index-friendly LIKE - and("$lowerCol LIKE ?", "${prefix.lowercase()}%") + // Prefix-searchable fields match case-insensitively through their pre-lowercased column; + // everything else matches the stored value as-is. + val col = lowerCol ?: fieldColumns[field] ?: continue + val value = if (lowerCol != null) prefix.lowercase() else prefix + + if (value.isEmpty()) { + // An empty prefix means "has a value", which is what `LIKE '%'` used to express. + and("$col IS NOT NULL") + continue + } + + /* + * The range bounds are what make this use the column's index: SQLite only optimises LIKE + * into a range scan when case_sensitive_like is on or the column collates NOCASE, and + * neither holds here, so a bare LIKE scans the whole table. The escaped LIKE stays as the + * semantic guard -- it is what rejects a literal '_' or '%' in the prefix, which are valid + * identifier characters that an unescaped pattern would treat as wildcards. + */ + val upperBound = exclusiveUpperBound(value) + if (upperBound != null) { + and("$col >= ? AND $col < ?", value, upperBound) } else { - // Fallback: case-sensitive prefix on the regular column - val col = fieldColumns[field] ?: continue - and("$col LIKE ?", "$prefix%") + and("$col >= ?", value) } + and("$col LIKE ? ESCAPE '\\'", "${escapeLikeLiteral(value)}%") } for ((field, mustExist) in query.presence) { diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt new file mode 100644 index 0000000000..0c5fa1d100 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt @@ -0,0 +1,138 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * Prefix matching on the SQLite-backed index. + * + * `_` and `%` are the cases that matter: `_` is a legal identifier character, so a user typing one + * must not be handed matches that merely have some other character in that position. + */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexPrefixTest { + data class Entry( + override val key: String, + override val sourceId: String, + val name: String, + val pkg: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_prefix" + override val fields = + listOf( + IndexField("name", prefixSearchable = true), + IndexField("pkg"), + ) + + override fun fieldValues(entry: Entry) = mapOf("name" to entry.name, "pkg" to entry.pkg) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.name}|${entry.pkg}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2], parts[3]) + } + } + + private val index = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + ) + + @After + fun tearDown() { + index.close() + } + + private fun namesMatching(prefix: String) = + index + .query(IndexQuery(prefixMatch = mapOf("name" to prefix), limit = 0)) + .map { it.name } + .toList() + + @Test + fun `underscore in a prefix matches only a literal underscore`() = + runTest { + index.insert(Entry("k1", "s", "my_field", "p")) + index.insert(Entry("k2", "s", "myXfield", "p")) + + assertThat(namesMatching("my_f")).containsExactly("my_field") + } + + @Test + fun `percent in a prefix matches only a literal percent`() = + runTest { + index.insert(Entry("k1", "s", "pct%value", "p")) + index.insert(Entry("k2", "s", "pctOther", "p")) + + assertThat(namesMatching("pct%")).containsExactly("pct%value") + } + + @Test + fun `backslash in a prefix matches only a literal backslash`() = + runTest { + index.insert(Entry("k1", "s", "a\\b", "p")) + index.insert(Entry("k2", "s", "axb", "p")) + + assertThat(namesMatching("a\\")).containsExactly("a\\b") + } + + @Test + fun `prefix search on a prefix-searchable field ignores case`() = + runTest { + index.insert(Entry("k1", "s", "ArrayList", "p")) + index.insert(Entry("k2", "s", "arraydeque", "p")) + + assertThat(namesMatching("array")).containsExactly("ArrayList", "arraydeque") + } + + @Test + fun `prefix search on a plain field respects case`() = + runTest { + index.insert(Entry("k1", "s", "n1", "com.foo")) + index.insert(Entry("k2", "s", "n2", "COM.BAR")) + + // Matches InMemoryIndex, which has always compared these with startsWith. + val pkgs = + index + .query(IndexQuery(prefixMatch = mapOf("pkg" to "com."), limit = 0)) + .map { it.pkg } + .toList() + assertThat(pkgs).containsExactly("com.foo") + } + + @Test + fun `an empty prefix matches every entry that has the field`() = + runTest { + index.insert(Entry("k1", "s", "Alpha", "p")) + index.insert(Entry("k2", "s", "Beta", "p")) + + assertThat(namesMatching("")).containsExactly("Alpha", "Beta") + } + + @Test + fun `a prefix ending at the highest character still matches`() = + runTest { + // No exclusive upper bound exists here, so only the lower bound can be emitted. + val high = "z${Char.MAX_VALUE}" + index.insert(Entry("k1", "s", "${high}tail", "p")) + index.insert(Entry("k2", "s", "zzz", "p")) + + assertThat(namesMatching(high)).containsExactly("${high}tail") + } +} From ec8c2a7d82c41570d8c0b06c89c491c991d096f2 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 09:05:37 +0000 Subject: [PATCH 07/48] style: spotless reformat, no functional change --- .../codeonthego/indexing/jvm/JvmSymbol.kt | 232 ++--- .../indexing/jvm/JvmSymbolDescriptor.kt | 808 +++++++++--------- 2 files changed, 552 insertions(+), 488 deletions(-) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt index aec88b542e..851c3526e8 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt @@ -3,43 +3,74 @@ package org.appdevforall.codeonthego.indexing.jvm import org.appdevforall.codeonthego.indexing.api.Indexable enum class JvmSymbolKind { - CLASS, INTERFACE, ENUM, ENUM_ENTRY, ANNOTATION_CLASS, - OBJECT, COMPANION_OBJECT, DATA_CLASS, VALUE_CLASS, - SEALED_CLASS, SEALED_INTERFACE, - FUNCTION, EXTENSION_FUNCTION, CONSTRUCTOR, - PROPERTY, EXTENSION_PROPERTY, FIELD, - TYPE_ALIAS; - - val isCallable: Boolean - get() = this in CALLABLE_KINDS - - val isClassifier: Boolean - get() = this in CLASSIFIER_KINDS - - val isExtension: Boolean - get() = this == EXTENSION_FUNCTION || this == EXTENSION_PROPERTY - - companion object { - val CALLABLE_KINDS = setOf( - FUNCTION, EXTENSION_FUNCTION, CONSTRUCTOR, - PROPERTY, EXTENSION_PROPERTY, FIELD, - ) - val CLASSIFIER_KINDS = setOf( - CLASS, INTERFACE, ENUM, ANNOTATION_CLASS, - OBJECT, COMPANION_OBJECT, DATA_CLASS, - VALUE_CLASS, SEALED_CLASS, SEALED_INTERFACE, - TYPE_ALIAS, - ) - } + CLASS, + INTERFACE, + ENUM, + ENUM_ENTRY, + ANNOTATION_CLASS, + OBJECT, + COMPANION_OBJECT, + DATA_CLASS, + VALUE_CLASS, + SEALED_CLASS, + SEALED_INTERFACE, + FUNCTION, + EXTENSION_FUNCTION, + CONSTRUCTOR, + PROPERTY, + EXTENSION_PROPERTY, + FIELD, + TYPE_ALIAS, + ; + + val isCallable: Boolean + get() = this in CALLABLE_KINDS + + val isClassifier: Boolean + get() = this in CLASSIFIER_KINDS + + val isExtension: Boolean + get() = this == EXTENSION_FUNCTION || this == EXTENSION_PROPERTY + + companion object { + val CALLABLE_KINDS = + setOf( + FUNCTION, + EXTENSION_FUNCTION, + CONSTRUCTOR, + PROPERTY, + EXTENSION_PROPERTY, + FIELD, + ) + val CLASSIFIER_KINDS = + setOf( + CLASS, + INTERFACE, + ENUM, + ANNOTATION_CLASS, + OBJECT, + COMPANION_OBJECT, + DATA_CLASS, + VALUE_CLASS, + SEALED_CLASS, + SEALED_INTERFACE, + TYPE_ALIAS, + ) + } } enum class JvmSourceLanguage { JAVA, KOTLIN } enum class JvmVisibility { - PUBLIC, PROTECTED, INTERNAL, PRIVATE, PACKAGE_PRIVATE; - - val isAccessibleOutsideClass: Boolean - get() = this == PUBLIC || this == PROTECTED || this == INTERNAL + PUBLIC, + PROTECTED, + INTERNAL, + PRIVATE, + PACKAGE_PRIVATE, + ; + + val isAccessibleOutsideClass: Boolean + get() = this == PUBLIC || this == PROTECTED || this == INTERNAL } /** @@ -56,7 +87,6 @@ enum class JvmVisibility { data class JvmSymbol( override val key: String, override val sourceId: String, - val name: String, val shortName: String, val packageName: String, @@ -64,44 +94,45 @@ data class JvmSymbol( val language: JvmSourceLanguage, val visibility: JvmVisibility = JvmVisibility.PUBLIC, val isDeprecated: Boolean = false, - val data: JvmSymbolInfo, ) : Indexable { - val fqName: String get() = name.toFqName() - val isTopLevel: Boolean - get() = data.containingClassName.isEmpty() + val isTopLevel: Boolean + get() = data.containingClassName.isEmpty() - val isExtension: Boolean - get() = kind.isExtension + val isExtension: Boolean + get() = kind.isExtension - val receiverTypeName: String? - get() = when (val d = data) { - is JvmFunctionInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } - is JvmFieldInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } - else -> null - } + val receiverTypeName: String? + get() = + when (val d = data) { + is JvmFunctionInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } + is JvmFieldInfo -> d.kotlin?.receiverTypeName?.takeIf { it.isNotEmpty() } + else -> null + } val receiverTypeFqName: String? get() = receiverTypeName?.toFqName() - val containingClassName: String - get() = data.containingClassName - - val returnTypeDisplay: String - get() = when (val d = data) { - is JvmFunctionInfo -> d.returnTypeDisplayName - is JvmFieldInfo -> d.typeDisplayName - else -> "" - } - - val signatureDisplay: String - get() = when (val d = data) { - is JvmFunctionInfo -> d.signatureDisplay - else -> "" - } + val containingClassName: String + get() = data.containingClassName + + val returnTypeDisplay: String + get() = + when (val d = data) { + is JvmFunctionInfo -> d.returnTypeDisplayName + is JvmFieldInfo -> d.typeDisplayName + else -> "" + } + + val signatureDisplay: String + get() = + when (val d = data) { + is JvmFunctionInfo -> d.signatureDisplay + else -> "" + } } /** @@ -109,11 +140,10 @@ data class JvmSymbol( * Every variant provides [containingClassName] (empty for top-level). */ sealed interface JvmSymbolInfo { - /** * The internal name of the containing class. */ - val containingClassName: String + val containingClassName: String /** * The fully qualified name of the containing class, in dot format. @@ -135,15 +165,15 @@ data class JvmClassInfo( ) : JvmSymbolInfo data class KotlinClassInfo( - val isData: Boolean = false, - val isValue: Boolean = false, - val isSealed: Boolean = false, - val isFunInterface: Boolean = false, - val isExpect: Boolean = false, - val isActual: Boolean = false, - val isExternal: Boolean = false, - val sealedSubclasses: List = emptyList(), - val companionObjectName: String = "", + val isData: Boolean = false, + val isValue: Boolean = false, + val isSealed: Boolean = false, + val isFunInterface: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isExternal: Boolean = false, + val sealedSubclasses: List = emptyList(), + val companionObjectName: String = "", ) data class JvmFunctionInfo( @@ -164,30 +194,30 @@ data class JvmFunctionInfo( } data class JvmParameterInfo( - val name: String, - val typeName: String, - val typeDisplayName: String, - val hasDefaultValue: Boolean = false, - val isCrossinline: Boolean = false, - val isNoinline: Boolean = false, - val isVararg: Boolean = false, + val name: String, + val typeName: String, + val typeDisplayName: String, + val hasDefaultValue: Boolean = false, + val isCrossinline: Boolean = false, + val isNoinline: Boolean = false, + val isVararg: Boolean = false, ) { val typeFqName: String get() = typeName.toFqName() } data class KotlinFunctionInfo( - val receiverTypeName: String = "", - val receiverTypeDisplayName: String = "", - val isSuspend: Boolean = false, - val isInline: Boolean = false, - val isInfix: Boolean = false, - val isOperator: Boolean = false, - val isTailrec: Boolean = false, - val isExternal: Boolean = false, - val isExpect: Boolean = false, - val isActual: Boolean = false, - val isReturnTypeNullable: Boolean = false, + val receiverTypeName: String = "", + val receiverTypeDisplayName: String = "", + val isSuspend: Boolean = false, + val isInline: Boolean = false, + val isInfix: Boolean = false, + val isOperator: Boolean = false, + val isTailrec: Boolean = false, + val isExternal: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isReturnTypeNullable: Boolean = false, ) { val receiverTypeFqName: String get() = receiverTypeName.toFqName() @@ -207,17 +237,17 @@ data class JvmFieldInfo( } data class KotlinPropertyInfo( - val receiverTypeName: String = "", - val receiverTypeDisplayName: String = "", - val isConst: Boolean = false, - val isLateinit: Boolean = false, - val hasGetter: Boolean = false, - val hasSetter: Boolean = false, - val isDelegated: Boolean = false, - val isExpect: Boolean = false, - val isActual: Boolean = false, - val isExternal: Boolean = false, - val isTypeNullable: Boolean = false, + val receiverTypeName: String = "", + val receiverTypeDisplayName: String = "", + val isConst: Boolean = false, + val isLateinit: Boolean = false, + val hasGetter: Boolean = false, + val hasSetter: Boolean = false, + val isDelegated: Boolean = false, + val isExpect: Boolean = false, + val isActual: Boolean = false, + val isExternal: Boolean = false, + val isTypeNullable: Boolean = false, ) { val receiverTypeFqName: String get() = receiverTypeName.toFqName() @@ -241,4 +271,4 @@ data class JvmTypeAliasInfo( private fun String.toFqName() = replace('/', '.') .replace('$', '.') - .replace('#', '.') \ No newline at end of file + .replace('#', '.') diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt index c414709759..2285da25ea 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt @@ -19,7 +19,6 @@ import org.appdevforall.codeonthego.indexing.jvm.proto.JvmSymbolProtos.JvmSymbol * Blob serialization uses Protobuf with `oneof` for type-specific data. */ object JvmSymbolDescriptor : IndexDescriptor { - const val KEY_NAME = "name" const val KEY_PACKAGE = "package" const val KEY_KIND = "kind" @@ -27,390 +26,425 @@ object JvmSymbolDescriptor : IndexDescriptor { const val KEY_CONTAINING_CLASS = "containingClass" const val KEY_LANGUAGE = "language" - override val name: String = "jvm_symbols" - - override val fields: List = listOf( - IndexField(name = KEY_NAME, prefixSearchable = true), - IndexField(name = KEY_PACKAGE), - IndexField(name = KEY_KIND), - IndexField(name = KEY_RECEIVER_TYPE), - IndexField(name = KEY_CONTAINING_CLASS), - IndexField(name = KEY_LANGUAGE), - ) - - override fun fieldValues(entry: JvmSymbol): Map = mapOf( - KEY_NAME to entry.shortName, - KEY_PACKAGE to entry.packageName, - KEY_KIND to entry.kind.name, - KEY_RECEIVER_TYPE to entry.receiverTypeName, - KEY_CONTAINING_CLASS to entry.containingClassName.ifEmpty { null }, - KEY_LANGUAGE to entry.language.name, - ) - - override fun serialize(entry: JvmSymbol): ByteArray = - toProto(entry).toByteArray() - - override fun deserialize(bytes: ByteArray): JvmSymbol = - fromProto(JvmSymbolData.parseFrom(bytes)) - - private fun toProto(s: JvmSymbol): JvmSymbolData { - val builder = JvmSymbolData.newBuilder() - .setName(s.name) - .setShortName(s.shortName) - .setPackageName(s.packageName) - .setSourceId(s.sourceId) - .setKind(kindToProto(s.kind)) - .setLanguage(languageToProto(s.language)) - .setVisibility(visibilityToProto(s.visibility)) - .setIsDeprecated(s.isDeprecated) - - when (val data = s.data) { - is JvmClassInfo -> builder.setClassData(classInfoToProto(data)) - is JvmFunctionInfo -> builder.setFunctionData(functionInfoToProto(data)) - is JvmFieldInfo -> builder.setFieldData(fieldInfoToProto(data)) - is JvmEnumEntryInfo -> builder.setEnumEntryData(enumEntryToProto(data)) - is JvmTypeAliasInfo -> builder.setTypeAliasData(typeAliasToProto(data)) - } - - return builder.build() - } - - private fun classInfoToProto(d: JvmClassInfo): JvmSymbolProtos.ClassData { - val builder = JvmSymbolProtos.ClassData.newBuilder() - .setContainingClassName(d.containingClassName) - .addAllSupertypeNames(d.supertypeNames) - .addAllTypeParameters(d.typeParameters) - .setIsAbstract(d.isAbstract) - .setIsFinal(d.isFinal) - .setIsInner(d.isInner) - .setIsStatic(d.isStatic) - - d.kotlin?.let { kd -> - builder.setKotlin( - JvmSymbolProtos.KotlinClassData.newBuilder() - .setIsData(kd.isData) - .setIsValue(kd.isValue) - .setIsSealed(kd.isSealed) - .setIsFunInterface(kd.isFunInterface) - .setIsExpect(kd.isExpect) - .setIsActual(kd.isActual) - .setIsExternal(kd.isExternal) - .addAllSealedSubclasses(kd.sealedSubclasses) - .setCompanionObjectName(kd.companionObjectName) - ) - } - - return builder.build() - } - - private fun functionInfoToProto(d: JvmFunctionInfo): JvmSymbolProtos.FunctionData { - val builder = JvmSymbolProtos.FunctionData.newBuilder() - .setContainingClassName(d.containingClassName) - .setReturnTypeName(d.returnTypeName) - .setReturnTypeDisplayName(d.returnTypeDisplayName) - .setParameterCount(d.parameterCount) - .addAllParameters(d.parameters.map { paramToProto(it) }) - .setSignatureDisplay(d.signatureDisplay) - .addAllTypeParameters(d.typeParameters) - .setIsStatic(d.isStatic) - .setIsAbstract(d.isAbstract) - .setIsFinal(d.isFinal) - - d.kotlin?.let { kd -> - builder.setKotlin( - JvmSymbolProtos.KotlinFunctionData.newBuilder() - .setReceiverTypeName(kd.receiverTypeName) - .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) - .setIsSuspend(kd.isSuspend) - .setIsInline(kd.isInline) - .setIsInfix(kd.isInfix) - .setIsOperator(kd.isOperator) - .setIsTailrec(kd.isTailrec) - .setIsExternal(kd.isExternal) - .setIsExpect(kd.isExpect) - .setIsActual(kd.isActual) - .setIsReturnTypeNullable(kd.isReturnTypeNullable) - ) - } - - return builder.build() - } - - private fun paramToProto(p: JvmParameterInfo): JvmSymbolProtos.ParameterData = - JvmSymbolProtos.ParameterData.newBuilder() - .setName(p.name) - .setTypeName(p.typeName) - .setTypeDisplayName(p.typeDisplayName) - .setHasDefaultValue(p.hasDefaultValue) - .setIsCrossinline(p.isCrossinline) - .setIsNoinline(p.isNoinline) - .setIsVararg(p.isVararg) - .build() - - private fun fieldInfoToProto(d: JvmFieldInfo): JvmSymbolProtos.FieldData { - val builder = JvmSymbolProtos.FieldData.newBuilder() - .setContainingClassName(d.containingClassName) - .setTypeName(d.typeName) - .setTypeDisplayName(d.typeDisplayName) - .setIsStatic(d.isStatic) - .setIsFinal(d.isFinal) - .setConstantValue(d.constantValue) - - d.kotlin?.let { kd -> - builder.setKotlin( - JvmSymbolProtos.KotlinPropertyData.newBuilder() - .setReceiverTypeName(kd.receiverTypeName) - .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) - .setIsConst(kd.isConst) - .setIsLateinit(kd.isLateinit) - .setHasGetter(kd.hasGetter) - .setHasSetter(kd.hasSetter) - .setIsDelegated(kd.isDelegated) - .setIsExpect(kd.isExpect) - .setIsActual(kd.isActual) - .setIsExternal(kd.isExternal) - .setIsTypeNullable(kd.isTypeNullable) - ) - } - - return builder.build() - } - - private fun enumEntryToProto(d: JvmEnumEntryInfo): JvmSymbolProtos.EnumEntryData = - JvmSymbolProtos.EnumEntryData.newBuilder() - .setContainingEnumName(d.containingClassName) - .setOrdinal(d.ordinal) - .build() - - private fun typeAliasToProto(d: JvmTypeAliasInfo): JvmSymbolProtos.TypeAliasData = - JvmSymbolProtos.TypeAliasData.newBuilder() - .setExpandedTypeName(d.expandedTypeName) - .setExpandedTypeDisplayName(d.expandedTypeDisplayName) - .addAllTypeParameters(d.typeParameters) - .build() - - private fun fromProto(p: JvmSymbolData): JvmSymbol { - val kind = kindFromProto(p.kind) - val data = dataFromProto(p) - - val key = when { - kind.isCallable && kind != JvmSymbolKind.PROPERTY - && kind != JvmSymbolKind.EXTENSION_PROPERTY - && kind != JvmSymbolKind.FIELD -> { - val params = (data as? JvmFunctionInfo) - ?.parameters - ?.joinToString(",") { it.typeName } - ?: "" - "${p.name}($params)" - } - else -> p.name - } - - return JvmSymbol( - key = key, - sourceId = p.sourceId, - name = p.name, - shortName = p.shortName, - packageName = p.packageName, - kind = kind, - language = languageFromProto(p.language), - visibility = visibilityFromProto(p.visibility), - isDeprecated = p.isDeprecated, - data = data, - ) - } - - private fun dataFromProto(p: JvmSymbolData): JvmSymbolInfo = when (p.dataCase) { - JvmSymbolData.DataCase.CLASS_DATA -> classInfoFromProto(p.classData) - JvmSymbolData.DataCase.FUNCTION_DATA -> functionInfoFromProto(p.functionData) - JvmSymbolData.DataCase.FIELD_DATA -> fieldInfoFromProto(p.fieldData) - JvmSymbolData.DataCase.ENUM_ENTRY_DATA -> enumEntryFromProto(p.enumEntryData) - JvmSymbolData.DataCase.TYPE_ALIAS_DATA -> typeAliasFromProto(p.typeAliasData) - else -> JvmClassInfo() // fallback - } - - private fun classInfoFromProto(p: JvmSymbolProtos.ClassData): JvmClassInfo { - val kotlin = if (p.hasKotlin()) { - val kd = p.kotlin - KotlinClassInfo( - isData = kd.isData, - isValue = kd.isValue, - isSealed = kd.isSealed, - isFunInterface = kd.isFunInterface, - isExpect = kd.isExpect, - isActual = kd.isActual, - isExternal = kd.isExternal, - sealedSubclasses = kd.sealedSubclassesList.toList(), - companionObjectName = kd.companionObjectName, - ) - } else null - - return JvmClassInfo( - containingClassName = p.containingClassName, - supertypeNames = p.supertypeNamesList.toList(), - typeParameters = p.typeParametersList.toList(), - isAbstract = p.isAbstract, - isFinal = p.isFinal, - isInner = p.isInner, - isStatic = p.isStatic, - kotlin = kotlin, - ) - } - - private fun functionInfoFromProto(p: JvmSymbolProtos.FunctionData): JvmFunctionInfo { - val kotlin = if (p.hasKotlin()) { - val kd = p.kotlin - KotlinFunctionInfo( - receiverTypeName = kd.receiverTypeName, - receiverTypeDisplayName = kd.receiverTypeDisplayName, - isSuspend = kd.isSuspend, - isInline = kd.isInline, - isInfix = kd.isInfix, - isOperator = kd.isOperator, - isTailrec = kd.isTailrec, - isExternal = kd.isExternal, - isExpect = kd.isExpect, - isActual = kd.isActual, - isReturnTypeNullable = kd.isReturnTypeNullable, - ) - } else null - - return JvmFunctionInfo( - containingClassName = p.containingClassName, - returnTypeName = p.returnTypeName, - returnTypeDisplayName = p.returnTypeDisplayName, - parameterCount = p.parameterCount, - parameters = p.parametersList.map { paramFromProto(it) }, - signatureDisplay = p.signatureDisplay, - typeParameters = p.typeParametersList.toList(), - isStatic = p.isStatic, - isAbstract = p.isAbstract, - isFinal = p.isFinal, - kotlin = kotlin, - ) - } - - private fun paramFromProto(p: JvmSymbolProtos.ParameterData): JvmParameterInfo = - JvmParameterInfo( - name = p.name, - typeName = p.typeName, - typeDisplayName = p.typeDisplayName, - hasDefaultValue = p.hasDefaultValue, - isCrossinline = p.isCrossinline, - isNoinline = p.isNoinline, - isVararg = p.isVararg, - ) - - private fun fieldInfoFromProto(p: JvmSymbolProtos.FieldData): JvmFieldInfo { - val kotlin = if (p.hasKotlin()) { - val kd = p.kotlin - KotlinPropertyInfo( - receiverTypeName = kd.receiverTypeName, - receiverTypeDisplayName = kd.receiverTypeDisplayName, - isConst = kd.isConst, - isLateinit = kd.isLateinit, - hasGetter = kd.hasGetter, - hasSetter = kd.hasSetter, - isDelegated = kd.isDelegated, - isExpect = kd.isExpect, - isActual = kd.isActual, - isExternal = kd.isExternal, - isTypeNullable = kd.isTypeNullable, - ) - } else null - - return JvmFieldInfo( - containingClassName = p.containingClassName, - typeName = p.typeName, - typeDisplayName = p.typeDisplayName, - isStatic = p.isStatic, - isFinal = p.isFinal, - constantValue = p.constantValue, - kotlin = kotlin, - ) - } - - private fun enumEntryFromProto(p: JvmSymbolProtos.EnumEntryData): JvmEnumEntryInfo = - JvmEnumEntryInfo( - containingClassName = p.containingEnumName, - ordinal = p.ordinal, - ) - - private fun typeAliasFromProto(p: JvmSymbolProtos.TypeAliasData): JvmTypeAliasInfo = - JvmTypeAliasInfo( - expandedTypeName = p.expandedTypeName, - expandedTypeDisplayName = p.expandedTypeDisplayName, - typeParameters = p.typeParametersList.toList(), - ) - - private fun kindToProto(k: JvmSymbolKind) = when (k) { - JvmSymbolKind.CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_CLASS - JvmSymbolKind.INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE - JvmSymbolKind.ENUM -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM - JvmSymbolKind.ENUM_ENTRY -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY - JvmSymbolKind.ANNOTATION_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS - JvmSymbolKind.OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT - JvmSymbolKind.COMPANION_OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT - JvmSymbolKind.DATA_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS - JvmSymbolKind.VALUE_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS - JvmSymbolKind.SEALED_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS - JvmSymbolKind.SEALED_INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE - JvmSymbolKind.FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION - JvmSymbolKind.EXTENSION_FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION - JvmSymbolKind.CONSTRUCTOR -> JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR - JvmSymbolKind.PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY - JvmSymbolKind.EXTENSION_PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY - JvmSymbolKind.FIELD -> JvmSymbolProtos.JvmSymbolKind.KIND_FIELD - JvmSymbolKind.TYPE_ALIAS -> JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS - } - - private fun kindFromProto(k: JvmSymbolProtos.JvmSymbolKind) = when (k) { - JvmSymbolProtos.JvmSymbolKind.KIND_CLASS -> JvmSymbolKind.CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE -> JvmSymbolKind.INTERFACE - JvmSymbolProtos.JvmSymbolKind.KIND_ENUM -> JvmSymbolKind.ENUM - JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY -> JvmSymbolKind.ENUM_ENTRY - JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS -> JvmSymbolKind.ANNOTATION_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT -> JvmSymbolKind.OBJECT - JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT -> JvmSymbolKind.COMPANION_OBJECT - JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS -> JvmSymbolKind.DATA_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS -> JvmSymbolKind.VALUE_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS -> JvmSymbolKind.SEALED_CLASS - JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE -> JvmSymbolKind.SEALED_INTERFACE - JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION -> JvmSymbolKind.FUNCTION - JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION -> JvmSymbolKind.EXTENSION_FUNCTION - JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR -> JvmSymbolKind.CONSTRUCTOR - JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY -> JvmSymbolKind.PROPERTY - JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY -> JvmSymbolKind.EXTENSION_PROPERTY - JvmSymbolProtos.JvmSymbolKind.KIND_FIELD -> JvmSymbolKind.FIELD - JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS -> JvmSymbolKind.TYPE_ALIAS - else -> JvmSymbolKind.CLASS - } - - private fun languageToProto(l: JvmSourceLanguage) = when (l) { - JvmSourceLanguage.JAVA -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA - JvmSourceLanguage.KOTLIN -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN - } - - private fun languageFromProto(l: JvmSymbolProtos.JvmSourceLanguage) = when (l) { - JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA -> JvmSourceLanguage.JAVA - JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN -> JvmSourceLanguage.KOTLIN - else -> JvmSourceLanguage.JAVA - } - - private fun visibilityToProto(v: JvmVisibility) = when (v) { - JvmVisibility.PUBLIC -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC - JvmVisibility.PROTECTED -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED - JvmVisibility.INTERNAL -> JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL - JvmVisibility.PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE - JvmVisibility.PACKAGE_PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE - } - - private fun visibilityFromProto(v: JvmSymbolProtos.JvmVisibility) = when (v) { - JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC -> JvmVisibility.PUBLIC - JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED -> JvmVisibility.PROTECTED - JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL -> JvmVisibility.INTERNAL - JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE -> JvmVisibility.PRIVATE - JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE -> JvmVisibility.PACKAGE_PRIVATE - else -> JvmVisibility.PUBLIC - } + override val name: String = "jvm_symbols" + + override val fields: List = + listOf( + IndexField(name = KEY_NAME, prefixSearchable = true), + IndexField(name = KEY_PACKAGE), + IndexField(name = KEY_KIND), + IndexField(name = KEY_RECEIVER_TYPE), + IndexField(name = KEY_CONTAINING_CLASS), + IndexField(name = KEY_LANGUAGE), + ) + + override fun fieldValues(entry: JvmSymbol): Map = + mapOf( + KEY_NAME to entry.shortName, + KEY_PACKAGE to entry.packageName, + KEY_KIND to entry.kind.name, + KEY_RECEIVER_TYPE to entry.receiverTypeName, + KEY_CONTAINING_CLASS to entry.containingClassName.ifEmpty { null }, + KEY_LANGUAGE to entry.language.name, + ) + + override fun serialize(entry: JvmSymbol): ByteArray = toProto(entry).toByteArray() + + override fun deserialize(bytes: ByteArray): JvmSymbol = fromProto(JvmSymbolData.parseFrom(bytes)) + + private fun toProto(s: JvmSymbol): JvmSymbolData { + val builder = + JvmSymbolData + .newBuilder() + .setName(s.name) + .setShortName(s.shortName) + .setPackageName(s.packageName) + .setSourceId(s.sourceId) + .setKind(kindToProto(s.kind)) + .setLanguage(languageToProto(s.language)) + .setVisibility(visibilityToProto(s.visibility)) + .setIsDeprecated(s.isDeprecated) + + when (val data = s.data) { + is JvmClassInfo -> builder.setClassData(classInfoToProto(data)) + is JvmFunctionInfo -> builder.setFunctionData(functionInfoToProto(data)) + is JvmFieldInfo -> builder.setFieldData(fieldInfoToProto(data)) + is JvmEnumEntryInfo -> builder.setEnumEntryData(enumEntryToProto(data)) + is JvmTypeAliasInfo -> builder.setTypeAliasData(typeAliasToProto(data)) + } + + return builder.build() + } + + private fun classInfoToProto(d: JvmClassInfo): JvmSymbolProtos.ClassData { + val builder = + JvmSymbolProtos.ClassData + .newBuilder() + .setContainingClassName(d.containingClassName) + .addAllSupertypeNames(d.supertypeNames) + .addAllTypeParameters(d.typeParameters) + .setIsAbstract(d.isAbstract) + .setIsFinal(d.isFinal) + .setIsInner(d.isInner) + .setIsStatic(d.isStatic) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinClassData + .newBuilder() + .setIsData(kd.isData) + .setIsValue(kd.isValue) + .setIsSealed(kd.isSealed) + .setIsFunInterface(kd.isFunInterface) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsExternal(kd.isExternal) + .addAllSealedSubclasses(kd.sealedSubclasses) + .setCompanionObjectName(kd.companionObjectName), + ) + } + + return builder.build() + } + + private fun functionInfoToProto(d: JvmFunctionInfo): JvmSymbolProtos.FunctionData { + val builder = + JvmSymbolProtos.FunctionData + .newBuilder() + .setContainingClassName(d.containingClassName) + .setReturnTypeName(d.returnTypeName) + .setReturnTypeDisplayName(d.returnTypeDisplayName) + .setParameterCount(d.parameterCount) + .addAllParameters(d.parameters.map { paramToProto(it) }) + .setSignatureDisplay(d.signatureDisplay) + .addAllTypeParameters(d.typeParameters) + .setIsStatic(d.isStatic) + .setIsAbstract(d.isAbstract) + .setIsFinal(d.isFinal) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinFunctionData + .newBuilder() + .setReceiverTypeName(kd.receiverTypeName) + .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) + .setIsSuspend(kd.isSuspend) + .setIsInline(kd.isInline) + .setIsInfix(kd.isInfix) + .setIsOperator(kd.isOperator) + .setIsTailrec(kd.isTailrec) + .setIsExternal(kd.isExternal) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsReturnTypeNullable(kd.isReturnTypeNullable), + ) + } + + return builder.build() + } + + private fun paramToProto(p: JvmParameterInfo): JvmSymbolProtos.ParameterData = + JvmSymbolProtos.ParameterData + .newBuilder() + .setName(p.name) + .setTypeName(p.typeName) + .setTypeDisplayName(p.typeDisplayName) + .setHasDefaultValue(p.hasDefaultValue) + .setIsCrossinline(p.isCrossinline) + .setIsNoinline(p.isNoinline) + .setIsVararg(p.isVararg) + .build() + + private fun fieldInfoToProto(d: JvmFieldInfo): JvmSymbolProtos.FieldData { + val builder = + JvmSymbolProtos.FieldData + .newBuilder() + .setContainingClassName(d.containingClassName) + .setTypeName(d.typeName) + .setTypeDisplayName(d.typeDisplayName) + .setIsStatic(d.isStatic) + .setIsFinal(d.isFinal) + .setConstantValue(d.constantValue) + + d.kotlin?.let { kd -> + builder.setKotlin( + JvmSymbolProtos.KotlinPropertyData + .newBuilder() + .setReceiverTypeName(kd.receiverTypeName) + .setReceiverTypeDisplayName(kd.receiverTypeDisplayName) + .setIsConst(kd.isConst) + .setIsLateinit(kd.isLateinit) + .setHasGetter(kd.hasGetter) + .setHasSetter(kd.hasSetter) + .setIsDelegated(kd.isDelegated) + .setIsExpect(kd.isExpect) + .setIsActual(kd.isActual) + .setIsExternal(kd.isExternal) + .setIsTypeNullable(kd.isTypeNullable), + ) + } + + return builder.build() + } + + private fun enumEntryToProto(d: JvmEnumEntryInfo): JvmSymbolProtos.EnumEntryData = + JvmSymbolProtos.EnumEntryData + .newBuilder() + .setContainingEnumName(d.containingClassName) + .setOrdinal(d.ordinal) + .build() + + private fun typeAliasToProto(d: JvmTypeAliasInfo): JvmSymbolProtos.TypeAliasData = + JvmSymbolProtos.TypeAliasData + .newBuilder() + .setExpandedTypeName(d.expandedTypeName) + .setExpandedTypeDisplayName(d.expandedTypeDisplayName) + .addAllTypeParameters(d.typeParameters) + .build() + + private fun fromProto(p: JvmSymbolData): JvmSymbol { + val kind = kindFromProto(p.kind) + val data = dataFromProto(p) + + val key = + when { + kind.isCallable && kind != JvmSymbolKind.PROPERTY && + kind != JvmSymbolKind.EXTENSION_PROPERTY && + kind != JvmSymbolKind.FIELD -> { + val params = + (data as? JvmFunctionInfo) + ?.parameters + ?.joinToString(",") { it.typeName } + ?: "" + "${p.name}($params)" + } + + else -> { + p.name + } + } + + return JvmSymbol( + key = key, + sourceId = p.sourceId, + name = p.name, + shortName = p.shortName, + packageName = p.packageName, + kind = kind, + language = languageFromProto(p.language), + visibility = visibilityFromProto(p.visibility), + isDeprecated = p.isDeprecated, + data = data, + ) + } + + private fun dataFromProto(p: JvmSymbolData): JvmSymbolInfo = + when (p.dataCase) { + JvmSymbolData.DataCase.CLASS_DATA -> classInfoFromProto(p.classData) + JvmSymbolData.DataCase.FUNCTION_DATA -> functionInfoFromProto(p.functionData) + JvmSymbolData.DataCase.FIELD_DATA -> fieldInfoFromProto(p.fieldData) + JvmSymbolData.DataCase.ENUM_ENTRY_DATA -> enumEntryFromProto(p.enumEntryData) + JvmSymbolData.DataCase.TYPE_ALIAS_DATA -> typeAliasFromProto(p.typeAliasData) + else -> JvmClassInfo() // fallback + } + + private fun classInfoFromProto(p: JvmSymbolProtos.ClassData): JvmClassInfo { + val kotlin = + if (p.hasKotlin()) { + val kd = p.kotlin + KotlinClassInfo( + isData = kd.isData, + isValue = kd.isValue, + isSealed = kd.isSealed, + isFunInterface = kd.isFunInterface, + isExpect = kd.isExpect, + isActual = kd.isActual, + isExternal = kd.isExternal, + sealedSubclasses = kd.sealedSubclassesList.toList(), + companionObjectName = kd.companionObjectName, + ) + } else { + null + } + + return JvmClassInfo( + containingClassName = p.containingClassName, + supertypeNames = p.supertypeNamesList.toList(), + typeParameters = p.typeParametersList.toList(), + isAbstract = p.isAbstract, + isFinal = p.isFinal, + isInner = p.isInner, + isStatic = p.isStatic, + kotlin = kotlin, + ) + } + + private fun functionInfoFromProto(p: JvmSymbolProtos.FunctionData): JvmFunctionInfo { + val kotlin = + if (p.hasKotlin()) { + val kd = p.kotlin + KotlinFunctionInfo( + receiverTypeName = kd.receiverTypeName, + receiverTypeDisplayName = kd.receiverTypeDisplayName, + isSuspend = kd.isSuspend, + isInline = kd.isInline, + isInfix = kd.isInfix, + isOperator = kd.isOperator, + isTailrec = kd.isTailrec, + isExternal = kd.isExternal, + isExpect = kd.isExpect, + isActual = kd.isActual, + isReturnTypeNullable = kd.isReturnTypeNullable, + ) + } else { + null + } + + return JvmFunctionInfo( + containingClassName = p.containingClassName, + returnTypeName = p.returnTypeName, + returnTypeDisplayName = p.returnTypeDisplayName, + parameterCount = p.parameterCount, + parameters = p.parametersList.map { paramFromProto(it) }, + signatureDisplay = p.signatureDisplay, + typeParameters = p.typeParametersList.toList(), + isStatic = p.isStatic, + isAbstract = p.isAbstract, + isFinal = p.isFinal, + kotlin = kotlin, + ) + } + + private fun paramFromProto(p: JvmSymbolProtos.ParameterData): JvmParameterInfo = + JvmParameterInfo( + name = p.name, + typeName = p.typeName, + typeDisplayName = p.typeDisplayName, + hasDefaultValue = p.hasDefaultValue, + isCrossinline = p.isCrossinline, + isNoinline = p.isNoinline, + isVararg = p.isVararg, + ) + + private fun fieldInfoFromProto(p: JvmSymbolProtos.FieldData): JvmFieldInfo { + val kotlin = + if (p.hasKotlin()) { + val kd = p.kotlin + KotlinPropertyInfo( + receiverTypeName = kd.receiverTypeName, + receiverTypeDisplayName = kd.receiverTypeDisplayName, + isConst = kd.isConst, + isLateinit = kd.isLateinit, + hasGetter = kd.hasGetter, + hasSetter = kd.hasSetter, + isDelegated = kd.isDelegated, + isExpect = kd.isExpect, + isActual = kd.isActual, + isExternal = kd.isExternal, + isTypeNullable = kd.isTypeNullable, + ) + } else { + null + } + + return JvmFieldInfo( + containingClassName = p.containingClassName, + typeName = p.typeName, + typeDisplayName = p.typeDisplayName, + isStatic = p.isStatic, + isFinal = p.isFinal, + constantValue = p.constantValue, + kotlin = kotlin, + ) + } + + private fun enumEntryFromProto(p: JvmSymbolProtos.EnumEntryData): JvmEnumEntryInfo = + JvmEnumEntryInfo( + containingClassName = p.containingEnumName, + ordinal = p.ordinal, + ) + + private fun typeAliasFromProto(p: JvmSymbolProtos.TypeAliasData): JvmTypeAliasInfo = + JvmTypeAliasInfo( + expandedTypeName = p.expandedTypeName, + expandedTypeDisplayName = p.expandedTypeDisplayName, + typeParameters = p.typeParametersList.toList(), + ) + + private fun kindToProto(k: JvmSymbolKind) = + when (k) { + JvmSymbolKind.CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_CLASS + JvmSymbolKind.INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE + JvmSymbolKind.ENUM -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM + JvmSymbolKind.ENUM_ENTRY -> JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY + JvmSymbolKind.ANNOTATION_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS + JvmSymbolKind.OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT + JvmSymbolKind.COMPANION_OBJECT -> JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT + JvmSymbolKind.DATA_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS + JvmSymbolKind.VALUE_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS + JvmSymbolKind.SEALED_CLASS -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS + JvmSymbolKind.SEALED_INTERFACE -> JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE + JvmSymbolKind.FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION + JvmSymbolKind.EXTENSION_FUNCTION -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION + JvmSymbolKind.CONSTRUCTOR -> JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR + JvmSymbolKind.PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY + JvmSymbolKind.EXTENSION_PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY + JvmSymbolKind.FIELD -> JvmSymbolProtos.JvmSymbolKind.KIND_FIELD + JvmSymbolKind.TYPE_ALIAS -> JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS + } + + private fun kindFromProto(k: JvmSymbolProtos.JvmSymbolKind) = + when (k) { + JvmSymbolProtos.JvmSymbolKind.KIND_CLASS -> JvmSymbolKind.CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_INTERFACE -> JvmSymbolKind.INTERFACE + JvmSymbolProtos.JvmSymbolKind.KIND_ENUM -> JvmSymbolKind.ENUM + JvmSymbolProtos.JvmSymbolKind.KIND_ENUM_ENTRY -> JvmSymbolKind.ENUM_ENTRY + JvmSymbolProtos.JvmSymbolKind.KIND_ANNOTATION_CLASS -> JvmSymbolKind.ANNOTATION_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_OBJECT -> JvmSymbolKind.OBJECT + JvmSymbolProtos.JvmSymbolKind.KIND_COMPANION_OBJECT -> JvmSymbolKind.COMPANION_OBJECT + JvmSymbolProtos.JvmSymbolKind.KIND_DATA_CLASS -> JvmSymbolKind.DATA_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_VALUE_CLASS -> JvmSymbolKind.VALUE_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_CLASS -> JvmSymbolKind.SEALED_CLASS + JvmSymbolProtos.JvmSymbolKind.KIND_SEALED_INTERFACE -> JvmSymbolKind.SEALED_INTERFACE + JvmSymbolProtos.JvmSymbolKind.KIND_FUNCTION -> JvmSymbolKind.FUNCTION + JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_FUNCTION -> JvmSymbolKind.EXTENSION_FUNCTION + JvmSymbolProtos.JvmSymbolKind.KIND_CONSTRUCTOR -> JvmSymbolKind.CONSTRUCTOR + JvmSymbolProtos.JvmSymbolKind.KIND_PROPERTY -> JvmSymbolKind.PROPERTY + JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY -> JvmSymbolKind.EXTENSION_PROPERTY + JvmSymbolProtos.JvmSymbolKind.KIND_FIELD -> JvmSymbolKind.FIELD + JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS -> JvmSymbolKind.TYPE_ALIAS + else -> JvmSymbolKind.CLASS + } + + private fun languageToProto(l: JvmSourceLanguage) = + when (l) { + JvmSourceLanguage.JAVA -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA + JvmSourceLanguage.KOTLIN -> JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN + } + + private fun languageFromProto(l: JvmSymbolProtos.JvmSourceLanguage) = + when (l) { + JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_JAVA -> JvmSourceLanguage.JAVA + JvmSymbolProtos.JvmSourceLanguage.LANGUAGE_KOTLIN -> JvmSourceLanguage.KOTLIN + else -> JvmSourceLanguage.JAVA + } + + private fun visibilityToProto(v: JvmVisibility) = + when (v) { + JvmVisibility.PUBLIC -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC + JvmVisibility.PROTECTED -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED + JvmVisibility.INTERNAL -> JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL + JvmVisibility.PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE + JvmVisibility.PACKAGE_PRIVATE -> JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE + } + + private fun visibilityFromProto(v: JvmSymbolProtos.JvmVisibility) = + when (v) { + JvmSymbolProtos.JvmVisibility.VISIBILITY_PUBLIC -> JvmVisibility.PUBLIC + JvmSymbolProtos.JvmVisibility.VISIBILITY_PROTECTED -> JvmVisibility.PROTECTED + JvmSymbolProtos.JvmVisibility.VISIBILITY_INTERNAL -> JvmVisibility.INTERNAL + JvmSymbolProtos.JvmVisibility.VISIBILITY_PRIVATE -> JvmVisibility.PRIVATE + JvmSymbolProtos.JvmVisibility.VISIBILITY_PACKAGE_PRIVATE -> JvmVisibility.PACKAGE_PRIVATE + else -> JvmVisibility.PUBLIC + } } From 268f35265a5470f6b865be3c32957966adec8783 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 13:57:36 +0000 Subject: [PATCH 08/48] style: spotless reformat, no functional change --- .../lsp/kotlin/completion/KotlinCompletions.kt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 09906f6a6c..5f05e9cb71 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -861,22 +861,39 @@ private fun KaSession.kindOf(symbol: KaSymbol): CompletionItemKind = private fun KaSession.kindOf(symbol: JvmSymbol): CompletionItemKind = when (symbol.kind) { JvmSymbolKind.CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.INTERFACE -> CompletionItemKind.INTERFACE + JvmSymbolKind.ENUM -> CompletionItemKind.ENUM + JvmSymbolKind.ENUM_ENTRY -> CompletionItemKind.ENUM_MEMBER + JvmSymbolKind.ANNOTATION_CLASS -> CompletionItemKind.ANNOTATION_TYPE + JvmSymbolKind.OBJECT -> CompletionItemKind.CLASS + JvmSymbolKind.COMPANION_OBJECT -> CompletionItemKind.CLASS + JvmSymbolKind.DATA_CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.VALUE_CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.SEALED_CLASS -> CompletionItemKind.CLASS + JvmSymbolKind.SEALED_INTERFACE -> CompletionItemKind.INTERFACE + JvmSymbolKind.FUNCTION -> CompletionItemKind.FUNCTION + JvmSymbolKind.EXTENSION_FUNCTION -> CompletionItemKind.FUNCTION + JvmSymbolKind.CONSTRUCTOR -> CompletionItemKind.CONSTRUCTOR + JvmSymbolKind.PROPERTY -> CompletionItemKind.PROPERTY + JvmSymbolKind.EXTENSION_PROPERTY -> CompletionItemKind.PROPERTY + JvmSymbolKind.FIELD -> CompletionItemKind.FIELD + JvmSymbolKind.TYPE_ALIAS -> CompletionItemKind.CLASS } From 63e0b1d1230def2481f6cb59114c5a79f467f923 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 13:58:11 +0000 Subject: [PATCH 09/48] ADFA-6228: Index facades, package-private classes, and Kotlin nested classes correctly Three gaps between what the scanners record and what a classpath actually holds. Each would have shown up as a silent narrowing when class lookup moves off the classpath trie and onto this index. Facades were missing entirely. Metadata for `FileFacade` and `MultiFileClassFacade` describes a package, not a class, so `FooKt` -- the class Java calls a Kotlin file's top-level functions through -- had no entry at all. It gets one now, under a new `FILE_FACADE` kind so the two languages can disagree about it: Java needs to import it, Kotlin must never offer it as a type. That is why the kind is in `JVM_CLASS_KINDS` and not in `CLASSIFIER_KINDS`, and why `isJvmClass` asks whether a class file exists rather than whether Java would want it. Part classes (`FooKt__Part`) stay out; their declarations belong to the facade and neither language names the part. Package-private classes were dropped. The trie held every top-level class in a JAR regardless of visibility, so same-package completion would have lost candidates. They are recorded with `PACKAGE_PRIVATE` visibility for callers to filter on. Private classes stay out, and members are still gated on the class being public or protected. Kotlin nested classes were wrong twice over. Kotlin metadata separates a nested class from its outer one with '.', not '$', so splitting only on '$' left `Regex.Companion` as the short name, which no prefix search for `Companion` could match. `containingClassName` was never set either, so every nested Kotlin class reported `isTopLevel == true` -- and a top-level filter is exactly what a class-name query needs. --- .../indexing/jvm/JarSymbolScanner.kt | 9 +- .../codeonthego/indexing/jvm/JvmSymbol.kt | 38 ++++++++ .../indexing/jvm/JvmSymbolDescriptor.kt | 2 + .../indexing/jvm/KotlinMetadataScanner.kt | 66 +++++++++++++- .../jvm/JarSymbolScannerVisibilityTest.kt | 63 +++++++++++++ .../jvm/KotlinMetadataScannerShapeTest.kt | 91 +++++++++++++++++++ .../src/main/proto/jvm_symbol.proto | 3 + .../kotlin/completion/KotlinCompletions.kt | 4 + 8 files changed, 270 insertions(+), 6 deletions(-) create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt index e0054d0de8..2abe775ea3 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt @@ -81,7 +81,14 @@ object JarSymbolScanner { } override fun visitEnd() { - if (!isPublicOrProtected(classAccess)) return + /* + * Package-private classes are indexed, not dropped: they are real classpath entries and + * same-package code may legitimately reference them, so the caller decides by visibility + * whether to offer one. Private classes stay out - nothing outside the declaring class can + * name them. Members remain gated on the class being public or protected, so a + * package-private class contributes its own name and nothing else. + */ + if (hasFlag(classAccess, Opcodes.ACC_PRIVATE)) return val isAnonymous = isInnerClass && diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt index 851c3526e8..7cc2a16dcd 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt @@ -21,6 +21,16 @@ enum class JvmSymbolKind { EXTENSION_PROPERTY, FIELD, TYPE_ALIAS, + + /** + * The synthetic class a Kotlin file's top-level declarations compile into, named after the file + * (`Foo.kt` -> `FooKt`). + * + * Deliberately not in [CLASSIFIER_KINDS]: Kotlin code names the declarations inside it and never + * the class, so offering it as a Kotlin type would be wrong. It is in [JVM_CLASS_KINDS] because a + * class file by that name really exists, and Java calls the top-level functions through it. + */ + FILE_FACADE, ; val isCallable: Boolean @@ -29,6 +39,10 @@ enum class JvmSymbolKind { val isClassifier: Boolean get() = this in CLASSIFIER_KINDS + /** Whether a class file exists under this symbol's name. */ + val isJvmClass: Boolean + get() = this in JVM_CLASS_KINDS + val isExtension: Boolean get() = this == EXTENSION_FUNCTION || this == EXTENSION_PROPERTY @@ -42,6 +56,8 @@ enum class JvmSymbolKind { EXTENSION_PROPERTY, FIELD, ) + + /** Kinds that declare a type in the source language. */ val CLASSIFIER_KINDS = setOf( CLASS, @@ -56,6 +72,28 @@ enum class JvmSymbolKind { SEALED_INTERFACE, TYPE_ALIAS, ) + + /** + * Kinds backed by an actual class file, which is what a classpath scan finds and what an + * import has to resolve to. + * + * This is not [CLASSIFIER_KINDS]: a type alias declares a type but compiles to no class, and a + * file facade is a class that declares no type. + */ + val JVM_CLASS_KINDS = + setOf( + CLASS, + INTERFACE, + ENUM, + ANNOTATION_CLASS, + OBJECT, + COMPANION_OBJECT, + DATA_CLASS, + VALUE_CLASS, + SEALED_CLASS, + SEALED_INTERFACE, + FILE_FACADE, + ) } } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt index 2285da25ea..fa41b59f9a 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt @@ -391,6 +391,7 @@ object JvmSymbolDescriptor : IndexDescriptor { JvmSymbolKind.EXTENSION_PROPERTY -> JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY JvmSymbolKind.FIELD -> JvmSymbolProtos.JvmSymbolKind.KIND_FIELD JvmSymbolKind.TYPE_ALIAS -> JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS + JvmSymbolKind.FILE_FACADE -> JvmSymbolProtos.JvmSymbolKind.KIND_FILE_FACADE } private fun kindFromProto(k: JvmSymbolProtos.JvmSymbolKind) = @@ -413,6 +414,7 @@ object JvmSymbolDescriptor : IndexDescriptor { JvmSymbolProtos.JvmSymbolKind.KIND_EXTENSION_PROPERTY -> JvmSymbolKind.EXTENSION_PROPERTY JvmSymbolProtos.JvmSymbolKind.KIND_FIELD -> JvmSymbolKind.FIELD JvmSymbolProtos.JvmSymbolKind.KIND_TYPE_ALIAS -> JvmSymbolKind.TYPE_ALIAS + JvmSymbolProtos.JvmSymbolKind.KIND_FILE_FACADE -> JvmSymbolKind.FILE_FACADE else -> JvmSymbolKind.CLASS } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt index 3121ff9793..017789c4da 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -69,19 +69,60 @@ object KotlinMetadataScanner { } is KotlinClassMetadata.FileFacade -> { - extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) + facadeSymbol(collector, sourceId) + + extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) } + /* + * A part class is a compiler-generated fragment of a facade ("FooKt__Part"). Its + * declarations belong to the facade, so they are extracted, but neither language ever names + * the part class itself, so it is deliberately left out. + */ is KotlinClassMetadata.MultiFileClassPart -> { extractFromPackage(metadata.kmPackage, collector.packageName, sourceId) } + /* + * The facade of a @JvmMultifileClass. It declares nothing of its own -- the parts hold the + * declarations -- but Java calls the top-level functions through it, so the class still has + * to be findable. + */ + is KotlinClassMetadata.MultiFileClassFacade -> { + facadeSymbol(collector, sourceId) + } + else -> { null } } } + /** + * The symbol for a facade class itself, as opposed to the declarations it carries. + * + * Metadata for a facade describes a package rather than a class, so without this the class is + * absent from the index entirely and a Java file importing `FooKt` has nothing to resolve + * against. + */ + private fun facadeSymbol( + collector: MetadataCollector, + sourceId: String, + ): List { + val internalName = collector.internalName.ifEmpty { return emptyList() } + return listOf( + JvmSymbol( + key = internalName, + sourceId = sourceId, + name = internalName, + shortName = internalName.substringAfterLast('/'), + packageName = collector.packageName, + kind = JvmSymbolKind.FILE_FACADE, + language = JvmSourceLanguage.KOTLIN, + data = JvmClassInfo(internalName = internalName), + ), + ) + } + private fun extractFromClass( klass: KmClass, sourceId: String, @@ -92,10 +133,19 @@ object KotlinMetadataScanner { className .substringBeforeLast('/') .replace('/', '.') - val shortName = - className - .substringAfterLast('/') - .substringAfterLast('$') + /* + * Kotlin metadata separates a nested class from its outer one with '.', not '$', so + * "com/example/Outer.Inner" has to be split on both. Splitting only on '$' left the short name + * as "Outer.Inner", which no prefix search for "Inner" could match. + */ + val simpleNames = className.substringAfterLast('/') + val shortName = simpleNames.substringAfterLast('.').substringAfterLast('$') + val containingClassName = + if (simpleNames.contains('.')) { + className.substringBeforeLast('.') + } else { + "" + } val kind = when (klass.kind) { @@ -128,6 +178,8 @@ object KotlinMetadataScanner { visibility = kmVisibility(klass.visibility), data = JvmClassInfo( + internalName = className, + containingClassName = containingClassName, supertypeNames = supertypes, typeParameters = klass.typeParameters.map { it.name }, isAbstract = klass.modality == Modality.ABSTRACT, @@ -375,6 +427,9 @@ object KotlinMetadataScanner { var metadataHeader: Metadata? = null var packageName = "" + /** The visited class's internal name, e.g. `com/example/FooKt`. */ + var internalName = "" + private var metadataKind: Int? = null private var metadataVersion: IntArray? = null private var data1: Array? = null @@ -391,6 +446,7 @@ object KotlinMetadataScanner { superName: String?, interfaces: Array?, ) { + internalName = name val lastSlash = name.lastIndexOf('/') packageName = if (lastSlash >= 0) name.substring(0, lastSlash).replace('/', '.') else "" } diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt new file mode 100644 index 0000000000..19f2664cad --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt @@ -0,0 +1,63 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * Which classes [JarSymbolScanner] admits, by visibility. + * + * The classpath trie this index replaces held every top-level class in a JAR regardless of + * visibility, so dropping package-private ones would silently narrow completion. + */ +@RunWith(JUnit4::class) +class JarSymbolScannerVisibilityTest { + private fun classBytes( + internalName: String, + access: Int, + ): ByteArray = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, access, internalName, null, "java/lang/Object", null) + visitEnd() + }.toByteArray() + + private fun scan( + internalName: String, + access: Int, + ): List = JarSymbolScanner.parseClassFile(classBytes(internalName, access).inputStream(), "test.jar") + + @Test + fun `a public class is indexed`() { + val symbols = scan("com/example/Pub", Opcodes.ACC_PUBLIC) + + assertThat(symbols.map { it.shortName }).containsExactly("Pub") + assertThat(symbols.single().visibility).isEqualTo(JvmVisibility.PUBLIC) + } + + @Test + fun `a package-private class is indexed and keeps its visibility`() { + val symbols = scan("com/example/Pkg", 0) + + assertThat(symbols.map { it.shortName }).containsExactly("Pkg") + assertThat(symbols.single().visibility).isEqualTo(JvmVisibility.PACKAGE_PRIVATE) + } + + @Test + fun `a private nested class is not indexed`() { + // Nothing outside the declaring class can name it, so it is never a completion candidate. + assertThat(scan("com/example/Outer\$Hidden", Opcodes.ACC_PRIVATE)).isEmpty() + } + + @Test + fun `a nested class records its outer class`() { + val nested = scan("com/example/Outer\$Inner", Opcodes.ACC_PUBLIC).single() + + assertThat(nested.shortName).isEqualTo("Inner") + assertThat(nested.isTopLevel).isFalse() + assertThat(nested.data.containingClassFqName).isEqualTo("com.example.Outer") + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt new file mode 100644 index 0000000000..1da9a8694b --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt @@ -0,0 +1,91 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.junit.Assume.assumeTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.util.jar.JarFile + +/** + * [KotlinMetadataScanner] against the real Kotlin standard library, which is the only convenient + * source of genuine `@kotlin.Metadata` in the shapes that matter: a file facade, a multi-file facade + * and a nested class. + */ +@RunWith(JUnit4::class) +class KotlinMetadataScannerShapeTest { + private val stdlibJar: File? = + runCatching { + File( + Unit::class.java.protectionDomain.codeSource.location + .toURI(), + ) + }.getOrNull() + ?.takeIf { it.isFile && it.name.endsWith(".jar") } + + private fun symbolsFor(entryName: String): List { + val jar = stdlibJar + assumeTrue("kotlin-stdlib jar not resolvable from the test classpath", jar != null) + return JarFile(jar).use { file -> + val entry = file.getJarEntry(entryName) + assumeTrue("$entryName missing from ${jar!!.name}", entry != null) + file.getInputStream(entry).use { KotlinMetadataScanner.parseKotlinClass(it, "stdlib") } + } ?: emptyList() + } + + @Test + fun `a nested class reports its simple name, not the outer-qualified one`() { + val companion = symbolsFor("kotlin/text/Regex\$Companion.class").first { it.kind.isClassifier } + + // Kotlin metadata spells this "kotlin/text/Regex.Companion", so splitting on '$' alone left + // the short name as "Regex.Companion" and no prefix search could reach it. + assertThat(companion.shortName).isEqualTo("Companion") + } + + @Test + fun `a nested class is not reported as top level`() { + val companion = symbolsFor("kotlin/text/Regex\$Companion.class").first { it.kind.isClassifier } + + assertThat(companion.isTopLevel).isFalse() + assertThat(companion.data.containingClassFqName).isEqualTo("kotlin.text.Regex") + } + + @Test + fun `a file facade is indexed as a class in its own right`() { + val symbols = symbolsFor("kotlin/io/CloseableKt.class") + + val facade = symbols.singleOrNull { it.kind == JvmSymbolKind.FILE_FACADE } + assertThat(facade).isNotNull() + assertThat(facade!!.shortName).isEqualTo("CloseableKt") + assertThat(facade.fqName).isEqualTo("kotlin.io.CloseableKt") + assertThat(facade.packageName).isEqualTo("kotlin.io") + } + + @Test + fun `a multi-file facade is indexed even though it declares nothing itself`() { + val symbols = symbolsFor("kotlin/collections/CollectionsKt.class") + + val facade = symbols.singleOrNull { it.kind == JvmSymbolKind.FILE_FACADE } + assertThat(facade).isNotNull() + assertThat(facade!!.fqName).isEqualTo("kotlin.collections.CollectionsKt") + } + + @Test + fun `a facade is a JVM class but not a classifier`() { + val facade = + symbolsFor("kotlin/io/CloseableKt.class").single { it.kind == JvmSymbolKind.FILE_FACADE } + + // Java calls top-level functions through the facade, so it must be findable as a class. + assertThat(facade.kind.isJvmClass).isTrue() + // Kotlin refers to the declarations, never the facade, so it must not be offered as a type. + assertThat(facade.kind.isClassifier).isFalse() + } + + @Test + fun `the declarations inside a facade are still indexed alongside it`() { + val symbols = symbolsFor("kotlin/io/CloseableKt.class") + + assertThat(symbols.any { it.kind.isCallable }).isTrue() + } +} diff --git a/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto index 7f7e5517e6..85f18e3a97 100644 --- a/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto +++ b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto @@ -192,6 +192,9 @@ enum JvmSymbolKind { KIND_EXTENSION_PROPERTY = 16; KIND_FIELD = 17; KIND_TYPE_ALIAS = 18; + // The synthetic class a Kotlin file's top-level declarations compile into ("FooKt"). + // Java can name it; Kotlin never should. + KIND_FILE_FACADE = 19; } enum JvmSourceLanguage { diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 5f05e9cb71..3d356de350 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -895,6 +895,10 @@ private fun KaSession.kindOf(symbol: JvmSymbol): CompletionItemKind = JvmSymbolKind.FIELD -> CompletionItemKind.FIELD JvmSymbolKind.TYPE_ALIAS -> CompletionItemKind.CLASS + + // Kotlin completion should never reach a facade -- it is not a classifier, and the + // declarations it holds are offered individually -- but the mapping has to be total. + JvmSymbolKind.FILE_FACADE -> CompletionItemKind.CLASS } private fun partialIdentifier(prefix: String): String = prefix.takeLastWhile { char -> Character.isJavaIdentifierPart(char) } From 85c42ef34e9913afde2fb0fa0033e181dcc1e318 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 13:59:58 +0000 Subject: [PATCH 10/48] ADFA-6228: Stop indexing anonymous classes and multi-release module-info as classes Both found by the differential test added here, which compares what the index offers against what `ModuleProject.compileClasspathClasses` held. The oracle is built from the old path -- `JarFsClasspathReader` filtered to top-level classes, which is literally what filled the trie -- so it cannot be satisfied by the new code agreeing with itself. Kotlin metadata marks a local or anonymous class with a leading `.` and nests with `$` once compiled, not only with `.` in source order. Handling just the source form left `AbstractMap$keys$1` looking like a top-level class whose simple name was `1`. Against kotlin-stdlib that was 65 entries the trie never held, all of which would have surfaced as completion noise. `module-info.class` was skipped by comparing the whole JAR entry path, so a multi-release JAR carrying it under `META-INF/versions/9/` slipped through as a class. Compared on the file name now. Against kotlin-stdlib the two sets are now equal but for 88 `@JvmMultifileClass` parts, which the index withholds deliberately: their declarations belong to the facade, which is indexed, and neither language names the part. Nothing else is missing and nothing extra is offered. --- .../indexing/jvm/CombinedJarScanner.kt | 5 +- .../indexing/jvm/KotlinMetadataScanner.kt | 23 +++- .../jvm/ClasspathTrieDifferentialTest.kt | 106 ++++++++++++++++++ 3 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt index 1a6289d091..e2d0fdecc5 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt @@ -66,7 +66,10 @@ object CombinedJarScanner { while (entries.hasMoreElements()) { val entry = entries.nextElement() if (!entry.name.endsWith(".class")) continue - if (entry.name == "module-info.class" || entry.name == "package-info.class") continue + // Compared on the file name, not the full entry path: a multi-release JAR carries these + // under META-INF/versions//, where an exact-path match lets them through as classes. + val entryFileName = entry.name.substringAfterLast('/') + if (entryFileName == "module-info.class" || entryFileName == "package-info.class") continue // emit stays outside the try so cancellation is not swallowed. val symbols = diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt index 017789c4da..ed073685db 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -134,15 +134,26 @@ object KotlinMetadataScanner { .substringBeforeLast('/') .replace('/', '.') /* - * Kotlin metadata separates a nested class from its outer one with '.', not '$', so - * "com/example/Outer.Inner" has to be split on both. Splitting only on '$' left the short name - * as "Outer.Inner", which no prefix search for "Inner" could match. + * A leading '.' marks a local or anonymous class. The classpath trie never held these -- it + * treated any name containing '$' as not top level -- and neither language names them, so + * offering one as a completion would be noise. JarSymbolScanner drops its equivalents too. + */ + if (className.startsWith('.')) { + return symbols + } + + /* + * Kotlin metadata nests with '.' in source order and with '$' once compiled, so a nested class + * has to be split on either. Splitting on '$' alone left the short name as "Outer.Inner", which + * no prefix search for "Inner" could match; splitting on '.' alone left a compiled-nested class + * looking top level. */ val simpleNames = className.substringAfterLast('/') - val shortName = simpleNames.substringAfterLast('.').substringAfterLast('$') + val lastSeparator = maxOf(simpleNames.lastIndexOf('.'), simpleNames.lastIndexOf('$')) + val shortName = if (lastSeparator >= 0) simpleNames.substring(lastSeparator + 1) else simpleNames val containingClassName = - if (simpleNames.contains('.')) { - className.substringBeforeLast('.') + if (lastSeparator >= 0) { + className.substring(0, className.length - (simpleNames.length - lastSeparator)) } else { "" } diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt new file mode 100644 index 0000000000..30d29ed5ef --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt @@ -0,0 +1,106 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import com.itsaky.androidide.projects.classpath.JarFsClasspathReader +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.toSet +import kotlinx.coroutines.runBlocking +import org.junit.Assume.assumeTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.nio.file.Paths + +/** + * The index must offer exactly the classes the classpath trie held. + * + * The oracle is built from the **old** path: `JarFsClasspathReader` filtered to top-level classes is + * literally what filled `ModuleProject.compileClasspathClasses`. Building it from the index instead + * would let the new code agree with itself and pin nothing. + * + * It runs against a real JAR because the cases that matter -- Kotlin file facades, multi-file class + * parts, package-private classes, anonymous and nested types, multi-release entries -- do not occur + * in a hand-built fixture. Two defects were found this way: anonymous Kotlin classes surfacing as + * top-level classes named `1`, and `module-info` from a multi-release JAR indexed as a class. + */ +@RunWith(JUnit4::class) +class ClasspathTrieDifferentialTest { + private val jar: File? = + runCatching { + File( + Unit::class.java.protectionDomain.codeSource.location + .toURI(), + ) + }.getOrNull() + ?.takeIf { it.isFile && it.name.endsWith(".jar") } + + /** What the trie held: every top-level class the classpath reader found. */ + private fun trieNames(jar: File): Set = + JarFsClasspathReader() + .listClasses(listOf(jar)) + .asSequence() + .filter { it.isTopLevel } + .map { it.name } + .toSet() + + /** What the index offers for the same JAR. */ + private fun indexNames(jar: File): Set = + runBlocking { + CombinedJarScanner + .scan(Paths.get(jar.absolutePath), "differential") + .filter { it.kind.isJvmClass && it.isTopLevel } + .map { it.fqName } + .toSet() + } + + /** + * A fragment of a `@JvmMultifileClass` facade, e.g. `LazyKt__LazyJVMKt`. + * + * The only thing the index deliberately withholds: the declarations belong to the facade, which + * is indexed, and neither Java nor Kotlin names the part. + */ + private fun isMultiFilePart(qualifiedName: String) = qualifiedName.substringAfterLast('.').contains("__") + + private fun requireJar(): File { + assumeTrue("kotlin-stdlib not resolvable from the test classpath", jar != null) + return jar!! + } + + @Test + fun `every class the trie held is offered by the index, bar multi-file parts`() { + val jar = requireJar() + + val missing = trieNames(jar) - indexNames(jar) + + assertThat(missing.filterNot(::isMultiFilePart)).isEmpty() + } + + @Test + fun `the index offers nothing the trie did not hold`() { + val jar = requireJar() + + // Anonymous Kotlin classes and multi-release `module-info` entries both used to land here. + assertThat(indexNames(jar) - trieNames(jar)).isEmpty() + } + + @Test + fun `the withheld classes really are multi-file parts, and there are some`() { + val jar = requireJar() + + val missing = trieNames(jar) - indexNames(jar) + + // Guards the exception itself: if the corpus stopped containing parts, the first test would + // start passing for the wrong reason. + assertThat(missing).isNotEmpty() + assertThat(missing.all(::isMultiFilePart)).isTrue() + } + + @Test + fun `the corpus is large enough for the comparison to mean something`() { + val jar = requireJar() + + assertThat(trieNames(jar).size).isGreaterThan(500) + } +} From 9ef4088b609c0dd615434fab8d043cd8605d55fc Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 11:26:17 +0000 Subject: [PATCH 11/48] ADFA-6228: Cap unimported-symbol completion by items delivered, not rows fetched UNIMPORTED_SYMBOL_LIMIT was the query limit for each of three indexes, while `addCompletionItem` then discarded symbols in the current package, symbols failing the visibility check, and member-level callables. The limit was spent on rows before any of that ran, so a prefix whose leading rows were all rejected showed nothing while valid matches sat past row 100. The cap now counts items added, and it is one total rather than three. Three lists of 100 pre-filter was never a considered number; what a user can scan in a popup is. Each index keeps a separate, larger fetch budget so filtering cannot starve the result. Sized from measurement rather than taste: scanning kotlin-stdlib 2.3.0 (8,704 indexed symbols), 29% of entries are member-level callables that `buildUnimportedSymbolItem` drops outright, and the worst prefix probed -- `get` -- kept only 80 of 236 matches, a 34% survival rate before the package and visibility filters run at all. 3x the display cap covers that; the remaining filters eat the margin. Not unbounded, because an unrestricted prefix query over the symbol table would run on every keystroke. Sources are suppliers, not sequences, because `SQLiteIndex.query` runs its query and collects the rows before returning -- a lazy `filter`/`take` over what it returns would defer the filtering but not the fetch. Passing a supplier is what actually stops an index being queried once the cap is met. One behaviour change worth naming: with a shared cap, whichever index is queried last loses when the cap binds. The order is now source, generated, library, so what gets crowded out is a dependency's symbols rather than the user's own. What this does not do is decide *which* symbols win. There is no ranking pass here; within an index that is still SQL order. The collection loop is extracted so it can be tested; the wiring that calls it is covered by compilation and inspection, since driving the real path needs an analysis session and no completion harness exists. --- .../kotlin/completion/KotlinCompletions.kt | 90 +++++++++++--- .../kotlin/completion/CollectUpToLimitTest.kt | 117 ++++++++++++++++++ 2 files changed, 191 insertions(+), 16 deletions(-) create mode 100644 lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUpToLimitTest.kt diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 3d356de350..348f24d247 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -77,8 +77,28 @@ private const val KT_COMPLETION_PLACEHOLDER = "KT_COMPLETION_PLACEHOLDER" private val logger = LoggerFactory.getLogger("KotlinCompletions") -/** Max unimported symbols pulled from each index for scope completion (see [collectUnimportedSymbols]). */ -private const val UNIMPORTED_SYMBOL_LIMIT = 100 +/** + * Max unimported symbols [collectUnimportedSymbols] adds to one completion result, across every + * index rather than per index. + * + * This counts items the user is actually offered. It used to be the query limit, which counted rows + * fetched -- so a prefix whose first rows were all rejected by the package, visibility or kind + * filters produced nothing while valid matches sat just past the limit. + */ +private const val UNIMPORTED_SYMBOL_DISPLAY_LIMIT = 100 + +/** + * Rows each index may return before filtering, sized so ordinary filtering cannot starve the + * result. + * + * Measured against kotlin-stdlib 2.3.0 (8,704 indexed symbols): 29% of entries are member-level + * callables, which [buildUnimportedSymbolItem] drops outright, and the worst realistic prefix of + * those probed (`get`) kept only 80 of 236 matches -- a 34% survival rate before the package and + * visibility filters even run. Three times the display limit covers that worst case; the package + * and visibility filters then eat into the margin. It is deliberately not unbounded: an unrestricted + * prefix query over the symbol table runs on every keystroke. + */ +private const val UNIMPORTED_SYMBOL_FETCH_BUDGET = 3 * UNIMPORTED_SYMBOL_DISPLAY_LIMIT /** * The [ScheduledCancelChecker] for the completion running on this thread, set for the duration of @@ -412,10 +432,10 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) val useSiteModule = this.useSiteModule val visibilityChecker = env.symbolVisibilityChecker - fun addCompletionItem(symbol: JvmSymbol) { + fun addCompletionItem(symbol: JvmSymbol): Boolean { abortIfCancelled() - if (symbol.packageName == currentPackage) return + if (symbol.packageName == currentPackage) return false val isVisible = visibilityChecker.isVisible( @@ -424,22 +444,60 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) useSitePackage = currentPackage, ) - if (!isVisible) return + if (!isVisible) return false - buildUnimportedSymbolItem(symbol)?.let { to += it } + val item = buildUnimportedSymbolItem(symbol) ?: return false + to += item + return true } - env.libraryIndex - ?.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_LIMIT) - ?.forEach(::addCompletionItem) - - env.sourceIndex - ?.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_LIMIT) - ?.forEach(::addCompletionItem) + /* + * Source first, library last. One shared cap across the indexes means whichever is queried last + * loses when the cap binds, and a symbol from the user's own project is likelier to be the one + * they are reaching for than one from a dependency. + */ + val indexes = listOfNotNull(env.sourceIndex, env.generatedIndex, env.libraryIndex) + + collectUpToLimit( + limit = UNIMPORTED_SYMBOL_DISPLAY_LIMIT, + sources = indexes.map { index -> { index.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_FETCH_BUDGET) } }, + accept = ::addCompletionItem, + ) +} - env.generatedIndex - ?.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_LIMIT) - ?.forEach(::addCompletionItem) +/** + * Offers items from [sources] in order to [accept], stopping once [limit] of them have been + * accepted, and returns how many were. + * + * The limit counts *accepted* items, which is the whole point: capping the fetch instead lets + * rejected entries consume the budget, so a query whose leading rows are all filtered out yields + * nothing while valid matches sit just beyond the cap. + * + * Sources are suppliers, not sequences, because `SQLiteIndex.query` runs its query and collects the + * rows before returning -- the sequence it hands back is already a list. Taking a supplier is what + * keeps a source that the limit makes unnecessary from being queried at all; how many rows a source + * fetches when it is reached remains the job of the limit passed to the query itself. + */ +internal fun collectUpToLimit( + limit: Int, + sources: List<() -> Sequence>, + accept: (T) -> Boolean, +): Int { + var accepted = 0 + for (source in sources) { + if (accepted >= limit) { + break + } + for (item in source()) { + if (accepted >= limit) { + break + } + if (accept(item)) { + accepted++ + } + } + } + return accepted } context(ctx: AnalysisContext) diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUpToLimitTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUpToLimitTest.kt new file mode 100644 index 0000000000..9d88189263 --- /dev/null +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUpToLimitTest.kt @@ -0,0 +1,117 @@ +package com.itsaky.androidide.lsp.kotlin.completion + +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * [collectUpToLimit] is the cap that unimported-symbol completion runs on. + * + * The behaviour under test is that the cap counts items the user is offered, not rows the index + * returned. Capping the fetch instead is what made a prefix whose leading rows are all rejected -- + * by the current-package, visibility or member-kind filters -- produce nothing while valid matches + * sat just past the cap. + */ +@RunWith(JUnit4::class) +class CollectUpToLimitTest { + private fun source(vararg items: String): () -> Sequence = { items.asSequence() } + + @Test + fun `rejected items do not consume the limit`() { + val offered = (1..200).map { "reject$it" } + listOf("keepA", "keepB", "keepC") + val kept = mutableListOf() + + val accepted = + collectUpToLimit(limit = 3, sources = listOf(source(*offered.toTypedArray()))) { item -> + if (item.startsWith("keep")) { + kept += item + true + } else { + false + } + } + + assertThat(accepted).isEqualTo(3) + assertThat(kept).containsExactly("keepA", "keepB", "keepC") + } + + @Test + fun `collection stops once the limit is reached`() { + val kept = mutableListOf() + + val accepted = + collectUpToLimit(limit = 2, sources = listOf(source("a", "b", "c", "d"))) { item -> + kept += item + true + } + + assertThat(accepted).isEqualTo(2) + assertThat(kept).containsExactly("a", "b") + } + + @Test + fun `a later source is never queried once the limit is reached`() { + var secondSourceQueried = false + val second: () -> Sequence = { + secondSourceQueried = true + sequenceOf("x") + } + + collectUpToLimit(limit = 2, sources = listOf(source("a", "b"), second)) { true } + + // The supplier matters: an index query runs eagerly, so taking a sequence here would have + // issued the query regardless of the limit. + assertThat(secondSourceQueried).isFalse() + } + + @Test + fun `later sources fill the remainder when earlier ones fall short`() { + val kept = mutableListOf() + + val accepted = + collectUpToLimit( + limit = 4, + sources = listOf(source("a"), source("b", "c"), source("d", "e")), + ) { item -> + kept += item + true + } + + assertThat(accepted).isEqualTo(4) + assertThat(kept).containsExactly("a", "b", "c", "d").inOrder() + } + + @Test + fun `every source is drained when the limit is never reached`() { + val kept = mutableListOf() + + val accepted = + collectUpToLimit(limit = 100, sources = listOf(source("a"), source("b"))) { item -> + kept += item + true + } + + assertThat(accepted).isEqualTo(2) + assertThat(kept).containsExactly("a", "b").inOrder() + } + + @Test + fun `a source that yields nothing acceptable does not block the next one`() { + val kept = mutableListOf() + + collectUpToLimit( + limit = 2, + sources = listOf(source("reject1", "reject2"), source("keepA", "keepB")), + ) { item -> + if (item.startsWith("keep")) { + kept += item + true + } else { + false + } + } + + assertThat(kept).containsExactly("keepA", "keepB") + } +} From 7963cae6d74009083ef5f704b6c1455f298009bd Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 18 Sep 2026 11:34:42 +0000 Subject: [PATCH 12/48] ADFA-6228: Filter by kind in the query, so the unbounded package lookups stop being unbounded `findClassifiersInPackage` and `findTopLevelCallablesInPackage` queried with `limit = 0` and filtered the result, which is fetch-all-then-filter: a package holds far more members than top-level declarations, so keeping a handful of classifiers meant materialising every symbol in the package. Survivable while a trie in memory was the primary path; not once these are answering completion against the symbol table. `IndexQuery.anyOf` expresses `field IN (values)`, which is what pushing a kind filter down requires. It is language-neutral -- set membership on an indexed column -- so it adds no surface that only one language can use. An empty collection matches nothing, for the same reason an empty `sourceIds` does. Top-level is expressible too: the descriptor already stores a null containing class for it, so `notExists` covers it without a new field. `findBySimpleName` takes an optional kind set for the same reason. An exact-name lookup otherwise returns every method and field sharing the name, and its only caller wanted classifiers. `findSymbolBySimpleName` now passes its limit to each index instead of applying it to the concatenation, which had made every call fetch both indexes in full. `AddImportAction` keeps `limit = 0` deliberately: capping it would hide an importable class from the chooser, and with the kind restriction the row count is the number of classes with that simple name rather than a scan. The restriction also keeps Kotlin from being offered a file facade, which is not a classifier. Its comment claiming an unbounded scan is updated, since that is no longer what it does. --- .../codeonthego/indexing/InMemoryIndex.kt | 6 + .../codeonthego/indexing/SQLiteIndex.kt | 12 ++ .../codeonthego/indexing/api/Query.kt | 18 +++ .../indexing/SQLiteIndexScopeTest.kt | 41 +++++ .../indexing/jvm/JvmSymbolIndex.kt | 34 +++- .../jvm/JvmSymbolIndexKindScopeTest.kt | 148 ++++++++++++++++++ .../lsp/kotlin/actions/AddImportAction.kt | 14 +- .../kotlin/compiler/index/KtSymbolIndex.kt | 13 +- 8 files changed, 276 insertions(+), 10 deletions(-) create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index d0d945d11d..697567d3bb 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -183,6 +183,12 @@ class InMemoryIndex( candidates = intersect(candidates, fieldMap[value]) } + for ((field, values) in query.anyOf) { + val fieldMap = fieldMaps[field] ?: return@read emptySequence() + val matching = values.flatMapTo(mutableSetOf()) { fieldMap[it].orEmpty() } + candidates = intersect(candidates, matching) + } + for ((field, prefix) in query.prefixMatch) { val buckets = prefixBuckets[field] val matching: Set = diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 601200a3eb..fb1c72e640 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -495,6 +495,18 @@ class SQLiteIndex( and("$col = ?", value) } + for ((field, values) in query.anyOf) { + val col = fieldColumns[field] ?: continue + if (values.isEmpty()) { + // Scoped to nothing, as opposed to unscoped: no row can satisfy it. + and("0 = 1") + continue + } + val distinct = values.distinct() + val placeholders = distinct.joinToString(",") { "?" } + and("$col IN ($placeholders)", *distinct.toTypedArray()) + } + for ((field, prefix) in query.prefixMatch) { val lowerCol = prefixColumns[field] // Prefix-searchable fields match case-insensitively through their pre-lowercased column; diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt index f02c5e205a..617c31db5c 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -12,6 +12,14 @@ data class IndexQuery( val exactMatch: Map = emptyMap(), /** Prefix match predicates: field name → prefix (case-insensitive). */ val prefixMatch: Map = emptyMap(), + /** + * Set-membership predicates: field name to the values that field may take. + * + * An empty collection matches nothing, on the same reasoning as [sourceIds]. This exists so a + * caller wanting several values of one field -- every classifier kind, say -- can say so in the + * query instead of fetching every value and discarding most of them. + */ + val anyOf: Map> = emptyMap(), /** * Presence predicates: field name → whether the field must be * non-null (true) or null (false). @@ -52,6 +60,7 @@ class IndexQueryBuilder { private val exact = mutableMapOf() private val prefix = mutableMapOf() private val pres = mutableMapOf() + private val anyOfValues = mutableMapOf>() var sourceId: String? = null var sourceIds: Collection? = null var key: String? = null @@ -73,6 +82,14 @@ class IndexQueryBuilder { prefix[field] = value } + /** Field must hold one of [values]. */ + fun anyOf( + field: String, + values: Collection, + ) { + anyOfValues[field] = values + } + /** Field must be non-null. */ fun exists(field: String) { pres[field] = true @@ -87,6 +104,7 @@ class IndexQueryBuilder { IndexQuery( exactMatch = exact.toMap(), prefixMatch = prefix.toMap(), + anyOf = anyOfValues.toMap(), presence = pres.toMap(), sourceId = sourceId, sourceIds = sourceIds, diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt index 3b4c99cb76..72711efffb 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt @@ -150,4 +150,45 @@ class SQLiteIndexScopeTest { assertThat(index.distinctValues("group").toList()).containsExactly("one", "two") } + + @Test + fun `anyOf matches any of the listed values`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + index.insert(Entry("k2", "jarA", "Beta", group = "two")) + index.insert(Entry("k3", "jarA", "Gamma", group = "three")) + + val keys = + index + .query(IndexQuery(anyOf = mapOf("group" to listOf("one", "three")), limit = 0)) + .map { it.key } + .toList() + + assertThat(keys).containsExactly("k1", "k3") + } + + @Test + fun `an empty anyOf matches nothing`() = + runTest { + index.insert(Entry("k1", "jarA", "Alpha", group = "one")) + + // Same distinction as an empty source scope: restricted to nothing, not unrestricted. + assertThat(index.query(IndexQuery(anyOf = mapOf("group" to emptyList()), limit = 0)).toList()) + .isEmpty() + } + + @Test + fun `anyOf narrows the rows the limit is spent on`() = + runTest { + repeat(20) { index.insert(Entry("other$it", "jarA", "Cls$it", group = "unwanted")) } + index.insert(Entry("wanted", "jarA", "ClsWanted", group = "wanted")) + + val keys = + index + .query(IndexQuery(anyOf = mapOf("group" to listOf("wanted")), limit = 1)) + .map { it.key } + .toList() + + assertThat(keys).containsExactly("wanted") + } } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index b9ac3170db..be46dd372b 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -10,6 +10,7 @@ import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.WritableIndex import org.appdevforall.codeonthego.indexing.api.indexQuery import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_CONTAINING_CLASS +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_KIND import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_NAME import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_PACKAGE import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor.KEY_RECEIVER_TYPE @@ -28,6 +29,10 @@ open class JvmSymbolIndex( WritableIndex by backing, Closeable { companion object { + /** Kind names as the descriptor stores them, for set-membership predicates. */ + private val CLASSIFIER_KIND_NAMES = JvmSymbolKind.CLASSIFIER_KINDS.map { it.name } + private val CALLABLE_KIND_NAMES = JvmSymbolKind.CALLABLE_KINDS.map { it.name } + const val DB_NAME_DEFAULT = "jvm_symbol_index.db" const val INDEX_NAME_LIBRARY = "jvm-library-cache" @@ -111,6 +116,14 @@ open class JvmSymbolIndex( }, ) + /** + * Top-level callables declared in [packageName]. + * + * The kind and top-level conditions are part of the query rather than a filter over its result: + * a package holds far more members than top-level callables, so filtering afterwards meant + * fetching every symbol in the package to keep a handful. Top-level is expressible because the + * descriptor stores a null containing class for it. + */ fun findTopLevelCallablesInPackage( packageName: String, namePrefix: String = "", @@ -119,11 +132,14 @@ open class JvmSymbolIndex( query( indexQuery { eq(KEY_PACKAGE, packageName) + anyOf(KEY_KIND, CALLABLE_KIND_NAMES) + notExists(KEY_CONTAINING_CLASS) if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) - this.limit = 0 + this.limit = limit }, - ).filter { it.kind.isCallable && it.isTopLevel }.take(limit) + ) + /** Classifiers declared in [packageName], filtered by kind in the query for the same reason. */ fun findClassifiersInPackage( packageName: String, namePrefix: String = "", @@ -132,10 +148,11 @@ open class JvmSymbolIndex( query( indexQuery { eq(KEY_PACKAGE, packageName) + anyOf(KEY_KIND, CLASSIFIER_KIND_NAMES) if (namePrefix.isNotEmpty()) prefix(KEY_NAME, namePrefix) - this.limit = 0 + this.limit = limit }, - ).filter { it.kind.isClassifier }.take(limit) + ) fun findMembersOf( classFqName: String, @@ -150,12 +167,21 @@ open class JvmSymbolIndex( }, ) + /** + * Symbols whose simple name is exactly [name], optionally restricted to [kinds]. + * + * Restricting by kind matters more than it looks: without it an exact-name lookup still returns + * every method and field sharing that name, and a caller wanting only classifiers pays for all + * of them. + */ fun findBySimpleName( name: String, limit: Int = 200, + kinds: Set? = null, ) = query( indexQuery { eq(KEY_NAME, name) + if (kinds != null) anyOf(KEY_KIND, kinds.map { it.name }) this.limit = limit }, ) diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt new file mode 100644 index 0000000000..838f318ace --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt @@ -0,0 +1,148 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * The package and simple-name lookups filter by kind in the query rather than over its result. + * + * A package holds far more members than top-level declarations, so a post-filter meant fetching + * every symbol in the package to keep a handful -- and the caller's limit was spent on rows it + * would then throw away. + */ +@RunWith(JUnit4::class) +class JvmSymbolIndexKindScopeTest { + private fun index(): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { + override fun visibleSourceIds(): Collection? = null + } + } + + private fun classifier( + simpleName: String, + pkg: String = "com.example", + ) = JvmSymbol( + key = "$pkg.$simpleName", + sourceId = "test.jar", + name = "$pkg.$simpleName", + shortName = simpleName, + packageName = pkg, + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(internalName = "$pkg/$simpleName"), + ) + + private fun member( + simpleName: String, + owner: String, + pkg: String = "com.example", + ) = JvmSymbol( + key = "$pkg.$owner#$simpleName", + sourceId = "test.jar", + name = "$pkg.$owner#$simpleName", + shortName = simpleName, + packageName = pkg, + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.JAVA, + data = JvmFunctionInfo(containingClassName = "$pkg/$owner"), + ) + + private fun topLevelFunction( + simpleName: String, + pkg: String = "com.example", + ) = JvmSymbol( + key = "$pkg#$simpleName", + sourceId = "test.jar", + name = "$pkg#$simpleName", + shortName = simpleName, + packageName = pkg, + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.KOTLIN, + data = JvmFunctionInfo(), + ) + + @Test + fun `classifiers in a package exclude that package's members`() = + runTest { + val index = index() + index.insert(classifier("Widget")) + repeat(50) { index.insert(member("helper$it", owner = "Widget")) } + + val found = index.findClassifiersInPackage("com.example").map { it.shortName }.toList() + + assertThat(found).containsExactly("Widget") + } + + @Test + fun `a classifier is still returned when members would have filled the limit`() = + runTest { + val index = index() + // Members are inserted first, so a post-filter would spend the limit on them. + repeat(50) { index.insert(member("helper$it", owner = "Widget")) } + index.insert(classifier("Widget")) + + val found = index.findClassifiersInPackage("com.example", limit = 5).map { it.shortName }.toList() + + assertThat(found).containsExactly("Widget") + } + + @Test + fun `top-level callables in a package exclude members of its classes`() = + runTest { + val index = index() + index.insert(topLevelFunction("launch")) + repeat(20) { index.insert(member("helper$it", owner = "Widget")) } + + val found = index.findTopLevelCallablesInPackage("com.example").map { it.shortName }.toList() + + assertThat(found).containsExactly("launch") + } + + @Test + fun `a simple-name lookup can be restricted to classifiers`() = + runTest { + val index = index() + index.insert(classifier("Widget")) + index.insert(member("Widget", owner = "Factory")) + + val all = index.findBySimpleName("Widget").toList() + val classifiersOnly = + index.findBySimpleName("Widget", kinds = JvmSymbolKind.CLASSIFIER_KINDS).toList() + + assertThat(all).hasSize(2) + assertThat(classifiersOnly.map { it.kind }).containsExactly(JvmSymbolKind.CLASS) + } + + @Test + fun `a file facade is not offered as a classifier`() = + runTest { + val index = index() + index.insert( + JvmSymbol( + key = "com/example/WidgetKt", + sourceId = "test.jar", + name = "com/example/WidgetKt", + shortName = "WidgetKt", + packageName = "com.example", + kind = JvmSymbolKind.FILE_FACADE, + language = JvmSourceLanguage.KOTLIN, + data = JvmClassInfo(internalName = "com/example/WidgetKt"), + ), + ) + + // Kotlin's import action asks for classifiers, and a facade is not one. + assertThat( + index.findBySimpleName("WidgetKt", kinds = JvmSymbolKind.CLASSIFIER_KINDS).toList(), + ).isEmpty() + // It is still findable as a class, which is what Java needs. + assertThat( + index.findBySimpleName("WidgetKt", kinds = JvmSymbolKind.JVM_CLASS_KINDS).toList(), + ).hasSize(1) + } +} diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt index 17bdd07f33..0092739633 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt @@ -27,6 +27,7 @@ import com.itsaky.androidide.utils.flashError import com.itsaky.androidide.utils.flashInfo import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind import java.nio.file.Path class AddImportAction : BaseKotlinCodeAction() { @@ -86,15 +87,20 @@ class AddImportAction : BaseKotlinCodeAction() { referenceName: String, ): ImportCandidates { /* - * Resolved before the file is pinned, not inside the pin: this is an unbounded SQLite scan that - * never reads the file, and a pin held across it freezes live-PSI refresh for the path - every + * Resolved before the file is pinned, not inside the pin: this is a SQLite read that never + * reads the file, and a pin held across it freezes live-PSI refresh for the path - every * concurrent acquirer joins the frozen instance and the refresh is only owed on release. * Materialized here too, so the index's lazy source-active filter cannot trail into the scope. + * + * Left unbounded deliberately: dropping a candidate here would hide an importable class from + * the chooser. It stays cheap because it is an exact-name lookup on an indexed column narrowed + * to classifier kinds, so the row count is the number of classes with that simple name, not a + * scan. The kind restriction also keeps Kotlin from being offered a file facade, which is not + * a classifier. */ val classifiers = env.ktSymbolIndex - .findSymbolBySimpleName(referenceName, limit = 0) - .filter { it.kind.isClassifier } + .findSymbolBySimpleName(referenceName, limit = 0, kinds = JvmSymbolKind.CLASSIFIER_KINDS) .toList() if (classifiers.isEmpty()) { diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt index 7c3625d422..c3e39373cc 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt @@ -24,6 +24,7 @@ import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.future.await import kotlinx.coroutines.launch import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolIndex +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind import org.appdevforall.codeonthego.indexing.jvm.KtFileMetadataIndex import org.checkerframework.checker.index.qual.NonNegative import org.jetbrains.kotlin.analysis.api.KaSession @@ -610,9 +611,17 @@ internal fun KtSymbolIndex.subpackageNames(packageFqn: String) = fileIndex.getSu * [org.appdevforall.codeonthego.indexing.api.ReadableIndex.query] ("If IndexQuery.limit is 0, all * matches are emitted"). A plain `take(limit)` would turn the common `limit = 0` call into * `take(0)`, silently yielding no results. + * + * The limit is passed to each index rather than applied only to the concatenation. Applying it + * afterwards made every call fetch both indexes in full, so a caller asking for ten paid for all of + * them. [kinds] narrows the fetch for the same reason: an exact-name lookup otherwise returns every + * method and field that happens to share the name. */ internal fun KtSymbolIndex.findSymbolBySimpleName( name: String, limit: Int, -) = (sourceIndex.findBySimpleName(name, 0) + libraryIndex.findBySimpleName(name, 0)) - .let { if (limit <= 0) it else it.take(limit) } + kinds: Set? = null, +) = ( + sourceIndex.findBySimpleName(name, limit, kinds) + + libraryIndex.findBySimpleName(name, limit, kinds) +).let { if (limit <= 0) it else it.take(limit) } From 80f867a206f8ff0af2f148cec860107216f57bea Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 14:18:34 +0000 Subject: [PATCH 13/48] style: spotless reformat, no functional change --- .../codeonthego/indexing/api/Core.kt | 90 +++++++++---------- .../indexing/jvm/JvmLibraryIndexingService.kt | 35 ++++---- .../indexing/jvm/KtFileMetadataIndex.kt | 33 +++---- 3 files changed, 80 insertions(+), 78 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt index b595e604ba..f91ba6a03c 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt @@ -14,16 +14,15 @@ package org.appdevforall.codeonthego.indexing.api * - For Python symbols: key = qualified name, source = .py file path */ interface Indexable { + /** Unique identifier within the index. */ + val key: String - /** Unique identifier within the index. */ - val key: String - - /** - * Identifies the origin of this entry. - * All entries sharing a [sourceId] can be removed atomically - * via [WritableIndex.removeBySource]. - */ - val sourceId: String + /** + * Identifies the origin of this entry. + * All entries sharing a [sourceId] can be removed atomically + * via [WritableIndex.removeBySource]. + */ + val sourceId: String } /** @@ -36,45 +35,44 @@ interface Indexable { * @param T The domain type being indexed. */ interface IndexDescriptor { + /** + * A unique name for this index type. Used as the table name + * in persistent storage and the namespace in composite indexes. + */ + val name: String - /** - * A unique name for this index type. Used as the table name - * in persistent storage and the namespace in composite indexes. - */ - val name: String - - /** - * The fields that should be queryable. - * Defines the "schema" for this index type. - * - * The persistent layer will create SQL columns and indexes - * for each declared field. - */ - val fields: List + /** + * The fields that should be queryable. + * Defines the "schema" for this index type. + * + * The persistent layer will create SQL columns and indexes + * for each declared field. + */ + val fields: List - /** - * Extract the queryable field values from an entry. - * - * The returned map's keys must be a subset of [fields]'s names. - * Null values mean the field is not applicable for this entry - * (e.g. receiverType is null for a non-extension function). - */ - fun fieldValues(entry: T): Map + /** + * Extract the queryable field values from an entry. + * + * The returned map's keys must be a subset of [fields]'s names. + * Null values mean the field is not applicable for this entry + * (e.g. receiverType is null for a non-extension function). + */ + fun fieldValues(entry: T): Map - /** - * Serialize an entry to bytes for persistent storage. - * - * Use whatever format is appropriate - protobuf, JSON, - * custom binary. Called once on insert; the bytes are - * stored opaquely. - */ - fun serialize(entry: T): ByteArray + /** + * Serialize an entry to bytes for persistent storage. + * + * Use whatever format is appropriate - protobuf, JSON, + * custom binary. Called once on insert; the bytes are + * stored opaquely. + */ + fun serialize(entry: T): ByteArray - /** - * Deserialize bytes back into an entry. - * Must be the inverse of [serialize]. - */ - fun deserialize(bytes: ByteArray): T + /** + * Deserialize bytes back into an entry. + * Must be the inverse of [serialize]. + */ + fun deserialize(bytes: ByteArray): T } /** @@ -86,6 +84,6 @@ interface IndexDescriptor { * the persistent layer creates SQL indexes. */ data class IndexField( - val name: String, - val prefixSearchable: Boolean = false, + val name: String, + val prefixSearchable: Boolean = false, ) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt index 8b53c393dd..e0ac4a22c9 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt @@ -40,7 +40,6 @@ val JVM_LIBRARY_SYMBOL_INDEX = IndexKey("jvm-library-symbols") class JvmLibraryIndexingService( private val context: Context, ) : IndexingService { - companion object { const val ID = "jvm-indexing-service" private val log = LoggerFactory.getLogger(JvmLibraryIndexingService::class.java) @@ -55,11 +54,12 @@ class JvmLibraryIndexingService( private val coroutineScope = CoroutineScope(Dispatchers.Default) override suspend fun initialize(registry: IndexRegistry) { - val jvmIndex = JvmSymbolIndex.createSqliteIndex( - context = context, - dbName = JvmSymbolIndex.DB_NAME_DEFAULT, - indexName = JvmSymbolIndex.INDEX_NAME_LIBRARY - ) + val jvmIndex = + JvmSymbolIndex.createSqliteIndex( + context = context, + dbName = JvmSymbolIndex.DB_NAME_DEFAULT, + indexName = JvmSymbolIndex.INDEX_NAME_LIBRARY, + ) this.libraryIndex = jvmIndex registry.register(JVM_LIBRARY_SYMBOL_INDEX, jvmIndex) @@ -81,15 +81,17 @@ class JvmLibraryIndexingService( } private suspend fun reindexLibraries() { - val index = this.libraryIndex ?: run { - log.warn("Not indexing libraries. Index not initialized.") - return - } + val index = + this.libraryIndex ?: run { + log.warn("Not indexing libraries. Index not initialized.") + return + } - val workspace = ProjectManagerImpl.getInstance().workspace ?: run { - log.warn("Not indexing libraries. Workspace model not available.") - return - } + val workspace = + ProjectManagerImpl.getInstance().workspace ?: run { + log.warn("Not indexing libraries. Workspace model not available.") + return + } val currentJars = workspace.subProjects @@ -104,8 +106,7 @@ class JvmLibraryIndexingService( addAll(project.getCompileClasspaths(excludeSourceGeneratedClassPath = true)) } - } - .filter { jar -> jar.exists() && isIndexableJar(jar.toPath()) } + }.filter { jar -> jar.exists() && isIndexableJar(jar.toPath()) } .map { jar -> jar.absolutePath } .toSet() @@ -148,4 +149,4 @@ class JvmLibraryIndexingService( val ext = path.extension.lowercase() return ext == "jar" || ext == "aar" } -} \ No newline at end of file +} diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt index e1ecee7391..080b6b5b70 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt @@ -14,9 +14,7 @@ import java.io.Closeable class KtFileMetadataIndex( private val backing: Index, ) : Closeable { - companion object { - /** * Creates a [KtFileMetadataIndex] backed by a SQLite database. * @@ -25,7 +23,7 @@ class KtFileMetadataIndex( */ fun sqliteBacked( context: Context, - dbName: String? = null + dbName: String? = null, ): KtFileMetadataIndex = KtFileMetadataIndex( SQLiteIndex( @@ -33,7 +31,7 @@ class KtFileMetadataIndex( context = context, dbName = dbName, name = "kt-file-metadata", - ) + ), ) } @@ -82,15 +80,14 @@ class KtFileMetadataIndex( indexQuery { eq(KEY_PACKAGE, packageFqName) limit = 0 - } + }, ) /** * Returns a [Sequence] of absolute file paths whose declared package exactly * matches [packageFqName]. */ - fun getFilePathsForPackage(packageFqName: String): Sequence = - getFilesForPackage(packageFqName).map { it.filePath } + fun getFilePathsForPackage(packageFqName: String): Sequence = getFilesForPackage(packageFqName).map { it.filePath } /** * Returns `true` if at least one file with package [packageFqName] is @@ -99,10 +96,13 @@ class KtFileMetadataIndex( * Pass an empty string for the root (default) package. */ fun packageExists(packageFqName: String): Boolean = - backing.query(indexQuery { - eq(KEY_PACKAGE, packageFqName) - limit = 1 - }).firstOrNull() != null + backing + .query( + indexQuery { + eq(KEY_PACKAGE, packageFqName) + limit = 1 + }, + ).firstOrNull() != null /** * Returns the simple names of the direct child packages of [packageFqName]. @@ -140,10 +140,13 @@ class KtFileMetadataIndex( * symbols have not yet been extracted ([KtFileMetadata.isIndexed] is `false`). */ fun getUnindexedFiles(): Sequence = - backing.query(indexQuery { - eq(KEY_IS_INDEXED, false.toString()) - limit = 0 - }).map { it.filePath } + backing + .query( + indexQuery { + eq(KEY_IS_INDEXED, false.toString()) + limit = 0 + }, + ).map { it.filePath } /** Remove all records from the index. */ suspend fun clear() = backing.clear() From 187a7c655e2a6d82aaca9c8d271c02baee7fe957 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 14:25:32 +0000 Subject: [PATCH 14/48] ADFA-6228: Identify index rows by source and key, and resolve shared keys to the smallest source id --- .../codeonthego/indexing/FilteredIndex.kt | 18 +- .../codeonthego/indexing/InMemoryIndex.kt | 198 ++++++++++-------- .../codeonthego/indexing/SQLiteIndex.kt | 36 ++-- .../codeonthego/indexing/api/Core.kt | 9 +- .../codeonthego/indexing/api/Index.kt | 7 +- .../codeonthego/indexing/api/Query.kt | 16 +- .../indexing/FilteredIndexLimitTest.kt | 10 + .../codeonthego/indexing/FilteredIndexTest.kt | 20 ++ .../codeonthego/indexing/InMemoryIndexTest.kt | 79 +++++++ .../indexing/SQLiteIndexIdentityTest.kt | 153 ++++++++++++++ .../indexing/jvm/KtFileMetadataIndex.kt | 6 +- 11 files changed, 434 insertions(+), 118 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt index d05c086f24..18434e9c39 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -1,5 +1,7 @@ package org.appdevforall.codeonthego.indexing +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable import org.appdevforall.codeonthego.indexing.api.ReadableIndex @@ -105,10 +107,18 @@ open class FilteredIndex( return query.copy(sourceIds = scoped) } - override suspend fun get(key: String): T? { - val entry = backing.get(key) ?: return null - return if (isActive(entry.sourceId)) entry else null - } + /** + * Returns the visible entry for [key], the one with the smallest source id among the visible + * sources that have it. + * + * This is a scoped key query rather than a check on the backing index's own [get]: that returns + * the smallest source id overall, and when that source is inactive a visible entry for the same + * key would be missed. + */ + override suspend fun get(key: String): T? = + withContext(Dispatchers.IO) { + backing.query(scopedToActive(IndexQuery.byKey(key))).firstOrNull() + } override suspend fun containsSource(sourceId: String): Boolean = isActive(sourceId) && backing.containsSource(sourceId) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index 697567d3bb..7070661b24 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -16,11 +16,15 @@ import kotlin.concurrent.write * Optimized for small-to-medium datasets (source files, typically * hundreds to low thousands of entries) that change frequently. * + * An entry is identified by its source and key together, matching [SQLiteIndex]: the same key in + * two sources is two entries, and a key lookup matching several returns the smallest source id. + * * Data layout: - * - [primaryMap]: key → entry (O(1) point lookup) - * - [sourceMap]: sourceId → set of keys (O(1) bulk removal) - * - [fieldMaps]: fieldName → (fieldValue → set of keys) (equality filter) - * - [prefixBuckets]: fieldName → (lowercased first char → list of (value, key)) + * - [rows]: (sourceId, key) -> entry + * - [sourceMap]: sourceId -> rows (O(1) bulk removal) + * - [keyMap]: key -> rows (key lookup) + * - [fieldMaps]: fieldName -> (fieldValue -> rows) (equality filter) + * - [prefixBuckets]: fieldName -> (lowercased first char -> list of (value, row)) * Provides a ~36-way partition for prefix search. * * All mutations go through [lock] in write mode for consistency @@ -33,16 +37,22 @@ class InMemoryIndex( override val descriptor: IndexDescriptor, override val name: String = "memory:${descriptor.name}", ) : Index { - private val primaryMap = ConcurrentHashMap(256) - private val sourceMap = ConcurrentHashMap>(32) - private val fieldMaps = ConcurrentHashMap>>() + private val rows = ConcurrentHashMap(256) + private val sourceMap = ConcurrentHashMap>(32) + private val keyMap = ConcurrentHashMap>(256) + private val fieldMaps = ConcurrentHashMap>>() private val prefixBuckets = ConcurrentHashMap>>() private val lock = ReentrantReadWriteLock() + private data class RowId( + val sourceId: String, + val key: String, + ) + private data class PrefixEntry( val lowerValue: String, - val key: String, + val row: RowId, ) init { @@ -55,14 +65,19 @@ class InMemoryIndex( } override fun query(query: IndexQuery): Sequence { - val keys = resolveMatchingKeys(query) + val matching = resolveMatchingRows(query) + val ordered = if (query.key != null) matching.sortedBy { it.sourceId } else matching val limit = if (query.limit <= 0) Int.MAX_VALUE else query.limit - return keys - .mapNotNull { primaryMap[it] } + return ordered + .asSequence() + .mapNotNull { rows[it] } .take(limit) } - override suspend fun get(key: String): T? = primaryMap[key] + override suspend fun get(key: String): T? = + lock.read { + keyMap[key]?.minByOrNull { it.sourceId }?.let { rows[it] } + } override suspend fun containsSource(sourceId: String): Boolean = sourceMap.containsKey(sourceId) @@ -85,11 +100,11 @@ class InMemoryIndex( return lock .read { val values = LinkedHashSet() - for (key in resolveMatchingKeys(query)) { + for (row in resolveMatchingRows(query)) { if (values.size >= limit) { break } - val entry = primaryMap[key] ?: continue + val entry = rows[row] ?: continue descriptor.fieldValues(entry)[fieldName]?.let { values.add(it) } } values.toList() @@ -115,7 +130,7 @@ class InMemoryIndex( * Remove every entry belonging to any of [sourceIds]. * * Acquires the write lock once and removes each source under it, so the whole - * batch is atomic with respect to concurrent readers and writers — there is no + * batch is atomic with respect to concurrent readers and writers - there is no * intermediate state in which only some of the sources have been removed. */ override suspend fun removeBySources(sourceIds: Collection) = @@ -130,45 +145,46 @@ class InMemoryIndex( * indexes. Caller MUST already hold the write lock; this method does not lock. */ private fun removeBySourceLocked(sourceId: String) { - val keys = sourceMap.remove(sourceId) ?: return - for (key in keys) { - val entry = primaryMap.remove(key) ?: continue - removeFromSecondaryIndexes(entry) + val sourceRows = sourceMap.remove(sourceId) ?: return + for (row in sourceRows) { + val entry = rows.remove(row) ?: continue + keyMap[row.key]?.let { keyRows -> + keyRows.remove(row) + if (keyRows.isEmpty()) keyMap.remove(row.key) + } + removeFromSecondaryIndexes(row, entry) } } override suspend fun clear() = lock.write { - primaryMap.clear() + rows.clear() sourceMap.clear() + keyMap.clear() fieldMaps.values.forEach { it.clear() } prefixBuckets.values.forEach { it.clear() } } - val size: Int get() = primaryMap.size + val size: Int get() = rows.size val sourceCount: Int get() = sourceMap.size /** - * Resolves the set of keys matching the query by intersecting - * the results of each predicate. + * Resolves the rows matching the query by intersecting the results of each predicate. * - * Starts with the most selective predicate to minimize the - * intersection set. + * Every predicate is ANDed, the key included, as SQLite does. A predicate naming a value no row + * has matches nothing rather than being ignored. The result is materialized under the read lock + * so callers never iterate a set a concurrent writer is mutating. */ - private fun resolveMatchingKeys(query: IndexQuery): Sequence = + private fun resolveMatchingRows(query: IndexQuery): List = lock.read { - var candidates: Set? = null + var candidates: Set? = null if (query.key != null) { - return@read if (primaryMap.containsKey(query.key)) { - sequenceOf(query.key) - } else { - emptySequence() - } + candidates = intersect(candidates, keyMap[query.key].orEmpty()) } if (query.sourceId != null) { - candidates = intersect(candidates, sourceMap[query.sourceId]) + candidates = intersect(candidates, sourceMap[query.sourceId].orEmpty()) } val sourceIds = query.sourceIds @@ -179,84 +195,87 @@ class InMemoryIndex( } for ((field, value) in query.exactMatch) { - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - candidates = intersect(candidates, fieldMap[value]) + val fieldMap = fieldMaps[field] ?: return@read emptyList() + candidates = intersect(candidates, fieldMap[value].orEmpty()) } for ((field, values) in query.anyOf) { - val fieldMap = fieldMaps[field] ?: return@read emptySequence() + val fieldMap = fieldMaps[field] ?: return@read emptyList() val matching = values.flatMapTo(mutableSetOf()) { fieldMap[it].orEmpty() } candidates = intersect(candidates, matching) } for ((field, prefix) in query.prefixMatch) { - val buckets = prefixBuckets[field] - val matching: Set = - if (buckets != null) { - // Prefix-searchable: case-insensitive match via the lowercased buckets, - // mirroring SQLite's `lowerCol LIKE 'prefix%'`. - val lowerPrefix = prefix.lowercase() - val firstChar = lowerPrefix.firstOrNull() - if (firstChar == null) { - // Empty prefix == "field present", matching SQLite's `LIKE '%'` - // (which excludes rows where the column IS NULL). - buckets.values.flatMapTo(mutableSetOf()) { entries -> entries.map { it.key } } - } else { - val bucket = buckets[firstChar] ?: return@read emptySequence() - bucket - .asSequence() - .filter { it.lowerValue.startsWith(lowerPrefix) } - .map { it.key } - .toSet() - } - } else { - // Not prefix-searchable: fall back to a case-sensitive prefix scan of the - // regular field map, mirroring SQLite's `col LIKE 'prefix%'` fallback. - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - fieldMap.entries - .asSequence() - .filter { (value, _) -> value.startsWith(prefix) } - .flatMap { (_, keys) -> keys.asSequence() } - .toSet() - } - + val matching = rowsWithPrefix(field, prefix) ?: return@read emptyList() candidates = intersect(candidates, matching) } for ((field, mustExist) in query.presence) { - val fieldMap = fieldMaps[field] ?: return@read emptySequence() - val allKeysWithField = fieldMap.values.flatMapTo(mutableSetOf()) { it } + val fieldMap = fieldMaps[field] ?: return@read emptyList() + val rowsWithField = fieldMap.values.flatMapTo(mutableSetOf()) { it } candidates = if (mustExist) { - intersect(candidates, allKeysWithField) + intersect(candidates, rowsWithField) } else { - // Keys that DON'T have this field - val allKeys = primaryMap.keys.toMutableSet() - allKeys.removeAll(allKeysWithField) - intersect(candidates, allKeys) + intersect(candidates, rows.keys - rowsWithField) } } - candidates?.asSequence() ?: primaryMap.keys.asSequence() + (candidates ?: rows.keys).toList() + } + + /** + * Rows whose [field] starts with [prefix], or `null` when [field] is not declared. + * + * Prefix-searchable fields match case-insensitively via the lowercased buckets, mirroring + * SQLite's lowercased column; any other field falls back to a case-sensitive scan of its values. + * An empty prefix means "field present", which excludes rows where the field is null. + */ + private fun rowsWithPrefix( + field: String, + prefix: String, + ): Set? { + val buckets = prefixBuckets[field] + if (buckets == null) { + val fieldMap = fieldMaps[field] ?: return null + return fieldMap.entries + .asSequence() + .filter { (value, _) -> value.startsWith(prefix) } + .flatMap { (_, valueRows) -> valueRows.asSequence() } + .toSet() + } + + val lowerPrefix = prefix.lowercase() + val firstChar = lowerPrefix.firstOrNull() + if (firstChar == null) { + return buckets.values.flatMapTo(mutableSetOf()) { entries -> entries.map { it.row } } } + return buckets[firstChar] + .orEmpty() + .asSequence() + .filter { it.lowerValue.startsWith(lowerPrefix) } + .map { it.row } + .toSet() + } private fun intersect( - current: Set?, - other: Set?, - ): Set? { - if (other == null) return current + current: Set?, + other: Set, + ): Set { if (current == null) return other return current.intersect(other) } private fun insertSingleLocked(entry: T) { - val existing = primaryMap[entry.key] + val row = RowId(entry.sourceId, entry.key) + val existing = rows[row] if (existing != null) { - removeFromSecondaryIndexes(existing) + removeFromSecondaryIndexes(row, existing) } - primaryMap[entry.key] = entry - sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(entry.key) + rows[row] = entry + sourceMap.getOrPut(entry.sourceId) { mutableSetOf() }.add(row) + keyMap.getOrPut(entry.key) { mutableSetOf() }.add(row) val fields = descriptor.fieldValues(entry) for ((fieldName, value) in fields) { @@ -264,7 +283,7 @@ class InMemoryIndex( fieldMaps[fieldName] ?.getOrPut(value) { mutableSetOf() } - ?.add(entry.key) + ?.add(row) val buckets = prefixBuckets[fieldName] if (buckets != null) { @@ -272,25 +291,28 @@ class InMemoryIndex( val firstChar = lower.firstOrNull() ?: continue buckets .getOrPut(firstChar) { mutableListOf() } - .add(PrefixEntry(lower, entry.key)) + .add(PrefixEntry(lower, row)) } } } - private fun removeFromSecondaryIndexes(entry: T) { + /** Drops [row] from the field and prefix maps; [sourceMap] and [keyMap] are the caller's job. */ + private fun removeFromSecondaryIndexes( + row: RowId, + entry: T, + ) { val fields = descriptor.fieldValues(entry) for ((fieldName, value) in fields) { if (value == null) continue - fieldMaps[fieldName]?.get(value)?.remove(entry.key) + fieldMaps[fieldName]?.get(value)?.remove(row) val buckets = prefixBuckets[fieldName] if (buckets != null) { val lower = value.lowercase() val firstChar = lower.firstOrNull() ?: continue - buckets[firstChar]?.removeAll { it.key == entry.key } + buckets[firstChar]?.removeAll { it.row == row } } } - // Note: sourceMap is handled by the caller } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index fb1c72e640..7252b695be 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -25,20 +25,25 @@ import kotlin.collections.iterator * Creates a table dynamically based on the [IndexDescriptor]: * ``` * CREATE TABLE IF NOT EXISTS {name} ( - * _key TEXT PRIMARY KEY, + * _key TEXT NOT NULL, * _source_id TEXT NOT NULL, * f_{field1} TEXT, * f_{field1}_lower TEXT, -- if prefix-searchable * f_{field2} TEXT, * ... - * _payload BLOB NOT NULL + * _payload BLOB NOT NULL, + * PRIMARY KEY (_source_id, _key) * ); * ``` * + * A row is identified by its source and key together, so the same key (a class present in two + * JARs, say) has one row per source. The primary key leads with `_source_id`, which is what serves + * source-scoped reads and bulk removal. + * * SQL indexes are created on: - * - `_source_id` (for bulk removal) + * - `(_key, _source_id)` (for key lookups, already ordered by source) * - Each `f_{field}` (for equality filter) - * - Each `f_{field}_lower` (for prefix search via `LIKE 'prefix%'`) + * - Each `f_{field}_lower` (for prefix search) * * Uses WAL journal mode for concurrent read/write performance. * Inserts are batched inside transactions for throughput. @@ -82,13 +87,13 @@ class SQLiteIndex( private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") - // Field column names: "f_{fieldName}" + /** Field column names: `f_{fieldName}`. */ private val fieldColumns = descriptor.fields.associate { field -> field.name to "f_${field.name}" } - // Prefix-searchable fields also get a "_lower" column + /** Prefix-searchable fields also get a lowercased `f_{fieldName}_lower` column. */ private val prefixColumns = descriptor.fields .filter { it.prefixSearchable } @@ -162,7 +167,7 @@ class SQLiteIndex( ifOpen(null) { val cursor = db.query( - "SELECT _payload FROM $tableName WHERE _key = ? LIMIT 1", + "SELECT _payload FROM $tableName WHERE _key = ? ORDER BY _source_id LIMIT 1", arrayOf(key), ) cursor.use { @@ -324,7 +329,7 @@ class SQLiteIndex( private fun createTable(db: SupportSQLiteDatabase) { val columns = buildString { - append("_key TEXT PRIMARY KEY, ") + append("_key TEXT NOT NULL, ") append("_source_id TEXT NOT NULL, ") for (field in descriptor.fields) { @@ -337,14 +342,14 @@ class SQLiteIndex( } } - append("_payload BLOB NOT NULL") + append("_payload BLOB NOT NULL, ") + append("PRIMARY KEY (_source_id, _key)") } db.execSQL("CREATE TABLE IF NOT EXISTS $tableName ($columns)") - // Indexes db.execSQL( - "CREATE INDEX IF NOT EXISTS idx_${tableName}_source ON $tableName(_source_id)", + "CREATE INDEX IF NOT EXISTS idx_${tableName}_key ON $tableName(_key, _source_id)", ) for (field in descriptor.fields) { @@ -431,13 +436,17 @@ class SQLiteIndex( * Returns a single `null` chunk when the query is unscoped, and no chunks at all when it is * scoped to an empty set -- the caller then runs no statement and yields nothing, which is the * difference between "any source" and "none of them". + * + * Chunks come in ascending source id order. Together with the per-statement `ORDER BY` on key + * queries, that is what makes a limited key lookup return the smallest source id overall rather + * than the smallest within whichever chunk happened to run first. */ private fun sourceIdChunks(query: IndexQuery): List?> { val sourceIds = query.sourceIds ?: return listOf(null) if (sourceIds.isEmpty()) { return emptyList() } - return sourceIds.distinct().chunked(SOURCE_ID_CHUNK_SIZE) + return sourceIds.toSortedSet().chunked(SOURCE_ID_CHUNK_SIZE) } private fun buildSelectQuery( @@ -453,6 +462,9 @@ class SQLiteIndex( append(" WHERE ") append(where) } + if (query.key != null) { + append(" ORDER BY _source_id") + } if (limit != Int.MAX_VALUE) { append(" LIMIT $limit") } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt index f91ba6a03c..06b04f79a0 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt @@ -3,9 +3,10 @@ package org.appdevforall.codeonthego.indexing.api /** * Any object that can be stored in an index. * - * The only requirements are a unique key (for deduplication and - * point lookups) and a source identifier (for bulk operations - * when the source changes). + * The only requirements are a key (for deduplication and point + * lookups) and a source identifier (for bulk operations when the + * source changes). An entry is identified by the two together: the + * same key may appear once in each of several sources. * * What constitutes a "key" and "source" depends entirely on * the consumer: @@ -14,7 +15,7 @@ package org.appdevforall.codeonthego.indexing.api * - For Python symbols: key = qualified name, source = .py file path */ interface Indexable { - /** Unique identifier within the index. */ + /** Identifies the entry within its source. */ val key: String /** diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index 03d5ba12ce..472d40653c 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -23,6 +23,9 @@ interface ReadableIndex { /** * Point lookup by key. Returns null if not found. + * + * When several sources hold an entry for [key], returns the one with the smallest source id, so + * the answer does not depend on insertion order. */ suspend fun get(key: String): T? @@ -69,8 +72,8 @@ interface WritableIndex { * Insert entries from a [Sequence]. * * Entries are consumed lazily from the sequence and batched - * internally for throughput. If an entry with the same key - * already exists, it is replaced. + * internally for throughput. If an entry with the same source + * and key already exists, it is replaced. */ suspend fun insertAll(entries: Sequence) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt index 617c31db5c..59a68b0735 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -8,9 +8,9 @@ package org.appdevforall.codeonthego.indexing.api * for Kotlin symbols, Android resources, Python declarations, etc. */ data class IndexQuery( - /** Exact match predicates: field name → expected value. */ + /** Exact match predicates: field name -> expected value. */ val exactMatch: Map = emptyMap(), - /** Prefix match predicates: field name → prefix (case-insensitive). */ + /** Prefix match predicates: field name -> prefix (case-insensitive). */ val prefixMatch: Map = emptyMap(), /** * Set-membership predicates: field name to the values that field may take. @@ -21,7 +21,7 @@ data class IndexQuery( */ val anyOf: Map> = emptyMap(), /** - * Presence predicates: field name → whether the field must be + * Presence predicates: field name -> whether the field must be * non-null (true) or null (false). */ val presence: Map = emptyMap(), @@ -36,7 +36,13 @@ data class IndexQuery( * rows that were only going to be discarded. */ val sourceIds: Collection? = null, - /** Filter by key (exact). */ + /** + * Filter by key (exact). + * + * Several sources can hold the same key. A key query returns its matches in ascending source id + * order, so with a limit of 1 it yields the one with the smallest source id among the sources in + * scope. + */ val key: String? = null, /** Maximum number of results. 0 = unlimited (use with care). */ val limit: Int = 200, @@ -45,7 +51,7 @@ data class IndexQuery( /** Match everything up to [limit]. */ val ALL = IndexQuery() - /** Exact key lookup. */ + /** Exact key lookup: the entry with the smallest source id among those holding [key]. */ fun byKey(key: String) = IndexQuery(key = key, limit = 1) /** All entries from a specific source. */ diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt index 5b9c76d1b5..f2d2e57fff 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt @@ -142,4 +142,14 @@ class FilteredIndexLimitTest { assertThat(activeOnly("jarA").distinctValues("value").toList()).containsExactly("Alpha") } + + @Test + fun `a point lookup finds the active source's row when an inactive source shares the key`() = + runTest { + backing.insert(Entry("shared", "jarA", "Inactive")) + backing.insert(Entry("shared", "jarB", "Active")) + + assertThat(activeOnly("jarB").get("shared")!!.value).isEqualTo("Active") + assertThat(activeOnly("jarB").query(IndexQuery.byKey("shared")).single().value).isEqualTo("Active") + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt index 09d9036bac..f899530617 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexTest.kt @@ -176,4 +176,24 @@ class FilteredIndexTest { filtered.close() assertThat(filtered.activeSources()).isEmpty() } + + @Test + fun `byKey returns nothing for a key only in an inactive source`() = + runTest { + val (_, filtered) = setupBackingAndFiltered() + filtered.activateSource("src1") + + assertThat(filtered.query(IndexQuery.byKey("k3")).toList()).isEmpty() + } + + @Test + fun `get returns the active source's entry when an inactive source shares the key`() = + runTest { + val backing = InMemoryIndex(descriptor) + backing.insert(Entry("shared", "src1", "inactive")) + backing.insert(Entry("shared", "src2", "active")) + val filtered = FilteredIndex(backing).apply { activateSource("src2") } + + assertThat(filtered.get("shared")!!.value).isEqualTo("active") + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt index 658afbeff9..768af61e98 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt @@ -333,4 +333,83 @@ class InMemoryIndexTest { assertThat(index.distinctValues("category").toList()).containsExactly("beta") } + + @Test + fun `the same key in two sources keeps an entry for each`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "FromOne")) + index.insert(entry("k1", "src2", "FromTwo")) + + val fromOne = index.query(IndexQuery(key = "k1", sourceIds = listOf("src1"), limit = 0)).single() + val fromTwo = index.query(IndexQuery(key = "k1", sourceIds = listOf("src2"), limit = 0)).single() + assertThat(fromOne.name).isEqualTo("FromOne") + assertThat(fromTwo.name).isEqualTo("FromTwo") + assertThat(index.size).isEqualTo(2) + } + + @Test + fun `get returns the entry with the smallest source id when several sources share the key`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src2", "FromTwo")) + index.insert(entry("k1", "src1", "FromOne")) + index.insert(entry("k1", "src3", "FromThree")) + + assertThat(index.get("k1")!!.sourceId).isEqualTo("src1") + assertThat(index.query(IndexQuery.byKey("k1")).single().sourceId).isEqualTo("src1") + } + + @Test + fun `removing one source keeps the same key in another source`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "FromOne", category = "alpha")) + index.insert(entry("k1", "src2", "FromTwo", category = "alpha")) + index.removeBySource("src1") + + assertThat(index.get("k1")!!.sourceId).isEqualTo("src2") + assertThat(index.query(IndexQuery(exactMatch = mapOf("category" to "alpha"), limit = 0)).single().sourceId) + .isEqualTo("src2") + } + + @Test + fun `a key query still applies its source scope`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + assertThat(index.query(IndexQuery(key = "k1", sourceIds = listOf("src2"), limit = 0)).toList()).isEmpty() + assertThat(index.query(IndexQuery(key = "k1", sourceId = "src2", limit = 0)).toList()).isEmpty() + } + + @Test + fun `a key query still applies its field predicates`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + + assertThat(index.query(IndexQuery(key = "k1", exactMatch = mapOf("category" to "beta"), limit = 0)).toList()) + .isEmpty() + assertThat(index.query(IndexQuery(key = "k1", exactMatch = mapOf("category" to "alpha"), limit = 0)).toList()) + .hasSize(1) + } + + @Test + fun `an exact match on a value no entry has matches nothing`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo", category = "alpha")) + + assertThat(index.query(IndexQuery(exactMatch = mapOf("category" to "missing"), limit = 0)).toList()).isEmpty() + } + + @Test + fun `a query for a source with no entries matches nothing`() = + runTest { + val index = makeIndex() + index.insert(entry("k1", "src1", "Foo")) + + assertThat(index.query(IndexQuery.bySource("src999")).toList()).isEmpty() + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt new file mode 100644 index 0000000000..923ee5085f --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt @@ -0,0 +1,153 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** Row identity on the SQLite-backed index: a row is identified by its source and key together. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexIdentityTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_identity" + override val fields = listOf(IndexField("value", prefixSearchable = true)) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + private val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME) + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + @Test + fun `the same key in two sources keeps a row for each`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + index.insert(Entry("com.example.Foo", "jarB", "fromB")) + + assertThat(valueOfKeyIn("com.example.Foo", "jarA")).isEqualTo("fromA") + assertThat(valueOfKeyIn("com.example.Foo", "jarB")).isEqualTo("fromB") + assertThat(index.size()).isEqualTo(2) + } + + @Test + fun `re-inserting the same source and key replaces the row`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "original")) + index.insert(Entry("com.example.Foo", "jarA", "updated")) + + assertThat(valueOfKeyIn("com.example.Foo", "jarA")).isEqualTo("updated") + assertThat(index.size()).isEqualTo(1) + } + + @Test + fun `get returns the row with the smallest source id when several sources share the key`() = + runTest { + index.insert(Entry("com.example.Foo", "jarB", "fromB")) + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + index.insert(Entry("com.example.Foo", "jarC", "fromC")) + + assertThat(index.get("com.example.Foo")!!.sourceId).isEqualTo("jarA") + assertThat(index.query(IndexQuery.byKey("com.example.Foo")).single().sourceId).isEqualTo("jarA") + } + + @Test + fun `a scoped key lookup returns the smallest source id across source chunks`() = + runTest { + // More sources than one IN clause holds, requested in descending order, so the smallest id + // sits in the last chunk the caller's order would produce. + val sourceIds = (0 until 2000).map { "jar%04d".format(it) } + index.insertAll(sourceIds.asSequence().map { Entry("com.example.Foo", it, it) }) + + val found = index.query(IndexQuery(key = "com.example.Foo", sourceIds = sourceIds.reversed(), limit = 1)) + + assertThat(found.single().sourceId).isEqualTo("jar0000") + } + + @Test + fun `a key lookup scoped to one source ignores the same key elsewhere`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + index.insert(Entry("com.example.Foo", "jarB", "fromB")) + + val found = index.query(IndexQuery(key = "com.example.Foo", sourceIds = listOf("jarB"), limit = 1)) + + assertThat(found.single().value).isEqualTo("fromB") + } + + @Test + fun `source id lookups are served by the primary key`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + + val plan = explain("SELECT _payload FROM test_identity WHERE _source_id = ?", "jarA") + + assertThat(plan).contains("sqlite_autoindex_test_identity_1") + } + + @Test + fun `key lookups are served by an index`() = + runTest { + index.insert(Entry("com.example.Foo", "jarA", "fromA")) + + val plan = explain("SELECT _payload FROM test_identity WHERE _key = ?", "com.example.Foo") + + assertThat(plan).contains("USING INDEX") + assertThat(plan).doesNotContain("SCAN") + } + + private fun valueOfKeyIn( + key: String, + sourceId: String, + ) = index + .query(IndexQuery(key = key, sourceIds = listOf(sourceId), limit = 0)) + .single() + .value + + /** Returns the query plan for [sql], read over a second connection because the index exposes none. */ + private fun explain( + sql: String, + vararg args: String, + ): String { + val path = context.getDatabasePath(DB_NAME).path + return SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> + db.rawQuery("EXPLAIN QUERY PLAN $sql", args).use { cursor -> + val detail = cursor.getColumnIndexOrThrow("detail") + buildList { while (cursor.moveToNext()) add(cursor.getString(detail)) }.joinToString("\n") + } + } + } + + private companion object { + const val DB_NAME = "identity_test.db" + } +} diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt index 080b6b5b70..713e66792e 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt @@ -38,8 +38,8 @@ class KtFileMetadataIndex( /** * Insert or replace the metadata record for a single file. * - * Because [KtFileMetadata.key] == [KtFileMetadata.filePath], the - * underlying `CONFLICT_REPLACE` strategy ensures this is a true upsert. + * Because [KtFileMetadata.key] and [KtFileMetadata.sourceId] are both the + * file path, a file has at most one row and an insert replaces it. */ suspend fun upsert(metadata: KtFileMetadata) = backing.insert(metadata) @@ -55,7 +55,7 @@ class KtFileMetadataIndex( * batched, transactional operation. * * Equivalent to calling [remove] once per path, but issues the deletes as one - * transaction instead of N — see [Index.removeBySources]. Paths not present in + * transaction instead of N - see [Index.removeBySources]. Paths not present in * the index are ignored. */ suspend fun removeAll(filePaths: Collection) = backing.removeBySources(filePaths) From 30fb0c92b364b6df286886e3dacd74b79501f8f2 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 14:29:06 +0000 Subject: [PATCH 15/48] ADFA-6228: Version the index format and rebuild on any mismatch --- .../codeonthego/indexing/SQLiteIndex.kt | 40 +++++++- .../indexing/FilteredIndexLimitTest.kt | 1 + .../indexing/SQLiteIndexIdentityTest.kt | 2 +- .../indexing/SQLiteIndexPrefixTest.kt | 1 + .../indexing/SQLiteIndexScopeTest.kt | 1 + .../indexing/SQLiteIndexVersionTest.kt | 99 +++++++++++++++++++ .../indexing/jvm/JvmSymbolIndex.kt | 12 +++ .../indexing/jvm/KtFileMetadataIndex.kt | 1 + 8 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexVersionTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 7252b695be..5cde79460e 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -59,12 +59,17 @@ import kotlin.collections.iterator * @param dbName Database file name. Pass `null` to create an in-memory database * that is discarded when closed. Different index types can share * a database (each gets its own table) or use separate files. + * @param formatVersion Version of the stored format: the schema and whatever produced the rows. + * It belongs to the database file, so opening a file stored at any other version + * discards every table in it; each index sharing the file recreates its own table + * when it opens. Indexes sharing a file must therefore pass the same version. * @param batchSize Number of rows per INSERT transaction. */ class SQLiteIndex( override val descriptor: IndexDescriptor, context: Context, dbName: String?, + formatVersion: Int, override val name: String = "sqlite:${descriptor.name}", private val batchSize: Int = 500, ) : Index { @@ -83,6 +88,10 @@ class SQLiteIndex( * chunks are disjoint on `_source_id`, so no row can be returned by two of them. */ private const val SOURCE_ID_CHUNK_SIZE = 900 + + /** Every table in a database except SQLite's and Android's own bookkeeping tables. */ + private const val USER_TABLES_QUERY = + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata'" } private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") @@ -110,7 +119,7 @@ class SQLiteIndex( .builder(context) .name(dbName) .callback( - object : SupportSQLiteOpenHelper.Callback(1) { + object : SupportSQLiteOpenHelper.Callback(formatVersion) { override fun onCreate(db: SupportSQLiteDatabase) { createTable(db) } @@ -120,12 +129,15 @@ class SQLiteIndex( oldVersion: Int, newVersion: Int, ) { - // TODO: Add migration support - db.execSQL("DROP TABLE IF EXISTS $tableName") - createTable(db) + rebuild(db) } - override fun onOpen(db: SupportSQLiteDatabase) { + override fun onDowngrade( + db: SupportSQLiteDatabase, + oldVersion: Int, + newVersion: Int, + ) { + rebuild(db) } }, ).build() @@ -326,6 +338,24 @@ class SQLiteIndex( } } + /** + * Discards every table in [db] and recreates this index's own. + * + * The index is a cache of what the scanners produce, so a format change is handled by + * rebuilding rather than migrating: rows written by an older scanner are wrong, not just + * differently shaped. + */ + private fun rebuild(db: SupportSQLiteDatabase) { + val tables = + db.query(USER_TABLES_QUERY).use { + buildList { while (it.moveToNext()) add(it.getString(0)) } + } + for (table in tables) { + db.execSQL("DROP TABLE IF EXISTS \"$table\"") + } + createTable(db) + } + private fun createTable(db: SupportSQLiteDatabase) { val columns = buildString { diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt index f2d2e57fff..e51a6d7262 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndexLimitTest.kt @@ -48,6 +48,7 @@ class FilteredIndexLimitTest { descriptor = descriptor, context = ApplicationProvider.getApplicationContext(), dbName = null, + formatVersion = 1, ) @After diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt index 923ee5085f..b535699ad2 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt @@ -40,7 +40,7 @@ class SQLiteIndexIdentityTest { private val context = ApplicationProvider.getApplicationContext() - private val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME) + private val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) @After fun tearDown() { diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt index 0c5fa1d100..b330772f4a 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt @@ -52,6 +52,7 @@ class SQLiteIndexPrefixTest { descriptor = descriptor, context = ApplicationProvider.getApplicationContext(), dbName = null, + formatVersion = 1, ) @After diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt index 72711efffb..219e15fe25 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt @@ -51,6 +51,7 @@ class SQLiteIndexScopeTest { descriptor = descriptor, context = ApplicationProvider.getApplicationContext(), dbName = null, + formatVersion = 1, ) @After diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexVersionTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexVersionTest.kt new file mode 100644 index 0000000000..1961ccfcc2 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexVersionTest.kt @@ -0,0 +1,99 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** The stored format version decides whether an existing database file's rows are kept. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexVersionTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_version" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + private fun open(version: Int) = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = version) + + @Test + fun `reopening at the same version keeps the rows`() = + runTest { + open(version = 1).use { it.insert(Entry("k1", "jarA", "v")) } + + open(version = 1).use { assertThat(it.containsSource("jarA")).isTrue() } + } + + @Test + fun `reopening at a newer version drops the old rows`() = + runTest { + open(version = 1).use { it.insert(Entry("k1", "jarA", "v")) } + + open(version = 2).use { + assertThat(it.containsSource("jarA")).isFalse() + assertThat(it.size()).isEqualTo(0) + } + } + + @Test + fun `reopening at an older version drops the newer rows`() = + runTest { + open(version = 2).use { it.insert(Entry("k1", "jarA", "v")) } + + open(version = 1).use { assertThat(it.size()).isEqualTo(0) } + } + + @Test + fun `a version change replaces a table created with the single-column key`() = + runTest { + val path = context.getDatabasePath(DB_NAME) + path.parentFile!!.mkdirs() + SQLiteDatabase.openOrCreateDatabase(path, null).use { db -> + db.execSQL( + "CREATE TABLE test_version (_key TEXT PRIMARY KEY, _source_id TEXT NOT NULL, f_value TEXT, _payload BLOB NOT NULL)", + ) + db.version = 1 + } + + open(version = 2).use { + it.insert(Entry("k1", "jarA", "fromA")) + it.insert(Entry("k1", "jarB", "fromB")) + + assertThat(it.size()).isEqualTo(2) + } + } + + private companion object { + const val DB_NAME = "version_test.db" + } +} diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index be46dd372b..d3b81a5279 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -36,6 +36,17 @@ open class JvmSymbolIndex( const val DB_NAME_DEFAULT = "jvm_symbol_index.db" const val INDEX_NAME_LIBRARY = "jvm-library-cache" + /** + * Storage format version of the SQLite-backed JVM symbol and Kotlin file metadata indexes. + * + * Bump it whenever the index schema or any scanner's output changes: an install holding a + * different version drops its rows and re-indexes every source, which is the only way rows + * produced by an older scanner get replaced. [KtFileMetadataIndex] shares it because a + * source file's symbols are re-indexed only when its metadata row says so; dropping the + * symbols alone would leave files recorded as indexed with no symbols. + */ + const val FORMAT_VERSION = 2 + /** * Create (or get) a JVM symbol index backed by SQLite. * @@ -53,6 +64,7 @@ open class JvmSymbolIndex( descriptor = JvmSymbolDescriptor, context = context, dbName = dbName, + formatVersion = FORMAT_VERSION, name = indexName, ) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt index 713e66792e..44e5eed353 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KtFileMetadataIndex.kt @@ -30,6 +30,7 @@ class KtFileMetadataIndex( descriptor = KtFileMetadataDescriptor, context = context, dbName = dbName, + formatVersion = JvmSymbolIndex.FORMAT_VERSION, name = "kt-file-metadata", ), ) From fd48adfbeb74e878e64ffdf1ebebae6ac8a4689e Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 14:31:25 +0000 Subject: [PATCH 16/48] ADFA-6228: Enable WAL on the SQLite index, which androidx leaves off by default --- .../codeonthego/indexing/SQLiteIndex.kt | 11 +-- .../indexing/SQLiteIndexJournalTest.kt | 71 +++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexJournalTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 5cde79460e..fa0a3d1882 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -45,7 +45,8 @@ import kotlin.collections.iterator * - Each `f_{field}` (for equality filter) * - Each `f_{field}_lower` (for prefix search) * - * Uses WAL journal mode for concurrent read/write performance. + * File-backed databases use WAL journal mode: a commit appends to the write-ahead log instead + * of writing a rollback journal and the main file. In-memory databases have no journal to set. * Inserts are batched inside transactions for throughput. * * [query] and [distinctValues] eagerly collect results and return a @@ -142,10 +143,10 @@ class SQLiteIndex( }, ).build() - db = - FrameworkSQLiteOpenHelperFactory() - .create(config) - .writableDatabase + val helper = FrameworkSQLiteOpenHelperFactory().create(config) + // androidx disables WAL unless asked, and it only takes effect if set before the first open. + helper.setWriteAheadLoggingEnabled(true) + db = helper.writableDatabase // Ensure table exists (for shared databases) createTable(db) diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexJournalTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexJournalTest.kt new file mode 100644 index 0000000000..c7ab19a3d1 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexJournalTest.kt @@ -0,0 +1,71 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexJournalTest { + data class Entry( + override val key: String, + override val sourceId: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_journal" + override val fields = emptyList() + + override fun fieldValues(entry: Entry) = emptyMap() + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + context.deleteDatabase(DB_NAME) + } + + @Test + fun `the database uses write-ahead logging`() { + val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) + + /* + * Probed over a read-only connection because a writable one would apply its own journal mode + * to the file on open, and WAL, once set, is what every later connection reports. + */ + val path = context.getDatabasePath(DB_NAME).path + val mode = + try { + SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> + db.rawQuery("PRAGMA journal_mode", null).use { cursor -> + cursor.moveToFirst() + cursor.getString(0) + } + } + } finally { + index.close() + } + + assertThat(mode).isEqualTo("wal") + } + + private companion object { + const val DB_NAME = "journal_test.db" + } +} From e570ffdfe44e289445e8c3cfdfdb9e8ea61d4b31 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 14:39:32 +0000 Subject: [PATCH 17/48] ADFA-6228: Re-index a library JAR whose size or modification time changed at the same path --- .../codeonthego/indexing/InMemoryIndex.kt | 22 ++- .../codeonthego/indexing/SQLiteIndex.kt | 102 +++++++++++--- .../codeonthego/indexing/api/Index.kt | 21 ++- .../indexing/util/BackgroundIndexer.kt | 30 +++- .../codeonthego/indexing/InMemoryIndexTest.kt | 17 +++ .../indexing/SQLiteIndexFingerprintTest.kt | 128 ++++++++++++++++++ .../indexing/util/BackgroundIndexerTest.kt | 34 +++++ .../indexing/jvm/JvmLibraryIndexingService.kt | 31 +++-- .../indexing/jvm/JvmSymbolIndex.kt | 16 ++- .../indexing/jvm/JarFingerprintTest.kt | 38 ++++++ 10 files changed, 392 insertions(+), 47 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexFingerprintTest.kt create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarFingerprintTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index 7070661b24..0428e4f3e5 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -26,6 +26,7 @@ import kotlin.concurrent.write * - [fieldMaps]: fieldName -> (fieldValue -> rows) (equality filter) * - [prefixBuckets]: fieldName -> (lowercased first char -> list of (value, row)) * Provides a ~36-way partition for prefix search. + * - [fingerprints]: sourceId -> fingerprint recorded by [insertSource] * * All mutations go through [lock] in write mode for consistency * across the multiple maps. Reads use read mode. @@ -42,6 +43,7 @@ class InMemoryIndex( private val keyMap = ConcurrentHashMap>(256) private val fieldMaps = ConcurrentHashMap>>() private val prefixBuckets = ConcurrentHashMap>>() + private val fingerprints = ConcurrentHashMap() private val lock = ReentrantReadWriteLock() @@ -121,6 +123,20 @@ class InMemoryIndex( override suspend fun insert(entry: T) = lock.write { insertSingleLocked(entry) } + /** Inserts under one write lock, so readers never see the fingerprint without every entry. */ + override suspend fun insertSource( + sourceId: String, + fingerprint: String, + entries: Sequence, + ) = lock.write { + for (entry in entries) { + insertSingleLocked(entry) + } + fingerprints[sourceId] = fingerprint + } + + override suspend fun sourceFingerprint(sourceId: String): String? = fingerprints[sourceId] + override suspend fun removeBySource(sourceId: String) = lock.write { removeBySourceLocked(sourceId) @@ -141,10 +157,11 @@ class InMemoryIndex( } /** - * Remove all entries for [sourceId] from the primary, source, and secondary - * indexes. Caller MUST already hold the write lock; this method does not lock. + * Remove all entries and the fingerprint for [sourceId] from every map. + * Caller MUST already hold the write lock; this method does not lock. */ private fun removeBySourceLocked(sourceId: String) { + fingerprints.remove(sourceId) val sourceRows = sourceMap.remove(sourceId) ?: return for (row in sourceRows) { val entry = rows.remove(row) ?: continue @@ -161,6 +178,7 @@ class InMemoryIndex( rows.clear() sourceMap.clear() keyMap.clear() + fingerprints.clear() fieldMaps.values.forEach { it.clear() } prefixBuckets.values.forEach { it.clear() } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index fa0a3d1882..0624adadf0 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -97,6 +97,9 @@ class SQLiteIndex( private val tableName = descriptor.name.replace(Regex("[^a-zA-Z0-9_]"), "_") + /** One row per source that was indexed with a fingerprint, see [insertSource]. */ + private val sourcesTableName = "${tableName}_sources" + /** Field column names: `f_{fieldName}`. */ private val fieldColumns = descriptor.fields.associate { field -> @@ -250,37 +253,60 @@ class SQLiteIndex( } } - override suspend fun insertAll(entries: Sequence) = + override suspend fun insertAll(entries: Sequence) = insertBatched(entries, fingerprint = null) + + override suspend fun insertSource( + sourceId: String, + fingerprint: String, + entries: Sequence, + ) = insertBatched(entries, SourceFingerprint(sourceId, fingerprint)) + + override suspend fun sourceFingerprint(sourceId: String): String? = withContext(Dispatchers.IO) { - val batch = mutableListOf() - for (entry in entries) { - batch.add(entry) - if (batch.size >= batchSize) { - ifOpen { insertBatchLocked(batch) } - batch.clear() - } + ifOpen(null) { + db + .query( + "SELECT _fingerprint FROM $sourcesTableName WHERE _source_id = ?", + arrayOf(sourceId), + ).use { if (it.moveToFirst()) it.getString(0) else null } } - if (batch.isNotEmpty()) { - ifOpen { insertBatchLocked(batch) } + } + + /** + * Inserts [entries] in transactions of [batchSize] rows, taking the lock per batch so reads can + * interleave with a long insert. A [fingerprint] goes into the last transaction, which runs + * even when there are no entries, so an empty source is still recorded as indexed. + */ + private suspend fun insertBatched( + entries: Sequence, + fingerprint: SourceFingerprint?, + ) = withContext(Dispatchers.IO) { + val batch = mutableListOf() + for (entry in entries) { + batch.add(entry) + if (batch.size >= batchSize) { + ifOpen { insertBatchLocked(batch, fingerprint = null) } + batch.clear() } } + if (batch.isNotEmpty() || fingerprint != null) { + ifOpen { insertBatchLocked(batch, fingerprint) } + } + } override suspend fun insert(entry: T) = withContext(Dispatchers.IO) { - ifOpen { insertBatchLocked(listOf(entry)) } + ifOpen { insertBatchLocked(listOf(entry), fingerprint = null) } } - override suspend fun removeBySource(sourceId: String) = - withContext(Dispatchers.IO) { - ifOpen { db.execSQL("DELETE FROM $tableName WHERE _source_id = ?", arrayOf(sourceId)) } - } + override suspend fun removeBySource(sourceId: String) = removeBySources(listOf(sourceId)) /** - * Remove every row whose `_source_id` is in [sourceIds] using a single SQLite - * transaction. The ids are split into chunks of at most [DELETE_CHUNK_SIZE] so - * each `DELETE ... IN (?, ?, ...)` stays within SQLite's bound-parameter limit; - * all chunks run inside the one transaction, so the batch commits atomically - * (an empty [sourceIds] is a no-op and opens no transaction). + * Remove every row whose `_source_id` is in [sourceIds], with those sources' fingerprints, + * using a single SQLite transaction. The ids are split into chunks of at most + * [DELETE_CHUNK_SIZE] so each `DELETE ... IN (?, ?, ...)` stays within SQLite's + * bound-parameter limit; all chunks run inside the one transaction, so the batch commits + * atomically (an empty [sourceIds] is a no-op and opens no transaction). * * @param sourceIds Source ids whose rows should be deleted. */ @@ -296,6 +322,10 @@ class SQLiteIndex( "DELETE FROM $tableName WHERE _source_id IN ($placeholders)", chunk.toTypedArray(), ) + db.execSQL( + "DELETE FROM $sourcesTableName WHERE _source_id IN ($placeholders)", + chunk.toTypedArray(), + ) } db.setTransactionSuccessful() } finally { @@ -306,7 +336,16 @@ class SQLiteIndex( override suspend fun clear() = withContext(Dispatchers.IO) { - ifOpen { db.execSQL("DELETE FROM $tableName") } + ifOpen { + db.beginTransaction() + try { + db.execSQL("DELETE FROM $tableName") + db.execSQL("DELETE FROM $sourcesTableName") + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } } override fun close() { @@ -378,6 +417,9 @@ class SQLiteIndex( } db.execSQL("CREATE TABLE IF NOT EXISTS $tableName ($columns)") + db.execSQL( + "CREATE TABLE IF NOT EXISTS $sourcesTableName (_source_id TEXT NOT NULL PRIMARY KEY, _fingerprint TEXT NOT NULL)", + ) db.execSQL( "CREATE INDEX IF NOT EXISTS idx_${tableName}_key ON $tableName(_key, _source_id)", @@ -398,7 +440,10 @@ class SQLiteIndex( } } - private fun insertBatchLocked(entries: List) { + private fun insertBatchLocked( + entries: List, + fingerprint: SourceFingerprint?, + ) { db.beginTransaction() try { for (entry in entries) { @@ -427,12 +472,25 @@ class SQLiteIndex( cv, ) } + if (fingerprint != null) { + val cv = + ContentValues().apply { + put("_source_id", fingerprint.sourceId) + put("_fingerprint", fingerprint.value) + } + db.insert(sourcesTableName, SQLiteDatabase.CONFLICT_REPLACE, cv) + } db.setTransactionSuccessful() } finally { db.endTransaction() } } + private data class SourceFingerprint( + val sourceId: String, + val value: String, + ) + private data class SqlQuery( val sql: String, val args: List, diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index 472d40653c..41fc289fa0 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -83,7 +83,7 @@ interface WritableIndex { suspend fun insert(entry: T) /** - * Remove all entries from the given source. + * Remove all entries from the given source, and its fingerprint if one is recorded. */ suspend fun removeBySource(sourceId: String) @@ -98,7 +98,7 @@ interface WritableIndex { suspend fun removeBySources(sourceIds: Collection) /** - * Remove all entries. + * Remove all entries and fingerprints. */ suspend fun clear() } @@ -118,5 +118,22 @@ interface Index : /** The descriptor governing serialization and field extraction. */ val descriptor: IndexDescriptor + /** + * Inserts [entries], all from [sourceId], and records [fingerprint] as that source's. + * + * The fingerprint is opaque to the index; it is what lets a caller tell whether the source has + * changed since it was indexed. It is committed with the last batch of rows, so it is present + * only once every entry is stored: an insert that fails or is cancelled part-way leaves none. + * Removing the source's entries removes it too. + */ + suspend fun insertSource( + sourceId: String, + fingerprint: String, + entries: Sequence, + ) + + /** Returns the fingerprint recorded for [sourceId] by [insertSource], or `null` if there is none. */ + suspend fun sourceFingerprint(sourceId: String): String? + override fun close() {} } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt index ff46835f3d..2418524d21 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt @@ -75,13 +75,18 @@ class BackgroundIndexer( /** * Index a single source. The [provider] returns a [Flow] that is * collected on [Dispatchers.IO] and inserted in batches through - * [Index.insertAll]. + * [Index.insertAll]; with a [fingerprint], the last batch goes through + * [Index.insertSource] so the fingerprint is recorded only on success. * - * If [skipIfExists] is true and the source is already indexed, - * this is a no-op. + * If [skipIfExists] is true and the source is already indexed, this is a + * no-op. With a [fingerprint], "already indexed" means the index holds + * that same fingerprint for the source, so a source whose content changed + * at the same id is re-indexed; without one, it means the index holds any + * entry from the source. * * @param sourceId Identifies the source. * @param skipIfExists Skip if already indexed. + * @param fingerprint Identifies the source's current content, or `null` to not track it. * @param provider Lambda returning a [Flow] of entries. Resources the flow opens * are released when collection completes, fails or is cancelled. * @return The launched job. @@ -89,6 +94,7 @@ class BackgroundIndexer( fun indexSource( sourceId: String, skipIfExists: Boolean = true, + fingerprint: String? = null, provider: (sourceId: String) -> Flow, ): Job { // Cancel any in-flight job for this source @@ -97,7 +103,7 @@ class BackgroundIndexer( val job = scope.launch { try { - if (skipIfExists && index.containsSource(sourceId)) { + if (skipIfExists && isIndexed(sourceId, fingerprint)) { log.debug("Skipping already-indexed: {}", sourceId) progressListener?.onProgress(sourceId, IndexingEvent.Skipped) return@launch @@ -139,7 +145,11 @@ class BackgroundIndexer( // also leaves a partial source that containsSource() treats as indexed. error = e } - if (batch.isNotEmpty()) index.insertAll(batch.asSequence()) + if (fingerprint != null && error == null) { + index.insertSource(sourceId, fingerprint, batch.asSequence()) + } else if (batch.isNotEmpty()) { + index.insertAll(batch.asSequence()) + } error?.let { throw it } } @@ -162,6 +172,16 @@ class BackgroundIndexer( return job } + private suspend fun isIndexed( + sourceId: String, + fingerprint: String?, + ): Boolean = + if (fingerprint != null) { + index.sourceFingerprint(sourceId) == fingerprint + } else { + index.containsSource(sourceId) + } + /** * Index multiple sources sequentially in the background. * diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt index 768af61e98..f87e4a9407 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt @@ -412,4 +412,21 @@ class InMemoryIndexTest { assertThat(index.query(IndexQuery.bySource("src999")).toList()).isEmpty() } + + @Test + fun `insertSource records the fingerprint and removing the source drops it`() = + runTest { + val index = makeIndex() + index.insertSource("src1", "100:1", sequenceOf(entry("k1", "src1", "Foo"))) + index.insertSource("src2", "200:2", emptySequence()) + + assertThat(index.sourceFingerprint("src1")).isEqualTo("100:1") + assertThat(index.sourceFingerprint("src2")).isEqualTo("200:2") + + index.removeBySource("src1") + index.removeBySource("src2") + + assertThat(index.sourceFingerprint("src1")).isNull() + assertThat(index.sourceFingerprint("src2")).isNull() + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexFingerprintTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexFingerprintTest.kt new file mode 100644 index 0000000000..5c4c39fa96 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexFingerprintTest.kt @@ -0,0 +1,128 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** Per-source fingerprints on the SQLite-backed index, stored and dropped with the source's rows. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexFingerprintTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_fingerprint" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + private fun open() = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) + + private val index = open() + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + @Test + fun `insertSource records the fingerprint and it survives reopening`() = + runTest { + index.insertSource("jarA", "100:1", sequenceOf(Entry("k1", "jarA", "v"))) + index.close() + + open().use { assertThat(it.sourceFingerprint("jarA")).isEqualTo("100:1") } + } + + @Test + fun `a source with no entries still records its fingerprint`() = + runTest { + index.insertSource("jarA", "100:1", emptySequence()) + + assertThat(index.sourceFingerprint("jarA")).isEqualTo("100:1") + } + + @Test + fun `removing a source drops its fingerprint`() = + runTest { + index.insertSource("jarA", "100:1", sequenceOf(Entry("k1", "jarA", "v"))) + index.insertSource("jarB", "200:2", sequenceOf(Entry("k1", "jarB", "v"))) + index.insertSource("jarC", "300:3", sequenceOf(Entry("k1", "jarC", "v"))) + + index.removeBySource("jarA") + index.removeBySources(listOf("jarB")) + + assertThat(index.sourceFingerprint("jarA")).isNull() + assertThat(index.sourceFingerprint("jarB")).isNull() + assertThat(index.sourceFingerprint("jarC")).isEqualTo("300:3") + } + + @Test + fun `clear drops every fingerprint`() = + runTest { + index.insertSource("jarA", "100:1", sequenceOf(Entry("k1", "jarA", "v"))) + + index.clear() + + assertThat(index.sourceFingerprint("jarA")).isNull() + } + + @Test + fun `an insert that fails part-way records no fingerprint`() = + runTest { + // Past the first batch, so some rows are already committed when the source fails. + val failing = + sequence { + repeat(700) { yield(Entry("k$it", "jarA", "v")) } + throw IllegalStateException("corrupt JAR") + } + + runCatching { index.insertSource("jarA", "100:1", failing) } + + assertThat(index.containsSource("jarA")).isTrue() + assertThat(index.sourceFingerprint("jarA")).isNull() + } + + @Test + fun `a JAR rebuilt at the same path is re-indexed and an unchanged one is skipped`() = + runTest { + val indexer = BackgroundIndexer(index) + + indexer.indexSource("jarA", fingerprint = "100:1") { flowOf(Entry("k1", it, "v1")) }.join() + indexer.indexSource("jarA", fingerprint = "100:1") { flowOf(Entry("k1", it, "skipped")) }.join() + assertThat(index.get("k1")!!.value).isEqualTo("v1") + + indexer.indexSource("jarA", fingerprint = "140:2") { flowOf(Entry("k2", it, "v2")) }.join() + assertThat(index.get("k1")).isNull() + assertThat(index.get("k2")!!.value).isEqualTo("v2") + } + + private companion object { + const val DB_NAME = "fingerprint_test.db" + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt index fb06569b55..5ebe8e76ab 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt @@ -365,4 +365,38 @@ class BackgroundIndexerTest { assertThat(indexer.activeJobCount).isEqualTo(0) } + + @Test + fun `an unchanged fingerprint skips re-indexing`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "original")) }.join() + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "updated")) }.join() + + assertThat(index.get("k1")!!.value).isEqualTo("original") + } + + @Test + fun `a changed fingerprint re-indexes the source`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "original")) }.join() + indexer.indexSource("src1", fingerprint = "120:2") { flowOf(Entry("k1", it, "rebuilt")) }.join() + + assertThat(index.get("k1")!!.value).isEqualTo("rebuilt") + assertThat(index.sourceFingerprint("src1")).isEqualTo("120:2") + } + + @Test + fun `a fingerprint re-indexes a source that was indexed without one`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + indexer.indexSource("src1") { flowOf(Entry("k1", it, "original")) }.join() + indexer.indexSource("src1", fingerprint = "100:1") { flowOf(Entry("k1", it, "fingerprinted")) }.join() + + assertThat(index.get("k1")!!.value).isEqualTo("fingerprinted") + } } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt index e0ac4a22c9..5d318355da 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt @@ -17,6 +17,7 @@ import org.appdevforall.codeonthego.indexing.service.IndexingService import org.greenrobot.eventbus.Subscribe import org.greenrobot.eventbus.ThreadMode import org.slf4j.LoggerFactory +import java.io.File import java.nio.file.Path import java.nio.file.Paths import kotlin.io.path.extension @@ -112,28 +113,30 @@ class JvmLibraryIndexingService( log.info("{} JARs on classpath", currentJars.size) - // Step 1: Set the active set - this is instant. - // JARs not in the set become invisible to queries. - // JARs in the set that are already cached become - // visible immediately. + /* + * Step 1: Set the active set - this is instant. JARs not in the set become invisible to + * queries; JARs in the set that are already cached become visible immediately. + */ index.setActiveSources(currentJars) - // Step 2: Index any JARs not yet in the cache. - // Already-cached JARs are skipped (cheap existence check). - // Newly cached JARs are automatically visible because - // they're already in the active set. + /* + * Step 2: Index any JAR that is not cached, or was cached with a different fingerprint: + * a JAR rebuilt at the same path (a snapshot, a local file dependency) is re-scanned. + * Newly cached JARs are automatically visible because they're already in the active set. + */ var newCount = 0 for (jarPath in currentJars) { - if (!index.isCached(jarPath)) { + val fingerprint = jarFingerprint(File(jarPath)) + if (index.sourceFingerprint(jarPath) != fingerprint) { newCount++ - index.indexSource(jarPath, skipIfExists = true) { sourceId -> + index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> CombinedJarScanner.scan(Paths.get(jarPath), sourceId) } } } if (newCount > 0) { - log.info("{} new JARs submitted for background indexing", newCount) + log.info("{} new or changed JARs submitted for background indexing", newCount) } else { log.info("All JARs already cached, nothing to index") } @@ -150,3 +153,9 @@ class JvmLibraryIndexingService( return ext == "jar" || ext == "aar" } } + +/** + * Identifies the content of [jar] by its size and last-modified time, which a rewrite of the file + * changes without the cost of reading it. Stats the file, so never call it on the main thread. + */ +internal fun jarFingerprint(jar: File): String = "${jar.length()}:${jar.lastModified()}" diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index d3b81a5279..bb4a5f6682 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -76,23 +76,29 @@ open class JvmSymbolIndex( /** * Index a single source. The [provider] returns a [Flow] that is * collected on [Dispatchers.IO] and inserted in batches through - * [Index.insertAll]. + * [Index.insertAll], or [Index.insertSource] when a [fingerprint] is given. * - * If [skipIfExists] is true and the source is already indexed, - * this is a no-op. + * If [skipIfExists] is true and the source is already indexed (with the + * same [fingerprint], when one is given), this is a no-op. * * @param sourceId Identifies the source. * @param skipIfExists Skip if already indexed. + * @param fingerprint Identifies the source's current content, or `null` to not track it. * @param provider Lambda returning a [Flow] of entries. Resources the flow opens * (e.g. a JAR) are released when collection completes, fails or is * cancelled, so Flow operators such as `.filter {}` are safe. - * @return The launched job. + * @return The launched job, or the already-running job for the same fingerprint if one exists. + * @see BackgroundIndexer.indexSource */ fun indexSource( sourceId: String, skipIfExists: Boolean = true, + fingerprint: String? = null, provider: (sourceId: String) -> Flow, - ): Job = indexer.indexSource(sourceId, skipIfExists, provider) + ): Job = indexer.indexSource(sourceId, skipIfExists, fingerprint, provider) + + /** Returns the fingerprint [sourceId] was last indexed with, or `null` if it has none. */ + suspend fun sourceFingerprint(sourceId: String): String? = backing.sourceFingerprint(sourceId) /** * Find symbols matching the given prefix. diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarFingerprintTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarFingerprintTest.kt new file mode 100644 index 0000000000..14d53d6d7e --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarFingerprintTest.kt @@ -0,0 +1,38 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class JarFingerprintTest { + @get:Rule + val temp = TemporaryFolder() + + @Test + fun `rewriting a JAR with a different size changes its fingerprint`() { + val jar = temp.newFile("lib.jar").apply { writeBytes(ByteArray(10)) } + val before = jarFingerprint(jar) + val mtime = jar.lastModified() + + jar.writeBytes(ByteArray(20)) + jar.setLastModified(mtime) + + assertThat(jarFingerprint(jar)).isNotEqualTo(before) + } + + @Test + fun `rewriting a JAR at the same size changes its fingerprint through the modification time`() { + val jar = temp.newFile("lib.jar").apply { writeBytes(ByteArray(10)) } + jar.setLastModified(1_000_000L) + val before = jarFingerprint(jar) + + jar.writeBytes(ByteArray(10) { 1 }) + jar.setLastModified(2_000_000L) + + assertThat(jarFingerprint(jar)).isNotEqualTo(before) + } +} From 1188ce0e120cbe1ddb046176fa39ae6adcf34044 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 14:55:31 +0000 Subject: [PATCH 18/48] ADFA-6228: Explain index queries through the SQL the index itself builds --- .../codeonthego/indexing/SQLiteIndex.kt | 69 +++++++++++++++---- .../indexing/SQLiteIndexIdentityTest.kt | 19 +---- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 0624adadf0..3cb3b0925a 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -4,6 +4,7 @@ import android.content.ContentValues import android.content.Context import android.database.sqlite.SQLiteDatabase import android.os.Looper +import androidx.annotation.VisibleForTesting import androidx.sqlite.db.SupportSQLiteDatabase import androidx.sqlite.db.SupportSQLiteOpenHelper import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory @@ -230,19 +231,7 @@ class SQLiteIndex( break } - val (where, args) = buildWhereClause(query, chunk) - val sql = - buildString { - append("SELECT DISTINCT $col FROM $tableName WHERE $col IS NOT NULL") - if (where.isNotEmpty()) { - append(" AND ") - append(where) - } - if (limit != Int.MAX_VALUE) { - append(" LIMIT ${limit - values.size}") - } - } - + val (sql, args) = buildDistinctQuery(col, query, chunk, limit - values.size) db.query(sql, args.toTypedArray()).use { while (it.moveToNext()) { values.add(it.getString(0)) @@ -562,6 +551,60 @@ class SQLiteIndex( return SqlQuery(sql, args) } + private fun buildDistinctQuery( + column: String, + query: IndexQuery, + sourceIdChunk: List?, + limit: Int, + ): SqlQuery { + val (where, args) = buildWhereClause(query, sourceIdChunk) + val sql = + buildString { + append("SELECT DISTINCT $column FROM $tableName WHERE $column IS NOT NULL") + if (where.isNotEmpty()) { + append(" AND ") + append(where) + } + if (limit != Int.MAX_VALUE) { + append(" LIMIT $limit") + } + } + return SqlQuery(sql, args) + } + + /** Returns the plan of the first statement [query] runs, one plan row per line. */ + @VisibleForTesting + internal fun explainQuery(query: IndexQuery): String { + val select = buildSelectQuery(query, firstSourceIdChunk(query), effectiveLimit(query)) + return explain(select) + } + + /** Returns the plan of the first statement [distinctValues] runs for [fieldName] and [query]. */ + @VisibleForTesting + internal fun explainDistinctValues( + fieldName: String, + query: IndexQuery, + ): String { + val col = fieldColumns[fieldName] ?: throw IllegalArgumentException("Unknown field: $fieldName") + return explain(buildDistinctQuery(col, query, firstSourceIdChunk(query), effectiveLimit(query))) + } + + private fun firstSourceIdChunk(query: IndexQuery): List? { + val chunks = sourceIdChunks(query) + require(chunks.isNotEmpty()) { "A query scoped to no sources runs no statement" } + return chunks.first() + } + + private fun explain(query: SqlQuery): String = + runBlocking { + ifOpen("") { + db.query("EXPLAIN QUERY PLAN ${query.sql}", query.args.toTypedArray()).use { cursor -> + val detail = cursor.getColumnIndexOrThrow("detail") + buildList { while (cursor.moveToNext()) add(cursor.getString(detail)) }.joinToString("\n") + } + } + } + /** * Builds the shared `WHERE` body for [query], restricted to [sourceIdChunk] when the query is * source-scoped. Returns the clause without the `WHERE` keyword so both the row select and the diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt index b535699ad2..0c9b13764f 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexIdentityTest.kt @@ -1,7 +1,6 @@ package org.appdevforall.codeonthego.indexing import android.content.Context -import android.database.sqlite.SQLiteDatabase import androidx.test.core.app.ApplicationProvider import com.google.common.truth.Truth.assertThat import kotlinx.coroutines.test.runTest @@ -109,7 +108,7 @@ class SQLiteIndexIdentityTest { runTest { index.insert(Entry("com.example.Foo", "jarA", "fromA")) - val plan = explain("SELECT _payload FROM test_identity WHERE _source_id = ?", "jarA") + val plan = index.explainQuery(IndexQuery.bySource("jarA")) assertThat(plan).contains("sqlite_autoindex_test_identity_1") } @@ -119,7 +118,7 @@ class SQLiteIndexIdentityTest { runTest { index.insert(Entry("com.example.Foo", "jarA", "fromA")) - val plan = explain("SELECT _payload FROM test_identity WHERE _key = ?", "com.example.Foo") + val plan = index.explainQuery(IndexQuery.byKey("com.example.Foo")) assertThat(plan).contains("USING INDEX") assertThat(plan).doesNotContain("SCAN") @@ -133,20 +132,6 @@ class SQLiteIndexIdentityTest { .single() .value - /** Returns the query plan for [sql], read over a second connection because the index exposes none. */ - private fun explain( - sql: String, - vararg args: String, - ): String { - val path = context.getDatabasePath(DB_NAME).path - return SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> - db.rawQuery("EXPLAIN QUERY PLAN $sql", args).use { cursor -> - val detail = cursor.getColumnIndexOrThrow("detail") - buildList { while (cursor.moveToNext()) add(cursor.getString(detail)) }.joinToString("\n") - } - } - } - private companion object { const val DB_NAME = "identity_test.db" } From 39787d825c2944c1d8c8a605ad4768da2d2cf65a Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 15:06:09 +0000 Subject: [PATCH 19/48] ADFA-6228: Keep scope and kind terms out of index selection when a query has a selective predicate --- .../codeonthego/indexing/SQLiteIndex.kt | 56 +++++- .../codeonthego/indexing/api/Core.kt | 5 + .../indexing/SQLiteIndexQueryPlanTest.kt | 182 ++++++++++++++++++ .../indexing/SQLiteIndexScopeTest.kt | 29 ++- .../indexing/jvm/JvmSymbolDescriptor.kt | 4 +- 5 files changed, 264 insertions(+), 12 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 3cb3b0925a..fb1c027b0e 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -46,6 +46,10 @@ import kotlin.collections.iterator * - Each `f_{field}` (for equality filter) * - Each `f_{field}_lower` (for prefix search) * + * When a query has a selective predicate (a key, a prefix, or a match on a + * [selective][org.appdevforall.codeonthego.indexing.api.IndexField.selective] field), its source + * scope and its other terms only filter rows and never choose the index. + * * File-backed databases use WAL journal mode: a commit appends to the write-ahead log instead * of writing a rollback journal and the main file. In-memory databases have no journal to set. * Inserts are batched inside transactions for throughput. @@ -113,6 +117,12 @@ class SQLiteIndex( .filter { it.prefixSearchable } .associate { it.name to "f_${it.name}_lower" } + /** Fields whose value matches may choose the SQL index a query is served by. */ + private val selectiveFields = + descriptor.fields + .filter { it.selective } + .mapTo(HashSet()) { it.name } + private val mutex = Mutex() @Volatile private var closed = false @@ -558,9 +568,10 @@ class SQLiteIndex( limit: Int, ): SqlQuery { val (where, args) = buildWhereClause(query, sourceIdChunk) + val filter = filterOnlyMarker(query) val sql = buildString { - append("SELECT DISTINCT $column FROM $tableName WHERE $column IS NOT NULL") + append("SELECT DISTINCT $column FROM $tableName WHERE $filter$column IS NOT NULL") if (where.isNotEmpty()) { append(" AND ") append(where) @@ -605,6 +616,31 @@ class SQLiteIndex( } } + /** + * Whether [query] has a predicate its own SQL index narrows to few rows: a key, a non-empty + * prefix, or a value match on a selective field. + */ + private fun hasSelectivePredicate(query: IndexQuery): Boolean = + query.key != null || + query.exactMatch.keys.any { it in selectiveFields } || + query.anyOf.any { (field, values) -> field in selectiveFields && values.isNotEmpty() } || + query.prefixMatch.any { (field, prefix) -> field in fieldColumns && prefix.isNotEmpty() } + + /** + * The unary `+` that keeps a term out of SQL index selection when [query] has a selective + * predicate, and nothing otherwise. The term is still evaluated against every candidate row. + * + * Without statistics SQLite rates an `IN` list or an equality on any indexed column as good as a + * range, so a scope over hundreds of sources, or a kind, would otherwise choose the index and + * leave the selective predicate to be checked row by row. A query with no selective predicate + * keeps every term eligible, so its scope can still be served by the primary key. + * + * The trade-off: any non-empty prefix wins over the scope, even a one-character prefix under a + * single-source scope, where the scope would be narrower. Production scopes span hundreds of + * sources, so the prefix is the right choice for the queries that matter. + */ + private fun filterOnlyMarker(query: IndexQuery) = if (hasSelectivePredicate(query)) "+" else "" + /** * Builds the shared `WHERE` body for [query], restricted to [sourceIdChunk] when the query is * source-scoped. Returns the clause without the `WHERE` keyword so both the row select and the @@ -626,17 +662,21 @@ class SQLiteIndex( args.addAll(values) } + val filter = filterOnlyMarker(query) + + fun filterUnlessSelective(field: String) = if (field in selectiveFields) "" else filter + query.key?.let { and("_key = ?", it) } - query.sourceId?.let { and("_source_id = ?", it) } + query.sourceId?.let { and("${filter}_source_id = ?", it) } if (sourceIdChunk != null) { val placeholders = sourceIdChunk.joinToString(",") { "?" } - and("_source_id IN ($placeholders)", *sourceIdChunk.toTypedArray()) + and("${filter}_source_id IN ($placeholders)", *sourceIdChunk.toTypedArray()) } for ((field, value) in query.exactMatch) { val col = fieldColumns[field] ?: continue - and("$col = ?", value) + and("${filterUnlessSelective(field)}$col = ?", value) } for ((field, values) in query.anyOf) { @@ -648,7 +688,7 @@ class SQLiteIndex( } val distinct = values.distinct() val placeholders = distinct.joinToString(",") { "?" } - and("$col IN ($placeholders)", *distinct.toTypedArray()) + and("${filterUnlessSelective(field)}$col IN ($placeholders)", *distinct.toTypedArray()) } for ((field, prefix) in query.prefixMatch) { @@ -660,7 +700,7 @@ class SQLiteIndex( if (value.isEmpty()) { // An empty prefix means "has a value", which is what `LIKE '%'` used to express. - and("$col IS NOT NULL") + and("$filter$col IS NOT NULL") continue } @@ -683,9 +723,9 @@ class SQLiteIndex( for ((field, mustExist) in query.presence) { val col = fieldColumns[field] ?: continue if (mustExist) { - and("$col IS NOT NULL") + and("$filter$col IS NOT NULL") } else { - and("$col IS NULL") + and("$filter$col IS NULL") } } diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt index 06b04f79a0..dc5ba74bcd 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Core.kt @@ -83,8 +83,13 @@ interface IndexDescriptor { * @param prefixSearchable Whether this field supports prefix queries * (e.g. name prefix for completions). Affects how * the persistent layer creates SQL indexes. + * @param selective Whether matching this field to a value narrows a query to few entries. + * Pass `false` for a field with few distinct values, such as a kind or a + * flag: a match on it then only filters a query that has a selective + * predicate, instead of choosing the SQL index the query is served by. */ data class IndexField( val name: String, val prefixSearchable: Boolean = false, + val selective: Boolean = true, ) diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt new file mode 100644 index 0000000000..03ff9d9c3e --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt @@ -0,0 +1,182 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.indexQuery +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * Which SQL index serves each query shape the JVM symbol index issues. + * + * Every production query is scoped to the active sources, and without statistics SQLite rates an + * `IN` list or an equality on any indexed column as good as a range, so these pin that the + * selective predicate drives the plan and the scope is only a filter. + */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexQueryPlanTest { + data class Entry( + override val key: String, + override val sourceId: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_plan" + override val fields = + listOf( + IndexField(NAME, prefixSearchable = true), + IndexField(PACKAGE), + IndexField(KIND, selective = false), + IndexField(CONTAINING_CLASS), + ) + + override fun fieldValues(entry: Entry) = emptyMap() + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + private val index = SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1) + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + @Test + fun `a scoped name prefix query is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + prefix(NAME, "get") + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(NAME_INDEX) + } + + @Test + fun `a scoped name prefix query restricted to several kinds is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + prefix(NAME, "get") + anyOf(KIND, listOf("CLASS", "INTERFACE")) + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(NAME_INDEX) + } + + @Test + fun `a scoped name prefix query restricted to one kind is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + prefix(NAME, "get") + eq(KIND, "CLASS") + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(NAME_INDEX) + } + + @Test + fun `a scoped exact name query restricted to one kind is served by the name index`() { + val plan = + index.explainQuery( + indexQuery { + eq(NAME, "Foo") + anyOf(KIND, listOf("CLASS")) + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains("$EXACT_NAME_INDEX (") + } + + @Test + fun `scoped distinct packages under a prefix are served by the package index`() { + val plan = + index.explainDistinctValues( + PACKAGE, + indexQuery { + prefix(PACKAGE, "com.example") + sourceIds = SOURCES + limit = 0 + }, + ) + + assertThat(plan).contains(PACKAGE_INDEX) + } + + @Test + fun `scoped top-level callables in a package are served by the package index`() { + val plan = + index.explainQuery( + indexQuery { + eq(PACKAGE, "com.example") + anyOf(KIND, listOf("FUNCTION", "PROPERTY")) + notExists(CONTAINING_CLASS) + sourceIds = SOURCES + }, + ) + + assertThat(plan).contains(PACKAGE_INDEX) + } + + @Test + fun `a scoped key lookup is served by the key index`() { + val plan = + index.explainQuery( + indexQuery { + key = "com.example.Foo" + sourceIds = SOURCES + limit = 1 + }, + ) + + assertThat(plan).contains(KEY_INDEX) + } + + @Test + fun `a query scoped by source alone is served by the primary key`() { + val plan = index.explainQuery(indexQuery { sourceIds = SOURCES }) + + assertThat(plan).contains(PRIMARY_KEY_INDEX) + } + + private companion object { + const val DB_NAME = "plan_test.db" + + const val NAME = "name" + const val PACKAGE = "package" + const val KIND = "kind" + const val CONTAINING_CLASS = "containingClass" + + const val NAME_INDEX = "idx_test_plan_f_name_lower" + const val EXACT_NAME_INDEX = "idx_test_plan_f_name" + const val PACKAGE_INDEX = "idx_test_plan_f_package" + const val KEY_INDEX = "idx_test_plan_key" + const val PRIMARY_KEY_INDEX = "sqlite_autoindex_test_plan_1" + + val SOURCES = (0 until 300).map { "/libs/lib$it.jar" } + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt index 219e15fe25..ca8d5e9d8c 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt @@ -21,6 +21,7 @@ class SQLiteIndexScopeTest { override val sourceId: String, val value: String, val group: String = "", + val kind: String = "", ) : Indexable private val descriptor = @@ -30,19 +31,22 @@ class SQLiteIndexScopeTest { listOf( IndexField("value", prefixSearchable = true), IndexField("group"), + IndexField("kind", selective = false), ) override fun fieldValues(entry: Entry) = mapOf( "value" to entry.value, "group" to entry.group, + "kind" to entry.kind, ) - override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}|${entry.group}".toByteArray() + override fun serialize(entry: Entry) = + listOf(entry.key, entry.sourceId, entry.value, entry.group, entry.kind).joinToString("|").toByteArray() override fun deserialize(bytes: ByteArray): Entry { val parts = String(bytes).split("|") - return Entry(parts[0], parts[1], parts[2], parts[3]) + return Entry(parts[0], parts[1], parts[2], parts[3], parts[4]) } } @@ -192,4 +196,25 @@ class SQLiteIndexScopeTest { assertThat(keys).containsExactly("wanted") } + + @Test + fun `scope and kind terms that only filter still exclude non-matching rows`() = + runTest { + index.insert(Entry("match", "jarA", "Widget1", kind = "CLASS")) + index.insert(Entry("otherKind", "jarA", "Widget2", kind = "METHOD")) + index.insert(Entry("otherSource", "jarB", "Widget3", kind = "CLASS")) + index.insert(Entry("otherName", "jarA", "Gadget", kind = "CLASS")) + + val byAnyOf = + IndexQuery( + prefixMatch = mapOf("value" to "Widget"), + anyOf = mapOf("kind" to listOf("CLASS")), + sourceIds = listOf("jarA"), + limit = 0, + ) + val byExact = byAnyOf.copy(anyOf = emptyMap(), exactMatch = mapOf("kind" to "CLASS")) + + assertThat(index.query(byAnyOf).map { it.key }.toList()).containsExactly("match") + assertThat(index.query(byExact).map { it.key }.toList()).containsExactly("match") + } } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt index fa41b59f9a..9c73ae7be0 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt @@ -32,10 +32,10 @@ object JvmSymbolDescriptor : IndexDescriptor { listOf( IndexField(name = KEY_NAME, prefixSearchable = true), IndexField(name = KEY_PACKAGE), - IndexField(name = KEY_KIND), + IndexField(name = KEY_KIND, selective = false), IndexField(name = KEY_RECEIVER_TYPE), IndexField(name = KEY_CONTAINING_CLASS), - IndexField(name = KEY_LANGUAGE), + IndexField(name = KEY_LANGUAGE, selective = false), ) override fun fieldValues(entry: JvmSymbol): Map = From 6ce2c9de9ebed5c2d7d0f38074fcb56a0b032781 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 15:16:49 +0000 Subject: [PATCH 20/48] style: spotless reformat, no functional change --- .../jvm/JvmGeneratedIndexingService.kt | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt index 5a8c5ee4eb..63fff55a04 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt @@ -36,7 +36,6 @@ val JVM_GENERATED_SYMBOL_INDEX = IndexKey("jvm-generated-symbols class JvmGeneratedIndexingService( private val context: Context, ) : IndexingService { - companion object { const val ID = "jvm-generated-indexing-service" private const val DB_NAME = "jvm_generated_symbol_index.db" @@ -53,11 +52,12 @@ class JvmGeneratedIndexingService( private val coroutineScope = CoroutineScope(Dispatchers.Default) override suspend fun initialize(registry: IndexRegistry) { - val index = JvmSymbolIndex.createSqliteIndex( - context = context, - dbName = DB_NAME, - indexName = INDEX_NAME, - ) + val index = + JvmSymbolIndex.createSqliteIndex( + context = context, + dbName = DB_NAME, + indexName = INDEX_NAME, + ) this.generatedIndex = index registry.register(JVM_GENERATED_SYMBOL_INDEX, index) @@ -82,15 +82,17 @@ class JvmGeneratedIndexingService( } private suspend fun reindexGeneratedJars(forceReindex: Boolean) { - val index = this.generatedIndex ?: run { - log.warn("Not indexing generated JARs — index not initialized.") - return - } + val index = + this.generatedIndex ?: run { + log.warn("Not indexing generated JARs — index not initialized.") + return + } - val workspace = ProjectManagerImpl.getInstance().workspace ?: run { - log.warn("Not indexing generated JARs — workspace model not available.") - return - } + val workspace = + ProjectManagerImpl.getInstance().workspace ?: run { + log.warn("Not indexing generated JARs — workspace model not available.") + return + } val generatedJars = workspace.subProjects From be1e3826377d22255a5ee691ec8c3356fdf30561 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 15:21:07 +0000 Subject: [PATCH 21/48] ADFA-6228: Collect query planner statistics once a bulk indexing pass finishes --- .../codeonthego/indexing/SQLiteIndex.kt | 58 ++++++++++++++- .../codeonthego/indexing/api/Index.kt | 6 ++ .../indexing/SQLiteIndexAnalysisLimitTest.kt | 38 ++++++++++ .../indexing/SQLiteIndexQueryPlanTest.kt | 27 +++++++ .../jvm/JvmGeneratedIndexingService.kt | 33 ++++----- .../indexing/jvm/JvmLibraryIndexingService.kt | 28 ++++---- .../indexing/jvm/JvmSymbolIndex.kt | 15 ++++ .../jvm/JvmSymbolIndexOptimizeTest.kt | 72 +++++++++++++++++++ 8 files changed, 247 insertions(+), 30 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexAnalysisLimitTest.kt create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexOptimizeTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index fb1c027b0e..7127f5ca0c 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -48,7 +48,8 @@ import kotlin.collections.iterator * * When a query has a selective predicate (a key, a prefix, or a match on a * [selective][org.appdevforall.codeonthego.indexing.api.IndexField.selective] field), its source - * scope and its other terms only filter rows and never choose the index. + * scope and its other terms only filter rows and never choose the index. [optimize] collects the + * planner statistics for everything else. * * File-backed databases use WAL journal mode: a commit appends to the write-ahead log instead * of writing a rollback journal and the main file. In-memory databases have no journal to set. @@ -95,6 +96,29 @@ class SQLiteIndex( */ private const val SOURCE_ID_CHUNK_SIZE = 900 + /** Rows [optimize] samples per SQL index, which bounds its cost on a large table. */ + private const val ANALYSIS_LIMIT = 1000 + + /** The first SQLite version with `PRAGMA analysis_limit`. */ + private val ANALYSIS_LIMIT_SINCE = listOf(3, 32, 0) + + /** + * Whether SQLite [version] (as `sqlite_version()` reports it, e.g. `3.32.2`) supports + * `PRAGMA analysis_limit`. Compares the numeric components, so `3.9` is older than `3.32`; + * a version that does not parse counts as unsupported. + */ + @VisibleForTesting + internal fun supportsAnalysisLimit(version: String): Boolean { + val parts = version.split('.').map { it.toIntOrNull() ?: return false } + for (i in ANALYSIS_LIMIT_SINCE.indices) { + val part = parts.getOrElse(i) { 0 } + if (part != ANALYSIS_LIMIT_SINCE[i]) { + return part > ANALYSIS_LIMIT_SINCE[i] + } + } + return true + } + /** Every table in a database except SQLite's and Android's own bookkeeping tables. */ private const val USER_TABLES_QUERY = "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'android_metadata'" @@ -347,6 +371,38 @@ class SQLiteIndex( } } + /** + * Collects the statistics SQLite's query planner uses to choose between this table's indexes. + * + * This runs `ANALYZE` rather than `PRAGMA optimize`, which decides per table whether to analyze + * and, in older SQLite versions, only considers tables queried on the same connection: reads run + * on the pool's other connections, so right after indexing it could analyze nothing. The + * analysis limit samples each index instead of reading all of it. Both statements share one + * transaction so they run on the same connection. + * + * On SQLite older than 3.32 (API 30 and below) this does nothing. Those versions ignore the + * unknown analysis limit and read every index in full, which on a large table holds the lock + * that queries wait on for too long. Queries there rely on their selective predicates alone. + */ + override suspend fun optimize() = + withContext(Dispatchers.IO) { + ifOpen { + val version = db.query("SELECT sqlite_version()").use { if (it.moveToFirst()) it.getString(0) else "" } + if (!supportsAnalysisLimit(version)) { + log.debug("Not analyzing {}: SQLite {} has no analysis limit", tableName, version) + return@ifOpen + } + db.beginTransaction() + try { + db.query("PRAGMA analysis_limit = $ANALYSIS_LIMIT").close() + db.execSQL("ANALYZE $tableName") + db.setTransactionSuccessful() + } finally { + db.endTransaction() + } + } + } + override fun close() { if (Looper.getMainLooper() == Looper.myLooper()) { log.warn( diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index 41fc289fa0..5199a36d06 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -135,5 +135,11 @@ interface Index : /** Returns the fingerprint recorded for [sourceId] by [insertSource], or `null` if there is none. */ suspend fun sourceFingerprint(sourceId: String): String? + /** + * Refreshes whatever the index keeps to plan its queries, after a bulk change to its entries. + * Does nothing by default. + */ + suspend fun optimize() {} + override fun close() {} } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexAnalysisLimitTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexAnalysisLimitTest.kt new file mode 100644 index 0000000000..d9d7c6479d --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexAnalysisLimitTest.kt @@ -0,0 +1,38 @@ +package org.appdevforall.codeonthego.indexing + +import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** Which SQLite versions [SQLiteIndex.optimize] analyzes on: those with `PRAGMA analysis_limit`, 3.32.0 and later. */ +@RunWith(JUnit4::class) +class SQLiteIndexAnalysisLimitTest { + @Test + fun `versions from 3_32_0 on support the analysis limit`() { + for (version in listOf("3.32.0", "3.32.2", "3.39.2", "3.51.2", "4.0")) { + assertWithMessage(version).that(SQLiteIndex.supportsAnalysisLimit(version)).isTrue() + } + } + + @Test + fun `versions before 3_32_0 do not`() { + for (version in listOf("3.31.1", "3.28.0", "3.22.0", "2.99.99")) { + assertWithMessage(version).that(SQLiteIndex.supportsAnalysisLimit(version)).isFalse() + } + } + + @Test + fun `versions compare by number, not as text`() { + // As text "3.9.0" sorts after "3.32.0"; as a version it is older. + assertThat(SQLiteIndex.supportsAnalysisLimit("3.9.0")).isFalse() + } + + @Test + fun `an unparseable version does not`() { + for (version in listOf("", "3.x", "unknown")) { + assertWithMessage(version).that(SQLiteIndex.supportsAnalysisLimit(version)).isFalse() + } + } +} diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt index 03ff9d9c3e..9fda84f4f4 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexQueryPlanTest.kt @@ -1,8 +1,10 @@ package org.appdevforall.codeonthego.indexing import android.content.Context +import android.database.sqlite.SQLiteDatabase import androidx.test.core.app.ApplicationProvider import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.test.runTest import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexField import org.appdevforall.codeonthego.indexing.api.Indexable @@ -163,6 +165,31 @@ class SQLiteIndexQueryPlanTest { assertThat(plan).contains(PRIMARY_KEY_INDEX) } + @Test + fun `optimize collects planner statistics for the table's indexes`() = + runTest { + index.insertAll((0 until 10).asSequence().map { Entry("com.example.Foo$it", "/libs/lib$it.jar") }) + + index.optimize() + + assertThat(indexesWithStatistics()).containsAtLeast(KEY_INDEX, NAME_INDEX, PACKAGE_INDEX) + } + + /** Reads `sqlite_stat1` over a second connection, since the index exposes no statistics. */ + private fun indexesWithStatistics(): List { + val path = context.getDatabasePath(DB_NAME).path + return SQLiteDatabase.openDatabase(path, null, SQLiteDatabase.OPEN_READONLY).use { db -> + val hasStatTable = + db.rawQuery("SELECT 1 FROM sqlite_master WHERE name = 'sqlite_stat1'", null).use { it.moveToFirst() } + if (!hasStatTable) { + return@use emptyList() + } + db.rawQuery("SELECT idx FROM sqlite_stat1 WHERE tbl = 'test_plan'", null).use { cursor -> + buildList { while (cursor.moveToNext()) add(cursor.getString(0)) } + } + } + } + private companion object { const val DB_NAME = "plan_test.db" diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt index 63fff55a04..bd677ecac2 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt @@ -6,6 +6,7 @@ import com.itsaky.androidide.projects.api.ModuleProject import com.itsaky.androidide.tasks.cancelIfActive import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -65,9 +66,8 @@ class JvmGeneratedIndexingService( // Kick off an initial index pass for any already-built JARs. coroutineScope.launch { - indexingMutex.withLock { - reindexGeneratedJars(forceReindex = false) - } + val jobs = indexingMutex.withLock { reindexGeneratedJars(forceReindex = false) } + generatedIndex?.optimizeAfter(jobs) } } @@ -75,23 +75,23 @@ class JvmGeneratedIndexingService( // Generated JARs (especially R.jar) always change after a build — // their field values are regenerated. Force a full re-index. coroutineScope.launch { - indexingMutex.withLock { - reindexGeneratedJars(forceReindex = true) - } + val jobs = indexingMutex.withLock { reindexGeneratedJars(forceReindex = true) } + generatedIndex?.optimizeAfter(jobs) } } - private suspend fun reindexGeneratedJars(forceReindex: Boolean) { + /** Submits every generated JAR that needs indexing and returns the submitted jobs. */ + private suspend fun reindexGeneratedJars(forceReindex: Boolean): List { val index = this.generatedIndex ?: run { log.warn("Not indexing generated JARs — index not initialized.") - return + return emptyList() } val workspace = ProjectManagerImpl.getInstance().workspace ?: run { log.warn("Not indexing generated JARs — workspace model not available.") - return + return emptyList() } val generatedJars = @@ -109,21 +109,22 @@ class JvmGeneratedIndexingService( // Make exactly these JARs visible; remove stale ones from scope. index.setActiveSources(generatedJars) - var submitted = 0 + val jobs = mutableListOf() for (jarPath in generatedJars) { if (forceReindex || !index.isCached(jarPath)) { - submitted++ - index.indexSource(jarPath, skipIfExists = false) { sourceId -> - CombinedJarScanner.scan(Paths.get(jarPath), sourceId) - } + jobs += + index.indexSource(jarPath, skipIfExists = false) { sourceId -> + CombinedJarScanner.scan(Paths.get(jarPath), sourceId) + } } } - if (submitted > 0) { - log.info("{} generated JARs submitted for background indexing (force={})", submitted, forceReindex) + if (jobs.isNotEmpty()) { + log.info("{} generated JARs submitted for background indexing (force={})", jobs.size, forceReindex) } else { log.info("All generated JARs already cached, nothing to index") } + return jobs } override fun close() { diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt index 5d318355da..e9b0228e17 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt @@ -8,6 +8,7 @@ import com.itsaky.androidide.projects.models.bootClassPaths import com.itsaky.androidide.tasks.cancelIfActive import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -75,23 +76,23 @@ class JvmLibraryIndexingService( fun refresh() { coroutineScope.launch { - indexingMutex.withLock { - reindexLibraries() - } + val jobs = indexingMutex.withLock { reindexLibraries() } + libraryIndex?.optimizeAfter(jobs) } } - private suspend fun reindexLibraries() { + /** Submits every library JAR that needs indexing and returns the submitted jobs. */ + private suspend fun reindexLibraries(): List { val index = this.libraryIndex ?: run { log.warn("Not indexing libraries. Index not initialized.") - return + return emptyList() } val workspace = ProjectManagerImpl.getInstance().workspace ?: run { log.warn("Not indexing libraries. Workspace model not available.") - return + return emptyList() } val currentJars = @@ -124,22 +125,23 @@ class JvmLibraryIndexingService( * a JAR rebuilt at the same path (a snapshot, a local file dependency) is re-scanned. * Newly cached JARs are automatically visible because they're already in the active set. */ - var newCount = 0 + val jobs = mutableListOf() for (jarPath in currentJars) { val fingerprint = jarFingerprint(File(jarPath)) if (index.sourceFingerprint(jarPath) != fingerprint) { - newCount++ - index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> - CombinedJarScanner.scan(Paths.get(jarPath), sourceId) - } + jobs += + index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> + CombinedJarScanner.scan(Paths.get(jarPath), sourceId) + } } } - if (newCount > 0) { - log.info("{} new or changed JARs submitted for background indexing", newCount) + if (jobs.isNotEmpty()) { + log.info("{} new or changed JARs submitted for background indexing", jobs.size) } else { log.info("All JARs already cached, nothing to index") } + return jobs } override fun close() { diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index bb4a5f6682..9d05f618b0 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -4,6 +4,7 @@ import android.content.Context import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.joinAll import org.appdevforall.codeonthego.indexing.FilteredIndex import org.appdevforall.codeonthego.indexing.SQLiteIndex import org.appdevforall.codeonthego.indexing.api.Index @@ -210,6 +211,20 @@ open class JvmSymbolIndex( suspend fun awaitIndexing() = indexer.awaitAll() + /** + * Optimizes the backing index once every job in [jobs] has finished, or does nothing if [jobs] + * is empty. + * + * Whoever submits a pass of sources calls this with the jobs it submitted, because only it knows + * when the pass is over: the indexer can sit idle between two submissions while the submitter + * works out what to submit next, so an idle indexer is not a finished pass. + */ + suspend fun optimizeAfter(jobs: Collection) { + if (jobs.isEmpty()) return + jobs.joinAll() + backing.optimize() + } + override fun close() { indexer.close() super.close() diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexOptimizeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexOptimizeTest.kt new file mode 100644 index 0000000000..fb93bf48a1 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexOptimizeTest.kt @@ -0,0 +1,72 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.util.concurrent.CopyOnWriteArrayList + +/** When an indexing pass optimizes the index behind a [JvmSymbolIndex]. */ +@RunWith(JUnit4::class) +class JvmSymbolIndexOptimizeTest { + /** Records which of the test's sources the index held each time it was optimized. */ + private class RecordingIndex( + private val backing: Index, + ) : Index by backing { + val sourcesAtOptimize = CopyOnWriteArrayList>() + + override suspend fun optimize() { + sourcesAtOptimize += listOf(FIRST, SECOND).filter { backing.containsSource(it) }.toSet() + } + } + + @Test + fun `a pass with a gap between submissions optimizes once, after every job`() = + runTest { + val backing = RecordingIndex(InMemoryIndex(JvmSymbolDescriptor)) + val index = JvmSymbolIndex(backing, BackgroundIndexer(backing)) + + val first = index.indexSource(FIRST, skipIfExists = false) { flowOf(classifier("Foo", it)) } + // The gap: the first job finishes before the second is submitted, leaving the indexer idle mid-pass. + first.join() + val second = index.indexSource(SECOND, skipIfExists = false) { flowOf(classifier("Bar", it)) } + index.optimizeAfter(listOf(first, second)) + + assertThat(backing.sourcesAtOptimize).isEqualTo(listOf(setOf(FIRST, SECOND))) + } + + @Test + fun `a pass that submits nothing does not optimize`() = + runTest { + val backing = RecordingIndex(InMemoryIndex(JvmSymbolDescriptor)) + val index = JvmSymbolIndex(backing, BackgroundIndexer(backing)) + + index.optimizeAfter(emptyList()) + + assertThat(backing.sourcesAtOptimize).isEmpty() + } + + private fun classifier( + simpleName: String, + sourceId: String, + ) = JvmSymbol( + key = "com.example.$simpleName", + sourceId = sourceId, + name = "com.example.$simpleName", + shortName = simpleName, + packageName = "com.example", + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(internalName = "com/example/$simpleName"), + ) + + private companion object { + const val FIRST = "/libs/first.jar" + const val SECOND = "/libs/second.jar" + } +} From 031ac27e01b7fd11269186ad7a32ecc9d7326509 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:00:11 +0000 Subject: [PATCH 22/48] ADFA-6228: Copy IndexQueryBuilder.anyOf's collection instead of keeping it by reference --- .../codeonthego/indexing/api/Query.kt | 19 +++++++++++++++---- .../indexing/api/IndexQueryBuilderTest.kt | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/api/IndexQueryBuilderTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt index 59a68b0735..48354a70d3 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -10,7 +10,12 @@ package org.appdevforall.codeonthego.indexing.api data class IndexQuery( /** Exact match predicates: field name -> expected value. */ val exactMatch: Map = emptyMap(), - /** Prefix match predicates: field name -> prefix (case-insensitive). */ + /** + * Prefix match predicates: field name -> prefix. + * + * Case-insensitive on a field declared [prefixSearchable][IndexField.prefixSearchable]; + * case-sensitive on any other field. + */ val prefixMatch: Map = emptyMap(), /** * Set-membership predicates: field name to the values that field may take. @@ -68,6 +73,8 @@ class IndexQueryBuilder { private val pres = mutableMapOf() private val anyOfValues = mutableMapOf>() var sourceId: String? = null + + /** See [IndexQuery.sourceIds]. */ var sourceIds: Collection? = null var key: String? = null var limit: Int = 200 @@ -80,7 +87,7 @@ class IndexQueryBuilder { exact[field] = value } - /** Prefix match on a field (case-insensitive). */ + /** See [IndexQuery.prefixMatch]. */ fun prefix( field: String, value: String, @@ -88,12 +95,16 @@ class IndexQueryBuilder { prefix[field] = value } - /** Field must hold one of [values]. */ + /** + * Field must hold one of [values]. + * + * Copies [values] so a later mutation of the caller's collection cannot change the built query. + */ fun anyOf( field: String, values: Collection, ) { - anyOfValues[field] = values + anyOfValues[field] = values.toList() } /** Field must be non-null. */ diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/api/IndexQueryBuilderTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/api/IndexQueryBuilderTest.kt new file mode 100644 index 0000000000..90e76d6064 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/api/IndexQueryBuilderTest.kt @@ -0,0 +1,19 @@ +package org.appdevforall.codeonthego.indexing.api + +import com.google.common.truth.Truth.assertThat +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +@RunWith(JUnit4::class) +class IndexQueryBuilderTest { + @Test + fun `anyOf is unaffected by later mutation of the caller's collection`() { + val values = mutableListOf("CLASS") + + val query = indexQuery { anyOf("kind", values) } + values.add("INTERFACE") + + assertThat(query.anyOf["kind"]).containsExactly("CLASS") + } +} From 3eb4129b6d899362d80fc3e6972d53655c72a67d Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:00:24 +0000 Subject: [PATCH 23/48] ADFA-6228: Make FilteredIndex.activeSources() final and trim its stale KDoc visibleSourceIds() is the only extension point since filtering reads it, not isActive(); activeSources() itself is no longer meant to be overridden. --- .../org/appdevforall/codeonthego/indexing/FilteredIndex.kt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt index 18434e9c39..51ab70ec1d 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/FilteredIndex.kt @@ -59,15 +59,12 @@ open class FilteredIndex( /** * Returns the current set of active source IDs. */ - open fun activeSources(): Set = activeSources.toSet() + fun activeSources(): Set = activeSources.toSet() /** * The source IDs whose entries are visible, or `null` if every source is. * - * This, rather than [isActive], is the point to override to change what the filter admits. - * Scoping is pushed into the query, so the filter has to be able to *describe* its scope and not - * merely test one ID against it -- an override of [isActive] alone could not be honoured, and - * would silently hide every entry instead. + * Override this to change what the filter admits; `null` admits every source. */ protected open fun visibleSourceIds(): Collection? = activeSources From f5807539b7cc7dc1d396bdef2662029d5dfc486d Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:00:37 +0000 Subject: [PATCH 24/48] ADFA-6228: Document distinctValues' limit as capping rather than only zero meaning unlimited --- .../kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt index 5199a36d06..c405768a64 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Index.kt @@ -52,8 +52,8 @@ interface ReadableIndex { * for enumerating a large field -- every package on a classpath, say -- where fetching the * matching entries and reducing them in memory would defeat the point. * - * Values are deduplicated across the whole result, and their order is unspecified. If - * [IndexQuery.limit] is 0, every distinct value is emitted. + * Values are deduplicated across the whole result, and their order is unspecified. A positive + * [IndexQuery.limit] caps how many distinct values are emitted; 0 or negative means unlimited. * * @param fieldName Must be one of the fields declared in the [IndexDescriptor]. * @param query Restricts which entries contribute a value. From 013cad83ee634d5a72191df4f4ceaae930383663 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:01:21 +0000 Subject: [PATCH 25/48] ADFA-6228: Fix chunked query correctness and the prefix range's upper bound - Make the source-id chunk size an internal constructor parameter so a small size can exercise the multi-statement path in tests; Robolectric's SQLite allows far more than 999 bound parameters, so the existing 2100-id test passed even with chunking deleted. - Query each distinctValues chunk with the full limit instead of the remaining budget: a value repeating across chunks could otherwise exhaust that budget before a later chunk's still-unseen values were read. - Increment exclusiveUpperBound by Unicode code point instead of raw UTF-16 code unit: incrementing only a supplementary character's trailing low surrogate produced a bound below the prefix itself, so a matching row was excluded outright. - Fix the escaped LIKE guard's stale rationale: the range already rejects a literal '_' or '%' substituted for another character; the LIKE guard only matters when the range's bound is carried or absent. --- .../codeonthego/indexing/SQLiteIndex.kt | 78 +++++++++--- .../indexing/SQLiteIndexPrefixTest.kt | 60 ++++++++- .../indexing/SQLiteIndexScopeTest.kt | 120 +++++++++++++++++- 3 files changed, 240 insertions(+), 18 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 7127f5ca0c..5ffa528f8e 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -71,6 +71,8 @@ import kotlin.collections.iterator * discards every table in it; each index sharing the file recreates its own table * when it opens. Indexes sharing a file must therefore pass the same version. * @param batchSize Number of rows per INSERT transaction. + * @param sourceIdChunkSize Max number of `_source_id` placeholders per source-scoped statement. + * See [SOURCE_ID_CHUNK_SIZE]. */ class SQLiteIndex( override val descriptor: IndexDescriptor, @@ -79,6 +81,7 @@ class SQLiteIndex( formatVersion: Int, override val name: String = "sqlite:${descriptor.name}", private val batchSize: Int = 500, + @param:VisibleForTesting internal val sourceIdChunkSize: Int = SOURCE_ID_CHUNK_SIZE, ) : Index { companion object { private val log = LoggerFactory.getLogger(SQLiteIndex::class.java) @@ -90,9 +93,15 @@ class SQLiteIndex( private const val DELETE_CHUNK_SIZE = 900 /** - * Max number of `_source_id` placeholders per source-scoped SELECT, under the same - * bound-parameter limit as [DELETE_CHUNK_SIZE]. A scoped query runs one statement per chunk; - * chunks are disjoint on `_source_id`, so no row can be returned by two of them. + * Default max number of `_source_id` placeholders per source-scoped statement (see + * [sourceIdChunkSize]), under the same bound-parameter limit as [DELETE_CHUNK_SIZE]. A scoped + * query runs one statement per chunk; chunks are disjoint on `_source_id`, so no row can be + * returned by two of them. + * + * The gap between 900 and SQLite's 999 limit is headroom for the statement's other bound + * parameters -- an `anyOf` or an exact-match value adds one placeholder each, on top of the + * chunk's own. A single statement combining a full chunk with a large `anyOf` must still stay + * under 999 placeholders in total. */ private const val SOURCE_ID_CHUNK_SIZE = 900 @@ -258,6 +267,13 @@ class SQLiteIndex( * Deduplicated here as well as in SQL: DISTINCT only applies within one statement, and * chunked source ids mean one statement per chunk. The same package name legitimately * appears in many JARs, so without this the caller would see it once per chunk. + * + * Each chunk is queried with the full limit, not the remaining budget: a value already + * seen in an earlier chunk does not free up SQL-side room in a later one, since a + * chunk-local `LIMIT` counts rows before dedup -- a value repeating across chunks would + * otherwise starve a later chunk's still-unseen values out of a shrinking budget. The + * overall cap is enforced separately, while collecting a chunk's rows: a chunk queried + * with the full limit can itself hold more distinct new values than remain in the budget. */ val values = LinkedHashSet() for (chunk in sourceIdChunks(query)) { @@ -265,9 +281,9 @@ class SQLiteIndex( break } - val (sql, args) = buildDistinctQuery(col, query, chunk, limit - values.size) + val (sql, args) = buildDistinctQuery(col, query, chunk, limit) db.query(sql, args.toTypedArray()).use { - while (it.moveToNext()) { + while (it.moveToNext() && values.size < limit) { values.add(it.getString(0)) } } @@ -555,14 +571,42 @@ class SQLiteIndex( /** * The smallest string greater than every string starting with [prefix], or `null` when no such - * bound exists because [prefix] ends in the highest representable characters. + * bound exists because [prefix] consists only of the highest representable character, U+FFFF. + * + * Increments the trailing character by code point, not by UTF-16 code unit: a supplementary + * character (one stored as a surrogate pair) is incremented as the single code point it + * represents. Incrementing only the low surrogate, in isolation, would produce a code unit pair + * SQLite decodes as an unrelated, smaller code point, putting the bound below [prefix] itself and + * making the range it feeds ([SQLiteIndex.buildWhereClause]) match nothing. When an increment + * would land in the surrogate range (`0xD800`-`0xDFFF`, which by itself encodes no character), the + * result steps over it to the first ordinary character above it, `0xE000`. */ private fun exclusiveUpperBound(prefix: String): String? { - for (i in prefix.length - 1 downTo 0) { - val c = prefix[i] - if (c != Char.MAX_VALUE) { - return prefix.substring(0, i) + (c + 1) + var end = prefix.length + while (end > 0) { + val isSupplementary = end >= 2 && Character.isSurrogatePair(prefix[end - 2], prefix[end - 1]) + val start = if (isSupplementary) end - 2 else end - 1 + val codePoint = + if (isSupplementary) { + Character.toCodePoint(prefix[end - 2], prefix[end - 1]) + } else { + prefix[end - 1].code + } + val maxAtThisPosition = if (isSupplementary) Character.MAX_CODE_POINT else Char.MAX_VALUE.code + + if (codePoint == maxAtThisPosition) { + end = start + continue } + + val incremented = codePoint + 1 + val next = + if (incremented in Character.MIN_SURROGATE.code..Character.MAX_SURROGATE.code) { + Character.MAX_SURROGATE.code + 1 + } else { + incremented + } + return prefix.substring(0, start) + String(Character.toChars(next)) } return null } @@ -590,7 +634,7 @@ class SQLiteIndex( if (sourceIds.isEmpty()) { return emptyList() } - return sourceIds.toSortedSet().chunked(SOURCE_ID_CHUNK_SIZE) + return sourceIds.toSortedSet().chunked(sourceIdChunkSize) } private fun buildSelectQuery( @@ -755,7 +799,7 @@ class SQLiteIndex( val value = if (lowerCol != null) prefix.lowercase() else prefix if (value.isEmpty()) { - // An empty prefix means "has a value", which is what `LIKE '%'` used to express. + // An empty prefix means the field must simply be present. and("$filter$col IS NOT NULL") continue } @@ -763,9 +807,13 @@ class SQLiteIndex( /* * The range bounds are what make this use the column's index: SQLite only optimises LIKE * into a range scan when case_sensitive_like is on or the column collates NOCASE, and - * neither holds here, so a bare LIKE scans the whole table. The escaped LIKE stays as the - * semantic guard -- it is what rejects a literal '_' or '%' in the prefix, which are valid - * identifier characters that an unescaped pattern would treat as wildcards. + * neither holds here, so a bare LIKE scans the whole table. The range already rejects a + * literal '_' or '%' substituted for another character -- e.g. prefix "my_f" against a row + * "myXfield" -- because that changes the value's ordering, not just what a LIKE wildcard + * would match. The escaped LIKE stays as a second guard for the rare case the range alone + * is not tight: [exclusiveUpperBound] returns `null` for a prefix with no upper bound, or a + * looser bound when its last character carries into an earlier one, and either lets through + * a value that only differs from the prefix after the point the range stops constraining it. */ val upperBound = exclusiveUpperBound(value) if (upperBound != null) { diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt index b330772f4a..d5b7a04e10 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexPrefixTest.kt @@ -127,13 +127,69 @@ class SQLiteIndexPrefixTest { } @Test - fun `a prefix ending at the highest character still matches`() = + fun `a prefix ending in the highest character still matches`() = runTest { - // No exclusive upper bound exists here, so only the lower bound can be emitted. + // The trailing U+FFFF carries into the preceding character, giving an upper bound of "{". val high = "z${Char.MAX_VALUE}" index.insert(Entry("k1", "s", "${high}tail", "p")) index.insert(Entry("k2", "s", "zzz", "p")) assertThat(namesMatching(high)).containsExactly("${high}tail") } + + @Test + fun `a prefix consisting only of the highest character still matches`() = + runTest { + // Every character carries here, so no exclusive upper bound exists at all: only the lower + // bound is emitted. + val high = "${Char.MAX_VALUE}" + index.insert(Entry("k1", "s", "${high}tail", "p")) + index.insert(Entry("k2", "s", "notIt", "p")) + + assertThat(namesMatching(high)).containsExactly("${high}tail") + } + + @Test + fun `a prefix ending just below the surrogate range steps over it`() = + runTest { + // Naively incrementing 0xD7FF by one lands on 0xD800, the first high surrogate, which + // encodes no character by itself; the bound must step over the whole surrogate block. + val prefix = "a\uD7FF" + index.insert(Entry("k1", "s", "${prefix}tail", "p")) + index.insert(Entry("k2", "s", "b", "p")) + + assertThat(namesMatching(prefix)).containsExactly("${prefix}tail") + } + + @Test + fun `the LIKE guard rejects a row the carried range bound alone would admit`() = + runTest { + /* + * The trailing U+FFFF carries, giving the prefix an upper bound of "b": a row whose + * second character is any code point at or above U+FFFF -- including a supplementary + * character, which sorts above the whole basic plane -- falls inside that range without + * actually starting with the prefix. Only the LIKE clause (built through + * escapeLikeLiteral) rejects it. + */ + val prefix = "a${Char.MAX_VALUE}" + index.insert(Entry("k1", "s", "${prefix}match", "p")) + index.insert(Entry("k2", "s", "a\uD800\uDC00reject", "p")) + + assertThat(namesMatching(prefix)).containsExactly("${prefix}match") + } + + @Test + fun `a prefix ending in a supplementary character increments the whole code point`() = + runTest { + /* + * U+1D7FF is the surrogate pair \uD835\uDFFF. Incrementing only the trailing low surrogate + * (the old, per-code-unit bug) produces \uD835\uE000, which SQLite decodes as a different, + * smaller code point than the prefix itself, so the range would match nothing. + */ + val prefix = "a\uD835\uDFFF" + index.insert(Entry("k1", "s", "${prefix}tail", "p")) + index.insert(Entry("k2", "s", "b", "p")) + + assertThat(namesMatching(prefix)).containsExactly("${prefix}tail") + } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt index ca8d5e9d8c..2225efe3b0 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexScopeTest.kt @@ -13,7 +13,7 @@ import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner -/** Source scoping and column projection on the SQLite-backed index. */ +/** Source scoping, chunked multi-statement queries, and column projection on the SQLite-backed index. */ @RunWith(RobolectricTestRunner::class) class SQLiteIndexScopeTest { data class Entry( @@ -63,6 +63,15 @@ class SQLiteIndexScopeTest { index.close() } + private fun smallChunkIndex(chunkSize: Int) = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + sourceIdChunkSize = chunkSize, + ) + @Test fun `query scoped to source ids returns only those sources`() = runTest { @@ -102,6 +111,115 @@ class SQLiteIndexScopeTest { assertThat(found.toSet()).hasSize(2100) } + @Test + fun `query scoped across multiple small chunks returns every match`() = + runTest { + /* + * Robolectric's native SQLite allows far more than 999 bound parameters, so the 2100-id + * test above passes even with chunking deleted. A small chunk size exercises the same + * multi-statement path on real hardware. + */ + smallChunkIndex(2).use { idx -> + val sourceIds = (0 until 5).map { "jar$it" } + idx.insertAll(sourceIds.asSequence().mapIndexed { i, src -> Entry("k$i", src, "Cls$i") }) + + val found = idx.query(IndexQuery(sourceIds = sourceIds, limit = 0)).map { it.key }.toList() + + assertThat(found.toSet()).containsExactly("k0", "k1", "k2", "k3", "k4") + } + } + + @Test + fun `limit is honored across chunk boundaries`() = + runTest { + // Three sources, one chunk each: the limit must be spent across chunks rather than per + // chunk, and the loop must stop querying once it is met. + smallChunkIndex(1).use { idx -> + idx.insert(Entry("k1", "s1", "Alpha")) + idx.insert(Entry("k2", "s1", "Alpha")) + idx.insert(Entry("k3", "s2", "Alpha")) + idx.insert(Entry("k4", "s2", "Alpha")) + idx.insert(Entry("k5", "s3", "Alpha")) + idx.insert(Entry("k6", "s3", "Alpha")) + + val found = + idx + .query(IndexQuery(sourceIds = listOf("s1", "s2", "s3"), limit = 3)) + .map { it.key } + .toList() + + assertThat(found).hasSize(3) + assertThat(found).containsAtLeast("k1", "k2") + assertThat(found).containsNoneOf("k5", "k6") + } + } + + @Test + fun `a source id repeated across chunks is not returned twice`() = + runTest { + smallChunkIndex(1).use { idx -> + idx.insert(Entry("ka", "a", "Alpha")) + idx.insert(Entry("kb", "b", "Beta")) + + val found = + idx + .query(IndexQuery(sourceIds = listOf("a", "a", "b"), limit = 0)) + .map { it.key } + .toList() + + assertThat(found).containsExactly("ka", "kb") + } + } + + @Test + fun `distinctValues does not starve a later chunk's new values on the shrinking limit`() = + runTest { + /* + * Old bug: each chunk was queried with `limit - values.size`, the remaining budget, so a + * value repeating across chunks could exhaust that budget before a later chunk's genuinely + * new values were ever read. Chunk one already contributes "g1"; chunk two repeats "g1" + * (ordered first, by source id and by value) ahead of the newly-seen "g3", so a + * budget-of-one second query returns the duplicate and misses "g3" entirely. + */ + smallChunkIndex(2).use { idx -> + idx.insert(Entry("k1", "s1", "V1", group = "g1")) + idx.insert(Entry("k2", "s2", "V2", group = "g2")) + idx.insert(Entry("k3", "s3", "V3", group = "g1")) + idx.insert(Entry("k4", "s4", "V4", group = "g3")) + + val groups = + idx + .distinctValues("group", IndexQuery(sourceIds = listOf("s1", "s2", "s3", "s4"), limit = 3)) + .toList() + + assertThat(groups).containsExactly("g1", "g2", "g3") + } + } + + @Test + fun `distinctValues stops adding once the limit is reached mid-chunk`() = + runTest { + /* + * Old bug: querying a chunk with the full limit fixed the starvation above, but nothing + * capped the total afterward -- the inner cursor loop added every row a chunk returned. A + * chunk holding more new distinct values than remain in the budget must stop partway + * through, not push the total past the limit. + */ + smallChunkIndex(2).use { idx -> + idx.insert(Entry("k1", "s1", "V1", group = "g1")) + idx.insert(Entry("k2", "s2", "V2", group = "g1")) + idx.insert(Entry("k3", "s3", "V3", group = "g2")) + idx.insert(Entry("k4", "s4", "V4", group = "g3")) + + val groups = + idx + .distinctValues("group", IndexQuery(sourceIds = listOf("s1", "s2", "s3", "s4"), limit = 2)) + .toList() + + assertThat(groups).hasSize(2) + } + } + @Test fun `scoped query applies the scope before the limit`() = runTest { From aa37db7f5c0e244a4722d9af487ad8d7b3fff0d3 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:25:23 +0000 Subject: [PATCH 26/48] ADFA-6228: Serialise re-indexing of the same source A new indexSource call now cancelAndJoin's the previous job for that source before deleting its rows, so a superseded pass can never write after the new one has already replaced them. A call whose fingerprint matches an already-running job is folded into that job instead of restarting it, and a job that loses the race no longer clears the newer job's active-job entry on its own completion. --- .../indexing/util/BackgroundIndexer.kt | 62 ++++++-- .../indexing/util/BackgroundIndexerTest.kt | 135 ++++++++++++++++++ 2 files changed, 185 insertions(+), 12 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt index 2418524d21..3ed300e028 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexer.kt @@ -2,6 +2,7 @@ package org.appdevforall.codeonthego.indexing.util import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob @@ -60,8 +61,9 @@ class BackgroundIndexer( private val index: Index, parentScope: CoroutineScope = CoroutineScope(Dispatchers.Default), ) : Closeable { - private val job = SupervisorJob(parentScope.coroutineContext[Job]) - private val scope = CoroutineScope(parentScope.coroutineContext + job) + /** Named apart from any per-source [Job] a call to [indexSource] launches, so it can't be shadowed. */ + private val supervisor = SupervisorJob(parentScope.coroutineContext[Job]) + private val scope = CoroutineScope(parentScope.coroutineContext + supervisor) companion object { private val log = LoggerFactory.getLogger(BackgroundIndexer::class.java) @@ -70,7 +72,13 @@ class BackgroundIndexer( var progressListener: IndexingProgressListener? = null - private val activeJobs = ConcurrentHashMap() + /** The job currently indexing a source, and the fingerprint it was submitted with. */ + private data class ActiveJob( + val job: Job, + val fingerprint: String?, + ) + + private val activeJobs = ConcurrentHashMap() /** * Index a single source. The [provider] returns a [Flow] that is @@ -84,12 +92,25 @@ class BackgroundIndexer( * at the same id is re-indexed; without one, it means the index holds any * entry from the source. * + * A source already being indexed by a still-running job submitted with the same + * [fingerprint] is left alone rather than restarted: that job will produce the same + * result, so cancelling and resubmitting it would only throw away its progress. A + * source being indexed under a different fingerprint (its content changed again before + * the previous pass finished) is handled by cancelling and joining that job before this + * one deletes its rows, so the two passes' writes can never interleave. + * + * The check for an existing job and the recording of this call's own job in [activeJobs] are + * not atomic with each other, so two overlapping calls for the same [sourceId] could each + * decide independently to proceed. That's safe only because every caller of [indexSource] for + * a given index submits through its own single-submitter mutex (see the indexing services' + * `indexingMutex`), never concurrently with itself. + * * @param sourceId Identifies the source. * @param skipIfExists Skip if already indexed. * @param fingerprint Identifies the source's current content, or `null` to not track it. * @param provider Lambda returning a [Flow] of entries. Resources the flow opens * are released when collection completes, fails or is cancelled. - * @return The launched job. + * @return The launched job, or the already-running one this call was folded into. */ fun indexSource( sourceId: String, @@ -97,12 +118,25 @@ class BackgroundIndexer( fingerprint: String? = null, provider: (sourceId: String) -> Flow, ): Job { - // Cancel any in-flight job for this source - activeJobs[sourceId]?.cancel() + val existing = activeJobs[sourceId] + if (existing != null && fingerprint != null && fingerprint == existing.fingerprint && existing.job.isActive) { + log.debug("Skipping resubmit of already-running: {}", sourceId) + progressListener?.onProgress(sourceId, IndexingEvent.Skipped) + return existing.job + } + + val previousJob = existing?.job + // Signalled as soon as this call supersedes it, rather than only once this job's own + // coroutine gets scheduled, so a superseded pass starts winding down immediately. + previousJob?.cancel() val job = - scope.launch { + scope.launch(start = CoroutineStart.LAZY) { try { + // Let a superseded pass for this source finish (or actually stop) before + // this one touches its rows, so the two passes' writes never interleave. + previousJob?.join() + if (skipIfExists && isIndexed(sourceId, fingerprint)) { log.debug("Skipping already-indexed: {}", sourceId) progressListener?.onProgress(sourceId, IndexingEvent.Skipped) @@ -164,11 +198,15 @@ class BackgroundIndexer( } } - activeJobs[sourceId] = job + val active = ActiveJob(job, fingerprint) + // Recorded before the job starts, so a concurrent call for the same source always sees + // this job rather than racing to observe it mid-launch. + activeJobs[sourceId] = active // Untrack on completion, not in the body: a job cancelled before it runs never reaches a // finally, and a handler added after completion runs at once. Remove only this job: a // replacement for the same source may already be registered. - job.invokeOnCompletion { activeJobs.remove(sourceId, job) } + job.invokeOnCompletion { activeJobs.remove(sourceId, active) } + job.start() return job } @@ -205,14 +243,14 @@ class BackgroundIndexer( * Cancel all in-flight indexing and wait for completion. */ suspend fun cancelAll() { - activeJobs.values.toList().forEach { it.cancelAndJoin() } + activeJobs.values.toList().forEach { it.job.cancelAndJoin() } } /** * Wait for all in-flight indexing to complete. */ suspend fun awaitAll() { - activeJobs.values.toList().joinAll() + activeJobs.values.map { it.job }.joinAll() } /** @@ -229,7 +267,7 @@ class BackgroundIndexer( ) } runBlocking { - job.cancelAndJoin() + supervisor.cancelAndJoin() } activeJobs.clear() } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt index 5ebe8e76ab..e5b426490b 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/util/BackgroundIndexerTest.kt @@ -26,6 +26,8 @@ import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 import java.io.IOException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger @RunWith(JUnit4::class) @@ -191,6 +193,19 @@ class BackgroundIndexerTest { assertThat(indexer.activeJobCount).isEqualTo(0) } + @Test + fun `activeJobCount drops to zero once a completed job's entry is removed`() = + runTest { + val (_, indexer) = makeIndexAndIndexer() + + indexer + .indexSource("src1", skipIfExists = false) { sourceId -> + flowOf(Entry("k1", sourceId, "v1")) + }.join() + + assertThat(indexer.activeJobCount).isEqualTo(0) + } + @Test fun `close cancels all active jobs`() = runTest { @@ -399,4 +414,124 @@ class BackgroundIndexerTest { assertThat(index.get("k1")!!.value).isEqualTo("fingerprinted") } + + /** Waits for the latch, failing the test (rather than silently proceeding) on timeout. */ + private fun CountDownLatch.awaitOrFail() { + assertThat(await(5, TimeUnit.SECONDS)).isTrue() + } + + @Test + fun `a new job waits for the previous job to finish before deleting its rows`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + val job1Started = CountDownLatch(1) + val releaseJob1 = CountDownLatch(1) + + val job1 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + job1Started.countDown() + releaseJob1.awaitOrFail() + // Long enough that, without cancelAndJoin, job2's in-memory write below finishes first. + Thread.sleep(50) + flowOf(Entry("k1", sourceId, "old1"), Entry("k2", sourceId, "old2")) + } + + job1Started.awaitOrFail() + + val job2 = + indexer.indexSource("src1", fingerprint = "200:2") { sourceId -> + flowOf(Entry("k3", sourceId, "new1")) + } + + releaseJob1.countDown() + job2.join() + job1.join() + + assertThat(index.get("k1")).isNull() + assertThat(index.get("k2")).isNull() + assertThat(index.get("k3")!!.value).isEqualTo("new1") + assertThat(index.sourceFingerprint("src1")).isEqualTo("200:2") + } + + @Test + fun `indexSource skips resubmitting a source with an active job at the same fingerprint`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + val job1Started = CountDownLatch(1) + val releaseJob1 = CountDownLatch(1) + var providerInvocations = 0 + + val job1 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + providerInvocations++ + job1Started.countDown() + releaseJob1.awaitOrFail() + flowOf(Entry("k1", sourceId, "v1")) + } + + job1Started.awaitOrFail() + + val job2 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + providerInvocations++ + flowOf(Entry("k1", sourceId, "should-not-run")) + } + + assertThat(job2).isSameInstanceAs(job1) + + releaseJob1.countDown() + job1.join() + + assertThat(providerInvocations).isEqualTo(1) + assertThat(index.get("k1")!!.value).isEqualTo("v1") + } + + @Test + fun `a superseded job finishing does not clear the new job's active-job entry`() = + runTest { + val (index, indexer) = makeIndexAndIndexer() + + val releaseJob1 = CountDownLatch(1) + val job2ProviderStarted = CountDownLatch(1) + val releaseJob2 = CountDownLatch(1) + + val job1 = + indexer.indexSource("src1", fingerprint = "100:1") { sourceId -> + releaseJob1.awaitOrFail() + flowOf(Entry("k1", sourceId, "old")) + } + + var job2ProviderInvocations = 0 + val job2 = + indexer.indexSource("src1", fingerprint = "200:2") { sourceId -> + job2ProviderInvocations++ + job2ProviderStarted.countDown() + releaseJob2.awaitOrFail() + flowOf(Entry("k2", sourceId, "new")) + } + + // Let the superseded job1 finish (its cancellation is cooperative-only here, since + // InMemoryIndex never checks it) while job2 is still mid-flight. + releaseJob1.countDown() + job1.join() + job2ProviderStarted.awaitOrFail() + + var job3ProviderInvocations = 0 + val job3 = + indexer.indexSource("src1", fingerprint = "200:2") { sourceId -> + job3ProviderInvocations++ + flowOf(Entry("k2", sourceId, "should-not-run")) + } + + assertThat(job3).isSameInstanceAs(job2) + assertThat(job3ProviderInvocations).isEqualTo(0) + + releaseJob2.countDown() + job2.join() + + assertThat(job2ProviderInvocations).isEqualTo(1) + assertThat(index.get("k2")!!.value).isEqualTo("new") + } } From a15202ab89d90e19c67e46047684f886fdcd8a06 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:25:45 +0000 Subject: [PATCH 27/48] ADFA-6228: Check for cancellation before each insert batch SQLiteIndex.insertBatched never checked cancellation, and its mutex's uncontended fast path never suspends, so a cancelled indexing job kept committing batches until its scan finished. Adding an explicit check before each transaction bounds how much a superseded job can still write once cancelled. --- .../codeonthego/indexing/SQLiteIndex.kt | 8 ++ .../indexing/SQLiteIndexCancellationTest.kt | 94 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexCancellationTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 5ffa528f8e..321a75e545 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -9,6 +9,7 @@ import androidx.sqlite.db.SupportSQLiteDatabase import androidx.sqlite.db.SupportSQLiteOpenHelper import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ensureActive import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -315,6 +316,11 @@ class SQLiteIndex( * Inserts [entries] in transactions of [batchSize] rows, taking the lock per batch so reads can * interleave with a long insert. A [fingerprint] goes into the last transaction, which runs * even when there are no entries, so an empty source is still recorded as indexed. + * + * Checks for cancellation before each transaction: [Mutex.withLock] only checks it while + * actually suspended waiting for the lock, and its uncontended fast path never suspends, so a + * caller cancelled mid-scan (see [org.appdevforall.codeonthego.indexing.util.BackgroundIndexer]) + * would otherwise keep committing batches indefinitely. */ private suspend fun insertBatched( entries: Sequence, @@ -324,11 +330,13 @@ class SQLiteIndex( for (entry in entries) { batch.add(entry) if (batch.size >= batchSize) { + ensureActive() ifOpen { insertBatchLocked(batch, fingerprint = null) } batch.clear() } } if (batch.isNotEmpty() || fingerprint != null) { + ensureActive() ifOpen { insertBatchLocked(batch, fingerprint) } } } diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexCancellationTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexCancellationTest.kt new file mode 100644 index 0000000000..afdb91aa02 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndexCancellationTest.kt @@ -0,0 +1,94 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.junit.After +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +/** A cancelled insert must stop committing batches rather than run its entries to completion. */ +@RunWith(RobolectricTestRunner::class) +class SQLiteIndexCancellationTest { + data class Entry( + override val key: String, + override val sourceId: String, + val value: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_cancellation" + override val fields = listOf(IndexField("value")) + + override fun fieldValues(entry: Entry) = mapOf("value" to entry.value) + + override fun serialize(entry: Entry) = "${entry.key}|${entry.sourceId}|${entry.value}".toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val context = ApplicationProvider.getApplicationContext() + + /** batchSize = 1 so every entry is its own transaction, isolating the per-batch check. */ + private val index = + SQLiteIndex(descriptor = descriptor, context = context, dbName = DB_NAME, formatVersion = 1, batchSize = 1) + + @After + fun tearDown() { + index.close() + context.deleteDatabase(DB_NAME) + } + + private fun CountDownLatch.awaitOrFail() { + assertThat(await(5, TimeUnit.SECONDS)).isTrue() + } + + @Test + fun `cancelling an insert stops it from committing batches produced after cancellation`() = + runTest { + val secondEntryReached = CountDownLatch(1) + val proceedWithSecondEntry = CountDownLatch(1) + val entries = + sequence { + yield(Entry("k1", "jarA", "v1")) + secondEntryReached.countDown() + proceedWithSecondEntry.awaitOrFail() + yield(Entry("k2", "jarA", "v2")) + } + + val job = CoroutineScope(Dispatchers.IO).launch { index.insertSource("jarA", "1:1", entries) } + + // The first batch (batchSize = 1) has already committed by the time the sequence + // asks to produce its second entry. + secondEntryReached.awaitOrFail() + + job.cancel() + proceedWithSecondEntry.countDown() + job.cancelAndJoin() + + assertThat(index.get("k1")!!.value).isEqualTo("v1") + assertThat(index.get("k2")).isNull() + // The fingerprint is only ever written alongside the source's last batch; a cancelled + // insert must not record one, or a later call would wrongly treat "jarA" as up to date. + assertThat(index.sourceFingerprint("jarA")).isNull() + } + + private companion object { + const val DB_NAME = "cancellation_test.db" + } +} From 5b9929fa9d9c788d454134a8b99061a52bc43c77 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:26:06 +0000 Subject: [PATCH 28/48] ADFA-6228: Fingerprint generated JARs instead of caching by path alone JvmGeneratedIndexingService skipped a JAR once any row existed for its path, so a JAR rewritten outside an in-app build, or left half-indexed by a killed process, stayed stale until the next build touched it. Reusing the library indexer's size+mtime fingerprint means the unconditional re-index on every build completion is no longer needed either: a JAR whose fingerprint didn't change is now skipped there too. Documents jarFingerprint's accepted limitation: a same-size rewrite within the filesystem's mtime granularity goes undetected. --- .../jvm/JvmGeneratedIndexingService.kt | 26 ++++++++++--------- .../indexing/jvm/JvmLibraryIndexingService.kt | 4 +++ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt index bd677ecac2..1244add296 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt @@ -14,6 +14,7 @@ import org.appdevforall.codeonthego.indexing.service.IndexKey import org.appdevforall.codeonthego.indexing.service.IndexRegistry import org.appdevforall.codeonthego.indexing.service.IndexingService import org.slf4j.LoggerFactory +import java.io.File import java.nio.file.Paths import kotlin.io.path.extension @@ -30,9 +31,11 @@ val JVM_GENERATED_SYMBOL_INDEX = IndexKey("jvm-generated-symbols * [IndexingService] that scans build-generated JARs (R.jar, etc.) and * maintains a dedicated [JvmSymbolIndex] for them. * - * Generated JARs are re-indexed unconditionally on every build completion - * because their contents change (new R-field values, new resource IDs) even - * when the set of JARs doesn't change. + * A build completion re-scans every generated JAR's size and modification time (see + * [jarFingerprint]) and only re-indexes those whose fingerprint changed, the same check + * [JvmLibraryIndexingService] uses. This also catches a generated JAR rewritten outside an + * in-app build, or left half-indexed by a killed process, which staying cached by path alone + * would otherwise miss until some later build happened to touch it again. */ class JvmGeneratedIndexingService( private val context: Context, @@ -66,22 +69,20 @@ class JvmGeneratedIndexingService( // Kick off an initial index pass for any already-built JARs. coroutineScope.launch { - val jobs = indexingMutex.withLock { reindexGeneratedJars(forceReindex = false) } + val jobs = indexingMutex.withLock { reindexGeneratedJars() } generatedIndex?.optimizeAfter(jobs) } } override suspend fun onBuildCompleted() { - // Generated JARs (especially R.jar) always change after a build — - // their field values are regenerated. Force a full re-index. coroutineScope.launch { - val jobs = indexingMutex.withLock { reindexGeneratedJars(forceReindex = true) } + val jobs = indexingMutex.withLock { reindexGeneratedJars() } generatedIndex?.optimizeAfter(jobs) } } - /** Submits every generated JAR that needs indexing and returns the submitted jobs. */ - private suspend fun reindexGeneratedJars(forceReindex: Boolean): List { + /** Submits every generated JAR whose fingerprint changed and returns the submitted jobs. */ + private suspend fun reindexGeneratedJars(): List { val index = this.generatedIndex ?: run { log.warn("Not indexing generated JARs — index not initialized.") @@ -111,16 +112,17 @@ class JvmGeneratedIndexingService( val jobs = mutableListOf() for (jarPath in generatedJars) { - if (forceReindex || !index.isCached(jarPath)) { + val fingerprint = jarFingerprint(File(jarPath)) + if (index.sourceFingerprint(jarPath) != fingerprint) { jobs += - index.indexSource(jarPath, skipIfExists = false) { sourceId -> + index.indexSource(jarPath, skipIfExists = true, fingerprint = fingerprint) { sourceId -> CombinedJarScanner.scan(Paths.get(jarPath), sourceId) } } } if (jobs.isNotEmpty()) { - log.info("{} generated JARs submitted for background indexing (force={})", jobs.size, forceReindex) + log.info("{} generated JARs submitted for background indexing", jobs.size) } else { log.info("All generated JARs already cached, nothing to index") } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt index e9b0228e17..0a4b1d1be8 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmLibraryIndexingService.kt @@ -159,5 +159,9 @@ class JvmLibraryIndexingService( /** * Identifies the content of [jar] by its size and last-modified time, which a rewrite of the file * changes without the cost of reading it. Stats the file, so never call it on the main thread. + * + * Accepted limitation: a rewrite that keeps the exact same size and lands within the + * filesystem's modification-time granularity (a second on some filesystems) produces the same + * fingerprint as the original, so that rewrite goes undetected. */ internal fun jarFingerprint(jar: File): String = "${jar.length()}:${jar.lastModified()}" From 5ddefc7d13e134cef08040280c8e0032e5e52c76 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 15:57:09 +0000 Subject: [PATCH 29/48] style: spotless reformat, no functional change --- .../fs/CachingJarFileSystemProvider.kt | 108 +++++++++--------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt b/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt index 8edfaf87a0..6194028655 100644 --- a/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt +++ b/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt @@ -33,67 +33,71 @@ import kotlin.io.path.pathString * @author Akash Yadav */ object CachingJarFileSystemProvider : JarFileSystemProvider() { - private val cachedFs = ConcurrentHashMap() + private val cachedFs = ConcurrentHashMap() - private val log = LoggerFactory.getLogger(CachingJarFileSystemProvider::class.java) + private val log = LoggerFactory.getLogger(CachingJarFileSystemProvider::class.java) - override fun createFs(path: Path, env: MutableMap?): ZipFileSystem { - val cached = cachedFs[path.normalize().pathString] - if (cached != null) { - return cached - } - return createAndCache(path, env) - } + override fun createFs( + path: Path, + env: MutableMap?, + ): ZipFileSystem { + val cached = cachedFs[path.normalize().pathString] + if (cached != null) { + return cached + } + return createAndCache(path, env) + } - fun newFileSystem(path: Path): FileSystem? { - return newFileSystem(path, mutableMapOf()) - } + fun newFileSystem(path: Path): FileSystem? = newFileSystem(path, mutableMapOf()) - fun clearCache() { - cachedFs.values.forEach(this::closeFs) - cachedFs.clear() - } + fun clearCache() { + cachedFs.values.forEach(this::closeFs) + cachedFs.clear() + } - fun clearCaches(predicate: (Path) -> Boolean) { - return clearCachesForPaths { predicate(Paths.get(it)) } - } + fun clearCaches(predicate: (Path) -> Boolean) = clearCachesForPaths { predicate(Paths.get(it)) } - fun clearCachesForPaths(predicate: (String) -> Boolean) { - val toRemove = - this.cachedFs.keys.mapNotNull { - return@mapNotNull if (predicate(it)) { - it - } else null - } + fun clearCachesForPaths(predicate: (String) -> Boolean) { + val toRemove = + this.cachedFs.keys.mapNotNull { + return@mapNotNull if (predicate(it)) { + it + } else { + null + } + } - if (toRemove.isNotEmpty()) { - toRemove.forEach(this::clearCache) - } - } + if (toRemove.isNotEmpty()) { + toRemove.forEach(this::clearCache) + } + } - fun clearCache(path: Path) { - clearCache(path.normalize().pathString) - } + fun clearCache(path: Path) { + clearCache(path.normalize().pathString) + } - fun clearCache(path: String) { - val fs = cachedFs.remove(path) - if (fs != null) { - log.debug("Clearing cached JAR file system for path: {}", path) - closeFs(fs) - } - } + fun clearCache(path: String) { + val fs = cachedFs.remove(path) + if (fs != null) { + log.debug("Clearing cached JAR file system for path: {}", path) + closeFs(fs) + } + } - private fun closeFs(fs: CachedJarFileSystem) { - try { - fs.doClose() - } catch (err: Throwable) { - log.error("Failed to close cached zip file system: {}", fs, err) - } - } + private fun closeFs(fs: CachedJarFileSystem) { + try { + fs.doClose() + } catch (err: Throwable) { + log.error("Failed to close cached zip file system: {}", fs, err) + } + } - private fun createAndCache(path: Path, env: MutableMap?): CachedJarFileSystem { - val fs = CachedJarFileSystem(this, path, env) - cachedFs[path.normalize().pathString] = fs - return fs - } + private fun createAndCache( + path: Path, + env: MutableMap?, + ): CachedJarFileSystem { + val fs = CachedJarFileSystem(this, path, env) + cachedFs[path.normalize().pathString] = fs + return fs + } } From 9ae7ec01059ddaaf909bf1c64d6f0cf068ecdf27 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 15:57:51 +0000 Subject: [PATCH 30/48] ADFA-6228: Open each cached JAR file system once under contention createFs read the cache and then put, so several callers racing on one path each built a file system. Only the last reached the map; the others were never closed while a walk still held them, and the JARs most exposed to it are the most shared ones - kotlin-stdlib is named by 76 of this repository's modules. --- .../fs/CachingJarFileSystemProvider.kt | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt b/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt index 6194028655..e049984017 100644 --- a/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt +++ b/subprojects/javac-services/src/main/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProvider.kt @@ -41,11 +41,15 @@ object CachingJarFileSystemProvider : JarFileSystemProvider() { path: Path, env: MutableMap?, ): ZipFileSystem { - val cached = cachedFs[path.normalize().pathString] - if (cached != null) { - return cached + /* + * computeIfAbsent, not get-then-put: project setup opens the same JAR from many threads at + * once, and a check-then-act would build a file system per racing caller. Only the last would + * reach the map, so the others are never closed while a walk still holds them -- and the JARs + * most exposed to it are the most shared ones. + */ + return cachedFs.computeIfAbsent(path.normalize().pathString) { + CachedJarFileSystem(this, path, env) } - return createAndCache(path, env) } fun newFileSystem(path: Path): FileSystem? = newFileSystem(path, mutableMapOf()) @@ -91,13 +95,4 @@ object CachingJarFileSystemProvider : JarFileSystemProvider() { log.error("Failed to close cached zip file system: {}", fs, err) } } - - private fun createAndCache( - path: Path, - env: MutableMap?, - ): CachedJarFileSystem { - val fs = CachedJarFileSystem(this, path, env) - cachedFs[path.normalize().pathString] = fs - return fs - } } From f8e08be1c30ed5d84f6124bc0ef9cdc3664b9cc6 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:04:51 +0000 Subject: [PATCH 31/48] ADFA-6228: Add a regression test for concurrent createFs 32 threads racing createFs for the same JAR path each get a distinct CachedJarFileSystem on the pre-fix get-then-put code; computeIfAbsent guarantees they all get the same instance. --- ...ingJarFileSystemProviderConcurrencyTest.kt | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 subprojects/javac-services/src/test/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProviderConcurrencyTest.kt diff --git a/subprojects/javac-services/src/test/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProviderConcurrencyTest.kt b/subprojects/javac-services/src/test/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProviderConcurrencyTest.kt new file mode 100644 index 0000000000..28e88b0086 --- /dev/null +++ b/subprojects/javac-services/src/test/java/com/itsaky/androidide/javac/services/fs/CachingJarFileSystemProviderConcurrencyTest.kt @@ -0,0 +1,100 @@ +/* + * This file is part of AndroidIDE. + * + * AndroidIDE is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * AndroidIDE is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with AndroidIDE. If not, see . + */ + +package com.itsaky.androidide.javac.services.fs + +import com.google.common.truth.Truth.assertThat +import org.junit.After +import org.junit.Before +import org.junit.Test +import java.io.File +import java.nio.file.FileSystem +import java.nio.file.Files +import java.nio.file.Path +import java.util.Collections +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +/** + * Concurrent [CachingJarFileSystemProvider.createFs] calls for the same JAR path must open that + * JAR's file system exactly once. + */ +class CachingJarFileSystemProviderConcurrencyTest { + private lateinit var tmpDir: File + private lateinit var jarPath: Path + + @Before + fun setUp() { + // The provider caches by normalized path; start clean so prior runs cannot mask the race. + CachingJarFileSystemProvider.clearCache() + + tmpDir = Files.createTempDirectory("createfs-race").toFile() + jarPath = File(tmpDir, "test.jar").toPath() + writeMinimalJar(jarPath) + } + + @After + fun tearDown() { + CachingJarFileSystemProvider.clearCache() + tmpDir.deleteRecursively() + } + + @Test + fun concurrentCreateFsOpensTheSameJarOnce() { + val threadCount = 32 + val ready = CountDownLatch(threadCount) + val start = CountDownLatch(1) + val results = Collections.synchronizedList(mutableListOf()) + val executor = Executors.newFixedThreadPool(threadCount) + + repeat(threadCount) { + executor.submit { + ready.countDown() + start.await() + results.add(checkNotNull(CachingJarFileSystemProvider.newFileSystem(jarPath))) + } + } + + assertThat(ready.await(5, TimeUnit.SECONDS)).isTrue() + start.countDown() + executor.shutdown() + assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue() + + assertThat(results).hasSize(threadCount) + /* + * Referential identity is the point: computeIfAbsent must hand every racing caller the + * same CachedJarFileSystem instance instead of each caller building and briefly holding + * its own, only for all but one to be overwritten and never closed. + */ + assertThat(results.toSet()).hasSize(1) + } +} + +private fun writeMinimalJar(path: Path) { + ZipOutputStream(Files.newOutputStream(path)).use { zip -> + zip.putNextEntry(ZipEntry("META-INF/MANIFEST.MF")) + zip.write("Manifest-Version: 1.0\n".toByteArray()) + zip.closeEntry() + + zip.putNextEntry(ZipEntry("com/example/Placeholder.class")) + zip.write(ByteArray(16)) + zip.closeEntry() + } +} From c70b364a409bc4e05b698ace1c6d4326dbf214d4 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:39:37 +0000 Subject: [PATCH 32/48] ADFA-6228: Delete the unused per-scanner JAR entry points --- .../codeonthego/indexing/jvm/JarSymbolScanner.kt | 8 +++----- .../codeonthego/indexing/jvm/KotlinMetadataScanner.kt | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt index 2abe775ea3..bf0dce9755 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt @@ -10,12 +10,10 @@ import org.jetbrains.org.objectweb.asm.Type import java.io.InputStream /** - * Parses class files with ASM into [JvmSymbol]s. + * Parses a Java class file with ASM into [JvmSymbol]s. * - * For Java class files, this gives complete information. - * For Kotlin class files, use [KotlinMetadataScanner] or - * [CombinedJarScanner] instead — ASM cannot see Kotlin-specific - * semantics like extensions, suspend, or nullable types. + * [CombinedJarScanner] routes Kotlin class files to [KotlinMetadataScanner] instead: ASM cannot see + * Kotlin-specific semantics like extensions, suspend, or nullable types. */ object JarSymbolScanner { internal fun parseClassFile( diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt index ed073685db..957c4e256d 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -34,10 +34,10 @@ import kotlin.metadata.modality import kotlin.metadata.visibility /** - * Parses Kotlin metadata from class files into [JvmSymbol]s - * with full Kotlin semantics (extensions, suspend, inline, etc.). + * Parses a Kotlin class file's `@Metadata` into [JvmSymbol]s with full Kotlin semantics + * (extensions, suspend, inline, etc.). * - * Skips non-Kotlin class files (no `@Metadata` annotation). + * Returns nothing for a class file without `@Metadata`. */ object KotlinMetadataScanner { private val log = LoggerFactory.getLogger(KotlinMetadataScanner::class.java) From fdedc440ac4634f962fe0c3e8b9766077427ea89 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:44:35 +0000 Subject: [PATCH 33/48] ADFA-6228: Take nested class visibility and staticness from InnerClasses The class file header cannot carry private, protected or static, so every private nested class was indexed as package-private and every static nested class as inner. Kotlin completion derived isNested from isInner, which would have flipped for static nested classes, so it now uses isTopLevel. --- .../indexing/jvm/JarSymbolScanner.kt | 18 +++ .../jvm/JarSymbolScannerVisibilityTest.kt | 115 +++++++++++++++++- .../kotlin/completion/KotlinCompletions.kt | 2 +- 3 files changed, 131 insertions(+), 4 deletions(-) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt index bf0dce9755..1cc9166ebe 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt @@ -16,6 +16,9 @@ import java.io.InputStream * Kotlin-specific semantics like extensions, suspend, or nullable types. */ object JarSymbolScanner { + private const val VISIBILITY_AND_STATIC_FLAGS = + Opcodes.ACC_PUBLIC or Opcodes.ACC_PRIVATE or Opcodes.ACC_PROTECTED or Opcodes.ACC_STATIC + internal fun parseClassFile( input: InputStream, sourceId: String, @@ -69,6 +72,21 @@ object JarSymbolScanner { isInnerClass = name.contains('$') } + /* + * JVMS 4.1 forbids ACC_PRIVATE, ACC_PROTECTED and ACC_STATIC in a class file header, so javac + * records a nested class's declared visibility and staticness only in its own InnerClasses + * entry. ASM visits that entry before any member, so the member gates below see the real flags. + */ + override fun visitInnerClass( + name: String, + outerName: String?, + innerName: String?, + access: Int, + ) { + if (name != className) return + classAccess = (classAccess and VISIBILITY_AND_STATIC_FLAGS.inv()) or (access and VISIBILITY_AND_STATIC_FLAGS) + } + override fun visitAnnotation( descriptor: String?, visible: Boolean, diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt index 19f2664cad..414d1ed96a 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt @@ -1,11 +1,15 @@ package org.appdevforall.codeonthego.indexing.jvm import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage import org.jetbrains.org.objectweb.asm.ClassWriter import org.jetbrains.org.objectweb.asm.Opcodes import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 +import java.io.InputStream +import java.io.OutputStream +import java.nio.file.Files /** * Which classes [JarSymbolScanner] admits, by visibility. @@ -28,7 +32,29 @@ class JarSymbolScannerVisibilityTest { private fun scan( internalName: String, access: Int, - ): List = JarSymbolScanner.parseClassFile(classBytes(internalName, access).inputStream(), "test.jar") + ): List = scan(classBytes(internalName, access)) + + private fun scan(bytes: ByteArray): List = JarSymbolScanner.parseClassFile(bytes.inputStream(), "test.jar") + + private val nestedClasses: Map by lazy { + compileJava( + "com.example.Outer", + """ + package com.example; + + public class Outer { + private static class Hidden {} + protected static class Guarded {} + public static class Open {} + class Pkg {} + } + """.trimIndent(), + ) + } + + private fun scanNested(simpleName: String): List = scan(nestedClasses.getValue("com/example/Outer\$$simpleName")) + + private fun nestedClass(simpleName: String): JvmSymbol = scanNested(simpleName).single { it.kind.isJvmClass } @Test fun `a public class is indexed`() { @@ -48,8 +74,53 @@ class JarSymbolScannerVisibilityTest { @Test fun `a private nested class is not indexed`() { - // Nothing outside the declaring class can name it, so it is never a completion candidate. - assertThat(scan("com/example/Outer\$Hidden", Opcodes.ACC_PRIVATE)).isEmpty() + /* + * JVMS 4.1 forbids ACC_PRIVATE in a class file header, so javac writes ACC_SUPER there and + * records `private` only in the InnerClasses attribute. + */ + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_SUPER, "com/example/Outer\$Hidden", null, "java/lang/Object", null) + visitInnerClass( + "com/example/Outer\$Hidden", + "com/example/Outer", + "Hidden", + Opcodes.ACC_PRIVATE or Opcodes.ACC_STATIC, + ) + visitEnd() + }.toByteArray() + + assertThat(scan(bytes)).isEmpty() + } + + @Test + fun `a private nested class compiled by javac is not indexed`() { + assertThat(scanNested("Hidden")).isEmpty() + } + + @Test + fun `a protected nested class compiled by javac is indexed as protected`() { + // javac widens the header of a protected nested class to public. + assertThat(nestedClass("Guarded").visibility).isEqualTo(JvmVisibility.PROTECTED) + } + + @Test + fun `a static nested class compiled by javac is static, not inner`() { + val info = nestedClass("Open").data as JvmClassInfo + + assertThat(info.isStatic).isTrue() + assertThat(info.isInner).isFalse() + } + + @Test + fun `an inner class compiled by javac keeps package-private visibility and is inner`() { + val symbol = nestedClass("Pkg") + val info = symbol.data as JvmClassInfo + + assertThat(symbol.visibility).isEqualTo(JvmVisibility.PACKAGE_PRIVATE) + assertThat(info.isInner).isTrue() + assertThat(info.isStatic).isFalse() } @Test @@ -61,3 +132,41 @@ class JarSymbolScannerVisibilityTest { assertThat(nested.data.containingClassFqName).isEqualTo("com.example.Outer") } } + +/** + * Compiles one Java source file with the test JVM's own javac, returning each produced class file + * by internal name. + * + * Reached through reflection because this Android module compiles against `android.jar`, which has + * no `javax.tools`; the JDK running the tests does. + */ +private fun compileJava( + className: String, + source: String, +): Map { + val workDir = Files.createTempDirectory("javac-fixture") + try { + val sourceFile = workDir.resolve("${className.substringAfterLast('.')}.java") + sourceFile.toFile().writeText(source) + val outDir = Files.createDirectory(workDir.resolve("out")) + + val compiler = Class.forName("javax.tools.ToolProvider").getMethod("getSystemJavaCompiler").invoke(null) + assertWithMessage("the test JVM has no system Java compiler").that(compiler).isNotNull() + val run = + Class + .forName("javax.tools.Tool") + .getMethod("run", InputStream::class.java, OutputStream::class.java, OutputStream::class.java, Array::class.java) + val args = arrayOf("--release", "11", "-d", outDir.toString(), sourceFile.toString()) + val exitCode = run.invoke(compiler, null, null, null, args) as Int + assertWithMessage("javac exit code for $className").that(exitCode).isEqualTo(0) + + return Files.walk(outDir).use { paths -> + paths + .filter { it.toString().endsWith(".class") } + .toList() + .associate { outDir.relativize(it).toString().removeSuffix(".class") to Files.readAllBytes(it) } + } + } finally { + workDir.toFile().deleteRecursively() + } +} diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 348f24d247..59f99f345a 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -575,7 +575,7 @@ private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionIt item.detail = symbol.fqName item.setClassCompletionData( className = symbol.fqName, - isNested = classInfo.isInner, + isNested = !symbol.isTopLevel, topLevelClass = classInfo.containingClassFqName, ) } From 54b50ce0bec513980e2afb9c8703025542f3551b Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:47:04 +0000 Subject: [PATCH 34/48] ADFA-6228: Skip META-INF entries so multi-release copies cannot overwrite base classes --- .../indexing/jvm/CombinedJarScanner.kt | 23 +++-- .../jvm/CombinedJarScannerMultiReleaseTest.kt | 94 +++++++++++++++++++ 2 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt index e2d0fdecc5..0ccfdd026d 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScanner.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow import org.jetbrains.kotlin.analysis.api.KaImplementationDetail import org.jetbrains.kotlin.analysis.api.impl.base.util.LibraryUtils +import org.jetbrains.kotlin.com.intellij.openapi.vfs.VfsUtilCore import org.jetbrains.kotlin.com.intellij.openapi.vfs.VirtualFile import org.jetbrains.org.objectweb.asm.AnnotationVisitor import org.jetbrains.org.objectweb.asm.ClassReader @@ -30,8 +31,8 @@ object CombinedJarScanner { sequence { val allFiles = LibraryUtils.getAllVirtualFilesFromRoot(rootVf, includeRoot = true) for (vf in allFiles) { - if (!vf.name.endsWith(".class")) continue - if (vf.name == "module-info.class" || vf.name == "package-info.class") continue + val relativePath = VfsUtilCore.getRelativePath(vf, rootVf)?.takeIf { it.isNotEmpty() } ?: vf.name + if (!isIndexableClassEntry(relativePath)) continue try { val bytes = vf.contentsToByteArray() val symbols = @@ -65,11 +66,7 @@ object CombinedJarScanner { val entries = jar.entries() while (entries.hasMoreElements()) { val entry = entries.nextElement() - if (!entry.name.endsWith(".class")) continue - // Compared on the file name, not the full entry path: a multi-release JAR carries these - // under META-INF/versions//, where an exact-path match lets them through as classes. - val entryFileName = entry.name.substringAfterLast('/') - if (entryFileName == "module-info.class" || entryFileName == "package-info.class") continue + if (!isIndexableClassEntry(entry.name)) continue // emit stays outside the try so cancellation is not swallowed. val symbols = @@ -96,6 +93,18 @@ object CombinedJarScanner { } } + /* + * Everything under META-INF is skipped. A multi-release JAR's META-INF/versions// copy of a + * class shares the base class's key within one source, so indexing it would replace the base + * row. + */ + private fun isIndexableClassEntry(relativePath: String): Boolean { + if (!relativePath.endsWith(".class")) return false + if (relativePath.startsWith("META-INF/")) return false + val fileName = relativePath.substringAfterLast('/') + return fileName != "module-info.class" && fileName != "package-info.class" + } + private fun hasKotlinMetadata(classBytes: ByteArray): Boolean { var found = false try { diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt new file mode 100644 index 0000000000..7d4c628a3f --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt @@ -0,0 +1,94 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.com.intellij.mock.MockApplication +import org.jetbrains.kotlin.com.intellij.openapi.application.ApplicationManager +import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer +import org.jetbrains.kotlin.com.intellij.openapi.vfs.impl.jar.CoreJarFileSystem +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.util.jar.Attributes +import java.util.jar.JarEntry +import java.util.jar.JarOutputStream +import java.util.jar.Manifest + +/** + * A multi-release JAR must contribute its base classes only. + * + * Its `META-INF/versions//` copy of a class shares the base class's key within the same source, + * so indexing both let the versioned copy overwrite the base row. + */ +@RunWith(JUnit4::class) +class CombinedJarScannerMultiReleaseTest { + @get:Rule + val temp = TemporaryFolder() + + private fun classWithMethod( + internalName: String, + methodName: String, + ): ByteArray = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, internalName, null, "java/lang/Object", null) + visitMethod(Opcodes.ACC_PUBLIC, methodName, "()V", null, null).visitEnd() + visitEnd() + }.toByteArray() + + private fun multiReleaseJar(): File { + val manifest = + Manifest().apply { + mainAttributes[Attributes.Name.MANIFEST_VERSION] = "1.0" + mainAttributes[Attributes.Name("Multi-Release")] = "true" + } + val jar = temp.newFile("multi-release.jar") + JarOutputStream(jar.outputStream(), manifest).use { out -> + fun put( + path: String, + bytes: ByteArray, + ) { + out.putNextEntry(JarEntry(path)) + out.write(bytes) + out.closeEntry() + } + put("com/example/Widget.class", classWithMethod("com/example/Widget", "base")) + put("META-INF/versions/11/com/example/Widget.class", classWithMethod("com/example/Widget", "versioned")) + put("META-INF/versions/11/com/example/Extra.class", classWithMethod("com/example/Extra", "extra")) + } + return jar + } + + private fun assertOnlyBaseClasses(symbols: List) { + assertThat(symbols.filter { it.kind.isJvmClass }.map { it.key }).containsExactly("com/example/Widget") + assertThat(symbols.filter { it.kind == JvmSymbolKind.FUNCTION }.map { it.shortName }).containsExactly("base") + } + + @Test + fun `a jar path scan skips versioned entries`() { + val jar = multiReleaseJar() + + assertOnlyBaseClasses(runBlocking { CombinedJarScanner.scan(jar.toPath(), "mr").toList() }) + } + + @Test + fun `a virtual file scan skips versioned entries`() { + val jar = multiReleaseJar() + val disposable = Disposer.newDisposable() + try { + // Reading a jar entry's contents consults the application's file size limits. + ApplicationManager.setApplication(MockApplication(disposable), disposable) + val root = CoreJarFileSystem().findFileByPath("${jar.absolutePath}!/")!! + + assertOnlyBaseClasses(CombinedJarScanner.scan(root, "mr").toList()) + } finally { + Disposer.dispose(disposable) + } + } +} From 619880a9d5780bed94a2b46c211d9e2fc5f405e7 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:50:17 +0000 Subject: [PATCH 35/48] ADFA-6228: Name Kotlin classes by their class file internal name --- .../indexing/jvm/KotlinMetadataScanner.kt | 29 ++++--- .../KotlinMetadataScannerDollarNameTest.kt | 77 +++++++++++++++++++ .../jvm/KotlinMetadataScannerShapeTest.kt | 26 +++++++ 3 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerDollarNameTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt index 957c4e256d..5fd5600858 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -65,7 +65,7 @@ object KotlinMetadataScanner { return when (metadata) { is KotlinClassMetadata.Class -> { - extractFromClass(metadata.kmClass, sourceId) + extractFromClass(metadata.kmClass, collector.internalName, sourceId) } is KotlinClassMetadata.FileFacade -> { @@ -123,37 +123,44 @@ object KotlinMetadataScanner { ) } + /** + * Symbols for [klass] and its members. + * + * [className] is the class file's internal name (`kotlin/text/Regex$Companion`), not the metadata + * name (`kotlin/text/Regex.Companion`): JarSymbolScanner and SourceFileIndexer key and nest by the + * former, and a containing-class lookup only matches when every scanner spells it the same way. + */ private fun extractFromClass( klass: KmClass, + className: String, sourceId: String, ): List { val symbols = mutableListOf() - val className = klass.name + val metadataName = klass.name val packageName = - className - .substringBeforeLast('/') + metadataName + .substringBeforeLast('/', "") .replace('/', '.') /* * A leading '.' marks a local or anonymous class. The classpath trie never held these -- it * treated any name containing '$' as not top level -- and neither language names them, so * offering one as a completion would be noise. JarSymbolScanner drops its equivalents too. */ - if (className.startsWith('.')) { + if (metadataName.startsWith('.')) { return symbols } /* - * Kotlin metadata nests with '.' in source order and with '$' once compiled, so a nested class - * has to be split on either. Splitting on '$' alone left the short name as "Outer.Inner", which - * no prefix search for "Inner" could match; splitting on '.' alone left a compiled-nested class - * looking top level. + * Metadata nests with '.', so here a '$' can only be part of a declared name. It is still split + * on because the classpath trie treated every '$' as nesting, and the index should hold the + * same top-level classes. */ - val simpleNames = className.substringAfterLast('/') + val simpleNames = metadataName.substringAfterLast('/') val lastSeparator = maxOf(simpleNames.lastIndexOf('.'), simpleNames.lastIndexOf('$')) val shortName = if (lastSeparator >= 0) simpleNames.substring(lastSeparator + 1) else simpleNames val containingClassName = if (lastSeparator >= 0) { - className.substring(0, className.length - (simpleNames.length - lastSeparator)) + className.substring(0, className.length - shortName.length - 1) } else { "" } diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerDollarNameTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerDollarNameTest.kt new file mode 100644 index 0000000000..7f9707e80f --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerDollarNameTest.kt @@ -0,0 +1,77 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import kotlin.metadata.ClassKind +import kotlin.metadata.KmClass +import kotlin.metadata.KmClassifier +import kotlin.metadata.KmType +import kotlin.metadata.Visibility +import kotlin.metadata.jvm.JvmMetadataVersion +import kotlin.metadata.jvm.KotlinClassMetadata +import kotlin.metadata.kind +import kotlin.metadata.visibility + +/** + * A '$' in a Kotlin class's metadata name is part of a declared name, never nesting, yet the index + * treats it as nesting so it holds the same top-level classes the classpath trie did. + */ +@RunWith(JUnit4::class) +class KotlinMetadataScannerDollarNameTest { + /** A class file for [name] carrying genuine `@kotlin.Metadata` written by kotlin-metadata-jvm. */ + private fun kotlinClassFile(name: String): ByteArray { + val kmClass = + KmClass().apply { + this.name = name + kind = ClassKind.CLASS + visibility = Visibility.PUBLIC + supertypes += KmType().apply { classifier = KmClassifier.Class("kotlin/Any") } + } + val metadata = KotlinClassMetadata.Class(kmClass, JvmMetadataVersion.LATEST_STABLE_SUPPORTED, 0).write() + + return ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC or Opcodes.ACC_SUPER, name, null, "java/lang/Object", null) + visitAnnotation("Lkotlin/Metadata;", true).apply { + visit("k", metadata.kind) + visit("mv", metadata.metadataVersion) + visit("xi", metadata.extraInt) + visitArray("d1").apply { + metadata.data1.forEach { visit(null, it) } + visitEnd() + } + visitArray("d2").apply { + metadata.data2.forEach { visit(null, it) } + visitEnd() + } + visitEnd() + } + visitEnd() + }.toByteArray() + } + + private fun classSymbol(name: String): JvmSymbol = + KotlinMetadataScanner + .parseKotlinClass(kotlinClassFile(name).inputStream(), "test.jar")!! + .single { it.kind.isClassifier } + + @Test + fun `a dollar in a Kotlin class name is treated as nesting`() { + val symbol = classSymbol("com/example/Odd\$Name") + + assertThat(symbol.shortName).isEqualTo("Name") + assertThat(symbol.containingClassName).isEqualTo("com/example/Odd") + } + + @Test + fun `a Kotlin class without a dollar is top level`() { + val symbol = classSymbol("com/example/Plain") + + assertThat(symbol.shortName).isEqualTo("Plain") + assertThat(symbol.isTopLevel).isTrue() + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt index 1da9a8694b..365038fd2c 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt @@ -51,6 +51,32 @@ class KotlinMetadataScannerShapeTest { assertThat(companion.data.containingClassFqName).isEqualTo("kotlin.text.Regex") } + private fun classifierFor(entryName: String): JvmSymbol = symbolsFor(entryName).single { it.kind.isClassifier } + + @Test + fun `a nested class is keyed by its class file name, as the Java scanner keys it`() { + val companion = classifierFor("kotlin/text/Regex\$Companion.class") + + // Metadata spells it "kotlin/text/Regex.Companion"; the class file and JarSymbolScanner use '$'. + assertThat(companion.key).isEqualTo("kotlin/text/Regex\$Companion") + assertThat((companion.data as JvmClassInfo).internalName).isEqualTo("kotlin/text/Regex\$Companion") + } + + @Test + fun `a doubly nested class names its containing class in class file form`() { + val companion = classifierFor("kotlin/text/Regex\$Serialized\$Companion.class") + + assertThat(companion.containingClassName).isEqualTo("kotlin/text/Regex\$Serialized") + } + + @Test + fun `members of a nested class name it in class file form`() { + val members = symbolsFor("kotlin/text/Regex\$Companion.class").filter { it.kind.isCallable } + + assertThat(members).isNotEmpty() + assertThat(members.map { it.containingClassName }.toSet()).containsExactly("kotlin/text/Regex\$Companion") + } + @Test fun `a file facade is indexed as a class in its own right`() { val symbols = symbolsFor("kotlin/io/CloseableKt.class") From 098bcafc90a166bfd2c611caf3a09178c1903c59 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:52:50 +0000 Subject: [PATCH 36/48] ADFA-6228: Offer no file facades or companion objects as unimported completions --- .../indexing/jvm/JvmSymbolIndex.kt | 4 + .../jvm/JvmSymbolIndexKindScopeTest.kt | 12 +++ .../kotlin/completion/KotlinCompletions.kt | 38 ++++++--- .../UnimportedSymbolCandidateTest.kt | 79 +++++++++++++++++++ 4 files changed, 124 insertions(+), 9 deletions(-) create mode 100644 lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index 9d05f618b0..995baf24eb 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -106,15 +106,19 @@ open class JvmSymbolIndex( * * @param prefix The prefix to search for. * @param limit The result limit. + * @param kinds When non-null, only symbols of these kinds, filtered in the query so that the + * limit is not spent on rows the caller would discard. * @see query */ fun findByPrefix( prefix: String, limit: Int = 200, + kinds: Set? = null, ): Sequence = query( indexQuery { prefix(KEY_NAME, prefix) + if (kinds != null) anyOf(KEY_KIND, kinds.map { it.name }) this.limit = limit }, ) diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt index 838f318ace..e51a84c955 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndexKindScopeTest.kt @@ -119,6 +119,18 @@ class JvmSymbolIndexKindScopeTest { assertThat(classifiersOnly.map { it.kind }).containsExactly(JvmSymbolKind.CLASS) } + @Test + fun `a prefix lookup can be restricted by kind`() = + runTest { + val index = index() + index.insert(classifier("Widget")) + index.insert(member("WidgetFactory", owner = "Factory")) + + val found = index.findByPrefix("Widget", kinds = JvmSymbolKind.CLASSIFIER_KINDS).toList() + + assertThat(found.map { it.shortName }).containsExactly("Widget") + } + @Test fun `a file facade is not offered as a classifier`() = runTest { diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 59f99f345a..cd8ece894a 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -460,7 +460,10 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) collectUpToLimit( limit = UNIMPORTED_SYMBOL_DISPLAY_LIMIT, - sources = indexes.map { index -> { index.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_FETCH_BUDGET) } }, + sources = + indexes.map { index -> + { index.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_FETCH_BUDGET, kinds = UNIMPORTED_SYMBOL_KINDS) } + }, accept = ::addCompletionItem, ) } @@ -500,13 +503,28 @@ internal fun collectUpToLimit( return accepted } +/** + * The kinds unimported-symbol completion can offer: classifiers and callables. + * + * A file facade is neither, and Kotlin cannot name one. A companion object is reached through its + * class, so offering every `Companion` in the index by that bare name only adds noise. + */ +internal val UNIMPORTED_SYMBOL_KINDS: Set = + (JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT) + JvmSymbolKind.CALLABLE_KINDS + +/** + * Whether [symbol] can be offered as an unimported completion, whose bare name Kotlin resolves once + * the item's auto-import is applied. + */ +internal fun isUnimportedSymbolCandidate(symbol: JvmSymbol): Boolean { + if (symbol.kind !in UNIMPORTED_SYMBOL_KINDS) return false + // A member callable is reached through its receiver, never imported by name. + return !symbol.kind.isCallable || symbol.isTopLevel || symbol.isExtension +} + context(ctx: AnalysisContext) private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionItem? { - if (symbol.kind.isCallable && !symbol.isTopLevel && !symbol.isExtension) { - // member-level, non-extension callable symbols should not be - // completed in scope completions - return null - } + if (!isUnimportedSymbolCandidate(symbol)) return null abortIfCancelled() @@ -580,7 +598,9 @@ private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionIt ) } - else -> {} + else -> { + return null + } } return item @@ -954,8 +974,8 @@ private fun KaSession.kindOf(symbol: JvmSymbol): CompletionItemKind = JvmSymbolKind.TYPE_ALIAS -> CompletionItemKind.CLASS - // Kotlin completion should never reach a facade -- it is not a classifier, and the - // declarations it holds are offered individually -- but the mapping has to be total. + // Unimported-symbol completion rejects facades before building an item, since Kotlin + // cannot name one; this branch only keeps the mapping total. JvmSymbolKind.FILE_FACADE -> CompletionItemKind.CLASS } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt new file mode 100644 index 0000000000..c11cdc75ef --- /dev/null +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt @@ -0,0 +1,79 @@ +package com.itsaky.androidide.lsp.kotlin.completion + +import com.google.common.truth.Truth.assertThat +import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * Which index symbols unimported-symbol completion may offer. + * + * Accepting an item inserts its bare name, so only a symbol Kotlin can resolve after the + * auto-import is worth offering. + */ +@RunWith(JUnit4::class) +class UnimportedSymbolCandidateTest { + private fun symbol( + shortName: String, + kind: JvmSymbolKind, + data: JvmSymbolInfo = JvmClassInfo(internalName = "kotlin/text/$shortName"), + ) = JvmSymbol( + key = "kotlin/text/$shortName", + sourceId = "stdlib.jar", + name = "kotlin/text/$shortName", + shortName = shortName, + packageName = "kotlin.text", + kind = kind, + language = JvmSourceLanguage.KOTLIN, + data = data, + ) + + @Test + fun `a file facade is not offered`() { + // Kotlin cannot name `StringsKt`, so inserting it would not resolve. + assertThat(isUnimportedSymbolCandidate(symbol("StringsKt", JvmSymbolKind.FILE_FACADE))).isFalse() + } + + @Test + fun `a companion object is not offered by its bare name`() { + val companion = + symbol( + "Companion", + JvmSymbolKind.COMPANION_OBJECT, + JvmClassInfo(internalName = "kotlin/text/Regex\$Companion", containingClassName = "kotlin/text/Regex"), + ) + + assertThat(isUnimportedSymbolCandidate(companion)).isFalse() + } + + @Test + fun `a class is offered`() { + assertThat(isUnimportedSymbolCandidate(symbol("Regex", JvmSymbolKind.CLASS))).isTrue() + } + + @Test + fun `a top-level function is offered`() { + assertThat(isUnimportedSymbolCandidate(symbol("trimIndent", JvmSymbolKind.FUNCTION, JvmFunctionInfo()))).isTrue() + } + + @Test + fun `a member function is not offered`() { + val member = + symbol("find", JvmSymbolKind.FUNCTION, JvmFunctionInfo(containingClassName = "kotlin/text/Regex")) + + assertThat(isUnimportedSymbolCandidate(member)).isFalse() + } + + @Test + fun `the prefix query asks for exactly the kinds that can be offered`() { + assertThat(UNIMPORTED_SYMBOL_KINDS).doesNotContain(JvmSymbolKind.FILE_FACADE) + assertThat(UNIMPORTED_SYMBOL_KINDS).doesNotContain(JvmSymbolKind.COMPANION_OBJECT) + assertThat(UNIMPORTED_SYMBOL_KINDS).containsAtLeastElementsIn(JvmSymbolKind.CALLABLE_KINDS) + } +} From 19c32f3f2966452832994b1c33372031bae4a405 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 16:55:27 +0000 Subject: [PATCH 37/48] ADFA-6228: Fail rather than skip the differential test, and add a Java corpus The kotlin-stdlib comparison found two defects when it was written: anonymous Kotlin classes surfacing as top-level classes named `1`, and `module-info` from a multi-release JAR indexed as a class. JUnit 4 adds package-private, anonymous and nested Java classes to the same oracle. --- .../jvm/ClasspathTrieDifferentialTest.kt | 92 +++++++++++-------- .../jvm/KotlinMetadataScannerShapeTest.kt | 26 +++--- 2 files changed, 66 insertions(+), 52 deletions(-) diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt index 30d29ed5ef..23e7f8f0c8 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/ClasspathTrieDifferentialTest.kt @@ -1,15 +1,15 @@ package org.appdevforall.codeonthego.indexing.jvm import com.google.common.truth.Truth.assertThat +import com.google.common.truth.Truth.assertWithMessage import com.itsaky.androidide.projects.classpath.JarFsClasspathReader import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.toSet +import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking -import org.junit.Assume.assumeTrue +import org.junit.Assume.assumeFalse import org.junit.Test import org.junit.runner.RunWith -import org.junit.runners.JUnit4 +import org.junit.runners.Parameterized import java.io.File import java.nio.file.Paths @@ -20,24 +20,41 @@ import java.nio.file.Paths * literally what filled `ModuleProject.compileClasspathClasses`. Building it from the index instead * would let the new code agree with itself and pin nothing. * - * It runs against a real JAR because the cases that matter -- Kotlin file facades, multi-file class - * parts, package-private classes, anonymous and nested types, multi-release entries -- do not occur - * in a hand-built fixture. Two defects were found this way: anonymous Kotlin classes surfacing as - * top-level classes named `1`, and `module-info` from a multi-release JAR indexed as a class. + * It runs against real JARs because the cases that matter do not occur in a hand-built fixture: + * Kotlin file facades, multi-file class parts and anonymous Kotlin classes in kotlin-stdlib, and + * package-private, anonymous and nested Java classes in JUnit. */ -@RunWith(JUnit4::class) -class ClasspathTrieDifferentialTest { - private val jar: File? = - runCatching { +@RunWith(Parameterized::class) +class ClasspathTrieDifferentialTest( + private val corpus: String, + private val anchor: Class<*>, +) { + companion object { + @JvmStatic + @Parameterized.Parameters(name = "{0}") + fun corpora(): List> = + listOf( + arrayOf("kotlin-stdlib", Unit::class.java), + arrayOf("junit", Test::class.java), + ) + } + + private val jar: File by lazy { + val location = File( - Unit::class.java.protectionDomain.codeSource.location + anchor.protectionDomain.codeSource.location .toURI(), ) - }.getOrNull() - ?.takeIf { it.isFile && it.name.endsWith(".jar") } + assertWithMessage("$corpus must resolve to a JAR on the test classpath, got $location") + .that(location.isFile && location.name.endsWith(".jar")) + .isTrue() + location + } + + private val isKotlinCorpus get() = anchor == Unit::class.java /** What the trie held: every top-level class the classpath reader found. */ - private fun trieNames(jar: File): Set = + private fun trieNames(): Set = JarFsClasspathReader() .listClasses(listOf(jar)) .asSequence() @@ -45,16 +62,17 @@ class ClasspathTrieDifferentialTest { .map { it.name } .toSet() - /** What the index offers for the same JAR. */ - private fun indexNames(jar: File): Set = + private fun indexClasses(): List = runBlocking { CombinedJarScanner .scan(Paths.get(jar.absolutePath), "differential") .filter { it.kind.isJvmClass && it.isTopLevel } - .map { it.fqName } - .toSet() + .toList() } + /** What the index offers for the same JAR. */ + private fun indexNames(): Set = indexClasses().map { it.fqName }.toSet() + /** * A fragment of a `@JvmMultifileClass` facade, e.g. `LazyKt__LazyJVMKt`. * @@ -63,44 +81,40 @@ class ClasspathTrieDifferentialTest { */ private fun isMultiFilePart(qualifiedName: String) = qualifiedName.substringAfterLast('.').contains("__") - private fun requireJar(): File { - assumeTrue("kotlin-stdlib not resolvable from the test classpath", jar != null) - return jar!! - } - @Test fun `every class the trie held is offered by the index, bar multi-file parts`() { - val jar = requireJar() - - val missing = trieNames(jar) - indexNames(jar) + val missing = trieNames() - indexNames() assertThat(missing.filterNot(::isMultiFilePart)).isEmpty() } @Test fun `the index offers nothing the trie did not hold`() { - val jar = requireJar() - - // Anonymous Kotlin classes and multi-release `module-info` entries both used to land here. - assertThat(indexNames(jar) - trieNames(jar)).isEmpty() + assertThat(indexNames() - trieNames()).isEmpty() } @Test - fun `the withheld classes really are multi-file parts, and there are some`() { - val jar = requireJar() - - val missing = trieNames(jar) - indexNames(jar) + fun `the withheld classes really are multi-file parts, and a Kotlin corpus has some`() { + val missing = trieNames() - indexNames() // Guards the exception itself: if the corpus stopped containing parts, the first test would // start passing for the wrong reason. - assertThat(missing).isNotEmpty() assertThat(missing.all(::isMultiFilePart)).isTrue() + if (isKotlinCorpus) assertThat(missing).isNotEmpty() } @Test fun `the corpus is large enough for the comparison to mean something`() { - val jar = requireJar() + assertThat(trieNames().size).isGreaterThan(if (isKotlinCorpus) 500 else 150) + } + + @Test + fun `a Java corpus holds the package-private, anonymous and nested classes it is here for`() { + assumeFalse("only the Java corpus is chosen for these shapes", isKotlinCorpus) + val classes = JarFsClasspathReader().listClasses(listOf(jar)) - assertThat(trieNames(jar).size).isGreaterThan(500) + assertThat(indexClasses().any { it.visibility == JvmVisibility.PACKAGE_PRIVATE }).isTrue() + assertThat(classes.any { it.isAnonymous }).isTrue() + assertThat(classes.any { it.isInner }).isTrue() } } diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt index 365038fd2c..a98748b63a 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt @@ -1,7 +1,7 @@ package org.appdevforall.codeonthego.indexing.jvm import com.google.common.truth.Truth.assertThat -import org.junit.Assume.assumeTrue +import com.google.common.truth.Truth.assertWithMessage import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 @@ -15,24 +15,26 @@ import java.util.jar.JarFile */ @RunWith(JUnit4::class) class KotlinMetadataScannerShapeTest { - private val stdlibJar: File? = - runCatching { + private val stdlibJar: File by lazy { + val location = File( Unit::class.java.protectionDomain.codeSource.location .toURI(), ) - }.getOrNull() - ?.takeIf { it.isFile && it.name.endsWith(".jar") } + assertWithMessage("kotlin-stdlib must resolve to a JAR on the test classpath, got $location") + .that(location.isFile && location.name.endsWith(".jar")) + .isTrue() + location + } - private fun symbolsFor(entryName: String): List { - val jar = stdlibJar - assumeTrue("kotlin-stdlib jar not resolvable from the test classpath", jar != null) - return JarFile(jar).use { file -> + private fun symbolsFor(entryName: String): List = + JarFile(stdlibJar).use { file -> val entry = file.getJarEntry(entryName) - assumeTrue("$entryName missing from ${jar!!.name}", entry != null) + assertWithMessage("$entryName in ${stdlibJar.name}").that(entry).isNotNull() file.getInputStream(entry).use { KotlinMetadataScanner.parseKotlinClass(it, "stdlib") } } ?: emptyList() - } + + private fun classifierFor(entryName: String): JvmSymbol = symbolsFor(entryName).single { it.kind.isClassifier } @Test fun `a nested class reports its simple name, not the outer-qualified one`() { @@ -51,8 +53,6 @@ class KotlinMetadataScannerShapeTest { assertThat(companion.data.containingClassFqName).isEqualTo("kotlin.text.Regex") } - private fun classifierFor(entryName: String): JvmSymbol = symbolsFor(entryName).single { it.kind.isClassifier } - @Test fun `a nested class is keyed by its class file name, as the Java scanner keys it`() { val companion = classifierFor("kotlin/text/Regex\$Companion.class") From d54f5b02bcb011009b8b8ea2dfeea6292a439560 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 17:26:25 +0000 Subject: [PATCH 38/48] ADFA-6228: Query exact unimported-symbol matches before prefix matches With one shared cap over three prefix-only queries (source, generated, library), a short prefix could let the source and generated indexes fill the cap before the library index was ever queried, crowding out an exact match on the class the user is actually typing. Collect an exact simple-name match from every index first, then fill the remainder of the cap with prefix matches, deduplicated by qualified name (by full key for a callable, since two overloads share a qualified name) so a class indexed from two JARs on the active source set is offered once. --- .../codeonthego/indexing/jvm/JvmSymbol.kt | 13 + .../kotlin/completion/KotlinCompletions.kt | 97 ++++++-- .../CollectUnimportedSymbolMatchesTest.kt | 225 ++++++++++++++++++ 3 files changed, 319 insertions(+), 16 deletions(-) create mode 100644 lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt index 7cc2a16dcd..2971de19de 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt @@ -173,6 +173,19 @@ data class JvmSymbol( } } +/** + * The identity two rows stand for "the same symbol" when deduplicating matches drawn from more than + * one source or index. + * + * A classifier's qualified name already identifies it uniquely, but a callable's does not -- two + * overloads share one -- so a callable is identified by its full [JvmSymbol.key] instead. This is why + * the same class indexed from two JARs on the active source set (composite-keyed by + * `(sourceId, key)`, so both rows exist) collapses to one match: both rows share this identity even + * though their `(sourceId, key)` pairs differ. + */ +val JvmSymbol.dedupeKey: String + get() = if (kind.isCallable) key else fqName + /** * Base for all type-specific symbol data. * Every variant provides [containingClassName] (empty for top-level). diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index cd8ece894a..2ccf33f03e 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -31,8 +31,10 @@ import io.github.rosemoe.sora.lang.completion.CompletionCancelledException import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolIndex import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind import org.appdevforall.codeonthego.indexing.jvm.JvmTypeAliasInfo +import org.appdevforall.codeonthego.indexing.jvm.dedupeKey import org.jetbrains.kotlin.analysis.api.KaExperimentalApi import org.jetbrains.kotlin.analysis.api.KaIdeApi import org.jetbrains.kotlin.analysis.api.KaSession @@ -81,22 +83,23 @@ private val logger = LoggerFactory.getLogger("KotlinCompletions") * Max unimported symbols [collectUnimportedSymbols] adds to one completion result, across every * index rather than per index. * - * This counts items the user is actually offered. It used to be the query limit, which counted rows - * fetched -- so a prefix whose first rows were all rejected by the package, visibility or kind - * filters produced nothing while valid matches sat just past the limit. + * This counts items the user is actually offered, not rows a query returned, so a prefix whose + * leading rows are all rejected by the package, visibility, or kind filters still yields whatever + * valid matches exist instead of coming up empty. */ private const val UNIMPORTED_SYMBOL_DISPLAY_LIMIT = 100 /** - * Rows each index may return before filtering, sized so ordinary filtering cannot starve the - * result. + * Rows each index's prefix query may return before filtering, sized so ordinary filtering cannot + * starve the result. * - * Measured against kotlin-stdlib 2.3.0 (8,704 indexed symbols): 29% of entries are member-level - * callables, which [buildUnimportedSymbolItem] drops outright, and the worst realistic prefix of - * those probed (`get`) kept only 80 of 236 matches -- a 34% survival rate before the package and - * visibility filters even run. Three times the display limit covers that worst case; the package - * and visibility filters then eat into the margin. It is deliberately not unbounded: an unrestricted - * prefix query over the symbol table runs on every keystroke. + * The query's `kinds` filter already excludes every kind unimported-symbol completion can never + * offer (a file facade, a companion object, anything neither classifier nor callable), but it + * cannot tell a top-level or extension callable from a member one -- that distinction is + * [isUnimportedSymbolCandidate]'s job, run after the fetch, and member callables dominate most + * packages. The budget covers that remaining rejection, plus the current-package and visibility + * filters, without being unbounded: an unrestricted prefix query over the symbol table runs on + * every keystroke. */ private const val UNIMPORTED_SYMBOL_FETCH_BUDGET = 3 * UNIMPORTED_SYMBOL_DISPLAY_LIMIT @@ -458,16 +461,74 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) */ val indexes = listOfNotNull(env.sourceIndex, env.generatedIndex, env.libraryIndex) - collectUpToLimit( + collectUnimportedSymbolMatches( + indexes = indexes, + partial = ctx.partial, + kinds = UNIMPORTED_SYMBOL_KINDS, limit = UNIMPORTED_SYMBOL_DISPLAY_LIMIT, - sources = - indexes.map { index -> - { index.findByPrefix(ctx.partial, limit = UNIMPORTED_SYMBOL_FETCH_BUDGET, kinds = UNIMPORTED_SYMBOL_KINDS) } - }, + fetchBudget = UNIMPORTED_SYMBOL_FETCH_BUDGET, accept = ::addCompletionItem, ) } +/** + * The unimported-symbol matches [collectUnimportedSymbols] considers for [partial] across [indexes], + * handed to [accept] in query order and capped at [limit] accepted symbols. + * + * An exact simple-name match from every index is collected before any index's prefix match. The + * exact stage is a small, indexed equality query, cheap enough to run unconditionally, and running + * it first keeps a cap the prefix stage fills on its own from crowding out an exact match sitting in + * a later index -- typically the library index, the last one queried, where the class the user is + * actually typing is likeliest to live. Within each stage, indexes are queried in [indexes] order. + * + * A match is deduplicated by [JvmSymbol] identity ([dedupeKey]), keeping the first *accepted* + * occurrence in that order and discarding the rest: since a row's key is unique only per source, the + * same class indexed from two JARs on the active source set otherwise yields two rows for what is + * one class. A key is recorded only once [accept] has taken it, not merely seen it, so a row rejected + * by [accept] (for instance, a module that cannot reach that row's source) does not block a later + * duplicate from a source [accept] does allow through. A rejected row's own source and key are + * remembered too, so the same physical row surfacing again from the prefix stage is skipped without + * a second call to [accept]. + */ +internal fun collectUnimportedSymbolMatches( + indexes: List, + partial: String, + kinds: Set, + limit: Int, + fetchBudget: Int, + accept: (JvmSymbol) -> Boolean, +): Int { + val seen = mutableSetOf() + val rejectedRows = mutableSetOf>() + + fun acceptFirstOccurrence(symbol: JvmSymbol): Boolean { + val key = symbol.dedupeKey + if (key in seen) return false + + /* + * Identifies this exact row (source and key), not the class it names: the exact and prefix + * stages can both return the very same row for one index, and a row `accept` already turned + * down is not worth asking about again. A different source sharing `key`'s dedup identity is + * untouched by this and still gets its own `accept` call. + */ + val row = symbol.sourceId to symbol.key + if (row in rejectedRows) return false + + if (!accept(symbol)) { + rejectedRows += row + return false + } + + seen += key + return true + } + + val exactSources = indexes.map { index -> { index.findBySimpleName(partial, limit = limit, kinds = kinds) } } + val prefixSources = indexes.map { index -> { index.findByPrefix(partial, limit = fetchBudget, kinds = kinds) } } + + return collectUpToLimit(limit = limit, sources = exactSources + prefixSources, accept = ::acceptFirstOccurrence) +} + /** * Offers items from [sources] in order to [accept], stopping once [limit] of them have been * accepted, and returns how many were. @@ -480,6 +541,10 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) * rows before returning -- the sequence it hands back is already a list. Taking a supplier is what * keeps a source that the limit makes unnecessary from being queried at all; how many rows a source * fetches when it is reached remains the job of the limit passed to the query itself. + * + * A [limit] of zero or less accepts nothing and returns 0 without querying any source. This is the + * opposite of an index query's own convention, where a limit of zero means unbounded -- this helper + * has no unbounded case, since every caller caps how many items a user is offered. */ internal fun collectUpToLimit( limit: Int, diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt new file mode 100644 index 0000000000..5875ba72ee --- /dev/null +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt @@ -0,0 +1,225 @@ +package com.itsaky.androidide.lsp.kotlin.completion + +import com.google.common.truth.Truth.assertThat +import kotlinx.coroutines.runBlocking +import org.appdevforall.codeonthego.indexing.InMemoryIndex +import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolDescriptor +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolIndex +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.util.BackgroundIndexer +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * [collectUnimportedSymbolMatches] is the ordering and deduplication [collectUnimportedSymbols] + * runs on top of the raw index queries. + */ +@RunWith(JUnit4::class) +class CollectUnimportedSymbolMatchesTest { + /** A [JvmSymbolIndex] over a fresh, unfiltered in-memory backing store. */ + private fun newIndex(): JvmSymbolIndex { + val backing = InMemoryIndex(JvmSymbolDescriptor) + return object : JvmSymbolIndex(backing, BackgroundIndexer(backing)) { + // The default filters to explicitly activated sources; these tests care about query + // ordering and dedup, not source activation, so admit everything inserted. + override fun visibleSourceIds(): Collection? = null + } + } + + private fun classSymbol( + sourceId: String, + shortName: String, + packageName: String = "com.example", + ): JvmSymbol { + val internalName = "${packageName.replace('.', '/')}/$shortName" + return JvmSymbol( + key = internalName, + sourceId = sourceId, + name = internalName, + shortName = shortName, + packageName = packageName, + kind = JvmSymbolKind.CLASS, + language = JvmSourceLanguage.JAVA, + data = JvmClassInfo(internalName = internalName), + ) + } + + private fun funSymbol( + sourceId: String, + key: String, + shortName: String, + packageName: String = "com.example", + ): JvmSymbol { + val internalName = "${packageName.replace('.', '/')}/Foo#$shortName" + return JvmSymbol( + key = key, + sourceId = sourceId, + name = internalName, + shortName = shortName, + packageName = packageName, + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.JAVA, + data = JvmFunctionInfo(), + ) + } + + private fun JvmSymbolIndex.insertBlocking(symbol: JvmSymbol) = runBlocking { insert(symbol) } + + @Test + fun `an exact library match survives a cap already filled by an earlier index's prefix matches`() { + val sourceIndex = newIndex() + val libraryIndex = newIndex() + + // A prefix match that is not an exact match, sitting in the index queried first. + sourceIndex.insertBlocking(classSymbol(sourceId = "proj", shortName = "FooBar")) + // The exact match the user is typing, sitting in the index queried last. + libraryIndex.insertBlocking(classSymbol(sourceId = "lib.jar", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex, libraryIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 1, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + // With a cap of one, a prefix-only scan (source, then library) would spend the cap on + // FooBar and never reach the library index at all. + assertThat(accepted.map { it.shortName }).containsExactly("Foo") + } + + @Test + fun `the same class indexed from two jars is offered once`() { + val libraryIndex = newIndex() + libraryIndex.insertBlocking(classSymbol(sourceId = "libA.jar", shortName = "Foo")) + libraryIndex.insertBlocking(classSymbol(sourceId = "libB.jar", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(libraryIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + assertThat(accepted).hasSize(1) + assertThat(accepted.single().fqName).isEqualTo("com.example.Foo") + } + + @Test + fun `a single exact match is not also offered via the prefix stage`() { + val sourceIndex = newIndex() + sourceIndex.insertBlocking(classSymbol(sourceId = "proj", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + assertThat(accepted).hasSize(1) + } + + @Test + fun `a duplicate rejected on its first occurrence does not block a later acceptable one`() { + val sourceIndex = newIndex() + val libraryIndex = newIndex() + + /* + * Same class, indexed from two JARs. The querying module can only reach one of them -- + * modelled here as `accept` rejecting rows from "unreachable.jar" -- and that source happens + * to be queried first. + */ + sourceIndex.insertBlocking(classSymbol(sourceId = "unreachable.jar", shortName = "Foo")) + libraryIndex.insertBlocking(classSymbol(sourceId = "reachable.jar", shortName = "Foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex, libraryIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { symbol -> + if (symbol.sourceId == "unreachable.jar") { + false + } else { + accepted += symbol + true + } + }, + ) + + // Marking the dedup key "seen" on the rejected row would silently swallow the reachable + // module's otherwise-valid completion. + assertThat(accepted.map { it.sourceId }).containsExactly("reachable.jar") + } + + @Test + fun `a row accept rejects in the exact stage is not asked about again in the prefix stage`() { + val sourceIndex = newIndex() + sourceIndex.insertBlocking(classSymbol(sourceId = "proj", shortName = "Foo")) + + var acceptCalls = 0 + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex), + partial = "Foo", + kinds = setOf(JvmSymbolKind.CLASS), + limit = 10, + fetchBudget = 10, + accept = { + acceptCalls++ + false + }, + ) + + // The same row (same source and key) is a match in both the exact and prefix stages; once + // `accept` has turned it down, asking again wastes the visibility check and item-building + // work it would otherwise repeat. + assertThat(acceptCalls).isEqualTo(1) + } + + @Test + fun `two overloads sharing a qualified name are both offered`() { + val sourceIndex = newIndex() + sourceIndex.insertBlocking(funSymbol(sourceId = "proj", key = "com/example/Foo#foo()", shortName = "foo")) + sourceIndex.insertBlocking(funSymbol(sourceId = "proj", key = "com/example/Foo#foo(int)", shortName = "foo")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(sourceIndex), + partial = "foo", + kinds = setOf(JvmSymbolKind.FUNCTION), + limit = 10, + fetchBudget = 10, + accept = { + accepted += it + true + }, + ) + + assertThat(accepted).hasSize(2) + } +} From 7a9c7575fe31bf1f05780cfb35aa59074bf63fdf Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 17:26:37 +0000 Subject: [PATCH 39/48] ADFA-6228: Deduplicate AddImportAction candidates by qualified name --- .../lsp/kotlin/actions/AddImportAction.kt | 38 +++++++++++++++++-- .../lsp/kotlin/actions/AddImportActionTest.kt | 33 ++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt index 0092739633..762f5924ce 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt @@ -27,7 +27,10 @@ import com.itsaky.androidide.utils.flashError import com.itsaky.androidide.utils.flashInfo import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.jvm.JvmVisibility +import org.appdevforall.codeonthego.indexing.jvm.dedupeKey import java.nio.file.Path class AddImportAction : BaseKotlinCodeAction() { @@ -49,9 +52,11 @@ class AddImportAction : BaseKotlinCodeAction() { return } - // Optimistic visibility: decide from the in-memory unresolved-reference marker only. The - // importable-classifier resolution runs in the background execAction; doing it here would be - // main-thread SQLite I/O, because fillMenu() calls prepare() synchronously on the UI thread. + /* + * Optimistic visibility: decide from the in-memory unresolved-reference marker only. The + * importable-classifier resolution runs in the background execAction; doing it here would be + * main-thread SQLite I/O, because fillMenu() calls prepare() synchronously on the UI thread. + */ val resolveReferenceActionDiagnostic = data.findDiagnosticExtra() if (resolveReferenceActionDiagnostic == null) { @@ -97,10 +102,17 @@ class AddImportAction : BaseKotlinCodeAction() { * to classifier kinds, so the row count is the number of classes with that simple name, not a * scan. The kind restriction also keeps Kotlin from being offered a file facade, which is not * a classifier. + * + * Deduplicated by qualified name, keeping the first occurrence in query order (source before + * library): the same class indexed from two JARs on the active source set is two rows here. + * The `associate` below already folds duplicate keys together for a classifier, so this list + * is redundant with it today; it is kept so this result's own identity does not depend on + * how it happens to get folded downstream. */ val classifiers = env.ktSymbolIndex .findSymbolBySimpleName(referenceName, limit = 0, kinds = JvmSymbolKind.CLASSIFIER_KINDS) + .distinctBy { it.dedupeKey } .toList() if (classifiers.isEmpty()) { @@ -117,7 +129,10 @@ class AddImportAction : BaseKotlinCodeAction() { val candidates = live.read { ktFile -> - classifiers.associate { it.fqName to insertImport(ktFile, it.fqName) } + val currentPackage = ktFile.packageFqName.asString().ifEmpty { null } + classifiers + .filter { it.isImportableFrom(currentPackage) } + .associate { it.fqName to insertImport(ktFile, it.fqName) } } if (live.isStale) { @@ -239,6 +254,21 @@ class AddImportAction : BaseKotlinCodeAction() { } } +/** + * Whether [this] classifier is one a file in [currentPackage] could actually import. + * + * A private declaration is never importable, and a package-private one only from its own package. + * Public, protected, and internal candidates pass through unchanged: telling them apart needs the + * use-site module that completion's visibility checker resolves, which is not available here, so + * this only covers the visibility distinctions that a package name settles on its own. + */ +private fun JvmSymbol.isImportableFrom(currentPackage: String?): Boolean = + when (visibility) { + JvmVisibility.PRIVATE -> false + JvmVisibility.PACKAGE_PRIVATE -> packageName == currentPackage + JvmVisibility.PUBLIC, JvmVisibility.PROTECTED, JvmVisibility.INTERNAL -> true + } + /** * The outcome of resolving import candidates. * diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt index bb00c5dbeb..f4076bf3a7 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt @@ -9,6 +9,7 @@ import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol import org.appdevforall.codeonthego.indexing.jvm.JvmSymbolKind +import org.appdevforall.codeonthego.indexing.jvm.JvmVisibility import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test @@ -47,6 +48,8 @@ class AddImportActionTest : KtLspTest() { private fun index(vararg symbols: JvmSymbol) = runBlocking { symbols.forEach { env.ktSymbolIndex.sourceIndex.insert(it) } } + private fun indexLibrary(vararg symbols: JvmSymbol) = runBlocking { symbols.forEach { env.ktSymbolIndex.libraryIndex.insert(it) } } + @Test fun `resolves a single classifier candidate by simple name`() { index(classSymbol("lib", "Foo")) @@ -80,6 +83,36 @@ class AddImportActionTest : KtLspTest() { assertEquals(setOf("a.Foo"), candidates.keys) } + /** + * Pins the contract, not a bug this test caught: `associate`'s own dedup-by-fqName already makes + * this pass without `computeImportCandidates`'s explicit `.distinctBy { it.dedupeKey }`, which + * exists as insurance against a future change to how candidates are folded into the chooser, not + * because this ever surfaced a duplicate. + */ + @Test + fun `import candidates list each qualified name once, even from two jars`() { + indexLibrary(classSymbol("lib", "Foo").copy(sourceId = "jarA.jar"), classSymbol("lib", "Foo").copy(sourceId = "jarB.jar")) + createSourceFile("Main.kt", "package p\nfun f(x: Foo) {}") + + val candidates = AddImportAction().computeImportCandidates(env, mainPath, "Foo").found() + + assertEquals(setOf("lib.Foo"), candidates.keys) + assertEquals(1, candidates.size) + } + + @Test + fun `drops a package-private library classifier when a public one shares the simple name`() { + indexLibrary( + classSymbol("androidx.recyclerview.widget", "AdapterHelper").copy(visibility = JvmVisibility.PACKAGE_PRIVATE), + classSymbol("com.example", "AdapterHelper"), + ) + createSourceFile("Main.kt", "package p\nfun f(x: AdapterHelper) {}") + + val candidates = AddImportAction().computeImportCandidates(env, mainPath, "AdapterHelper").found() + + assertEquals(setOf("com.example.AdapterHelper"), candidates.keys) + } + @Test fun `returns no candidates for an unknown reference`() { createSourceFile("Main.kt", "package p\nfun f() {}") From 4691308e3f94db5403194ba6f113489f4db5fa6e Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 17:26:51 +0000 Subject: [PATCH 40/48] ADFA-6228: Tighten the scanner and completion candidate tests --- .../jvm/CombinedJarScannerMultiReleaseTest.kt | 6 ++- .../jvm/JarSymbolScannerVisibilityTest.kt | 39 +++++++++++-------- .../UnimportedSymbolCandidateTest.kt | 22 +++++++++-- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt index 7d4c628a3f..0d6240a785 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/CombinedJarScannerMultiReleaseTest.kt @@ -81,13 +81,17 @@ class CombinedJarScannerMultiReleaseTest { fun `a virtual file scan skips versioned entries`() { val jar = multiReleaseJar() val disposable = Disposer.newDisposable() + val jarFileSystem = CoreJarFileSystem() try { // Reading a jar entry's contents consults the application's file size limits. ApplicationManager.setApplication(MockApplication(disposable), disposable) - val root = CoreJarFileSystem().findFileByPath("${jar.absolutePath}!/")!! + val root = jarFileSystem.findFileByPath("${jar.absolutePath}!/")!! assertOnlyBaseClasses(CombinedJarScanner.scan(root, "mr").toList()) } finally { + // The handler this test opened for the jar keeps its file handle open; releasing it here + // is what lets TemporaryFolder delete the jar afterwards. + jarFileSystem.clearHandlersCache() Disposer.dispose(disposable) } } diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt index 414d1ed96a..2d72f9fc9d 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt @@ -36,22 +36,6 @@ class JarSymbolScannerVisibilityTest { private fun scan(bytes: ByteArray): List = JarSymbolScanner.parseClassFile(bytes.inputStream(), "test.jar") - private val nestedClasses: Map by lazy { - compileJava( - "com.example.Outer", - """ - package com.example; - - public class Outer { - private static class Hidden {} - protected static class Guarded {} - public static class Open {} - class Pkg {} - } - """.trimIndent(), - ) - } - private fun scanNested(simpleName: String): List = scan(nestedClasses.getValue("com/example/Outer\$$simpleName")) private fun nestedClass(simpleName: String): JvmSymbol = scanNested(simpleName).single { it.kind.isJvmClass } @@ -131,6 +115,29 @@ class JarSymbolScannerVisibilityTest { assertThat(nested.isTopLevel).isFalse() assertThat(nested.data.containingClassFqName).isEqualTo("com.example.Outer") } + + companion object { + /** + * JUnit4 builds a fresh test instance per `@Test` method, so an instance-level `by lazy` + * would recompile this fixture once per test. A companion `by lazy` is shared by every + * instance in the class, so javac runs once for the whole class. + */ + private val nestedClasses: Map by lazy { + compileJava( + "com.example.Outer", + """ + package com.example; + + public class Outer { + private static class Hidden {} + protected static class Guarded {} + public static class Open {} + class Pkg {} + } + """.trimIndent(), + ) + } + } } /** diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt index c11cdc75ef..85e3f94d6b 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt @@ -2,6 +2,7 @@ package com.itsaky.androidide.lsp.kotlin.completion import com.google.common.truth.Truth.assertThat import org.appdevforall.codeonthego.indexing.jvm.JvmClassInfo +import org.appdevforall.codeonthego.indexing.jvm.JvmEnumEntryInfo import org.appdevforall.codeonthego.indexing.jvm.JvmFunctionInfo import org.appdevforall.codeonthego.indexing.jvm.JvmSourceLanguage import org.appdevforall.codeonthego.indexing.jvm.JvmSymbol @@ -71,9 +72,22 @@ class UnimportedSymbolCandidateTest { } @Test - fun `the prefix query asks for exactly the kinds that can be offered`() { - assertThat(UNIMPORTED_SYMBOL_KINDS).doesNotContain(JvmSymbolKind.FILE_FACADE) - assertThat(UNIMPORTED_SYMBOL_KINDS).doesNotContain(JvmSymbolKind.COMPANION_OBJECT) - assertThat(UNIMPORTED_SYMBOL_KINDS).containsAtLeastElementsIn(JvmSymbolKind.CALLABLE_KINDS) + fun `an enum entry is not offered`() { + // Inserting a bare enum entry name with no import does not resolve; it must be qualified by + // its enum class. + val entry = + symbol( + "REGEX_OPTION", + JvmSymbolKind.ENUM_ENTRY, + JvmEnumEntryInfo(containingClassName = "kotlin/text/RegexOption"), + ) + + assertThat(isUnimportedSymbolCandidate(entry)).isFalse() + } + + @Test + fun `the query asks for exactly the kinds that can be offered`() { + assertThat(UNIMPORTED_SYMBOL_KINDS) + .containsExactlyElementsIn(JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT + JvmSymbolKind.CALLABLE_KINDS) } } From 44d6ef0506a82c03989f662ce264efb2c1fa4774 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 23 Sep 2026 18:20:39 +0000 Subject: [PATCH 41/48] ADFA-6228: Convert pre-existing multi-line comments to block form and drop stray em dashes These lines predate this branch but sit in files it touches; none has an owning commit here to fold into. Also drops a stale internal ticket reference from one comment while rewording it. --- .../codeonthego/indexing/MergedIndex.kt | 7 ++++--- .../jvm/JvmGeneratedIndexingService.kt | 4 ++-- .../kotlin/compiler/index/KtSymbolIndex.kt | 20 +++++++++++-------- .../index/IndexWorkerBatchRemovalTest.kt | 4 ++-- .../compiler/index/IndexWorkerDisposalTest.kt | 16 +++++++++------ .../index/IndexWorkerPhaseLifecycleTest.kt | 8 +++++--- .../kotlin/fixtures/KtLspTestEnvironment.kt | 10 ++++++---- 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt index d0892be44f..433a551596 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/MergedIndex.kt @@ -8,9 +8,10 @@ import java.io.Closeable /** * Merges query results from multiple [ReadableIndex] instances. * - * Indexes are queried sequentially in the order they are provided. - * Duplicate keys (same entry present in more than one backing index) - * are deduplicated — the first occurrence wins. + * Indexes are queried sequentially in the order they are provided. A key already seen is skipped + * and the first occurrence wins, whether the repeat comes from a later backing index or from the + * same index yielding it again under a different source, which a composite source-and-key primary + * key allows. * * @param T The indexed type. * @param indexes The indexes to merge, in priority order. diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt index 1244add296..937b4d8666 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmGeneratedIndexingService.kt @@ -85,13 +85,13 @@ class JvmGeneratedIndexingService( private suspend fun reindexGeneratedJars(): List { val index = this.generatedIndex ?: run { - log.warn("Not indexing generated JARs — index not initialized.") + log.warn("Not indexing generated JARs - index not initialized.") return emptyList() } val workspace = ProjectManagerImpl.getInstance().workspace ?: run { - log.warn("Not indexing generated JARs — workspace model not available.") + log.warn("Not indexing generated JARs - workspace model not available.") return emptyList() } diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt index c3e39373cc..7ec5949de6 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/KtSymbolIndex.kt @@ -459,9 +459,11 @@ internal class KtSymbolIndex( null } - // Applied on the way out rather than during the pin: the version bump that arrived while the path - // was frozen still has to reach the FIR session. Skipped once the document is gone, since - // invalidateCurrent already unregistered it. + /* + * Applied on the way out rather than during the pin: the version bump that arrived while the path + * was frozen still has to reach the FIR session. Skipped once the document is gone, since + * invalidateCurrent already unregistered it. + */ if (refreshOwed && FileManager.isActive(path)) { scope.launch { refreshCurrentKtFile(path) } } @@ -584,11 +586,13 @@ internal class KtSymbolIndex( scanningJob?.cancelAndJoin() indexingJob?.join() - // Cancel AND JOIN the index's own scope. Beyond the main worker loop drained above, the - // debounced modifiedFileIndexer and queueOnFileChangedAsync coroutines also run - // project.read { PsiManager … }. Joining guarantees none survive into the caller's - // Disposer.dispose(...), which would otherwise crash with "Project is already disposed" - // (APPDEVFORALL-17R). This index owns `scope`. + /* + * Cancel AND JOIN the index's own scope. Beyond the main worker loop drained above, the + * debounced modifiedFileIndexer and queueOnFileChangedAsync coroutines also run + * project.read { PsiManager ... }. Joining guarantees none survive into the caller's + * Disposer.dispose(...), which would otherwise crash with "Project is already disposed" once + * one of them touches the project after it is gone. This index owns `scope`. + */ scope.coroutineContext[Job]?.cancelAndJoin() // Drain the refresh pool before disposal: refreshToCurrent runs project.read/write on it (same diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt index a2cc8b3111..097e4be316 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerBatchRemovalTest.kt @@ -26,7 +26,7 @@ import java.time.Instant * commands must collapse BOTH the symbol-index deletes AND the per-file metadata deletes * into ONE batched call each ([JvmSymbolIndex.removeBySources] / * [KtFileMetadataIndex.removeAll], a single SQLite transaction apiece) instead of N separate - * [Index.removeBySource] calls (N transactions — the Sentry N+1). + * [Index.removeBySource] calls (N transactions - the Sentry N+1). */ @RunWith(JUnit4::class) class IndexWorkerBatchRemovalTest { @@ -124,7 +124,7 @@ class IndexWorkerBatchRemovalTest { pushBack = { pushedBack.add(it) }, ) - // The fix: exactly one batched transaction, zero per-source deletes — for BOTH indexes. + // The fix: exactly one batched transaction, zero per-source deletes - for BOTH indexes. assertThat(counting.removeBySourcesBatches).isEqualTo(1) assertThat(counting.removeBySourceCalls).isEqualTo(0) assertThat(counting.removedInLargestBatch).containsExactlyElementsIn(paths) diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt index 0970b56ee7..dfabdb5c78 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerDisposalTest.kt @@ -47,15 +47,19 @@ class IndexWorkerDisposalTest : KtLspTest() { scope = CoroutineScope(Dispatchers.Default + SupervisorJob()), ) - // Dispose the project out from under the worker, exactly as happens on LSP shutdown - // (env.close() disposes the project via its parent Disposable, the production path). - // IntelliJ requires model teardown to run inside a write action. + /* + * Dispose the project out from under the worker, exactly as happens on LSP shutdown + * (env.close() disposes the project via its parent Disposable, the production path). + * IntelliJ requires model teardown to run inside a write action. + */ ApplicationManager.getApplication().runWriteAction { env.close() } assertThat(env.project.isDisposed).isTrue() - // Pre-fix, processing either command calls PsiManager.findFile on the disposed project - // and throws AssertionError("Project is already disposed"). Post-fix, the disposal guard - // breaks the loop, so start() returns cleanly instead of throwing. + /* + * Pre-fix, processing either command calls PsiManager.findFile on the disposed project + * and throws AssertionError("Project is already disposed"). Post-fix, the disposal guard + * breaks the loop, so start() returns cleanly instead of throwing. + */ worker.submitCommand(IndexCommand.ScanSourceFile(vf)) worker.submitCommand(IndexCommand.IndexSourceFile(vf)) worker.submitCommand(IndexCommand.Stop) diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt index c4ad9a174b..641cd6484a 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/compiler/index/IndexWorkerPhaseLifecycleTest.kt @@ -116,9 +116,11 @@ class IndexWorkerPhaseLifecycleTest { Memprof.sink = sink val worker = worker() - // The second SourceScanningStarted simulates KtSymbolIndex.refreshSources() restarting - // a scan whose first attempt was cancelled before it ever reached SourceScanningComplete, - // leaving the first scanPhase stale and open. + /* + * The second SourceScanningStarted simulates KtSymbolIndex.refreshSources() restarting + * a scan whose first attempt was cancelled before it ever reached SourceScanningComplete, + * leaving the first scanPhase stale and open. + */ worker.submitCommand(IndexCommand.SourceScanningStarted(pass = 1)) worker.submitCommand(IndexCommand.SourceScanningStarted(pass = 2)) worker.submitCommand(IndexCommand.Stop) diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt index e5e6c7c55a..eebbc340f1 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/fixtures/KtLspTestEnvironment.kt @@ -100,10 +100,12 @@ internal class KtLspTestEnvironment( try { initialize(::buildModules, ::buildKtSymbolIndex) } catch (failure: Throwable) { - // A throwing constructor never hands the instance to KtLspTestRule, so its finally has - // nothing to close and the refcounted, process-wide application environment leaks for the - // rest of the suite - one bad TestSourceModuleSpec would reproduce the suite-wide OOM. - // Release it here, where the half-built instance is still reachable. + /* + * A throwing constructor never hands the instance to KtLspTestRule, so its finally has + * nothing to close and the refcounted, process-wide application environment leaks for the + * rest of the suite - one bad TestSourceModuleSpec would reproduce the suite-wide OOM. + * Release it here, where the half-built instance is still reachable. + */ runCatching { closeInWriteAction() }.exceptionOrNull()?.let(failure::addSuppressed) throw failure } From 7ae09707644b3978b0ef915675638668b40bfe65 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 25 Sep 2026 13:13:53 +0000 Subject: [PATCH 42/48] ADFA-6228: Deduplicate Kotlin import candidates after the visibility filter --- .../lsp/kotlin/actions/AddImportAction.kt | 10 ++++------ .../lsp/kotlin/actions/AddImportActionTest.kt | 13 +++++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt index 762f5924ce..7525dcc0f4 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportAction.kt @@ -103,16 +103,13 @@ class AddImportAction : BaseKotlinCodeAction() { * scan. The kind restriction also keeps Kotlin from being offered a file facade, which is not * a classifier. * - * Deduplicated by qualified name, keeping the first occurrence in query order (source before - * library): the same class indexed from two JARs on the active source set is two rows here. - * The `associate` below already folds duplicate keys together for a classifier, so this list - * is redundant with it today; it is kept so this result's own identity does not depend on - * how it happens to get folded downstream. + * The same class indexed from two JARs on the active source set is two rows here, possibly + * with different visibility, so they are deduplicated below only after the visibility filter: + * deduplicating first could keep a hidden copy and drop the importable one. */ val classifiers = env.ktSymbolIndex .findSymbolBySimpleName(referenceName, limit = 0, kinds = JvmSymbolKind.CLASSIFIER_KINDS) - .distinctBy { it.dedupeKey } .toList() if (classifiers.isEmpty()) { @@ -132,6 +129,7 @@ class AddImportAction : BaseKotlinCodeAction() { val currentPackage = ktFile.packageFqName.asString().ifEmpty { null } classifiers .filter { it.isImportableFrom(currentPackage) } + .distinctBy { it.dedupeKey } .associate { it.fqName to insertImport(ktFile, it.fqName) } } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt index f4076bf3a7..f2d6eafe87 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/actions/AddImportActionTest.kt @@ -100,6 +100,19 @@ class AddImportActionTest : KtLspTest() { assertEquals(1, candidates.size) } + @Test + fun `a class hidden in one jar is still offered from a jar that makes it public`() { + indexLibrary( + classSymbol("lib", "Foo").copy(sourceId = "jarA.jar", visibility = JvmVisibility.PACKAGE_PRIVATE), + classSymbol("lib", "Foo").copy(sourceId = "jarB.jar"), + ) + createSourceFile("Main.kt", "package p\nfun f(x: Foo) {}") + + val candidates = AddImportAction().computeImportCandidates(env, mainPath, "Foo").found() + + assertEquals(setOf("lib.Foo"), candidates.keys) + } + @Test fun `drops a package-private library classifier when a public one shares the simple name`() { indexLibrary( From f168e1b501c5021db04ba1927bd7dcddf374c52c Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Fri, 25 Sep 2026 13:16:14 +0000 Subject: [PATCH 43/48] ADFA-6228: Read newer Kotlin metadata leniently instead of dropping the class --- .../indexing/jvm/KotlinMetadataScanner.kt | 29 ++++++--- .../jvm/KotlinMetadataScannerVersionTest.kt | 62 +++++++++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerVersionTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt index 5fd5600858..74373e1e14 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -55,13 +55,7 @@ object KotlinMetadataScanner { val header = collector.metadataHeader ?: return null - val metadata = - try { - KotlinClassMetadata.readStrict(header) - } catch (e: Exception) { - log.debug("Failed to read Kotlin metadata: {}", e.message) - return null - } + val metadata = readMetadata(header) ?: return null return when (metadata) { is KotlinClassMetadata.Class -> { @@ -97,6 +91,27 @@ object KotlinMetadataScanner { } } + /** + * Reads [header] strictly, falling back to a lenient read. + * + * A strict read rejects metadata written by a newer Kotlin than the bundled reader supports, which + * a dependency or the project's own output can carry. Dropping those classes would hide them from + * every lookup, while a lenient read still yields their declarations. + */ + private fun readMetadata(header: Metadata): KotlinClassMetadata? { + try { + return KotlinClassMetadata.readStrict(header) + } catch (e: Exception) { + log.debug("Reading Kotlin metadata leniently after a strict read failed: {}", e.message) + } + return try { + KotlinClassMetadata.readLenient(header) + } catch (e: Exception) { + log.debug("Failed to read Kotlin metadata: {}", e.message) + null + } + } + /** * The symbol for a facade class itself, as opposed to the declarations it carries. * diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerVersionTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerVersionTest.kt new file mode 100644 index 0000000000..58bd72755f --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerVersionTest.kt @@ -0,0 +1,62 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.AnnotationVisitor +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import java.io.File +import java.util.jar.JarFile + +/** A class compiled by a newer Kotlin than the bundled metadata reader supports is still indexed. */ +@RunWith(JUnit4::class) +class KotlinMetadataScannerVersionTest { + private fun stdlibClass(entryName: String): ByteArray { + val stdlib = + File( + Unit::class.java.protectionDomain.codeSource.location + .toURI(), + ) + return JarFile(stdlib).use { jar -> jar.getInputStream(jar.getJarEntry(entryName)).use { it.readBytes() } } + } + + /** Rewrites the `mv` (metadata version) of [classBytes]'s `@kotlin.Metadata` to [version]. */ + private fun withMetadataVersion( + classBytes: ByteArray, + version: IntArray, + ): ByteArray { + val writer = ClassWriter(0) + ClassReader(classBytes).accept( + object : ClassVisitor(Opcodes.ASM9, writer) { + override fun visitAnnotation( + descriptor: String?, + visible: Boolean, + ): AnnotationVisitor? { + val delegate = super.visitAnnotation(descriptor, visible) + if (descriptor != "Lkotlin/Metadata;") return delegate + return object : AnnotationVisitor(Opcodes.ASM9, delegate) { + override fun visit( + name: String?, + value: Any?, + ) = super.visit(name, if (name == "mv") version else value) + } + } + }, + 0, + ) + return writer.toByteArray() + } + + @Test + fun `a class with metadata newer than the reader supports is still indexed`() { + val newer = withMetadataVersion(stdlibClass("kotlin/Pair.class"), intArrayOf(99, 0, 0)) + + val symbols = KotlinMetadataScanner.parseKotlinClass(newer.inputStream(), "stdlib").orEmpty() + + assertThat(symbols.filter { it.kind.isClassifier }.map { it.key }).containsExactly("kotlin/Pair") + } +} From 4aa024de47c5250561406c94de8fdee005566890 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 7 Oct 2026 15:41:56 +0000 Subject: [PATCH 44/48] ADFA-6228: Key Kotlin extension callables by their receiver too Extension overloads that differ only by receiver, such as Iterable.first() and List.first(), shared one key, so each source kept only the last one. The key is now stored in the payload as well: rebuilding it on read could not recover the receiver, and completion dedupes callables by key. --- .../codeonthego/indexing/jvm/JvmSymbol.kt | 19 ++++++++++++++ .../indexing/jvm/JvmSymbolDescriptor.kt | 21 ++-------------- .../indexing/jvm/JvmSymbolIndex.kt | 2 +- .../indexing/jvm/KotlinMetadataScanner.kt | 4 +-- .../jvm/KotlinMetadataScannerShapeTest.kt | 25 +++++++++++++++++++ .../src/main/proto/jvm_symbol.proto | 3 +++ .../compiler/index/SourceFileIndexer.kt | 6 +++-- 7 files changed, 56 insertions(+), 24 deletions(-) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt index 2971de19de..cef123e0d9 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbol.kt @@ -186,6 +186,25 @@ data class JvmSymbol( val JvmSymbol.dedupeKey: String get() = if (kind.isCallable) key else fqName +/** + * Builds the [JvmSymbol.key] of a Kotlin or Java callable named [name]. + * + * [receiverTypeName] (an internal name, null for a non-extension) is part of the key because + * extension overloads can differ by receiver alone, such as `Iterable.first()` and + * `List.first()`, and rows sharing a key within one source replace each other. + * [parameterTypeNames] is null for a property, which has no parameter list. + */ +fun jvmCallableKey( + name: String, + receiverTypeName: String?, + parameterTypeNames: List?, +): String = + buildString { + append(name) + if (receiverTypeName != null) append('@').append(receiverTypeName.toFqName()) + parameterTypeNames?.joinTo(this, ",", "(", ")") + } + /** * Base for all type-specific symbol data. * Every variant provides [containingClassName] (empty for top-level). diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt index 9c73ae7be0..0988a03a9f 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolDescriptor.kt @@ -56,6 +56,7 @@ object JvmSymbolDescriptor : IndexDescriptor { val builder = JvmSymbolData .newBuilder() + .setKey(s.key) .setName(s.name) .setShortName(s.shortName) .setPackageName(s.packageName) @@ -206,26 +207,8 @@ object JvmSymbolDescriptor : IndexDescriptor { val kind = kindFromProto(p.kind) val data = dataFromProto(p) - val key = - when { - kind.isCallable && kind != JvmSymbolKind.PROPERTY && - kind != JvmSymbolKind.EXTENSION_PROPERTY && - kind != JvmSymbolKind.FIELD -> { - val params = - (data as? JvmFunctionInfo) - ?.parameters - ?.joinToString(",") { it.typeName } - ?: "" - "${p.name}($params)" - } - - else -> { - p.name - } - } - return JvmSymbol( - key = key, + key = p.key, sourceId = p.sourceId, name = p.name, shortName = p.shortName, diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index 995baf24eb..dcaa70b9bb 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -46,7 +46,7 @@ open class JvmSymbolIndex( * source file's symbols are re-indexed only when its metadata row says so; dropping the * symbols alone would leave files recorded as indexed with no symbols. */ - const val FORMAT_VERSION = 2 + const val FORMAT_VERSION = 3 /** * Create (or get) a JVM symbol index backed by SQLite. diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt index 74373e1e14..50ae6bdaac 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScanner.kt @@ -329,7 +329,7 @@ object KotlinMetadataScanner { } else { "$packageName#${fn.name}" } - val key = "$name(${parameters.joinToString(",") { it.typeFqName }})" + val key = jvmCallableKey(name, receiverType?.let { kmTypeToName(it) }, parameters.map { it.typeFqName }) val signatureDisplay = buildString { @@ -400,7 +400,7 @@ object KotlinMetadataScanner { } return JvmSymbol( - key = name, + key = jvmCallableKey(name, receiverType?.let { kmTypeToName(it) }, parameterTypeNames = null), sourceId = sourceId, name = name, shortName = prop.name, diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt index a98748b63a..ce6d1f17a4 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/KotlinMetadataScannerShapeTest.kt @@ -114,4 +114,29 @@ class KotlinMetadataScannerShapeTest { assertThat(symbols.any { it.kind.isCallable }).isTrue() } + + @Test + fun `extension overloads that differ only by receiver get distinct keys`() { + // Iterable.first() and List.first(): same name, same (empty) parameter list. + val firsts = noArgFirstExtensions() + + assertThat(firsts.map { it.receiverTypeName }).containsAtLeast("kotlin/collections/Iterable", "kotlin/collections/List") + assertThat(firsts.map { it.key }.toSet()).hasSize(firsts.size) + } + + @Test + fun `a callable's key survives a round trip through the index payload`() { + val firsts = noArgFirstExtensions() + + val restored = firsts.map { JvmSymbolDescriptor.deserialize(JvmSymbolDescriptor.serialize(it)) } + + assertThat(restored.map { it.key }).containsExactlyElementsIn(firsts.map { it.key }).inOrder() + } + + private fun noArgFirstExtensions(): List = + symbolsFor("kotlin/collections/CollectionsKt___CollectionsKt.class").filter { + it.kind == JvmSymbolKind.EXTENSION_FUNCTION && + it.shortName == "first" && + (it.data as JvmFunctionInfo).parameters.isEmpty() + } } diff --git a/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto index 85f18e3a97..742d870bb4 100644 --- a/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto +++ b/lsp/jvm-symbol-models/src/main/proto/jvm_symbol.proto @@ -18,6 +18,9 @@ message JvmSymbolData { JvmVisibility visibility = 7; bool is_deprecated = 8; + // The row key, stored because no other field can rebuild it. + string key = 9; + oneof data { ClassData class_data = 20; FunctionData function_data = 21; diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt index 313b9acc49..f6c9d6698d 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/index/SourceFileIndexer.kt @@ -27,6 +27,7 @@ import org.appdevforall.codeonthego.indexing.jvm.KotlinFunctionInfo import org.appdevforall.codeonthego.indexing.jvm.KotlinPropertyInfo import org.appdevforall.codeonthego.indexing.jvm.KtFileMetadata import org.appdevforall.codeonthego.indexing.jvm.KtFileMetadataIndex +import org.appdevforall.codeonthego.indexing.jvm.jvmCallableKey import org.jetbrains.kotlin.analysis.api.KaExperimentalApi import org.jetbrains.kotlin.analysis.api.KaImplementationDetail import org.jetbrains.kotlin.analysis.api.KaSession @@ -327,7 +328,8 @@ private fun KaSession.analyzeFunction( } else { "$pkg#$fnName" } - val key = "$qualifiedName(${parameters.joinToString(",") { it.typeFqName }})" + val key = + jvmCallableKey(qualifiedName, receiverType?.let { kaTypeInternalName(it) }, parameters.map { it.typeFqName }) val signatureDisplay = buildString { @@ -473,7 +475,7 @@ private fun KaSession.analyzeProperty( } return JvmSymbol( - key = qualifiedName, + key = jvmCallableKey(qualifiedName, receiverType?.let { kaTypeInternalName(it) }, parameterTypeNames = null), sourceId = filePath, name = qualifiedName, shortName = propName, From a1286b42b1b65046b11bd8c8b1d54af95e9a0164 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 7 Oct 2026 15:42:16 +0000 Subject: [PATCH 45/48] ADFA-6228: Import an unimported type alias chosen from Kotlin completion --- .../kotlin/completion/KotlinCompletions.kt | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 2ccf33f03e..474e4646e5 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -621,6 +621,12 @@ private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionIt ) item.overrideTypeText = symbol.returnTypeDisplay + // Every item imports its symbol when chosen; setClassCompletionData swaps in an equivalent handler. + item.additionalEditHandler = + KotlinAutoImportEditHandler( + analysisContext = ctx, + symbolToImport = symbol, + ) when (symbol.kind) { JvmSymbolKind.EXTENSION_FUNCTION, JvmSymbolKind.FUNCTION, JvmSymbolKind.CONSTRUCTOR -> { val data = symbol.data as JvmFunctionInfo @@ -630,24 +636,12 @@ private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionIt hasParams = data.parameterCount > 0, ) - item.additionalEditHandler = - KotlinAutoImportEditHandler( - analysisContext = ctx, - symbolToImport = symbol, - ) - if (symbol.kind == JvmSymbolKind.CONSTRUCTOR) { item.overrideTypeText = symbol.shortName } } - in JvmSymbolKind.CALLABLE_KINDS -> { - item.additionalEditHandler = - KotlinAutoImportEditHandler( - analysisContext = ctx, - symbolToImport = symbol, - ) - } + in JvmSymbolKind.CALLABLE_KINDS -> Unit JvmSymbolKind.TYPE_ALIAS -> { item.detail = (symbol.data as JvmTypeAliasInfo).expandedTypeFqName From bf346be40b4044db330c53d2f8ee54d326cce61f Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Wed, 7 Oct 2026 15:42:16 +0000 Subject: [PATCH 46/48] ADFA-6228: Leave member callables out of the unimported-symbol queries Constructors, Java fields and member functions were fetched and then always rejected, and with no ORDER BY they could fill the whole prefix budget, so a prefix like `get` lost real candidates such as stdlib extensions. --- .../indexing/jvm/JvmSymbolIndex.kt | 7 +- .../kotlin/completion/KotlinCompletions.kt | 64 ++++++++++++------- .../CollectUnimportedSymbolMatchesTest.kt | 34 ++++++++++ .../UnimportedSymbolCandidateTest.kt | 17 ++++- 4 files changed, 94 insertions(+), 28 deletions(-) diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index dcaa70b9bb..b6d0c884c4 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -108,17 +108,20 @@ open class JvmSymbolIndex( * @param limit The result limit. * @param kinds When non-null, only symbols of these kinds, filtered in the query so that the * limit is not spent on rows the caller would discard. + * @param topLevelOnly Whether to leave out symbols declared inside a class. * @see query */ fun findByPrefix( prefix: String, limit: Int = 200, kinds: Set? = null, + topLevelOnly: Boolean = false, ): Sequence = query( indexQuery { prefix(KEY_NAME, prefix) if (kinds != null) anyOf(KEY_KIND, kinds.map { it.name }) + if (topLevelOnly) notExists(KEY_CONTAINING_CLASS) this.limit = limit }, ) @@ -195,16 +198,18 @@ open class JvmSymbolIndex( * * Restricting by kind matters more than it looks: without it an exact-name lookup still returns * every method and field sharing that name, and a caller wanting only classifiers pays for all - * of them. + * of them. [topLevelOnly] likewise leaves out symbols declared inside a class. */ fun findBySimpleName( name: String, limit: Int = 200, kinds: Set? = null, + topLevelOnly: Boolean = false, ) = query( indexQuery { eq(KEY_NAME, name) if (kinds != null) anyOf(KEY_KIND, kinds.map { it.name }) + if (topLevelOnly) notExists(KEY_CONTAINING_CLASS) this.limit = limit }, ) diff --git a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt index 474e4646e5..bf257190c7 100644 --- a/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt +++ b/lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/completion/KotlinCompletions.kt @@ -93,13 +93,9 @@ private const val UNIMPORTED_SYMBOL_DISPLAY_LIMIT = 100 * Rows each index's prefix query may return before filtering, sized so ordinary filtering cannot * starve the result. * - * The query's `kinds` filter already excludes every kind unimported-symbol completion can never - * offer (a file facade, a companion object, anything neither classifier nor callable), but it - * cannot tell a top-level or extension callable from a member one -- that distinction is - * [isUnimportedSymbolCandidate]'s job, run after the fetch, and member callables dominate most - * packages. The budget covers that remaining rejection, plus the current-package and visibility - * filters, without being unbounded: an unrestricted prefix query over the symbol table runs on - * every keystroke. + * The queries already leave out every row [isUnimportedSymbolCandidate] would reject, so the budget + * only has to cover the current-package and visibility filters, without being unbounded: an + * unrestricted prefix query over the symbol table runs on every keystroke. */ private const val UNIMPORTED_SYMBOL_FETCH_BUDGET = 3 * UNIMPORTED_SYMBOL_DISPLAY_LIMIT @@ -465,6 +461,7 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) indexes = indexes, partial = ctx.partial, kinds = UNIMPORTED_SYMBOL_KINDS, + topLevelKinds = UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS, limit = UNIMPORTED_SYMBOL_DISPLAY_LIMIT, fetchBudget = UNIMPORTED_SYMBOL_FETCH_BUDGET, accept = ::addCompletionItem, @@ -489,11 +486,14 @@ private fun KaSession.collectUnimportedSymbols(to: MutableList) * duplicate from a source [accept] does allow through. A rejected row's own source and key are * remembered too, so the same physical row surfacing again from the prefix stage is skipped without * a second call to [accept]. + * + * [kinds] are matched wherever they are declared, [topLevelKinds] only at the top level. */ internal fun collectUnimportedSymbolMatches( indexes: List, partial: String, kinds: Set, + topLevelKinds: Set = emptySet(), limit: Int, fetchBudget: Int, accept: (JvmSymbol) -> Boolean, @@ -523,8 +523,16 @@ internal fun collectUnimportedSymbolMatches( return true } - val exactSources = indexes.map { index -> { index.findBySimpleName(partial, limit = limit, kinds = kinds) } } - val prefixSources = indexes.map { index -> { index.findByPrefix(partial, limit = fetchBudget, kinds = kinds) } } + fun exactMatches(index: JvmSymbolIndex): Sequence = + index.findBySimpleName(partial, limit = limit, kinds = kinds) + + index.findBySimpleName(partial, limit = limit, kinds = topLevelKinds, topLevelOnly = true) + + fun prefixMatches(index: JvmSymbolIndex): Sequence = + index.findByPrefix(partial, limit = fetchBudget, kinds = kinds) + + index.findByPrefix(partial, limit = fetchBudget, kinds = topLevelKinds, topLevelOnly = true) + + val exactSources = indexes.map { index -> { exactMatches(index) } } + val prefixSources = indexes.map { index -> { prefixMatches(index) } } return collectUpToLimit(limit = limit, sources = exactSources + prefixSources, accept = ::acceptFirstOccurrence) } @@ -569,23 +577,33 @@ internal fun collectUpToLimit( } /** - * The kinds unimported-symbol completion can offer: classifiers and callables. + * The kinds unimported-symbol completion offers wherever they are declared: classifiers and + * extensions. * - * A file facade is neither, and Kotlin cannot name one. A companion object is reached through its - * class, so offering every `Companion` in the index by that bare name only adds noise. + * A file facade is not a classifier, and Kotlin cannot name one. A companion object is reached + * through its class, so offering every `Companion` in the index by that bare name only adds noise. */ internal val UNIMPORTED_SYMBOL_KINDS: Set = - (JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT) + JvmSymbolKind.CALLABLE_KINDS + (JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT) + + JvmSymbolKind.EXTENSION_FUNCTION + + JvmSymbolKind.EXTENSION_PROPERTY + +/** + * The kinds unimported-symbol completion offers only when declared at the top level. + * + * A member is reached through its receiver, never imported by name. Constructors and Java fields + * are always members, so they are not here at all. + */ +internal val UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS: Set = + setOf(JvmSymbolKind.FUNCTION, JvmSymbolKind.PROPERTY) /** * Whether [symbol] can be offered as an unimported completion, whose bare name Kotlin resolves once * the item's auto-import is applied. */ -internal fun isUnimportedSymbolCandidate(symbol: JvmSymbol): Boolean { - if (symbol.kind !in UNIMPORTED_SYMBOL_KINDS) return false - // A member callable is reached through its receiver, never imported by name. - return !symbol.kind.isCallable || symbol.isTopLevel || symbol.isExtension -} +internal fun isUnimportedSymbolCandidate(symbol: JvmSymbol): Boolean = + symbol.kind in UNIMPORTED_SYMBOL_KINDS || + (symbol.kind in UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS && symbol.isTopLevel) context(ctx: AnalysisContext) private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionItem? { @@ -628,20 +646,18 @@ private fun KaSession.buildUnimportedSymbolItem(symbol: JvmSymbol): CompletionIt symbolToImport = symbol, ) when (symbol.kind) { - JvmSymbolKind.EXTENSION_FUNCTION, JvmSymbolKind.FUNCTION, JvmSymbolKind.CONSTRUCTOR -> { + JvmSymbolKind.EXTENSION_FUNCTION, JvmSymbolKind.FUNCTION -> { val data = symbol.data as JvmFunctionInfo item.detail = data.signatureDisplay item.setInsertTextForFunction( name = symbol.shortName, hasParams = data.parameterCount > 0, ) - - if (symbol.kind == JvmSymbolKind.CONSTRUCTOR) { - item.overrideTypeText = symbol.shortName - } } - in JvmSymbolKind.CALLABLE_KINDS -> Unit + in JvmSymbolKind.CALLABLE_KINDS -> { + Unit + } JvmSymbolKind.TYPE_ALIAS -> { item.detail = (symbol.data as JvmTypeAliasInfo).expandedTypeFqName diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt index 5875ba72ee..a4b3193c3f 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/CollectUnimportedSymbolMatchesTest.kt @@ -222,4 +222,38 @@ class CollectUnimportedSymbolMatchesTest { assertThat(accepted).hasSize(2) } + + @Test + fun `member functions do not use up the fetch budget before a top-level match`() { + val libraryIndex = newIndex() + for (suffix in listOf("A", "B", "C", "D", "E")) { + libraryIndex.insertBlocking(memberFunSymbol(shortName = "get$suffix")) + } + libraryIndex.insertBlocking(funSymbol(sourceId = "lib.jar", key = "com/example#getTop()", shortName = "getTop")) + + val accepted = mutableListOf() + collectUnimportedSymbolMatches( + indexes = listOf(libraryIndex), + partial = "get", + kinds = UNIMPORTED_SYMBOL_KINDS, + topLevelKinds = UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS, + limit = 10, + fetchBudget = 3, + accept = { symbol -> isUnimportedSymbolCandidate(symbol).also { if (it) accepted += symbol } }, + ) + + assertThat(accepted.map { it.shortName }).containsExactly("getTop") + } + + private fun memberFunSymbol(shortName: String): JvmSymbol = + JvmSymbol( + key = "com/example/Foo#$shortName()", + sourceId = "lib.jar", + name = "com/example/Foo#$shortName", + shortName = shortName, + packageName = "com.example", + kind = JvmSymbolKind.FUNCTION, + language = JvmSourceLanguage.JAVA, + data = JvmFunctionInfo(containingClassName = "com/example/Foo"), + ) } diff --git a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt index 85e3f94d6b..634b8f4dbd 100644 --- a/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt +++ b/lsp/kotlin/src/test/java/com/itsaky/androidide/lsp/kotlin/completion/UnimportedSymbolCandidateTest.kt @@ -86,8 +86,19 @@ class UnimportedSymbolCandidateTest { } @Test - fun `the query asks for exactly the kinds that can be offered`() { - assertThat(UNIMPORTED_SYMBOL_KINDS) - .containsExactlyElementsIn(JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT + JvmSymbolKind.CALLABLE_KINDS) + fun `a constructor is not offered`() { + val constructor = + symbol("Regex", JvmSymbolKind.CONSTRUCTOR, JvmFunctionInfo(containingClassName = "kotlin/text/Regex")) + + assertThat(isUnimportedSymbolCandidate(constructor)).isFalse() + } + + @Test + fun `the queries ask for exactly the kinds that can be offered`() { + assertThat(UNIMPORTED_SYMBOL_KINDS + UNIMPORTED_TOP_LEVEL_SYMBOL_KINDS) + .containsExactlyElementsIn( + JvmSymbolKind.CLASSIFIER_KINDS - JvmSymbolKind.COMPANION_OBJECT + + JvmSymbolKind.CALLABLE_KINDS - JvmSymbolKind.CONSTRUCTOR - JvmSymbolKind.FIELD, + ) } } From 973203f9f4b386e3c07f623218d32a61a915b35a Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Thu, 8 Oct 2026 13:53:21 +0000 Subject: [PATCH 47/48] ADFA-6228: Decide JVM class nesting from InnerClasses, not a '$' in the name Generated top-level classes such as Foo$$ViewBinder and $AutoValue_Foo were indexed as nested and never offered as top-level classes. Folded into the format version 3 bump this layer already makes. --- .../indexing/jvm/JarSymbolScanner.kt | 23 +++++--- .../indexing/jvm/JvmSymbolIndex.kt | 3 + .../jvm/JarSymbolScannerDollarNameTest.kt | 56 +++++++++++++++++++ .../jvm/JarSymbolScannerVisibilityTest.kt | 9 --- 4 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerDollarNameTest.kt diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt index 1cc9166ebe..ec3aa34599 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScanner.kt @@ -46,6 +46,7 @@ object JarSymbolScanner { private var superName: String? = null private var interfaces: Array? = null private var isInnerClass = false + private var outerClassName: String? = null private var classDeprecated = false override fun visit( @@ -57,7 +58,7 @@ object JarSymbolScanner { interfaces: Array?, ) { className = name - classFqName = name.replace('/', '.').replace('$', '.') + classFqName = name.replace('/', '.') classAccess = access this.superName = superName this.interfaces = interfaces @@ -67,15 +68,17 @@ object JarSymbolScanner { packageName = if (lastSlash >= 0) name.substring(0, lastSlash).replace('/', '.') else "" val afterPackage = if (lastSlash >= 0) name.substring(lastSlash + 1) else name - shortClassName = afterPackage.replace('$', '.') - - isInnerClass = name.contains('$') + shortClassName = afterPackage + isInnerClass = false + outerClassName = null } /* - * JVMS 4.1 forbids ACC_PRIVATE, ACC_PROTECTED and ACC_STATIC in a class file header, so javac - * records a nested class's declared visibility and staticness only in its own InnerClasses - * entry. ASM visits that entry before any member, so the member gates below see the real flags. + * JVMS 4.7.6 gives every nested class an InnerClasses entry for itself, and a top-level class + * none, so that entry, not a '$' in the name, decides nesting: `Foo$$ViewBinder` is top-level. + * JVMS 4.1 also forbids ACC_PRIVATE, ACC_PROTECTED and ACC_STATIC in a class file header, so + * the entry is the only record of a nested class's declared visibility and staticness. ASM + * visits it before any member, so the member gates below see the real flags. */ override fun visitInnerClass( name: String, @@ -84,6 +87,10 @@ object JarSymbolScanner { access: Int, ) { if (name != className) return + isInnerClass = true + outerClassName = outerName + classFqName = className.replace('/', '.').replace('$', '.') + shortClassName = shortClassName.replace('$', '.') classAccess = (classAccess and VISIBILITY_AND_STATIC_FLAGS.inv()) or (access and VISIBILITY_AND_STATIC_FLAGS) } @@ -128,7 +135,7 @@ object JarSymbolScanner { val containingClass = if (isInnerClass) { - className.substringBeforeLast('$') + outerClassName ?: className.substringBeforeLast('$') } else { "" } diff --git a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt index b6d0c884c4..cfff6beae7 100644 --- a/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt +++ b/lsp/jvm-symbol-index/src/main/kotlin/org/appdevforall/codeonthego/indexing/jvm/JvmSymbolIndex.kt @@ -45,6 +45,9 @@ open class JvmSymbolIndex( * produced by an older scanner get replaced. [KtFileMetadataIndex] shares it because a * source file's symbols are re-indexed only when its metadata row says so; dropping the * symbols alone would leave files recorded as indexed with no symbols. + * + * Version 3 keys extension callables by their receiver, stores the key in the payload, and + * stops treating a top-level class with '$' in its name as nested. */ const val FORMAT_VERSION = 3 diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerDollarNameTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerDollarNameTest.kt new file mode 100644 index 0000000000..89059ff5c8 --- /dev/null +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerDollarNameTest.kt @@ -0,0 +1,56 @@ +package org.appdevforall.codeonthego.indexing.jvm + +import com.google.common.truth.Truth.assertThat +import org.jetbrains.org.objectweb.asm.ClassWriter +import org.jetbrains.org.objectweb.asm.Opcodes +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 + +/** + * A '$' in a class file name does not make it nested: code generators emit top-level classes such + * as `Foo$$ViewBinder` and `$AutoValue_Foo`. Only the class's own InnerClasses entry does. + */ +@RunWith(JUnit4::class) +class JarSymbolScannerDollarNameTest { + private fun scanClass( + internalName: String, + outerName: String? = null, + innerName: String? = null, + ): JvmSymbol { + val bytes = + ClassWriter(0) + .apply { + visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, internalName, null, "java/lang/Object", null) + if (innerName != null) visitInnerClass(internalName, outerName, innerName, Opcodes.ACC_PUBLIC) + visitEnd() + }.toByteArray() + return JarSymbolScanner.parseClassFile(bytes.inputStream(), "test.jar").single() + } + + @Test + fun `a top-level class with a doubled dollar in its name is top level`() { + val binder = scanClass("com/example/Foo\$\$ViewBinder") + + assertThat(binder.isTopLevel).isTrue() + assertThat(binder.shortName).isEqualTo("Foo\$\$ViewBinder") + } + + @Test + fun `a top-level class whose name starts with a dollar is top level`() { + val autoValue = scanClass("com/example/\$AutoValue_Foo") + + assertThat(autoValue.isTopLevel).isTrue() + assertThat(autoValue.shortName).isEqualTo("\$AutoValue_Foo") + } + + @Test + fun `a class with its own InnerClasses entry is nested in its outer class`() { + val nested = scanClass("com/example/Outer\$Inner", outerName = "com/example/Outer", innerName = "Inner") + + assertThat(nested.isTopLevel).isFalse() + assertThat(nested.containingClassName).isEqualTo("com/example/Outer") + assertThat(nested.data.containingClassFqName).isEqualTo("com.example.Outer") + assertThat(nested.shortName).isEqualTo("Inner") + } +} diff --git a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt index 2d72f9fc9d..5cd3693cb6 100644 --- a/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt +++ b/lsp/jvm-symbol-index/src/test/kotlin/org/appdevforall/codeonthego/indexing/jvm/JarSymbolScannerVisibilityTest.kt @@ -107,15 +107,6 @@ class JarSymbolScannerVisibilityTest { assertThat(info.isStatic).isFalse() } - @Test - fun `a nested class records its outer class`() { - val nested = scan("com/example/Outer\$Inner", Opcodes.ACC_PUBLIC).single() - - assertThat(nested.shortName).isEqualTo("Inner") - assertThat(nested.isTopLevel).isFalse() - assertThat(nested.data.containingClassFqName).isEqualTo("com.example.Outer") - } - companion object { /** * JUnit4 builds a fresh test instance per `@Test` method, so an instance-level `by lazy` From c72fa572c1efeaef8cc7caf3db0b61cace199189 Mon Sep 17 00:00:00 2001 From: Akash Yadav Date: Thu, 8 Oct 2026 13:53:21 +0000 Subject: [PATCH 48/48] ADFA-6228: Reject an index query on a field the descriptor does not declare --- .../codeonthego/indexing/InMemoryIndex.kt | 7 +- .../codeonthego/indexing/SQLiteIndex.kt | 10 ++- .../codeonthego/indexing/api/Query.kt | 14 ++++ .../codeonthego/indexing/InMemoryIndexTest.kt | 8 +- .../indexing/UndeclaredFieldQueryTest.kt | 75 +++++++++++++++++++ 5 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/UndeclaredFieldQueryTest.kt diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt index 0428e4f3e5..01c8d19ec2 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndex.kt @@ -4,6 +4,7 @@ import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.requireDeclaredFields import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.locks.ReentrantReadWriteLock import kotlin.collections.iterator @@ -193,8 +194,9 @@ class InMemoryIndex( * has matches nothing rather than being ignored. The result is materialized under the read lock * so callers never iterate a set a concurrent writer is mutating. */ - private fun resolveMatchingRows(query: IndexQuery): List = - lock.read { + private fun resolveMatchingRows(query: IndexQuery): List { + query.requireDeclaredFields(descriptor) + return lock.read { var candidates: Set? = null if (query.key != null) { @@ -241,6 +243,7 @@ class InMemoryIndex( (candidates ?: rows.keys).toList() } + } /** * Rows whose [field] starts with [prefix], or `null` when [field] is not declared. diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt index 321a75e545..e44d05936e 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/SQLiteIndex.kt @@ -18,6 +18,7 @@ import org.appdevforall.codeonthego.indexing.api.Index import org.appdevforall.codeonthego.indexing.api.IndexDescriptor import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.requireDeclaredFields import org.slf4j.LoggerFactory import kotlin.collections.iterator @@ -758,6 +759,7 @@ class SQLiteIndex( query: IndexQuery, sourceIdChunk: List?, ): Pair> { + query.requireDeclaredFields(descriptor) val where = StringBuilder() val args = mutableListOf() @@ -783,12 +785,12 @@ class SQLiteIndex( } for ((field, value) in query.exactMatch) { - val col = fieldColumns[field] ?: continue + val col = fieldColumns.getValue(field) and("${filterUnlessSelective(field)}$col = ?", value) } for ((field, values) in query.anyOf) { - val col = fieldColumns[field] ?: continue + val col = fieldColumns.getValue(field) if (values.isEmpty()) { // Scoped to nothing, as opposed to unscoped: no row can satisfy it. and("0 = 1") @@ -803,7 +805,7 @@ class SQLiteIndex( val lowerCol = prefixColumns[field] // Prefix-searchable fields match case-insensitively through their pre-lowercased column; // everything else matches the stored value as-is. - val col = lowerCol ?: fieldColumns[field] ?: continue + val col = lowerCol ?: fieldColumns.getValue(field) val value = if (lowerCol != null) prefix.lowercase() else prefix if (value.isEmpty()) { @@ -833,7 +835,7 @@ class SQLiteIndex( } for ((field, mustExist) in query.presence) { - val col = fieldColumns[field] ?: continue + val col = fieldColumns.getValue(field) if (mustExist) { and("$filter$col IS NOT NULL") } else { diff --git a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt index 48354a70d3..85745e5eb0 100644 --- a/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt +++ b/lsp/indexing/src/main/kotlin/org/appdevforall/codeonthego/indexing/api/Query.kt @@ -64,6 +64,20 @@ data class IndexQuery( } } +/** + * Throws [IllegalArgumentException] if this query has a predicate on a field [descriptor] does not + * declare. + * + * Every index implementation checks this, so a misspelt field fails the same way everywhere rather + * than matching nothing in one implementation and every row in another. + */ +internal fun IndexQuery.requireDeclaredFields(descriptor: IndexDescriptor<*>) { + val declared = descriptor.fields.mapTo(HashSet()) { it.name } + val named = exactMatch.keys + prefixMatch.keys + anyOf.keys + presence.keys + val unknown = named - declared + require(unknown.isEmpty()) { "Unknown field(s) $unknown in a query on ${descriptor.name}" } +} + /** * DSL builder for [IndexQuery]. */ diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt index f87e4a9407..a9fa04b3f4 100644 --- a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/InMemoryIndexTest.kt @@ -7,6 +7,7 @@ import org.appdevforall.codeonthego.indexing.api.IndexField import org.appdevforall.codeonthego.indexing.api.IndexQuery import org.appdevforall.codeonthego.indexing.api.Indexable import org.appdevforall.codeonthego.indexing.api.indexQuery +import org.junit.Assert.assertThrows import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.JUnit4 @@ -257,13 +258,14 @@ class InMemoryIndexTest { } @Test - fun `query with unknown field returns empty sequence`() = + fun `query with unknown field is rejected`() = runTest { val index = makeIndex() index.insert(entry("k1", "src1", "Foo")) - val results = index.query(indexQuery { eq("nonexistentField", "value") }).toList() - assertThat(results).isEmpty() + assertThrows(IllegalArgumentException::class.java) { + index.query(indexQuery { eq("nonexistentField", "value") }).toList() + } } @Test diff --git a/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/UndeclaredFieldQueryTest.kt b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/UndeclaredFieldQueryTest.kt new file mode 100644 index 0000000000..0c7e7d9ef8 --- /dev/null +++ b/lsp/indexing/src/test/kotlin/org/appdevforall/codeonthego/indexing/UndeclaredFieldQueryTest.kt @@ -0,0 +1,75 @@ +package org.appdevforall.codeonthego.indexing + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import kotlinx.coroutines.test.runTest +import org.appdevforall.codeonthego.indexing.api.Index +import org.appdevforall.codeonthego.indexing.api.IndexDescriptor +import org.appdevforall.codeonthego.indexing.api.IndexField +import org.appdevforall.codeonthego.indexing.api.IndexQuery +import org.appdevforall.codeonthego.indexing.api.Indexable +import org.appdevforall.codeonthego.indexing.api.indexQuery +import org.junit.After +import org.junit.Assert.assertThrows +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * A query naming a field the descriptor does not declare is a caller bug, which SQLite used to hide + * by dropping the predicate and returning every row in scope. InMemoryIndexTest covers the in-memory + * index, which rejects it the same way. + */ +@RunWith(RobolectricTestRunner::class) +class UndeclaredFieldQueryTest { + data class Entry( + override val key: String, + override val sourceId: String, + val kind: String, + ) : Indexable + + private val descriptor = + object : IndexDescriptor { + override val name = "test_undeclared" + override val fields = listOf(IndexField("kind")) + + override fun fieldValues(entry: Entry) = mapOf("kind" to entry.kind) + + override fun serialize(entry: Entry) = listOf(entry.key, entry.sourceId, entry.kind).joinToString("|").toByteArray() + + override fun deserialize(bytes: ByteArray): Entry { + val parts = String(bytes).split("|") + return Entry(parts[0], parts[1], parts[2]) + } + } + + private val sqlite = + SQLiteIndex( + descriptor = descriptor, + context = ApplicationProvider.getApplicationContext(), + dbName = null, + formatVersion = 1, + ) + + @After + fun tearDown() { + sqlite.close() + } + + private fun assertRejects( + index: Index, + query: IndexQuery, + ) = runTest { + index.insert(Entry("k1", "jar", "CLASS")) + assertThrows(IllegalArgumentException::class.java) { index.query(query).toList() } + } + + @Test + fun `SQLite rejects an anyOf on an undeclared field`() = assertRejects(sqlite, indexQuery { anyOf("kinds", listOf("CLASS")) }) + + @Test + fun `SQLite rejects an exact match on an undeclared field`() = assertRejects(sqlite, indexQuery { eq("kinds", "CLASS") }) + + @Test + fun `SQLite rejects a presence check on an undeclared field`() = assertRejects(sqlite, indexQuery { notExists("kinds") }) +}