diff --git a/README.md b/README.md index 43c01b5c..9c551001 100644 --- a/README.md +++ b/README.md @@ -357,7 +357,8 @@ val results = client.search(SQLQuery("SELECT * FROM users WHERE age > 25")) // Type-safe queries with compile-time validation case class User(id: String, name: String, age: Int) val users: Source[User, NotUsed] = client.scrollAs[User]( - "SELECT id, name, age FROM users WHERE active = true" + "SELECT id, name, age FROM users WHERE active = true", + client.defaultScrollConfig // the macro needs an explicit config; this one carries elastic.scroll.* ) ``` diff --git a/build.sbt b/build.sbt index ac14c27d..26269006 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ ThisBuild / organization := "app.softnetwork" name := "softclient4es" -ThisBuild / version := "0.20.4" +ThisBuild / version := "0.21.0-SNAPSHOT" ThisBuild / scalaVersion := scala213 @@ -183,7 +183,9 @@ lazy val testkit = Project(id = "softclient4es-core-testkit", base = file("testk buildInfoKeys += BuildInfoKey("elasticVersion" -> elasticSearchVersion.value), buildInfoObject := "SoftClient4esCoreTestkitBuildInfo", organization := "app.softnetwork.elastic", - name := s"softclient4es-core-testkit" + name := s"softclient4es-core-testkit", + // the template compiles against logback (SlicedScrollCompletenessSpec log capture, #238) + libraryDependencies += "ch.qos.logback" % "logback-classic" % Versions.logback ) .enablePlugins(BuildInfoPlugin) .dependsOn( @@ -224,6 +226,9 @@ def testkitProject(esVersion: String, ss: Def.SettingsDefinition*): Project = { "org.apache.logging.log4j" % "log4j-api" % Versions.log4j, // "org.apache.logging.log4j" % "log4j-slf4j-impl" % Versions.log4j, "org.apache.logging.log4j" % "log4j-core" % Versions.log4j, + // SlicedScrollCompletenessSpec (#238) captures the client log through logback's + // ListAppender at compile time — declared, not inherited from persistence-core + "ch.qos.logback" % "logback-classic" % Versions.logback, "app.softnetwork.persistence" %% "persistence-core-testkit" % Versions.genericPersistence, "org.testcontainers" % "testcontainers-elasticsearch" % Versions.testContainers excludeAll (jacksonExclusions: _*), "org.testcontainers" % "testcontainers-minio" % Versions.testContainers, diff --git a/core/src/main/resources/softnetwork-elastic.conf b/core/src/main/resources/softnetwork-elastic.conf index b0e549c2..f0299512 100644 --- a/core/src/main/resources/softnetwork-elastic.conf +++ b/core/src/main/resources/softnetwork-elastic.conf @@ -34,6 +34,17 @@ elastic { connection-timeout = 5s socket-timeout = 30s + # Paged row extraction (scroll / PIT + search_after) + scroll { + # Rows per page. Larger pages cut round-trips linearly and raise in-flight memory linearly. + size = 1000 + size = ${?ELASTIC_SCROLL_SIZE} + # Ceiling on concurrent PIT slices for a no-ORDER-BY extraction (ES 7.15+). The effective + # count is min(primary shards, max-slices); 1 disables slicing (sequential paging). + max-slices = 8 + max-slices = ${?ELASTIC_SCROLL_MAX_SLICES} + } + # When enabled, result rows surface the Elasticsearch document id as an `_id` column. # Disabled by default: SQL results carry only the selected columns. include-document-id = false diff --git a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala index a1d4f805..7c2e8f3d 100644 --- a/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala +++ b/core/src/main/scala-2.12/app/softnetwork/elastic/client/ElasticConfig.scala @@ -42,6 +42,9 @@ import java.time.Duration * @param includeDocumentId * When enabled, result rows surface the Elasticsearch document id as an `_id` column (disabled * by default) + * @param scroll + * Paged row extraction settings (`elastic.scroll`: page size and the ceiling on concurrent PIT + * slices, #238) */ case class ElasticConfig( credentials: ElasticCredentials = ElasticCredentials(), @@ -51,7 +54,8 @@ case class ElasticConfig( socketTimeout: Duration, metrics: MetricsConfig, watcher: ElasticCredentials, - includeDocumentId: Boolean = false) + includeDocumentId: Boolean = false, + scroll: ScrollSettings = ScrollSettings()) object ElasticConfig extends StrictLogging { def apply(config: Config): ElasticConfig = { diff --git a/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala b/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala index 78eac170..590fc3eb 100644 --- a/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala +++ b/core/src/main/scala-2.13/app/softnetwork/elastic/client/ElasticConfig.scala @@ -42,6 +42,9 @@ import java.time.Duration * @param includeDocumentId * When enabled, result rows surface the Elasticsearch document id as an `_id` column (disabled * by default) + * @param scroll + * Paged row extraction settings (`elastic.scroll`: page size and the ceiling on concurrent PIT + * slices, #238) */ case class ElasticConfig( credentials: ElasticCredentials = ElasticCredentials(), @@ -51,7 +54,8 @@ case class ElasticConfig( socketTimeout: Duration, metrics: MetricsConfig, watcher: ElasticCredentials, - includeDocumentId: Boolean = false + includeDocumentId: Boolean = false, + scroll: ScrollSettings = ScrollSettings() ) object ElasticConfig extends StrictLogging { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala index 7972933b..8812a19a 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticClientApi.scala @@ -17,6 +17,7 @@ package app.softnetwork.elastic.client import app.softnetwork.common.ClientCompanion +import app.softnetwork.elastic.client.scroll.ScrollConfig import app.softnetwork.elastic.licensing.metrics.MetricsApi import com.typesafe.config.{Config, ConfigFactory} import org.slf4j.Logger @@ -65,4 +66,14 @@ trait ElasticClientApi * `elastic.include-document-id` (disabled by default). */ override protected def includeDocumentId: Boolean = elasticConfig.includeDocumentId + + /** Paged row extraction defaults come from `elastic.scroll` (#238): the page size is applied + * here, the slice ceiling is inherited through `maxSlices = None` so an explicit + * `ScrollConfig(...)` still honours the HOCON/env opt-out. A `def` — see + * [[ScrollApi.defaultScrollConfig]]. + */ + override def defaultScrollConfig: ScrollConfig = + ScrollConfig(scrollSize = elasticConfig.scroll.size) + + override protected def configuredMaxSlices: Int = elasticConfig.scroll.maxSlices } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala index fb5fb76c..ea72745f 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ElasticsearchVersion.scala @@ -92,6 +92,16 @@ object ElasticsearchVersion { isAtLeast(version, 7, 12) } + /** Check if PIT slicing is usable (ES >= 7.15). + * + * `slice` + `pit` in one search request exists from 7.15 (elastic/elasticsearch#74457); on + * 7.12–7.14 a PIT extraction pages sequentially (#238). Under a PIT, slices are contiguous + * doc-id ranges split first across shards — not the `_id` hash filter of sliced scroll. + */ + def supportsPitSlicing(version: String): Boolean = { + isAtLeast(version, 7, 15) + } + /** Check if version is ES 8+ */ def isEs8OrHigher(version: String): Boolean = { diff --git a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala index c1552941..98336a21 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/IndicesApi.scala @@ -195,6 +195,9 @@ trait IndicesApi extends ElasticClientHelpers { executeCreateIndex(index, settings, updatedMappings, aliases) match { case success @ ElasticSuccess(true) => + // #238 — a shard count cached for an expression this index is the stem of (a name probed + // before it existed, `orders*` before the load) must not survive the creation + invalidateShardCounts(Some(index)) logger.info(s"✅ Index '$index' created successfully") success case success @ ElasticSuccess(_) => @@ -230,16 +233,20 @@ trait IndicesApi extends ElasticClientHelpers { def updateSchema(index: String, schema: Schema): Unit = { schemaCache.put(index, (schema, System.currentTimeMillis())) + // #238 — ALTER TABLE may have reindexed into a different shard count + invalidateShardCounts(Some(index)) logger.debug(s"📦 Schema cache updated for '$index'") } def invalidateSchema(index: String): Unit = { schemaCache.remove(index) + invalidateShardCounts(Some(index)) // #238 — the sliced-paging shard counts follow the schema logger.info(s"🗑️ Schema cache invalidated for '$index'") } def invalidateAllSchemas(): Unit = { schemaCache.clear() + invalidateShardCounts() logger.info("🗑️ All schema caches invalidated") } diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala index ce705c7e..53c71929 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollApi.scala @@ -36,7 +36,7 @@ import app.softnetwork.elastic.sql.query.{ SelectStatement, SingleSearch } -import org.json4s.{Formats, JNothing} +import org.json4s.{Formats, JNothing, JValue} import org.json4s.jackson.JsonMethods.parse import scala.collection.immutable.ListMap @@ -79,16 +79,25 @@ import scala.util.{Failure, Success} * The implementation automatically selects the most efficient strategy: * * {{{ - * ┌─────────────────┬───────────────┬──────────────────────────────────┐ - * │ ES Version │ Aggregations │ Strategy │ - * ├─────────────────┼───────────────┼──────────────────────────────────┤ - * │ 7.12+ │ No │ PIT + search_after (recommended) │ - * │ 7.12+ │ Yes │ Classic scroll │ - * │ < 7.12 │ No │ search_after │ - * │ < 7.12 │ Yes │ Classic scroll │ - * └─────────────────┴───────────────┴──────────────────────────────────┘ + * ┌─────────────────┬───────────────┬──────────────────────────────────────────────────┐ + * │ ES Version │ Aggregations │ Strategy │ + * ├─────────────────┼───────────────┼──────────────────────────────────────────────────┤ + * │ 7.15+ │ No │ PIT + search_after, SLICED when no ORDER BY / no │ + * │ │ │ LIMIT: one reader per primary shard (#238) │ + * │ 7.12+ │ No │ PIT + search_after (recommended) │ + * │ 7.12+ │ Yes │ Classic scroll │ + * │ < 7.12 │ No │ search_after │ + * │ < 7.12 │ Yes │ Classic scroll │ + * └─────────────────┴───────────────┴──────────────────────────────────────────────────┘ * }}} * + * '''Sliced PIT paging''' (ES 7.15+, #238): a no-`ORDER BY`, no-`LIMIT` extraction opens ONE PIT + * and reads `min(primary shards, max-slices)` slices of it concurrently, merged page by page into + * the single stream the caller consumes. The slice count is resolved once per stream by + * [[ScrollApi]] (`ScrollMetrics.slices` reports it); the ceiling comes from + * [[ScrollConfig.maxSlices]] or, when unset, `elastic.scroll.max-slices` (`1` = sequential). Row + * order interleaves across slices; quota-capped results are an arbitrary subset. + * * [[ScrollConfig.preferSearchAfter]] ` = false` overrides the no-aggregation rows and forces * classic scroll on every version — an operational opt-out for clusters that restrict the PIT API. * Slower than the PIT path (9.55 s vs 6.15 s per 1M rows) but equally row-complete. @@ -122,17 +131,127 @@ import scala.util.{Failure, Success} * [[https://www.elastic.co/guide/en/elasticsearch/reference/7.10/point-in-time-api.html PIT API Documentation]] */ trait ScrollApi extends ElasticClientHelpers { - _: VersionApi with SearchApi => + _: VersionApi with SearchApi with SettingsApi => // ======================================================================== // MAIN SCROLL METHODS // ======================================================================== + /** The scroll configuration used when a caller passes none (#238): page size from + * `elastic.scroll.size`, slice ceiling inherited (`maxSlices = None`) from + * [[configuredMaxSlices]]. A `def`, never a `val`: `ScrollMetrics.startTime` is a constructor + * default and must be fresh per stream. Overridden by [[ElasticClientApi]] with the HOCON + * values; the base keeps the historical `ScrollConfig()` defaults. + */ + def defaultScrollConfig: ScrollConfig = ScrollConfig() + + /** The ceiling on concurrent PIT slices applied when a [[ScrollConfig]] leaves `maxSlices` unset + * (`elastic.scroll.max-slices`, #238). `1` is a complete opt-out. + */ + protected def configuredMaxSlices: Int = ScrollConfig.DefaultMaxSlices + + /** TTL, in milliseconds, of the primary-shard-count cache consulted by sliced PIT paging (#238). + * Override in a subclass to change (default: 5 minutes, the schema cache's TTL). One `_settings` + * round-trip per distinct index set per TTL instead of one per un-LIMITed row query. What is + * cached: a positive count, and a **privilege** failure (HTTP 401 / 403 — the one failure class + * that is deterministic), so an under-privileged user sees ONE WARN per TTL instead of one per + * query; a transient failure (timeout, 503, index not found, unparseable payload) and an + * expression that matches no index are NOT cached — the next extraction probes again. Staleness + * is performance-only: a count that no longer matches the index changes how the PIT is split, + * never which rows come back. Every schema-cache write or invalidation on this client + * (`createIndex`, `updateSchema`, `invalidateSchema` — `DROP TABLE`, REPL `refresh`) drops the + * counts naming that index — see [[invalidateShardCounts]] for the match rule — and + * `invalidateAllSchemas` drops all of them; DDL issued through another client, and an entry + * whose expression the index does not prefix, are seen after the TTL. + */ + protected def shardCountCacheTtlMs: Long = 5 * 60 * 1000L + + private val shardCountCache = + new java.util.concurrent.ConcurrentHashMap[String, (ElasticResult[Int], Long)]() + + /** Above this many entries a cache miss also purges the expired ones (keys are index SETS, so a + * long-lived server over date-suffixed or per-tenant indices would otherwise grow without + * bound). + */ + private val shardCountCachePurgeThreshold = 256 + + private def shardCountCacheable(result: ElasticResult[Int]): Boolean = result match { + case ElasticSuccess(shards) => shards > 0 + case ElasticFailure(err) => err.statusCode.exists(s => s == 401 || s == 403) + } + + /** [[SettingsApi.primaryShardCount]] behind the TTL cache. Keyed by the sorted distinct index + * expressions (an index name cannot contain `,`); resolved under the map's per-key lock + * (`compute`), so K concurrent cold extractions of one index set issue ONE round-trip and the + * result is stamped after the lookup. Returns the lookup result and whether it was served from + * the cache — the caller logs a cached failure at DEBUG, a fresh one at WARN. + */ + private[client] def cachedPrimaryShardCount( + indices: Seq[String] + ): (ElasticResult[Int], Boolean) = { + val key = indices.distinct.sorted.mkString(",") + val ttl = shardCountCacheTtlMs + var result: ElasticResult[Int] = null + var fromCache = true + shardCountCache.compute( + key, + (_: String, entry: (ElasticResult[Int], Long)) => { + if (entry != null && System.currentTimeMillis() - entry._2 < ttl) { + result = entry._1 + entry + } else { + fromCache = false + val fresh = primaryShardCount(indices) + result = fresh + if (shardCountCacheable(fresh)) { + val stored = fresh match { + case ElasticFailure(err) => + ElasticFailure(err.copy(cause = None)) // never pin a stack + case success => success + } + (stored, System.currentTimeMillis()) + } else null // not cached: a null mapping removes the (expired) entry + } + } + ) + if (!fromCache && shardCountCache.size() > shardCountCachePurgeThreshold) { + val now = System.currentTimeMillis() + shardCountCache + .entrySet() + .removeIf((e: java.util.Map.Entry[String, (ElasticResult[Int], Long)]) => + now - e.getValue._2 >= ttl + ) + } + (result, fromCache) + } + + /** Drop cached primary shard counts — called on every schema-cache write or invalidation. + * + * With an `index`, only the entries naming it are dropped: cache keys are the sorted, joined + * FROM expressions of a statement, so the match is made **per expression** and by + * case-insensitive PREFIX (an index name is lower-case on the wire, and a prefix catches the + * expressions an index name is the stem of — `orders` drops `orders`, `orders_2026`, `orders*` — + * as well as the sets containing them). Expressions that do NOT start with the index — an + * unrelated wildcard `logs-*` that the new index nevertheless matches, or an alias — keep their + * cached count until the TTL; that is a performance matter only (it changes how a PIT is split, + * never which rows come back). `None` drops everything. + */ + private[client] def invalidateShardCounts(index: Option[String] = None): Unit = index match { + case Some(i) if i.nonEmpty => + val prefix = i.toLowerCase + shardCountCache + .keySet() + .removeIf((k: String) => k.split(',').exists(_.toLowerCase.startsWith(prefix))) + () + case _ => + shardCountCache.clear() + } + /** Create a scrolling source with automatic strategy selection */ def scroll( statement: SearchStatement, - config: ScrollConfig = ScrollConfig() + config: ScrollConfig = defaultScrollConfig )(implicit system: ActorSystem, context: ConversionContext @@ -156,12 +275,15 @@ trait ScrollApi extends ElasticClientHelpers { // Single search case single: SingleSearch => + // #238 — an explicit LIMIT keeps the sequential PIT path on EVERY branch, including the + // window-enrichment branch below (createBaseQuery keeps the LIMIT — AC 6). + val config0 = if (single.limit.isDefined) config.copy(maxSlices = Some(1)) else config if ( single.windowFunctions.exists(_.isWindowing) && (!single.select.fields.forall( _.isAggregation ) || single.scriptFields.nonEmpty) ) - return scrollWithWindowEnrichment(single, config) + return scrollWithWindowEnrichment(single, config0) val requestedFields = extractOutputFieldNames(single) val elasticQuery = @@ -177,7 +299,7 @@ trait ScrollApi extends ElasticClientHelpers { single.sqlAggregations, // `_id` is injected at parse time only when it will be kept — the streamed rows // need no per-row strip on this hot path. - config.copy(retainDocumentId = keepsDocumentId(requestedFields)), + config0.copy(retainDocumentId = keepsDocumentId(requestedFields)), single.sorts.nonEmpty, requestedFields, single.nestedHitsMappings @@ -224,7 +346,11 @@ trait ScrollApi extends ElasticClientHelpers { /** Typed scroll source converting results into typed entities from an SQL query * * @note - * This method provides compile-time SQL validation via macros. + * This method provides compile-time SQL validation via macros. Macro applications do not + * support default arguments (scalac: "macro applications do not support named and/or default + * arguments"), so `config` must always be passed explicitly — use [[defaultScrollConfig]] (or + * a `.copy` of it) to keep the `elastic.scroll` settings; [[scrollAsUnchecked]] does apply it + * when omitted. * * @param sql * - SQL query @@ -273,7 +399,7 @@ trait ScrollApi extends ElasticClientHelpers { */ def scrollAsUnchecked[T]( sql: SelectStatement, - config: ScrollConfig = ScrollConfig() + config: ScrollConfig = defaultScrollConfig )(implicit system: ActorSystem, m: Manifest[T], @@ -289,19 +415,30 @@ trait ScrollApi extends ElasticClientHelpers { // PRIVATE METHODS // ======================================================================== + /** The query body parsed ONCE per stream — `JNothing` when it is not valid JSON (both readers + * treat that as "clause absent", their historical behaviour). Both the strategy decision (`aggs` + * / `aggregations`) and the slice decision (`sort`) read this tree instead of parsing the same + * string twice per extraction. It is a per-stream cost, not a per-page one. + */ + private def parseQueryBody(query: String): JValue = + try parse(query) + catch { case _: Exception => JNothing } + /** Determine the best scroll strategy based on the query */ - private def determineScrollStrategy( + private[client] def determineScrollStrategy( elasticQuery: ElasticQuery, aggregations: ListMap[String, SQLAggregation], - config: ScrollConfig + config: ScrollConfig, + queryBody: JValue ): ScrollStrategy = { // If aggregations are present, use classic scrolling if (aggregations.nonEmpty) { UseScroll } else { - // Check if the query contains aggregations in the JSON - if (hasAggregations(elasticQuery.query)) { + // Check if the query contains aggregations in the JSON: `sqlAggregations` carries the METRIC + // aggregations only, so a bucket-only GROUP BY reaches here with an empty map (load-bearing). + if (hasAggregations(queryBody)) { UseScroll } else if (!config.preferSearchAfter) { // Operational opt-out (#201): classic scroll is slower than fixed PIT (9.55 s vs @@ -326,13 +463,20 @@ trait ScrollApi extends ElasticClientHelpers { } } - /** Scroll with metrics tracking + /** Scroll with metrics tracking. + * + * The strategy and the slice count (#238) are resolved ONCE per stream, off the caller's thread: + * the shard lookup behind [[resolveSlices]] is blocking HTTP and `scroll(...)` is built eagerly + * by gateway / JDBC / `Await` callers. `requested` is the caller's configuration; the resolved + * `config` bound below is what every downstream read uses — `maxDocuments`, `scrollSize`, + * `metrics`, `logEvery` and the zero-row metrics fallback included. A strategy / version failure + * surfaces as a failed stream (it used to be a synchronous throw). */ private def scrollWithMetrics( elasticQuery: ElasticQuery, fieldAliases: ListMap[String, String], aggregations: ListMap[String, SQLAggregation], - config: ScrollConfig, + requested: ScrollConfig, hasSorts: Boolean = false, fields: Seq[String] = Seq.empty, nestedHits: Map[String, Seq[(String, String)]] = Map.empty @@ -343,61 +487,164 @@ trait ScrollApi extends ElasticClientHelpers { implicit val ec: ExecutionContext = system.dispatcher - val metricsPromise = Promise[ScrollMetrics]() - - scroll(elasticQuery, fieldAliases, aggregations, config, hasSorts, fields, nestedHits) - .take(config.maxDocuments.getOrElse(Long.MaxValue)) - .grouped(config.scrollSize) - .statefulMapConcat { () => - var metrics = config.metrics // Thread-safe as statefulMapConcat is single-threaded - batch => { - metrics = metrics.copy( - totalDocuments = metrics.totalDocuments + batch.size, - totalBatches = metrics.totalBatches + 1 - ) - - if (metrics.totalBatches % config.logEvery == 0) { - logger.info( - s"Scroll progress: ${metrics.totalDocuments} docs, " + - s"${metrics.totalBatches} batches, " + - s"${metrics.documentsPerSecond} docs/sec" + Source + .lazyFutureSource { () => + // resolved on first demand (never for a source that is built but not run), inside + // `blocking`: the version / `_settings` round-trips are blocking HTTP on the dispatcher + Future { + scala.concurrent.blocking { + // ONE parse of the request body for both decisions (strategy: `aggs`; slices: `sort`) + val queryBody = parseQueryBody(elasticQuery.query) + val strategy = determineScrollStrategy(elasticQuery, aggregations, requested, queryBody) + val slices = resolveSlices(elasticQuery, strategy, requested, hasSorts, queryBody) + ( + strategy, + requested.copy(slices = slices, metrics = requested.metrics.copy(slices = slices)) ) } - batch.map(doc => (doc, metrics)) - } + }.map { case (strategy, config) => + val metricsPromise = Promise[ScrollMetrics]() + + scroll( + elasticQuery, + fieldAliases, + aggregations, + config, + hasSorts, + fields, + nestedHits, + strategy + ) + .take(config.maxDocuments.getOrElse(Long.MaxValue)) + .grouped(config.scrollSize) + .statefulMapConcat { () => + var metrics = config.metrics // Thread-safe as statefulMapConcat is single-threaded + batch => { + metrics = metrics.copy( + totalDocuments = metrics.totalDocuments + batch.size, + totalBatches = metrics.totalBatches + 1 + ) + + if (metrics.totalBatches % config.logEvery == 0) { + logger.info( + s"Scroll progress: ${metrics.totalDocuments} docs, " + + s"${metrics.totalBatches} batches, " + + s"${metrics.documentsPerSecond} docs/sec" + ) + } + batch.map(doc => (doc, metrics)) + } + } + .alsoTo(Sink.lastOption.mapMaterializedValue { lastFuture => + lastFuture + .map(opt => + opt.map(_._2).getOrElse(config.metrics) + ) // Get final metrics or fallback to current + .onComplete { + case Success(finalMetrics) => + val completed = finalMetrics.complete + logger.info( + s"Scroll completed: ${completed.totalDocuments} docs in ${completed.duration}ms " + + s"(${completed.documentsPerSecond} docs/sec)" + ) + metricsPromise.success(completed) + case Failure(ex) => + logger.error("Failed to get final metrics", ex) + metricsPromise.failure(ex) + }(system.dispatcher) + }) + .mapMaterializedValue(_ => NotUsed) + } } - .alsoTo(Sink.lastOption.mapMaterializedValue { lastFuture => - lastFuture - .map(opt => - opt.map(_._2).getOrElse(config.metrics) - ) // Get final metrics or fallback to current - .onComplete { - case Success(finalMetrics) => - val completed = finalMetrics.complete - logger.info( - s"Scroll completed: ${completed.totalDocuments} docs in ${completed.duration}ms " + - s"(${completed.documentsPerSecond} docs/sec)" - ) - metricsPromise.success(completed) - case Failure(ex) => - logger.error("Failed to get final metrics", ex) - metricsPromise.failure(ex) - }(system.dispatcher) - }) .mapMaterializedValue(_ => NotUsed) } - private def hasAggregations(query: String): Boolean = { - try { - val json = parse(query) - (json \ "aggregations") != JNothing || (json \ "aggs") != JNothing - } catch { - case _: Exception => false + private def hasAggregations(queryBody: JValue): Boolean = + (queryBody \ "aggregations") != JNothing || (queryBody \ "aggs") != JNothing + + /** Belt and braces for the slice guard: a TOP-LEVEL `sort` in the request body. + * + * Redundant against today's single entry point — the bridge emits the top-level `sort` from + * `orderBy.sorts` and from nothing else, which is exactly what the `hasSorts` flag reads from + * the AST — so it can only agree with it. It is kept (free, now that the body is parsed once for + * the strategy) because the failure it guards is a WRONG ANSWER, not a slowdown: a query that + * asked for an order, sliced, comes back interleaved. Any future path that reaches + * [[scrollWithMetrics]] with a hand-written body stays correct by construction. + */ + private def hasSortClause(queryBody: JValue): Boolean = (queryBody \ "sort") != JNothing + + /** #238 — slices for this stream: 1 unless the strategy is [[UsePIT]], no sort is present (AST + * and JSON), the effective ceiling is above 1 and the cluster supports PIT slicing (7.15+). One + * per primary shard of the resolved indices, capped by the ceiling, never above the shard count. + * The guard order is load-bearing: no `_settings` round-trip on ORDER BY / LIMIT / opt-out / ES6 + * / classic-scroll paths. The shard count comes from [[cachedPrimaryShardCount]] (one + * `_settings` round-trip per index set per [[shardCountCacheTtlMs]]); a lookup failure degrades + * to sequential with a WARN — once per TTL for a privilege failure (DEBUG while the cached + * failure is replayed), on every extraction for a transient one. + */ + private[client] def resolveSlices( + elasticQuery: ElasticQuery, + strategy: ScrollStrategy, + config: ScrollConfig, + hasSorts: Boolean, + queryBody: JValue + ): Int = { + val ceiling = config.maxSlices.getOrElse(configuredMaxSlices) + if (strategy != UsePIT || hasSorts || ceiling <= 1 || hasSortClause(queryBody)) { + logger.debug( + s"PIT slicing not applicable (strategy $strategy, sorted $hasSorts, max-slices $ceiling); paging sequentially" + ) + 1 + } else { + val targets = elasticQuery.indices.mkString(",") + version match { + case ElasticSuccess(v) if ElasticsearchVersion.supportsPitSlicing(v) => + val (lookup, cached) = cachedPrimaryShardCount(elasticQuery.indices) + lookup match { + case ElasticSuccess(shards) => + val n = math.max(1, math.min(shards, ceiling)) + if (n > 1) { + logger.info( + s"Sliced PIT paging: $n slices over $shards primary shards (max-slices $ceiling, page ${config.scrollSize}) for $targets" + ) + } else { + logger.debug(s"PIT paging stays sequential: $shards primary shard(s) for $targets") + } + n + case ElasticFailure(err) if cached => + logger.debug( + s"Primary shard count for $targets still unresolved (cached failure: ${err.message}); paging sequentially" + ) + 1 + case ElasticFailure(err) => + // only a privilege failure is remembered (see shardCountCacheTtlMs); anything else + // is probed again by the next extraction, and says so + val why = err.statusCode match { + case Some(s) if s == 401 || s == 403 => + s"the lookup needs the view_index_metadata privilege on the indices (HTTP $s) — remembered for ${shardCountCacheTtlMs / 1000} s" + case _ => + "retried on the next extraction" + } + logger.warn( + s"Could not resolve the primary shard count for $targets (${err.message}); paging sequentially — $why; set elastic.scroll.max-slices = 1 (ELASTIC_SCROLL_MAX_SLICES) to skip the lookup" + ) + 1 + } + case ElasticSuccess(v) => + logger.debug(s"ES version $v has PIT but no PIT slicing (7.15+); paging sequentially") + 1 + case ElasticFailure(err) => + // unreachable in practice (UsePIT implies a cached, successful version) — degrade, never throw + logger.warn( + s"Could not read the Elasticsearch version (${err.message}); paging sequentially" + ) + 1 + } } } - /** Create a scrolling source for JSON query with automatic strategy + /** Create a scrolling source for JSON query with the resolved strategy */ private def scroll( elasticQuery: ElasticQuery, @@ -406,13 +653,12 @@ trait ScrollApi extends ElasticClientHelpers { config: ScrollConfig, hasSorts: Boolean, fields: Seq[String], - nestedHits: Map[String, Seq[(String, String)]] + nestedHits: Map[String, Seq[(String, String)]], + strategy: ScrollStrategy )(implicit system: ActorSystem, context: ConversionContext ): Source[ListMap[String, Any], NotUsed] = { - val strategy = determineScrollStrategy(elasticQuery, aggregations, config) - logger.info( s"Using scroll strategy: $strategy for query \n$elasticQuery" ) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/ScrollSettings.scala b/core/src/main/scala/app/softnetwork/elastic/client/ScrollSettings.scala new file mode 100644 index 00000000..4dee8343 --- /dev/null +++ b/core/src/main/scala/app/softnetwork/elastic/client/ScrollSettings.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import app.softnetwork.elastic.client.scroll.ScrollConfig + +/** Paged row extraction settings (`elastic.scroll` in HOCON, #238). + * + * @param size + * rows per page (`elastic.scroll.size`, `ELASTIC_SCROLL_SIZE`): larger pages cut round-trips + * linearly and raise in-flight memory linearly. Must be positive. Not validated here: on the PIT + * / search_after path Elasticsearch rejects a page larger than the index's `max_result_window` + * (10,000 by default), so keep `size` at or below it + * @param maxSlices + * ceiling on concurrent PIT slices for a no-ORDER-BY extraction (`elastic.scroll.max-slices`, + * `ELASTIC_SCROLL_MAX_SLICES`, ES 7.15+): the effective count is min(primary shards, + * max-slices); 1 disables slicing (sequential paging) + */ +case class ScrollSettings( + size: Int = 1000, + maxSlices: Int = ScrollConfig.DefaultMaxSlices +) { + require(size > 0, s"elastic.scroll.size must be positive (ELASTIC_SCROLL_SIZE), got $size") + require( + maxSlices >= 1, + s"elastic.scroll.max-slices must be >= 1 (ELASTIC_SCROLL_MAX_SLICES), got $maxSlices" + ) + + /** REST connection pool per route for the clients that page through slices: every concurrent + * slice plus the PIT open / close and `_settings` calls that share the route; the Apache default + * (10) is the floor. + */ + def restPoolPerRoute: Int = math.max(10, maxSlices + 2) + + /** REST connection pool total; the Apache default (30) is the floor. */ + def restPoolTotal: Int = math.max(30, math.min(restPoolPerRoute, Int.MaxValue / 3) * 3) +} diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index c04d61d1..e0f0a96b 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -1825,8 +1825,18 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { s"▶ Row query ${maxDocuments.fold("without LIMIT")(max => s"with LIMIT window $max above ${SearchApi.DefaultMaxResultWindow}")} — routing through scroll for row completeness:\n${sql .getOrElse(elasticQuery.query)}" ) + // #238 — an explicit LIMIT keeps the sequential PIT path (the `.drop(offset)` below needs a + // deterministic `_doc` order); the statement's LIMIT was stripped above, so ScrollApi's own + // clamp cannot see it and the clamp must be applied here. The no-LIMIT branch inherits the + // configured ceiling (`maxSlices = None`). scrollApi - .scroll(statement, ScrollConfig(maxDocuments = maxDocuments)) + .scroll( + statement, + scrollApi.defaultScrollConfig.copy( + maxDocuments = maxDocuments, + maxSlices = if (single.limit.isDefined) Some(1) else None + ) + ) .map(_._1) .drop(offset) .runWith(Sink.seq) diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala index cd76abeb..a96f214d 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SettingsApi.scala @@ -24,6 +24,8 @@ import app.softnetwork.elastic.client.result.{ } import com.google.gson.JsonParser +import scala.jdk.CollectionConverters._ + /** Settings management API. */ trait SettingsApi { _: IndicesApi => @@ -219,6 +221,49 @@ trait SettingsApi { _: IndicesApi => } } + /** #238 — primary shard count of a PIT over `indices`: the sum of `index.number_of_shards` over + * every concrete index the expressions resolve to (aliases, wildcards, data streams — one + * top-level key per concrete index, the shape `loadSettings` parses), deduplicated by concrete + * index name across expressions. Elasticsearch compares a slice `max` with the shard count of + * the WHOLE request (`SliceBuilder.toFilter`), so the sum is the right quantity. Failures are an + * `ElasticFailure` (the caller degrades to sequential paging — never a throw); zero keys (an + * empty match, `NopeClientApi`'s `"{}"`) sum to **0** — "nothing matched", which the caller + * clamps to one slice and never caches; cross-cluster expressions (`remote:index`) have no + * `_settings` route and are skipped. + */ + private[client] def primaryShardCount(indices: Seq[String]): ElasticResult[Int] = { + val perIndex = scala.collection.mutable.LinkedHashMap.empty[String, Int] + val failure = indices.distinct.iterator + .filter { expr => + val crossCluster = expr.contains(":") + if (crossCluster) { + logger.debug(s"Skipping shard lookup for cross-cluster expression '$expr'") + } + !crossCluster + } + .map { expr => + executeLoadSettings(expr).flatMap { json => + ElasticResult.attempt { + JsonParser.parseString(json).getAsJsonObject.entrySet().asScala.foreach { e => + val n = Option(e.getValue) + .filter(_.isJsonObject) + .map(_.getAsJsonObject) + .flatMap(o => Option(o.getAsJsonObject("settings"))) + .flatMap(s => Option(s.getAsJsonObject("index"))) + .flatMap(i => Option(i.get("number_of_shards"))) + .map(_.getAsString) // a string on every client; number-safe too + .filter(s => s.nonEmpty && s.forall(_.isDigit)) + .map(_.toInt) + .getOrElse(1) + perIndex.getOrElseUpdate(e.getKey, n) + } + } + } + } + .collectFirst { case f @ ElasticFailure(_) => f } + failure.getOrElse(ElasticSuccess(perIndex.values.sum)) + } + // ======================================================================== // METHODS TO IMPLEMENT // ======================================================================== diff --git a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala index 705054f8..4b8b1d63 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/extensions/CoreDqlExtension.scala @@ -283,8 +283,10 @@ class CoreDqlExtension extends ExtensionSpi { client: ElasticClientApi )(implicit system: ActorSystem): Future[ElasticResult[QueryResult]] = { implicit val context: ConversionContext = NativeContext + // The client's configured defaults (page size, slice ceiling — #238) plus the quota cap; the + // cap is `.take(max)` on the MERGED stream, so it binds on the total whatever the slicing. val source = - client.scroll(single, ScrollConfig(maxDocuments = Some(max.toLong))) + client.scroll(single, client.defaultScrollConfig.copy(maxDocuments = Some(max.toLong))) val warning = s"Result capped to $max rows (license quota). " + s"Add an explicit LIMIT <= $max, or upgrade for more rows." diff --git a/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala b/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala index ac837123..57da8898 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/scroll/package.scala @@ -16,6 +16,12 @@ package app.softnetwork.elastic.client +import akka.{Done, NotUsed} +import akka.stream.scaladsl.Source + +import scala.concurrent.ExecutionContext +import scala.util.Try + package object scroll { /** Scroll configuration @@ -34,14 +40,30 @@ package object scroll { // `_id` row key. True for window-enrichment base queries (the ordinal lookup matches rows // by document id), when `elastic.include-document-id` is enabled, or when the query // selects `_id` explicitly. False keeps the hot scroll path free of any per-row overhead. - retainDocumentId: Boolean = false + retainDocumentId: Boolean = false, + // #238 — ceiling on concurrent PIT slices for a no-ORDER-BY extraction. None = inherit the + // client's `elastic.scroll.max-slices` (so the HOCON/env opt-out reaches explicit configs + // too); Some(n) = explicit, n <= 1 pages sequentially. Honoured only on PIT + search_after + // (ES >= 7.15) without sorts; the effective count is min(primary shards, ceiling). + maxSlices: Option[Int] = None, + // Internal (set by ScrollApi, not by callers): the slice count resolved for this stream. + slices: Int = 1 ) + object ScrollConfig { + + /** Default ceiling — one slice per primary shard up to 8: under the REST client's per-route + * pool (sized from `elastic.scroll.max-slices` by the es7/es8/es9 companions), and an + * in-flight bound of about 2 x slices x scrollSize rows. + */ + val DefaultMaxSlices: Int = 8 + } + /** Scroll strategy based on query type */ sealed trait ScrollStrategy case object UsePIT - extends ScrollStrategy // Point In Time + search_after (ES 7.10+, best performance) + extends ScrollStrategy // Point In Time + search_after (ES 7.12+, best performance) case object UseScroll extends ScrollStrategy // Classic scroll (supports aggregations) case object UseSearchAfter extends ScrollStrategy // search_after only (efficient, no server state) @@ -52,11 +74,38 @@ package object scroll { totalDocuments: Long = 0, totalBatches: Long = 0, startTime: Long = System.currentTimeMillis(), - endTime: Option[Long] = None + endTime: Option[Long] = None, + slices: Int = 1 // #238 — PIT slices merged into this stream (1 = sequential) ) { def duration: Long = endTime.getOrElse(System.currentTimeMillis()) - startTime def documentsPerSecond: Double = totalDocuments.toDouble / (duration / 1000.0) def complete: ScrollMetrics = copy(endTime = Some(System.currentTimeMillis())) } + /** #238 — merge independent PIT slice PAGE sources into ONE backpressured stream. + * + * `onTerminate` fires exactly once: on completion, failure, or downstream cancellation. A + * failing slice fails the merged stream (it never truncates it — the #228/#209/#224 lesson). + * `ec` must be the system dispatcher: `onTerminate` closes the PIT with a blocking call. + * + * The helper deliberately has no `require`: nothing between a successful `openPit` and the + * attachment of `watchTermination` may throw (#202 — the single PIT owner rule). + */ + object SliceMerge { + def apply[T](slices: Seq[Source[T, NotUsed]])(onTerminate: Try[Done] => Unit)(implicit + ec: ExecutionContext + ): Source[T, NotUsed] = { + val merged: Source[T, NotUsed] = slices match { + case Seq() => Source.empty[T] + case Seq(single) => single + case many => + Source(many.toList).flatMapMerge(many.size, (s: Source[T, NotUsed]) => s) + } + merged.watchTermination() { (_, done) => + done.onComplete(onTerminate) + NotUsed + } + } + } + } diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ElasticsearchVersionSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ElasticsearchVersionSpec.scala index 63f317ad..16ed3d4d 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/ElasticsearchVersionSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/ElasticsearchVersionSpec.scala @@ -65,6 +65,29 @@ class ElasticsearchVersionSpec extends AnyWordSpec with Matchers { } } + "ElasticsearchVersion.supportsPitSlicing" should { + // slice + pit in ONE search request exists from 7.15 (elastic/elasticsearch#74457), not 7.10 + // as the #238 issue text said; 7.12–7.14 keep PIT but page sequentially. + "return true for ES >= 7.15" in { + ElasticsearchVersion.supportsPitSlicing("7.15.0") shouldBe true + ElasticsearchVersion.supportsPitSlicing("7.17.29") shouldBe true + ElasticsearchVersion.supportsPitSlicing("8.18.3") shouldBe true + ElasticsearchVersion.supportsPitSlicing("9.0.3") shouldBe true + } + + "return false for ES < 7.15 (PIT without slicing on 7.12-7.14)" in { + ElasticsearchVersion.supportsPitSlicing("7.14.2") shouldBe false + ElasticsearchVersion.supportsPitSlicing("7.12.0") shouldBe false + ElasticsearchVersion.supportsPitSlicing("7.10.0") shouldBe false + ElasticsearchVersion.supportsPitSlicing("6.8.23") shouldBe false + } + + "keep supportsPit at 7.12" in { + ElasticsearchVersion.supportsPit("7.12.0") shouldBe true + ElasticsearchVersion.supportsPit("7.11.2") shouldBe false + } + } + "ElasticsearchVersion.isEs8OrHigher" should { "return true for ES >= 8.0" in { ElasticsearchVersion.isEs8OrHigher("8.0.0") shouldBe true diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ParseCostProbeSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ParseCostProbeSpec.scala new file mode 100644 index 00000000..ce78d269 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/ParseCostProbeSpec.scala @@ -0,0 +1,106 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import org.json4s.jackson.JsonMethods.parse +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +/** #238 — cost of the ONE parse of the request body that `ScrollApi.scrollWithMetrics` performs per + * stream and shares between the strategy decision (`aggs` / `aggregations`) and the slice decision + * (`sort`). It replaced two separate parses of the same string, so the net change was `-1` parse + * per extraction; this spec records what that single parse costs and guards the order of + * magnitude. + * + * Measured 2026-08-20 (Apple silicon, JDK 17, json4s-jackson): p50 0.003 ms at 76 B, 0.005 ms at + * 350 B, 0.028 ms at 3.2 KB, 0.067 ms at 32 KB; p99 ≤ 0.15 ms everywhere — versus a budget of 1 ms + * and an extraction that then runs for seconds. + * + * The assertion is on the MEDIAN, not a tail: a p99/max on a shared CI box measures GC pauses, not + * parsing (a 9 ms `max` was observed on the 3.2 KB body while the 10× larger body maxed at 0.12 + * ms). The ceiling is deliberately ~15× the worst measured median — it catches a change that makes + * body inspection categorically expensive, never a slow machine. + */ +class ParseCostProbeSpec extends AnyFlatSpec with Matchers { + + private val Warmup = 500 + private val Runs = 500 + + /** ~15× the worst measured median (0.067 ms), in nanoseconds. */ + private val MedianCeilingNanos = 1000000L // 1 ms + + private def sourceFields(n: Int): String = + (1 to n).map(i => s""""field_$i"""").mkString(",") + + private def filters(n: Int): String = + (1 to n).map(i => s"""{"term":{"field_$i":{"value":"v$i"}}}""").mkString(",") + + private val small = + """{"query":{"match_all":{}},"_source":{"includes":["id","value"]},"size":1000}""" + + private val typical = + s"""{"query":{"bool":{"filter":[${filters(3)}],"must":[{"range":{"ts":{"gte":"2026-01-01"}}}]}}, + |"_source":{"includes":[${sourceFields( + 10 + )}]},"size":1000,"track_total_hits":false}""".stripMargin + + private val large = + s"""{"query":{"bool":{"filter":[${filters(50)}]}}, + |"_source":{"includes":[${sourceFields(100)}]}, + |"script_fields":{"s1":{"script":{"lang":"painless","source":"def x = doc['a'].value; return x != null ? x * 2 : null"}}}, + |"size":5000,"track_total_hits":false}""".stripMargin + + private val pathological = + s"""{"query":{"bool":{"filter":[${filters(500)}]}}, + |"_source":{"includes":[${sourceFields(1000)}]},"size":10000}""".stripMargin + + /** Median parse time in nanoseconds, reported on stdout with the tail for context. */ + private def medianParseNanos(label: String, body: String): Long = { + var w = 0 + while (w < Warmup) { parse(body); w += 1 } + val timings = Array.fill(Runs)(0L) + var i = 0 + while (i < Runs) { + val t0 = System.nanoTime() + parse(body) + timings(i) = System.nanoTime() - t0 + i += 1 + } + val sorted = timings.sorted + val p50 = sorted(Runs / 2) + def ms(n: Long): String = f"${n / 1000000.0}%.4f" + info( + s"$label: ${body.length} bytes — p50 ${ms(p50)} ms, p95 ${ms(sorted((Runs * 95) / 100))} ms, " + + s"max ${ms(sorted(Runs - 1))} ms" + ) + p50 + } + + "the per-stream request-body parse" should "stay far below the 1 ms budget on every realistic body" in { + Seq( + "small" -> small, + "typical" -> typical, + "large" -> large, + "pathological" -> pathological + ).foreach { case (label, body) => + val p50 = medianParseNanos(label, body) + withClue(s"median parse of the $label body (${body.length} bytes) — ") { + p50 should be < MedianCeilingNanos + } + } + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ScrollSettingsSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ScrollSettingsSpec.scala new file mode 100644 index 00000000..5e414b33 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/ScrollSettingsSpec.scala @@ -0,0 +1,205 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.scroll.{ScrollConfig, ScrollMetrics} +import app.softnetwork.elastic.sql.query.{SearchStatement, SelectStatement} +import com.typesafe.config.{Config, ConfigFactory} +import org.json4s.{DefaultFormats, Formats} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import java.util.concurrent.atomic.AtomicReference +import scala.collection.immutable.ListMap + +/** #238 — the `elastic.scroll { size, max-slices }` surface and how it reaches every scroll call + * site: the HOCON block (run under `sbt "+ core/test"` — kxbmap 0.4.4 on 2.12, 0.6.1 on 2.13), + * `ScrollApi.defaultScrollConfig` (a `def`), the client overrides, and the `scrollRows` routing + * clamp (`maxSlices = Some(1)` on an explicit LIMIT, `None` otherwise). + */ +class ScrollSettingsSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll { + + implicit val system: ActorSystem = ActorSystem("scroll-settings-spec") + implicit val context: ConversionContext = NativeContext + + private val testLogger: Logger = LoggerFactory.getLogger(getClass) + + override def afterAll(): Unit = { + system.terminate() + super.afterAll() + } + + // ---- HOCON ------------------------------------------------------------------------------- + + "ElasticConfig" should "read elastic.scroll.size and elastic.scroll.max-slices" in { + val cfg = + ElasticConfig(ConfigFactory.parseString("elastic.scroll { size = 5000, max-slices = 2 }")) + cfg.scroll.size shouldBe 5000 + cfg.scroll.maxSlices shouldBe 2 + } + + it should "default elastic.scroll to 1000 rows per page and a ceiling of 8 slices" in { + val cfg = ElasticConfig(ConfigFactory.parseString("elastic { credentials { host = \"x\" } }")) + cfg.scroll.size shouldBe 1000 + cfg.scroll.maxSlices shouldBe ScrollConfig.DefaultMaxSlices + ScrollConfig.DefaultMaxSlices shouldBe 8 + } + + // ---- defaultScrollConfig ------------------------------------------------------------------ + + /** A client whose HOCON comes from a string — exposes the protected ceiling for assertion. */ + private class ConfiguredClient(hocon: String) extends NopeClientApi { + override protected def logger: Logger = testLogger + override def config: Config = ConfigFactory.parseString(hocon) + def ceiling: Int = configuredMaxSlices + } + + "ScrollApi.defaultScrollConfig" should "be a def: every call carries fresh metrics" in { + val client = new ConfiguredClient("") + val a = client.defaultScrollConfig + Thread.sleep(2) + val b = client.defaultScrollConfig + (a.metrics ne b.metrics) shouldBe true + b.metrics.startTime should be >= a.metrics.startTime + } + + it should "apply the HOCON page size and leave the ceiling inherited (maxSlices = None)" in { + val client = new ConfiguredClient("elastic.scroll { size = 250, max-slices = 3 }") + val cfg = client.defaultScrollConfig + cfg.scrollSize shouldBe 250 + cfg.maxSlices shouldBe None + cfg.slices shouldBe 1 + client.ceiling shouldBe 3 + } + + it should "fall back to the reference defaults when nothing is configured" in { + val client = new ConfiguredClient("") + client.defaultScrollConfig.scrollSize shouldBe 1000 + client.ceiling shouldBe ScrollConfig.DefaultMaxSlices + } + + "ScrollSettings" should "reject a non-positive page size or a ceiling below 1 at config load" in { + an[IllegalArgumentException] should be thrownBy ScrollSettings(size = 0) + an[IllegalArgumentException] should be thrownBy ScrollSettings(maxSlices = 0) + the[IllegalArgumentException] thrownBy ScrollSettings(size = -5) should have message + "requirement failed: elastic.scroll.size must be positive (ELASTIC_SCROLL_SIZE), got -5" + // through the HOCON reader the load itself fails (kxbmap 0.6.1 surfaces the requirement + // message; 0.4.4 on 2.12 falls back to the companion apply and reports a generic error) + an[Exception] should be thrownBy ElasticConfig( + ConfigFactory.parseString("elastic.scroll { size = 0 }") + ) + } + + it should "derive the REST pool sizing from the slice ceiling with the Apache defaults as floor" in { + ScrollSettings().restPoolPerRoute shouldBe 10 + ScrollSettings().restPoolTotal shouldBe 30 + ScrollSettings(maxSlices = 16).restPoolPerRoute shouldBe 18 + ScrollSettings(maxSlices = 16).restPoolTotal shouldBe 54 + } + + "ScrollConfig" should "default maxSlices to None and slices to 1" in { + val cfg = ScrollConfig() + cfg.maxSlices shouldBe None + cfg.slices shouldBe 1 + ScrollMetrics().slices shouldBe 1 + } + + // ---- scrollRows routing --------------------------------------------------------------------- + + /** Records the config that reaches `scroll`. The override deliberately does NOT redeclare a + * default argument: a redeclared default would shadow `defaultScrollConfig` for no-argument + * calls through this subclass and hide what the production path actually sends. + */ + private class RecordingClient extends NopeClientApi { + override protected def logger: Logger = testLogger + val scrolledConfig = new AtomicReference[ScrollConfig]() + val scrolledStatement = new AtomicReference[SearchStatement]() + + override def scroll( + statement: SearchStatement, + config: ScrollConfig + )(implicit + system: ActorSystem, + context: ConversionContext + ): Source[(ListMap[String, Any], ScrollMetrics), NotUsed] = { + scrolledStatement.set(statement) + scrolledConfig.set(config) + Source.empty[(ListMap[String, Any], ScrollMetrics)] + } + } + + "SearchApi.scrollRows" should "clamp maxSlices to Some(1) on an explicit LIMIT above the one-shot window" in { + val client = new RecordingClient + client.search(SelectStatement("SELECT id FROM idx LIMIT 11000")) + val cfg = client.scrolledConfig.get() + cfg should not be null + cfg.maxSlices shouldBe Some(1) + cfg.maxDocuments shouldBe Some(11000L) + cfg.scrollSize shouldBe client.defaultScrollConfig.scrollSize + } + + it should "leave maxSlices = None on a no-LIMIT row query (the configured ceiling applies)" in { + val client = new RecordingClient + client.search(SelectStatement("SELECT id FROM idx")) + val cfg = client.scrolledConfig.get() + cfg should not be null + cfg.maxSlices shouldBe None + cfg.maxDocuments shouldBe None + } + + it should "reach scroll with the inherited defaultScrollConfig on the no-argument gateway path" in { + val client = new RecordingClient + client.scroll(SelectStatement("SELECT id FROM idx")) + val cfg = client.scrolledConfig.get() + cfg should not be null + cfg.maxSlices shouldBe None + cfg.scrollSize shouldBe 1000 + } + + // Macro applications do not support default arguments (scalac rejects an omitted `config` on + // scrollAs outright), so the typed path reaches the HOCON defaults only through an explicit + // `defaultScrollConfig`; the macro re-emits that argument verbatim into scrollAsUnchecked → scroll. + // macros-tests cannot host this case (it does not depend on core, where the real macro lives). + "ScrollApi.scrollAs" should "forward an explicit defaultScrollConfig through the macro expansion" in { + implicit val formats: Formats = DefaultFormats + val client = new RecordingClient + client.scrollAs[ScrollSettingsSpec.IdRow]("SELECT id FROM idx", client.defaultScrollConfig) + val cfg = client.scrolledConfig.get() + cfg should not be null + cfg.maxSlices shouldBe None + cfg.scrollSize shouldBe client.defaultScrollConfig.scrollSize + } + + it should "apply defaultScrollConfig on scrollAsUnchecked when the config is omitted" in { + implicit val formats: Formats = DefaultFormats + val client = new RecordingClient + client.scrollAsUnchecked[ScrollSettingsSpec.IdRow](SelectStatement("SELECT id FROM idx")) + val cfg = client.scrolledConfig.get() + cfg should not be null + cfg.maxSlices shouldBe None + cfg.scrollSize shouldBe 1000 + } +} + +object ScrollSettingsSpec { + case class IdRow(id: String) +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/ScrollSlicingSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/ScrollSlicingSpec.scala new file mode 100644 index 00000000..1b4d6a16 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/ScrollSlicingSpec.scala @@ -0,0 +1,520 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Sink, Source} +import app.softnetwork.elastic.client.result.{ + ElasticError, + ElasticFailure, + ElasticResult, + ElasticSuccess +} +import app.softnetwork.elastic.client.scroll.{ScrollConfig, ScrollMetrics} +import app.softnetwork.elastic.sql.query.{SQLAggregation, SelectStatement} +import org.mockito.{ArgumentMatchersSugar, MockitoSugar} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.Logger + +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ + +/** #238 — the slice POLICY, decided once per stream in core (`ScrollApi.resolveSlices`) and handed + * to the client through `ScrollConfig.slices` / `ScrollMetrics.slices`. No Docker: a + * [[NopeClientApi]] subclass fakes the version, the `_settings` payload and the three page + * sources, and records the `ScrollConfig` that reaches `pitSearchAfter`. + */ +class ScrollSlicingSpec + extends AnyFlatSpec + with Matchers + with BeforeAndAfterAll + with MockitoSugar + with ArgumentMatchersSugar { + + implicit val system: ActorSystem = ActorSystem("scroll-slicing-spec") + implicit val context: ConversionContext = NativeContext + + override def afterAll(): Unit = { + system.terminate() + super.afterAll() + } + + private def shards(index: String, n: Int): String = + s"""{"$index":{"settings":{"index":{"number_of_shards":"$n"}}}}""" + + private val defaultSettings: Map[String, ElasticResult[String]] = Map( + "idx_a" -> ElasticSuccess(shards("idx_a", 3)), + "idx_a_2026" -> ElasticSuccess(shards("idx_a_2026", 2)), // `idx_a` is a PREFIX of it + "idx_one" -> ElasticSuccess(shards("idx_one", 1)), + "idx_six" -> ElasticSuccess(shards("idx_six", 6)), + "*_six" -> ElasticSuccess(shards("idx_six", 6)), // a wildcard `idx_six` does NOT prefix + "idx_big" -> ElasticSuccess(shards("idx_big", 12)), + "idx_fail" -> ElasticFailure(ElasticError("settings unavailable")), + "idx_bad" -> ElasticSuccess("this is not json"), + "idx_forbidden" -> ElasticFailure( + ElasticError( + "action [indices:monitor/settings/get] is unauthorized", + cause = Some(new RuntimeException("security_exception")), + statusCode = Some(403) + ) + ) + ) + + /** Overrides `version` (NOT `executeVersion`: VersionApi caches the first answer), the + * `_settings` payload and the page sources. `rows` rows per stream, each carrying the slice + * count the client saw. + */ + private class SlicingClient( + val mockLogger: Logger, + esVersion: String = "8.18.3", + ceiling: Int = ScrollConfig.DefaultMaxSlices, + settings: Map[String, ElasticResult[String]] = defaultSettings, + rows: Int = 3, + ttlMs: Long = 5 * 60 * 1000L + ) extends NopeClientApi { + override protected def logger: Logger = mockLogger + override def version: ElasticResult[String] = ElasticSuccess(esVersion) + override protected def configuredMaxSlices: Int = ceiling + override protected def shardCountCacheTtlMs: Long = ttlMs + + val pitConfig = new AtomicReference[ScrollConfig]() + val classicConfig = new AtomicReference[ScrollConfig]() + val pitCalls = new AtomicInteger(0) + val settingsCalls = new AtomicInteger(0) + + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = { + settingsCalls.incrementAndGet() + settings.getOrElse(index, ElasticSuccess("{}")) + } + + private def page: Source[ListMap[String, Any], NotUsed] = + Source(List.tabulate(rows)(i => ListMap[String, Any]("id" -> s"doc-$i"))) + + override private[client] def pitSearchAfter( + elasticQuery: ElasticQuery, + fieldAliases: ListMap[String, String], + config: ScrollConfig, + hasSorts: Boolean + )(implicit + system: ActorSystem, + context: ConversionContext + ): Source[ListMap[String, Any], NotUsed] = { + pitCalls.incrementAndGet() + pitConfig.set(config) + page + } + + override private[client] def scrollClassic( + elasticQuery: ElasticQuery, + fieldAliases: ListMap[String, String], + aggregations: ListMap[String, SQLAggregation], + config: ScrollConfig + )(implicit + system: ActorSystem, + context: ConversionContext + ): Source[ListMap[String, Any], NotUsed] = { + classicConfig.set(config) + page + } + } + + private def run( + client: SlicingClient, + sql: String, + config: Option[ScrollConfig] = None + ): Seq[(ListMap[String, Any], ScrollMetrics)] = { + val source = config match { + case Some(c) => client.scroll(SelectStatement(sql), c) + case None => client.scroll(SelectStatement(sql)) + } + Await.result(source.runWith(Sink.seq), 30.seconds) + } + + private def slicedInfo(logger: Logger, times: Int): Unit = + verify(logger, org.mockito.Mockito.times(times)) + .info(argThat[String]((s: String) => s != null && s.startsWith("Sliced PIT paging:"))) + + private def shardWarn(logger: Logger, times: Int): Unit = + verify(logger, org.mockito.Mockito.times(times)) + .warn( + argThat[String]((s: String) => + s != null && s.startsWith("Could not resolve the primary shard count") + ) + ) + + /** The DEBUG replay of a cached privilege failure. */ + private def shardDebug(logger: Logger, times: Int): Unit = + verify(logger, org.mockito.Mockito.times(times)) + .debug( + argThat[String]((s: String) => + s != null && s.startsWith("Primary shard count for") && s.contains("still unresolved") + ) + ) + + // --------------------------------------------------------------------------------------------- + + "ScrollApi" should "slice a no-ORDER-BY PIT extraction once per primary shard (3 shards → 3)" in { + val client = new SlicingClient(mock[Logger]) + val rows = run(client, "SELECT id FROM idx_a") + rows should have size 3 + client.pitConfig.get().slices shouldBe 3 + client.pitConfig.get().metrics.slices shouldBe 3 + rows.head._2.slices shouldBe 3 + rows.last._2.slices shouldBe 3 + slicedInfo(client.mockLogger, 1) + } + + it should "never slice above the shard count (1 shard → 1, no INFO line)" in { + val client = new SlicingClient(mock[Logger]) + val rows = run(client, "SELECT id FROM idx_one") + client.pitConfig.get().slices shouldBe 1 + rows.head._2.slices shouldBe 1 + slicedInfo(client.mockLogger, 0) + } + + it should "cap at the default ceiling of 8 (12 shards → 8) and read 6 shards as 6" in { + val big = new SlicingClient(mock[Logger]) + run(big, "SELECT id FROM idx_big").head._2.slices shouldBe 8 + val six = new SlicingClient(mock[Logger]) + run(six, "SELECT id FROM idx_six").head._2.slices shouldBe 6 + } + + it should "honour an explicit maxSlices = Some(2) (still a whole-shard split on the ES side)" in { + val client = new SlicingClient(mock[Logger]) + val rows = run(client, "SELECT id FROM idx_a", Some(ScrollConfig(maxSlices = Some(2)))) + rows.head._2.slices shouldBe 2 + client.pitConfig.get().slices shouldBe 2 + } + + it should "page sequentially on maxSlices = Some(1)" in { + val client = new SlicingClient(mock[Logger]) + run( + client, + "SELECT id FROM idx_a", + Some(ScrollConfig(maxSlices = Some(1))) + ).head._2.slices shouldBe 1 + slicedInfo(client.mockLogger, 0) + } + + it should "let the configured ceiling reach an explicit config that leaves maxSlices = None (opt-out)" in { + val client = new SlicingClient(mock[Logger], ceiling = 1) + run( + client, + "SELECT id FROM idx_a", + Some(ScrollConfig(scrollSize = 100)) + ).head._2.slices shouldBe 1 + client.pitConfig.get().maxSlices shouldBe None + slicedInfo(client.mockLogger, 0) + } + + it should "page sequentially on ES 7.12–7.14 (PIT without slicing)" in { + val client = new SlicingClient(mock[Logger], esVersion = "7.14.0") + run(client, "SELECT id FROM idx_a").head._2.slices shouldBe 1 + client.pitCalls.get() shouldBe 1 + } + + it should "slice on ES 7.15" in { + val client = new SlicingClient(mock[Logger], esVersion = "7.15.0") + run(client, "SELECT id FROM idx_a").head._2.slices shouldBe 3 + } + + it should "keep ORDER BY sequential" in { + val client = new SlicingClient(mock[Logger]) + run(client, "SELECT id FROM idx_a ORDER BY value").head._2.slices shouldBe 1 + client.pitConfig.get().slices shouldBe 1 + } + + it should "keep an explicit LIMIT sequential" in { + val client = new SlicingClient(mock[Logger]) + run(client, "SELECT id FROM idx_a LIMIT 5").head._2.slices shouldBe 1 + client.pitConfig.get().maxSlices shouldBe Some(1) + } + + it should "keep a windowed statement with a LIMIT sequential (clamp before the window branch)" in { + val client = new SlicingClient(mock[Logger]) + run( + client, + "SELECT id, value, ROW_NUMBER() OVER (PARTITION BY id ORDER BY value) AS rn FROM idx_a LIMIT 5" + ) + val cfg = client.pitConfig.get() + cfg should not be null + cfg.maxSlices shouldBe Some(1) + cfg.slices shouldBe 1 + } + + it should "keep a windowed statement WITHOUT a LIMIT sliced" in { + val client = new SlicingClient(mock[Logger]) + run( + client, + "SELECT id, value, ROW_NUMBER() OVER (PARTITION BY id ORDER BY value) AS rn FROM idx_a" + ) + val cfg = client.pitConfig.get() + cfg should not be null + cfg.maxSlices shouldBe None + cfg.slices shouldBe 3 + } + + it should "degrade to sequential with exactly one WARN when the _settings lookup fails" in { + val client = new SlicingClient(mock[Logger]) + val rows = run(client, "SELECT id FROM idx_fail") + rows should have size 3 + rows.head._2.slices shouldBe 1 + shardWarn(client.mockLogger, 1) + slicedInfo(client.mockLogger, 0) + } + + it should "degrade to sequential with exactly one WARN on a malformed _settings payload, without throwing" in { + val client = new SlicingClient(mock[Logger]) + val rows = run(client, "SELECT id FROM idx_bad") + rows should have size 3 + rows.head._2.slices shouldBe 1 + shardWarn(client.mockLogger, 1) + } + + it should "never consult _settings nor slice when preferSearchAfter = false (classic scroll)" in { + val client = new SlicingClient(mock[Logger]) + val rows = run(client, "SELECT id FROM idx_a", Some(ScrollConfig(preferSearchAfter = false))) + rows.head._2.slices shouldBe 1 + client.pitCalls.get() shouldBe 0 + client.classicConfig.get().slices shouldBe 1 + slicedInfo(client.mockLogger, 0) + } + + it should "hand the resolved count to the client even when the stream yields zero rows" in { + // (named arg + a later local `rows` trips 2.12's forward-reference parsing — hence `emitted`) + val client = new SlicingClient(mock[Logger], rows = 0) + val emitted = run(client, "SELECT id FROM idx_a") + emitted shouldBe empty + client.pitConfig.get().slices shouldBe 3 + client.pitConfig.get().metrics.slices shouldBe 3 + slicedInfo(client.mockLogger, 1) + } + + it should "surface a version failure as a FAILED stream, never a synchronous throw out of scroll()" in { + val client = new SlicingClient(mock[Logger]) { + override def version: ElasticResult[String] = + ElasticFailure(ElasticError("cluster unreachable")) + } + // building the source must not throw (gateway / JDBC build it eagerly)… + val source = client.scroll(SelectStatement("SELECT id FROM idx_a")) + // …the failure surfaces when the stream runs + val failure = Await.result(source.runWith(Sink.seq).failed, 30.seconds) + failure.getMessage should include("Failed to get ES version") + } + + it should "resolve once per stream (one INFO line per extraction) and bind the quota on the merged total" in { + val client = new SlicingClient(mock[Logger], rows = 10) + val emitted = run(client, "SELECT id FROM idx_a", Some(ScrollConfig(maxDocuments = Some(4)))) + emitted should have size 4 + emitted.head._2.slices shouldBe 3 + slicedInfo(client.mockLogger, 1) + } + + // --------------------------------------------------------------------------------------------- + // Shard-count cache (#238, review decision 3c) + + it should "serve the primary shard count from the cache within the TTL (one _settings round-trip for two extractions)" in { + val client = new SlicingClient(mock[Logger]) + run(client, "SELECT id FROM idx_a").head._2.slices shouldBe 3 + run(client, "SELECT id FROM idx_a").head._2.slices shouldBe 3 + client.settingsCalls.get() shouldBe 1 + slicedInfo(client.mockLogger, 2) // the per-extraction INFO line is NOT cached away + } + + it should "key the cache by the (sorted, distinct) index set" in { + val client = new SlicingClient(mock[Logger]) + run(client, "SELECT id FROM idx_a").head._2.slices shouldBe 3 + run(client, "SELECT id FROM idx_six").head._2.slices shouldBe 6 + client.settingsCalls.get() shouldBe 2 + // a new SET is a new key (one lookup per expression) … + client.cachedPrimaryShardCount(Seq("idx_six", "idx_a", "idx_a")) shouldBe ( + ( + ElasticSuccess(9), + false + ) + ) + client.settingsCalls.get() shouldBe 4 + // … and the same set in another order / with duplicates is the same entry + client.cachedPrimaryShardCount(Seq("idx_a", "idx_six")) shouldBe ((ElasticSuccess(9), true)) + client.settingsCalls.get() shouldBe 4 + } + + it should "consult _settings again once the TTL has expired" in { + val client = new SlicingClient(mock[Logger], ttlMs = 0L) + run(client, "SELECT id FROM idx_a") + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 2 + } + + it should "NOT cache a transient lookup failure: every extraction probes again and WARNs (no RBAC blame)" in { + val client = new SlicingClient(mock[Logger]) + run(client, "SELECT id FROM idx_fail").head._2.slices shouldBe 1 + run(client, "SELECT id FROM idx_fail").head._2.slices shouldBe 1 + client.settingsCalls.get() shouldBe 2 + shardWarn(client.mockLogger, 2) + verify(client.mockLogger, org.mockito.Mockito.times(2)) + .warn( + argThat[String]((s: String) => s != null && s.contains("retried on the next extraction")) + ) + verify(client.mockLogger, org.mockito.Mockito.never()) + .warn(argThat[String]((s: String) => s != null && s.contains("view_index_metadata"))) + shardDebug(client.mockLogger, 0) + } + + it should "cache a PRIVILEGE failure (403): ONE WARN naming the privilege per TTL, the replay at DEBUG, no stack pinned" in { + val client = new SlicingClient(mock[Logger]) + run(client, "SELECT id FROM idx_forbidden").head._2.slices shouldBe 1 + run(client, "SELECT id FROM idx_forbidden").head._2.slices shouldBe 1 + client.settingsCalls.get() shouldBe 1 + shardWarn(client.mockLogger, 1) + verify(client.mockLogger) + .warn( + argThat[String]((s: String) => + s != null && s.contains("view_index_metadata") && s.contains("HTTP 403") + ) + ) + shardDebug(client.mockLogger, 1) + // the cached failure carries no Throwable (the original did) + val (cached, hit) = client.cachedPrimaryShardCount(Seq("idx_forbidden")) + hit shouldBe true + cached match { + case ElasticFailure(err) => err.cause shouldBe None + case other => fail(s"expected a cached failure, got $other") + } + } + + it should "NOT cache an expression that matches no index (a later CREATE must be seen at once)" in { + val client = new SlicingClient(mock[Logger]) + // `idx_empty` is not in the settings map → NopeClientApi-style "{}" → 0 shards → sequential + run(client, "SELECT id FROM idx_empty").head._2.slices shouldBe 1 + run(client, "SELECT id FROM idx_empty").head._2.slices shouldBe 1 + client.settingsCalls.get() shouldBe 2 + shardWarn(client.mockLogger, 0) // an empty match is not a failure + } + + it should "drop only the cached counts naming the index on invalidateSchema(index), by case-insensitive prefix" in { + val client = new SlicingClient(mock[Logger]) + run(client, "SELECT id FROM idx_a") + run(client, "SELECT id FROM idx_six") + client.settingsCalls.get() shouldBe 2 + + // an unrelated index leaves both entries in place … + client.invalidateSchema("idx_other") + run(client, "SELECT id FROM idx_a") + run(client, "SELECT id FROM idx_six") + client.settingsCalls.get() shouldBe 2 + + // … the named one drops ONLY its own entry … + client.invalidateSchema("idx_a") + run(client, "SELECT id FROM idx_six") + client.settingsCalls.get() shouldBe 2 + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 3 + + // … case-insensitively … + client.invalidateSchema("IDX_A") + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 4 + + // … as a PREFIX (the index is the stem of the cached expression) … + client.cachedPrimaryShardCount(Seq("idx_a_2026"))._2 shouldBe false + client.settingsCalls.get() shouldBe 5 + client.cachedPrimaryShardCount(Seq("idx_a_2026"))._2 shouldBe true + client.invalidateSchema("idx_a") + client.cachedPrimaryShardCount(Seq("idx_a_2026"))._2 shouldBe false + client.settingsCalls.get() shouldBe 6 + + // … and per EXPRESSION inside a multi-index key (order in the key is alphabetical, so a + // whole-key startsWith would miss this one) + client.cachedPrimaryShardCount(Seq("idx_a", "idx_six"))._2 shouldBe false + client.cachedPrimaryShardCount(Seq("idx_a", "idx_six"))._2 shouldBe true + client.invalidateSchema("idx_six") + client.cachedPrimaryShardCount(Seq("idx_a", "idx_six"))._2 shouldBe false + + // a key the index does not prefix survives (documented limitation: `logs-*`, aliases) + client.cachedPrimaryShardCount(Seq("*_six"))._2 shouldBe false + client.invalidateSchema("idx_six") + client.cachedPrimaryShardCount(Seq("*_six"))._2 shouldBe true + } + + it should "drop the counts naming the index on updateSchema(...) and a successful createIndex, everything on invalidateAllSchemas()" in { + val client = new SlicingClient(mock[Logger]) { + // NopeClientApi answers ElasticSuccess(false) ("not created"); make the creation succeed + override private[client] def executeCreateIndex( + index: String, + settings: String, + mappings: Option[String], + aliases: Seq[app.softnetwork.elastic.sql.schema.TableAlias] + ): ElasticResult[Boolean] = ElasticSuccess(true) + } + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 1 + + // updateSchema (ALTER TABLE may have reindexed into a different shard count) — targeted + client.updateSchema("idx_six", app.softnetwork.elastic.sql.schema.Table("idx_six", Nil)) + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 1 + client.updateSchema("idx_a", app.softnetwork.elastic.sql.schema.Table("idx_a", Nil)) + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 2 + + // createIndex — targeted: an unrelated creation leaves the entry, `idx_a…` drops it + client.createIndex("idx_new").get shouldBe true + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 2 + client.createIndex("idx_a_2026").get shouldBe true // idx_a is NOT a prefix of the cached key… + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 2 + client.createIndex("idx_a").get shouldBe true // …this one is + run(client, "SELECT id FROM idx_a") + client.settingsCalls.get() shouldBe 3 + + // invalidateAllSchemas() clears everything, whatever the key + run(client, "SELECT id FROM idx_six") + client.settingsCalls.get() shouldBe 4 + client.invalidateAllSchemas() + run(client, "SELECT id FROM idx_a") + run(client, "SELECT id FROM idx_six") + client.settingsCalls.get() shouldBe 6 + } + + it should "resolve a cold key ONCE under concurrency (one round-trip, one WARN for 8 simultaneous extractions)" in { + val client = new SlicingClient(mock[Logger]) + val pool = java.util.concurrent.Executors.newFixedThreadPool(8) + try { + val start = new java.util.concurrent.CountDownLatch(1) + val tasks = (1 to 8).map { _ => + pool.submit(new java.util.concurrent.Callable[(ElasticResult[Int], Boolean)] { + def call(): (ElasticResult[Int], Boolean) = { + start.await() + client.cachedPrimaryShardCount(Seq("idx_forbidden")) + } + }) + } + start.countDown() + val outcomes = tasks.map(_.get(30, java.util.concurrent.TimeUnit.SECONDS)) + client.settingsCalls.get() shouldBe 1 + outcomes.count(_._2 == false) shouldBe 1 // exactly one caller saw the miss + outcomes.forall(_._1.isInstanceOf[ElasticFailure]) shouldBe true + } finally pool.shutdownNow() + } +} diff --git a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala index d59965f9..3496849c 100644 --- a/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala +++ b/core/src/test/scala/app/softnetwork/elastic/client/SettingsApiSpec.scala @@ -449,6 +449,104 @@ class SettingsApiSpec } } + // #238 — the shard count a sliced PIT extraction is sized from. Payloads mirror the shape + // every client's executeLoadSettings returns (MockElasticClientApi): one top-level key per + // concrete index → settings → index → number_of_shards (a string). + "primaryShardCount" should { + + def shards(n: String): String = s"""{"settings":{"index":{"number_of_shards":"$n"}}}""" + + class ShardCountApi(responses: Map[String, ElasticResult[String]]) extends NopeClientApi { + override protected def logger: Logger = mockLogger + val calls = scala.collection.mutable.ListBuffer.empty[String] + override private[client] def executeLoadSettings(index: String): ElasticResult[String] = { + calls += index + responses.getOrElse(index, ElasticSuccess("{}")) + } + } + + "sum number_of_shards over the concrete indices of every expression" in { + val api = new ShardCountApi( + Map( + "idx_a" -> ElasticSuccess(s"""{"idx_a":${shards("3")}}"""), + "idx_b" -> ElasticSuccess(s"""{"idx_b":${shards("2")}}""") + ) + ) + api.primaryShardCount(Seq("idx_a", "idx_b")) shouldBe ElasticSuccess(5) + } + + "count a concrete index once when several expressions resolve to it" in { + val api = new ShardCountApi( + Map( + "idx_*" -> ElasticSuccess(s"""{"idx_a":${shards("3")},"idx_b":${shards("2")}}"""), + "idx_a" -> ElasticSuccess(s"""{"idx_a":${shards("3")}}""") + ) + ) + api.primaryShardCount(Seq("idx_*", "idx_a")) shouldBe ElasticSuccess(5) + } + + "look a duplicated expression up once" in { + val api = new ShardCountApi(Map("idx_a" -> ElasticSuccess(s"""{"idx_a":${shards("3")}}"""))) + api.primaryShardCount(Seq("idx_a", "idx_a")) shouldBe ElasticSuccess(3) + api.calls.toList shouldBe List("idx_a") + } + + "return 0 when the lookup resolves to no index (nothing matched — never cached upstream)" in { + val api = new ShardCountApi(Map("idx_a" -> ElasticSuccess("{}"))) + api.primaryShardCount(Seq("idx_a")) shouldBe ElasticSuccess(0) + } + + "count 1 per index when number_of_shards is missing, without throwing" in { + val api = new ShardCountApi( + Map( + "idx_a" -> ElasticSuccess( + """{"idx_a":{"settings":{"index":{"number_of_replicas":"1"}}}}""" + ), + "idx_b" -> ElasticSuccess("""{"idx_b":{"settings":{}}}""") + ) + ) + api.primaryShardCount(Seq("idx_a", "idx_b")) shouldBe ElasticSuccess(2) + } + + "count 1 for a non-numeric number_of_shards" in { + val api = + new ShardCountApi(Map("idx_a" -> ElasticSuccess(s"""{"idx_a":${shards("three")}}"""))) + api.primaryShardCount(Seq("idx_a")) shouldBe ElasticSuccess(1) + } + + "accept a numeric number_of_shards" in { + val api = new ShardCountApi( + Map( + "idx_a" -> ElasticSuccess("""{"idx_a":{"settings":{"index":{"number_of_shards":6}}}}""") + ) + ) + api.primaryShardCount(Seq("idx_a")) shouldBe ElasticSuccess(6) + } + + "pass an executeLoadSettings failure through as ElasticFailure" in { + val api = new ShardCountApi(Map("idx_a" -> ElasticFailure(ElasticError("boom")))) + val result = api.primaryShardCount(Seq("idx_a")) + result.isFailure shouldBe true + result.error.get.message shouldBe "boom" + } + + "turn a malformed payload into ElasticFailure, never a throw" in { + val api = new ShardCountApi(Map("idx_a" -> ElasticSuccess("not json at all"))) + api.primaryShardCount(Seq("idx_a")).isFailure shouldBe true + } + + "skip cross-cluster expressions at DEBUG and never look them up" in { + val api = new ShardCountApi(Map("idx_a" -> ElasticSuccess(s"""{"idx_a":${shards("3")}}"""))) + api.primaryShardCount(Seq("remote:idx", "idx_a")) shouldBe ElasticSuccess(3) + api.calls.toList shouldBe List("idx_a") + verify(mockLogger).debug("Skipping shard lookup for cross-cluster expression 'remote:idx'") + } + + "return 0 for an empty expression list (nothing to look up)" in { + new ShardCountApi(Map.empty).primaryShardCount(Seq.empty) shouldBe ElasticSuccess(0) + } + } + "loadSettings" should { "successfully load settings for existing index" in { diff --git a/core/src/test/scala/app/softnetwork/elastic/client/scroll/SliceMergeSpec.scala b/core/src/test/scala/app/softnetwork/elastic/client/scroll/SliceMergeSpec.scala new file mode 100644 index 00000000..5d52e109 --- /dev/null +++ b/core/src/test/scala/app/softnetwork/elastic/client/scroll/SliceMergeSpec.scala @@ -0,0 +1,116 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client.scroll + +import akka.{Done, NotUsed} +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Sink, Source} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.concurrent.Eventually +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.duration._ +import scala.util.{Failure, Success, Try} + +/** #238 — the merge helper behind sliced PIT paging: every page of every slice reaches the + * consumer, a failing slice fails the merged stream, and `onTerminate` (the single PIT close) + * fires exactly once on completion, failure and downstream cancellation. + */ +class SliceMergeSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll with Eventually { + + implicit val system: ActorSystem = ActorSystem("slice-merge-spec") + implicit val ec: ExecutionContext = system.dispatcher + + override def afterAll(): Unit = { + system.terminate() + super.afterAll() + } + + private def pages(slice: Int, n: Int): Source[Seq[Int], NotUsed] = + Source(List.tabulate(n)(p => Seq(slice * 100 + p * 10, slice * 100 + p * 10 + 1))) + + private class Probe { + val calls = new AtomicInteger(0) + val last = new AtomicReference[Try[Done]]() + val onTerminate: Try[Done] => Unit = { t => + last.set(t) // before the counter the tests wait on, so `last` is never read as null + calls.incrementAndGet() + } + } + + "SliceMerge" should "deliver every page of every slice and terminate once with Success" in { + val probe = new Probe + val merged = SliceMerge(Seq(pages(1, 3), pages(2, 3), pages(3, 3)))(probe.onTerminate) + val rows = Await.result(merged.mapConcat(identity).runWith(Sink.seq), 10.seconds) + rows should have size 18 + rows.toSet should have size 18 + eventually(timeout(5.seconds)) { + probe.calls.get() shouldBe 1 + } + probe.last.get() shouldBe Success(Done) + } + + it should "fail the merged stream when one slice fails, and terminate once with Failure" in { + val probe = new Probe + val boom = new IllegalStateException("slice 2 broke") + val merged = SliceMerge( + Seq(pages(1, 3), Source.failed[Seq[Int]](boom), pages(3, 3)) + )(probe.onTerminate) + val result = Try(Await.result(merged.mapConcat(identity).runWith(Sink.seq), 10.seconds)) + result.isFailure shouldBe true + result.failed.get.getMessage shouldBe "slice 2 broke" + eventually(timeout(5.seconds)) { + probe.calls.get() shouldBe 1 + } + probe.last.get() shouldBe a[Failure[_]] + } + + it should "terminate once when the consumer cancels early (.take)" in { + val probe = new Probe + val merged = SliceMerge(Seq(pages(1, 50), pages(2, 50), pages(3, 50)))(probe.onTerminate) + val rows = Await.result(merged.mapConcat(identity).take(5).runWith(Sink.seq), 10.seconds) + rows should have size 5 + eventually(timeout(5.seconds)) { + probe.calls.get() shouldBe 1 + } + probe.last.get() shouldBe Success(Done) + } + + it should "pass a single source through and terminate once" in { + val probe = new Probe + val merged = SliceMerge(Seq(pages(1, 4)))(probe.onTerminate) + val rows = Await.result(merged.mapConcat(identity).runWith(Sink.seq), 10.seconds) + rows shouldBe List(100, 101, 110, 111, 120, 121, 130, 131) + eventually(timeout(5.seconds)) { + probe.calls.get() shouldBe 1 + } + } + + it should "produce an empty stream for no slices and still terminate once" in { + val probe = new Probe + val merged = SliceMerge(Seq.empty[Source[Seq[Int], NotUsed]])(probe.onTerminate) + val rows = Await.result(merged.runWith(Sink.seq), 10.seconds) + rows shouldBe empty + eventually(timeout(5.seconds)) { + probe.calls.get() shouldBe 1 + } + probe.last.get() shouldBe Success(Done) + } +} diff --git a/documentation/client/common_principles.md b/documentation/client/common_principles.md index 330140c9..86c39322 100644 --- a/documentation/client/common_principles.md +++ b/documentation/client/common_principles.md @@ -488,6 +488,16 @@ elastic { # `_id` column. Disabled by default: SQL results carry only the selected columns. include-document-id = false + # Paged row extraction (scroll / PIT + search_after) — 0.21.0+ + scroll { + # Rows per page. Larger pages cut round-trips linearly and raise in-flight memory linearly. + size = 1000 + # Ceiling on concurrent PIT slices for a no-ORDER-BY extraction (ES 7.15+). The effective + # count is min(primary shards, max-slices); 1 disables slicing (sequential paging) for every + # ScrollConfig that leaves maxSlices unset — an explicit Some(n) still wins. + max-slices = 8 + } + # Cluster discovery discovery { enabled = false @@ -524,6 +534,12 @@ export ELASTIC_PORT=9243 # Surface the document id as an `_id` column on result rows export ELASTIC_INCLUDE_DOCUMENT_ID=true + +# Paged row extraction (0.21.0+): rows per page, and the ceiling on concurrent PIT slices +# for a no-ORDER-BY extraction (ES 7.15+); 1 disables slicing everywhere a ScrollConfig leaves +# maxSlices unset (the default) — an explicit ScrollConfig(maxSlices = Some(n)) still wins +export ELASTIC_SCROLL_SIZE=1000 +export ELASTIC_SCROLL_MAX_SLICES=8 ``` ### Loading Configuration diff --git a/documentation/client/scroll.md b/documentation/client/scroll.md index ee2fbe17..0d23519c 100644 --- a/documentation/client/scroll.md +++ b/documentation/client/scroll.md @@ -46,12 +46,18 @@ The API automatically selects the best strategy based on your query and the Elas **Strategy Selection Matrix:** -| ES Version | Aggregations | Strategy | -|-----------------|---------------|----------------------------------| -| 7.10+ | No | PIT + search_after (recommended) | -| 7.10+ | Yes | Classic scroll | -| < 7.10 | No | search_after | -| < 7.10 | Yes | Classic scroll | +| ES Version | Aggregations | Strategy | +|-----------------|---------------|--------------------------------------------------------------------------| +| 7.15+ | No | PIT + search_after, **sliced** when no `ORDER BY` / no `LIMIT`: one reader per primary shard (0.21.0+) | +| 7.12+ | No | PIT + search_after (recommended) | +| 7.12+ | Yes | Classic scroll | +| < 7.12 | No | search_after | +| < 7.12 | Yes | Classic scroll | + +PIT paging is gated at 7.12 (not 7.10, where the PIT API first appeared): the `_shard_doc` +tiebreaker Elasticsearch appends under a PIT only exists from 7.12, and without it a paged +extraction silently drops rows across shards. PIT *slicing* (`slice` + `pit` in one request) +exists from 7.15. --- @@ -60,7 +66,7 @@ The API automatically selects the best strategy based on your query and the Elas ```scala sealed trait ScrollStrategy -// Point In Time + search_after (ES 7.10+, best performance) +// Point In Time + search_after (ES 7.12+, best performance; sliced from 7.15) case object UsePIT extends ScrollStrategy // search_after only (efficient, no server state) @@ -106,7 +112,7 @@ def scrollAsUnchecked[T]( ### Point In Time (PIT) + search_after -**Best for:** ES 7.10+, large result sets, no aggregations +**Best for:** ES 7.12+, large result sets, no aggregations **Advantages:** - ✅ Consistent snapshot across pagination @@ -117,10 +123,10 @@ def scrollAsUnchecked[T]( **Limitations:** - ❌ Not supported with aggregations -- ❌ Requires ES 7.10+ +- ❌ Requires ES 7.12+ ```scala -// Automatically used for ES 7.10+ without aggregations +// Automatically used for ES 7.12+ without aggregations val query = SQLQuery( query = """ SELECT id, name, price @@ -136,9 +142,89 @@ client.scroll(query).runWith(Sink.seq) --- +### Sliced PIT paging (ES 7.15+) + +**Default since 0.21.0.** A no-`ORDER BY`, no-`LIMIT` extraction from an N-shard index opens **one** +PIT and reads `min(N, max-slices)` slices of it concurrently — one reader per primary shard — merged +page by page into the single stream you already consume. The sequential pipeline was latency-bound +(one round-trip per page, every page fanning out to every shard); slicing is what makes the wall +clock improve when shards and nodes are added. + +**How the slice count is derived (once per stream, in core):** + +| Condition | Slices | +|-----------|--------| +| Strategy is not PIT + search_after (aggregations, classic scroll opt-out, ES < 7.12) | 1 | +| The statement has an `ORDER BY` (AST or JSON `sort`) | 1 | +| The statement has an explicit `LIMIT` (incl. the `LIMIT > max_result_window` route) | 1 | +| Effective ceiling `maxSlices.getOrElse(elastic.scroll.max-slices)` is `<= 1` | 1 | +| ES < 7.15 (PIT without slicing) | 1 | +| Otherwise | `max(1, min(Σ number_of_shards of the resolved indices, ceiling))` | + +The shard count comes from `GET /_settings` (the indices a wildcard, alias or data stream +resolves to are summed and deduplicated) and is **cached per index set for 5 minutes** — the same +TTL as the schema cache (`shardCountCacheTtlMs`, overridable in a subclass) — so a workload of many +small un-LIMITed queries pays one round-trip per table per TTL, not one per query (concurrent cold +extractions of the same set share one lookup). What is remembered: a positive count, and a +**privilege** failure (HTTP 401/403 — the credentials lack `view_index_metadata`), for which the +extraction logs **one WARN per TTL** naming the privilege (DEBUG while the cached failure is +replayed). A transient failure (timeout, 503, index not found, unparseable payload) is **not** +cached — every extraction probes again and logs its WARN — and neither is an expression that +matches no index, so a table created right after is seen at once. The lookup never fails the +extraction: it degrades to sequential paging. A stale count is a performance matter only (it changes +how the PIT is split, never which rows come back); every schema-cache write or invalidation on the +same client — `createIndex`, `updateSchema`, `invalidateSchema` (REPL `refresh [table]`, `DROP +TABLE`) — drops the cached counts **naming that index**, and `invalidateAllSchemas()` drops all of +them. Cache keys are the statement's `FROM` expressions, so the match is made per expression and by +case-insensitive **prefix**: invalidating `orders` drops the entries for `orders`, `orders_2026`, +`orders*` and any multi-index set containing them, but not an unrelated wildcard (`logs-*`) or an +alias that the index merely happens to match — those, like DDL issued through another client, are +seen after the TTL. `ScrollMetrics.slices` reports the resolved count on every row. + +**Ceiling and opt-out:** + +```scala +// Per call — Some(n) is explicit, n <= 1 pages sequentially +client.scroll(query, ScrollConfig(maxSlices = Some(2))) + +// Everywhere — HOCON or environment; 1 disables slicing on every path, INCLUDING explicit +// ScrollConfig(...) values that leave maxSlices = None (the default) +// elastic.scroll.max-slices = 1 ELASTIC_SCROLL_MAX_SLICES=1 +``` + +`ScrollConfig.maxSlices` is an `Option` on purpose: `None` inherits the client's configured ceiling, +so the HOCON / environment switch is a real kill switch even for callers that build their own +`ScrollConfig`. The default ceiling is `ScrollConfig.DefaultMaxSlices = 8`: one slice per primary +shard up to 8, under the REST client's per-route pool — which the es7 / es8 / es9 clients now size +from `max-slices` (`max(10, max-slices + 2)` per route). Slices are never configured above the shard +count (an explicit ceiling below it is still a whole-shard split on the Elasticsearch side). + +**What changes for consumers:** + +- Row order of an un-ordered extraction **interleaves across slices** (it was incidentally + `_doc`-ordered before). Add an `ORDER BY` when order matters — ordered statements stay sequential. +- A licence-quota-capped result (`maxDocuments`) is now an **arbitrary subset** of the matching + rows, not a stable prefix. +- `SELECT *` column metadata derived from the first row (JDBC, Arrow) may vary between runs on + heterogeneous multi-shard indices. +- In-flight memory is bounded by about `2 × slices × scrollSize` rows plus `slices` raw page + responses (each slice keeps one page in flight while the previous one drains). +- The PIT is opened lazily (at materialization) and closed exactly once — on completion, failure + or cancellation — whatever the slice count; a failing slice fails the whole stream (never a + silently truncated result). A cancel that lands while the PIT-open round-trip is in flight is + also handled (the PIT is closed as soon as its id arrives); the residual window is microseconds. +- Above the ceiling the load is uneven: 12 shards with `max-slices = 8` gives four slices that + read two shards each and four that read one — the wall clock follows the two-shard slices. + "One reader per primary shard" holds up to the ceiling. +- The REST pool is per client and sized to the ceiling for ONE extraction; concurrent sliced + extractions on the same client share it and queue (they still complete — measured k = 4 at 34 s + vs 48 s sequential — but do not scale linearly). + +--- + ### search_after -**Best for:** ES < 7.10, large result sets, no aggregations +**Best for:** ES < 7.12, large result sets, no aggregations **Advantages:** - ✅ No server-side state @@ -152,7 +238,7 @@ client.scroll(query).runWith(Sink.seq) - ⚠️ No consistent snapshot (data can change between pages) ```scala -// Automatically used for ES < 7.10 without aggregations +// Automatically used for ES < 7.12 without aggregations val query = SQLQuery( query = """ SELECT id, name, price @@ -228,20 +314,34 @@ case class ScrollConfig( metrics: ScrollMetrics = ScrollMetrics(), // Retry configuration - retryConfig: RetryConfig = RetryConfig() + retryConfig: RetryConfig = RetryConfig(), + + // Ceiling on concurrent PIT slices for a no-ORDER-BY extraction (ES 7.15+, 0.21.0+). + // None = inherit the client's `elastic.scroll.max-slices`; Some(n) explicit, n <= 1 sequential + maxSlices: Option[Int] = None, + + // Internal (set by ScrollApi, not by callers): the slice count resolved for this stream + slices: Int = 1 ) ``` +When no configuration is passed, `client.scroll(query)` uses `client.defaultScrollConfig`: the page +size comes from `elastic.scroll.size` and the slice ceiling from `elastic.scroll.max-slices` +(see [Sliced PIT paging](#sliced-pit-paging-es-715)). An explicit `ScrollConfig(...)` keeps its +explicit fields but still inherits the configured ceiling through `maxSlices = None`. + **Configuration Options:** | Parameter | Type | Default | Description | |---------------------|-----------------|-------------------|----------------------------------------------| -| `scrollSize` | `Int` | `1000` | Number of documents per batch | -| `keepAlive` | `String` | `"1m"` | Scroll context timeout (classic scroll only) | -| `maxDocuments` | `Option[Long]` | `None` | Maximum documents to retrieve | +| `scrollSize` | `Int` | `1000` | Number of documents per batch (`elastic.scroll.size` when omitted) | +| `keepAlive` | `String` | `"1m"` | Scroll / PIT context keep-alive | +| `maxDocuments` | `Option[Long]` | `None` | Maximum documents to retrieve (binds on the merged total when sliced) | | `preferSearchAfter` | `Boolean` | `true` | Prefer search_after when available | | `logEvery` | `Int` | `10` | Log progress every N batches | | `metrics` | `ScrollMetrics` | `ScrollMetrics()` | Initial metrics state | +| `maxSlices` | `Option[Int]` | `None` | Ceiling on concurrent PIT slices; `None` inherits `elastic.scroll.max-slices` (default 8), `Some(1)` pages sequentially | +| `slices` | `Int` | `1` | Internal — the resolved slice count (read it on `ScrollMetrics.slices`) | --- @@ -252,7 +352,8 @@ case class ScrollMetrics( totalDocuments: Long = 0, totalBatches: Int = 0, startTime: Long = System.currentTimeMillis(), - endTime: Option[Long] = None + endTime: Option[Long] = None, + slices: Int = 1 // PIT slices merged into this stream (1 = sequential) ) { // Calculate duration in milliseconds def duration: Long = endTime.getOrElse(System.currentTimeMillis()) - startTime @@ -276,6 +377,7 @@ case class ScrollMetrics( | `totalBatches` | `Int` | Total batches processed | | `startTime` | `Long` | Start timestamp (milliseconds) | | `endTime` | `Option[Long]` | End timestamp (milliseconds) | +| `slices` | `Int` | PIT slices merged into this stream (1 = sequential) | | `duration` | `Long` | Total duration (milliseconds) | | `documentsPerSecond` | `Double` | Throughput rate | @@ -468,7 +570,9 @@ val query = WHERE category = 'electronics' """ -client.scrollAs[Product](query) +// scrollAs is a macro: the config must be passed explicitly (macro applications take no default +// arguments) — `client.defaultScrollConfig` carries the `elastic.scroll` settings +client.scrollAs[Product](query, client.defaultScrollConfig) .map { case (product, _) => ProductSummary( name = product.name, @@ -1361,8 +1465,8 @@ val query = SQLQuery( ) // Automatically uses: -// - PIT + search_after for ES 7.10+ (best performance) -// - search_after for ES < 7.10 +// - PIT + search_after for ES 7.12+ (best performance; sliced from 7.15) +// - search_after for ES < 7.12 // - Classic scroll for aggregations client.scroll(query).runWith(Sink.seq) @@ -1727,7 +1831,7 @@ The **Scroll API** provides: | Feature | PIT + search_after | search_after | Classic Scroll | |---------|-------------------|--------------|----------------| -| **ES Version** | 7.10+ | All | All | +| **ES Version** | 7.12+ (sliced 7.15+) | All | All | | **Aggregations** | ❌ | ❌ | ✅ | | **Consistent Snapshot** | ✅ | ❌ | ✅ | | **Deep Pagination** | ⭐⭐⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐ | @@ -1737,8 +1841,8 @@ The **Scroll API** provides: **When to Use:** -- **PIT + search_after**: ES 7.10+, large datasets, no aggregations (recommended) -- **search_after**: ES < 7.10, large datasets, no aggregations +- **PIT + search_after**: ES 7.12+, large datasets, no aggregations (recommended; sliced one-reader-per-shard from 7.15) +- **search_after**: ES < 7.12, large datasets, no aggregations - **Classic scroll**: Any version with aggregations, or when consistent snapshot is required **Best Practices:** diff --git a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala index e40d7881..d1102351 100644 --- a/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala +++ b/es6/jest/src/main/scala/app/softnetwork/elastic/client/jest/JestScrollApi.scala @@ -39,7 +39,7 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success, Try} trait JestScrollApi extends ScrollApi with JestClientHelpers { - _: JestVersionApi with JestSearchApi with JestClientCompanion => + _: JestVersionApi with JestSearchApi with JestSettingsApi with JestClientCompanion => /** Classic scroll (works for both hits and aggregations) */ diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSlicedScrollCompletenessSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSlicedScrollCompletenessSpec.scala new file mode 100644 index 00000000..2c4901c2 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientSlicedScrollCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientSlicedScrollCompletenessSpec extends SlicedScrollCompletenessSpec diff --git a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index fcb731c6..c7d83440 100644 --- a/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es6/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -1435,6 +1435,7 @@ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHelpers { _: RestHighLevelClientVersionApi with RestHighLevelClientSearchApi + with RestHighLevelClientSettingsApi with RestHighLevelClientCompanion => // ========================================================================== diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSlicedScrollCompletenessSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSlicedScrollCompletenessSpec.scala new file mode 100644 index 00000000..4c094e3d --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSlicedScrollCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientSlicedScrollCompletenessSpec extends SlicedScrollCompletenessSpec diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala index d25449df..87dd48fd 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientApi.scala @@ -92,7 +92,14 @@ import org.elasticsearch.action.support.{IndicesOptions, WriteRequest} import org.elasticsearch.action.support.master.AcknowledgedResponse import org.elasticsearch.action.update.{UpdateRequest, UpdateResponse} import org.elasticsearch.action.{ActionListener, DocWriteRequest, DocWriteResponse} -import org.elasticsearch.client.{GetAliasesResponse, Request, RequestOptions, Response} +import org.elasticsearch.client.{ + GetAliasesResponse, + Request, + RequestOptions, + Response, + ResponseException, + ResponseListener +} import org.elasticsearch.client.core.{CountRequest, CountResponse} import org.elasticsearch.client.enrich.{ DeletePolicyRequest, @@ -172,12 +179,14 @@ import org.elasticsearch.search.aggregations.metrics.{ } import org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder import org.elasticsearch.search.builder.{PointInTimeBuilder, SearchSourceBuilder} +import org.elasticsearch.search.slice.SliceBuilder import org.elasticsearch.search.sort.{FieldSortBuilder, SortOrder} import org.json4s.jackson.JsonMethods import org.json4s.DefaultFormats import java.io.IOException import java.nio.charset.StandardCharsets +import java.util.concurrent.atomic.AtomicBoolean import java.time.ZonedDateTime import scala.collection.immutable.ListMap import scala.jdk.CollectionConverters._ @@ -1467,6 +1476,7 @@ trait RestHighLevelClientBulkApi extends BulkApi with RestHighLevelClientHelpers trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHelpers { _: RestHighLevelClientSearchApi with RestHighLevelClientVersionApi + with RestHighLevelClientSettingsApi with RestHighLevelClientCompanion => // ========================================================================== @@ -1494,14 +1504,45 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel try { readResponseTree(apply().getLowLevelClient.performRequest(request)) } catch { - case ex: org.elasticsearch.client.ResponseException => - val status = Try(ex.getResponse.getStatusLine.getStatusCode).getOrElse(0) - if (status >= 400 && status < 500 && status != 408 && status != 429) { - throw new IllegalStateException(ex.getMessage, ex) - } - throw ex + case ex: ResponseException if isNonRetriable4xx(ex) => + throw new IllegalStateException(ex.getMessage, ex) } + /** Asynchronous sibling of [[executeSearchPage]] for the PIT paging path (#238): the request + * travels through `performRequestAsync`, so no dispatcher thread blocks on the wire; the + * (heap-buffered) entity is Jackson-parsed on `ec` — the caller's dispatcher, never the IO + * reactor. `performRequestAsync` reports an error status through `onFailure(ResponseException)` + * without any `CompletionException` wrapper, so retry semantics hold by construction; the same + * 4xx-except-408/429 statuses are made non-retriable as on the synchronous path. + */ + private def executeSearchPageAsync(request: Request)(implicit + ec: ExecutionContext + ): Future[JsonNode] = { + val promise = Promise[Response]() + apply().getLowLevelClient.performRequestAsync( + request, + new ResponseListener { + override def onSuccess(response: Response): Unit = promise.trySuccess(response) + override def onFailure(exception: Exception): Unit = promise.tryFailure(exception) + } + ) + promise.future + .map(readResponseTree) + .recoverWith { + case re: ResponseException if isNonRetriable4xx(re) => + Future.failed(new IllegalStateException(re.getMessage, re)) + case jp: com.fasterxml.jackson.core.JsonProcessingException => + // an unparseable page body is an IOException subclass: a retry cannot help + Future.failed(new IllegalStateException(s"Unparseable page: ${jp.getMessage}", jp)) + } + } + + /** 4xx except 408 / 429: a permanent client error that must fail fast rather than be retried + * (`ResponseException` is an `IOException`, which `retryWithBackoff` would otherwise retry). + */ + private def isNonRetriable4xx(re: ResponseException): Boolean = + statusOf(re).exists(status => status >= 400 && status < 500 && status != 408 && status != 429) + /** Reasons of any failed shards on this page, if some shards failed. A page with failed shards is * silent row loss on a paging path — callers must fail loudly, mirroring the es8/es9 typed shard * check. @@ -1803,10 +1844,11 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel .mapConcat(identity) } - /** PIT + search_after for ES 7.10+ + /** PIT + search_after for ES 7.12+ — sliced (one reader per primary shard, #238) from 7.15 when + * core resolved `config.slices > 1`. * * @note - * Requires ES 7.10+. For ES 6.x, use searchAfterSource instead. + * Requires ES 7.12+. For ES 6.x, use searchAfterSource instead. */ private[client] def pitSearchAfter( elasticQuery: ElasticQuery, @@ -1819,146 +1861,204 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel ): Source[ListMap[String, Any], NotUsed] = { implicit val ec: ExecutionContext = system.dispatcher - // Open PIT - val pitIdFuture: Future[String] = openPit(elasticQuery.indices, config.keepAlive) + // #238 — core applied the whole slicing policy (strategy, sorts, gate, ceiling, shard count); + // the client applies the resolved count verbatim and never re-derives it (D1). + val sliceCount = math.max(1, config.slices) + // The PIT is opened LAZILY — at materialization, on first demand — so a source that is + // built but never materialized never opens a PIT. A consumer that cancels WHILE the open is + // in flight is caught by `outerDone` below: the inner source would never be materialized + // (its watchTermination never attached), so the PIT is closed right here instead. The + // residual window is the ordering of two callbacks on the dispatcher — microseconds. + val outerDone = new AtomicBoolean(false) Source - .futureSource { - pitIdFuture.map { pitId => - logger.info( - s"Opened PIT: ${pitId.take(20)}... for indices: ${elasticQuery.indices.mkString(", ")}" - ) - - Source - .unfoldAsync[Option[Array[Object]], Seq[ListMap[String, Any]]](None) { searchAfterOpt => - retryWithBackoff(config.retryConfig) { - Future { - searchAfterOpt match { - case None => - logger.info(s"Starting PIT search_after (pitId: ${pitId.take(20)}...)") - case Some(values) => - logger.debug( - s"Fetching next PIT search_after batch (after: ${if (values.length > 3) - s"[${values.take(3).mkString(", ")}...]" - else values.mkString(", ")})" - ) - } - - // Parse query - val xContentParser = XContentType.JSON - .xContent() - .createParser( - namedXContentRegistry, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - elasticQuery.query - ) - - val sourceBuilder = SearchSourceBuilder - .fromXContent(xContentParser) - .size(config.scrollSize) - // The paging path never reads hits.total — computing it costs ~30% of the - // ES-side CPU per page (#200) - .trackTotalHits(false) - - // Check if sorts already exist in the query - if (!hasSorts && sourceBuilder.sorts() == null) { - // _doc, NOT _shard_doc: a primary _shard_doc sort defeats the doc-id skip - // optimisation on ES 8 / Lucene 9 (#197); harmless but not needed on 7.x - // either. Under a PIT (>= 7.12) ES appends _shard_doc as an automatic - // tiebreaker, so _doc is a total order and row-complete across shards. - logger.debug( - "No sort fields in query for PIT search_after, adding default _doc sort." - ) - sourceBuilder.sort("_doc", SortOrder.ASC) - } else if (hasSorts && sourceBuilder.sorts() != null) { - // Sorts already present, check that a tie-breaker exists - val hasShardDocSort = sourceBuilder.sorts().asScala.exists { - case fieldSort: FieldSortBuilder => - fieldSort.getFieldName == "_shard_doc" || fieldSort.getFieldName == "_id" - case _ => - false - } - - if (!hasShardDocSort) { - // Add _id as tie-breaker - logger.debug("Adding _shard_doc as tie-breaker to existing sorts") - sourceBuilder.sort("_shard_doc", SortOrder.ASC) - } - } + .lazyFutureSource { () => + openPit(elasticQuery.indices, config.keepAlive).map { pitId => + if (outerDone.get) { + logger.info( + s"Stream cancelled while opening the PIT; closing PIT: ${pitId.take(20)}..." + ) + closePit(pitId) + Source.empty[ListMap[String, Any]] + } else { + logger.info( + s"Opened PIT: ${pitId.take(20)}... for indices: ${elasticQuery.indices.mkString(", ")} ($sliceCount slice(s))" + ) - // Add search_after - searchAfterOpt.foreach { searchAfter => - sourceBuilder.searchAfter(searchAfter) - } + // Set by onTerminate BEFORE the PIT is closed: every slice's next / in-flight step + // short-circuits to end-of-stream instead of racing the close with retried requests. + val terminated = new AtomicBoolean(false) + + /** One slice (or the whole PIT when `slice` is None) as a source of PAGES. */ + def pageSource(slice: Option[(Int, Int)]): Source[Seq[ListMap[String, Any]], NotUsed] = + Source.unfoldAsync[Option[Array[Object]], Seq[ListMap[String, Any]]](None) { + searchAfterOpt => + retryWithBackoff(config.retryConfig) { + // By-name: re-evaluated on every retry, so the terminated check stops retries too. + if (terminated.get) Future.successful(None) + else { + // The request is built on a dispatcher thread — NOT on the stream's interpreter + // thread (the XContent parse per page would serialise with the graph's own + // processing) — and a synchronous failure stays inside the Future chain (retry / + // stream failure), never an escape out of unfoldAsync. + Future { + searchAfterOpt match { + case None if slice.isEmpty => + logger.info(s"Starting PIT search_after (pitId: ${pitId.take(20)}...)") + case None => + // per slice: DEBUG, so a sliced extraction keeps ONE INFO line (AC 12) + logger.debug( + s"Starting PIT search_after (pitId: ${pitId + .take(20)}...${slice.fold("")(s => s", slice ${s._1}/${s._2}")})" + ) + case Some(values) => + logger.debug( + s"Fetching next PIT search_after batch (after: ${if (values.length > 3) + s"[${values.take(3).mkString(", ")}...]" + else values.mkString(", ")})" + ) + } - // Set PIT - val pitBuilder = new PointInTimeBuilder(pitId) - pitBuilder.setKeepAlive( - TimeValue.parseTimeValue(config.keepAlive, "pit_keep_alive") - ) - sourceBuilder.pointInTimeBuilder(pitBuilder) + // Parse query (per page: SearchSourceBuilder is a mutable builder) + val xContentParser = XContentType.JSON + .xContent() + .createParser( + namedXContentRegistry, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + elasticQuery.query + ) + + val sourceBuilder = SearchSourceBuilder + .fromXContent(xContentParser) + .size(config.scrollSize) + // The paging path never reads hits.total — computing it costs ~30% of + // the ES-side CPU per page (#200) + .trackTotalHits(false) + + // Check if sorts already exist in the query + if (!hasSorts && sourceBuilder.sorts() == null) { + // _doc, NOT _shard_doc: a primary _shard_doc sort defeats the doc-id + // skip optimisation on ES 8 / Lucene 9 (#197); harmless but not needed + // on 7.x either. Under a PIT (>= 7.12) ES appends _shard_doc as an + // automatic tiebreaker, so _doc is a total order and row-complete + // across shards. + sourceBuilder.sort("_doc", SortOrder.ASC) + } else if (hasSorts && sourceBuilder.sorts() != null) { + // Sorts already present, check that a tie-breaker exists + val hasShardDocSort = sourceBuilder.sorts().asScala.exists { + case fieldSort: FieldSortBuilder => + fieldSort.getFieldName == "_shard_doc" || fieldSort.getFieldName == "_id" + case _ => + false + } + + if (!hasShardDocSort) { + // Add _shard_doc as tie-breaker + sourceBuilder.sort("_shard_doc", SortOrder.ASC) + } + } - // Build request with PIT — no index in the path, the PIT owns the target - // (single parse #228: raw response bytes Jackson-parsed once) - val request = new Request("POST", "/_search") - request.addParameter("request_cache", "false") // Disable cache for PIT - request.setJsonEntity(Strings.toString(sourceBuilder)) + // Add search_after + searchAfterOpt.foreach { searchAfter => + sourceBuilder.searchAfter(searchAfter) + } - val tree = executeSearchPage(request) + // #238 — one slice of the PIT (doc-id range, split first across shards). + // No slice object when the count is 1: SliceBuilder rejects `max <= 1`. + slice.foreach { case (id, max) => + sourceBuilder.slice(new SliceBuilder(id, max)) + } - shardFailures(tree).foreach { reasons => - throw new IOException(s"PIT search_after failed: $reasons") - } + // Set PIT + val pitBuilder = new PointInTimeBuilder(pitId) + pitBuilder.setKeepAlive( + TimeValue.parseTimeValue(config.keepAlive, "pit_keep_alive") + ) + sourceBuilder.pointInTimeBuilder(pitBuilder) + + // Build request with PIT — no index in the path, the PIT owns the target + // (single parse #228: raw response bytes Jackson-parsed once) + val request = new Request("POST", "/_search") + request.addParameter("request_cache", "false") // Disable cache for PIT + request.setJsonEntity(Strings.toString(sourceBuilder)) + request + } + .flatMap { request => + // Asynchronous page fetch (#238): no dispatcher thread blocks on the wire. + executeSearchPageAsync(request) + } + .map { tree => + shardFailures(tree).foreach { reasons => + throw new IOException(s"PIT search_after failed: $reasons") + } + + val hitsArray = tree.path("hits").path("hits") + if (!hitsArray.isArray || hitsArray.size() == 0) { + None // end of this slice — watchTermination owns the single PIT close (#202) + } else { + // end-of-slice is decided on the RAW hits above: a page whose hits + // extracted to zero rows must never read as end-of-stream + val hits = + extractHitsOnly(tree, fieldAliases, config.retainDocumentId) + val lastHit = hitsArray.get(hitsArray.size() - 1) + val lastSort = lastHit.path("sort") + if (!lastSort.isArray || lastSort.size() == 0) { + // paging on without a cursor would refetch the same page forever + throw new IllegalStateException( + "search_after page returned hits without sort values — cannot continue paging" + ) + } + val nextSearchAfter = Some(sortValuesOf(lastSort)) - val hits = - extractHitsOnly(tree, fieldAliases, config.retainDocumentId) - - if (hits.isEmpty) { - None // end of stream — watchTermination owns the single PIT close (#202) - } else { - val hitsArray = tree.path("hits").path("hits") - val lastHit = hitsArray.get(hitsArray.size() - 1) - val lastSort = lastHit.path("sort") - if (!lastSort.isArray || lastSort.size() == 0) { - // paging on without a cursor would refetch the same page forever - throw new IllegalStateException( - "search_after page returned hits without sort values — cannot continue paging" - ) + logger.debug(s"Retrieved ${hits.size} hits, continuing with PIT") + Some((nextSearchAfter, hits)) + } + } } - val nextSearchAfter = Some(sortValuesOf(lastSort)) - - logger.debug(s"Retrieved ${hits.size} hits, continuing with PIT") - Some((nextSearchAfter, hits)) + }(system, logger).recoverWith { + case _ if terminated.get => + // a late failure after cancel / close (search_context_missing...): the + // stream is already over, drop it quietly + logger.debug("PIT page failed after the stream terminated; ignoring") + Future.successful(None) + case ex: Exception => + logger.error(s"PIT search_after failed after retries: ${ex.getMessage}", ex) + // fail the stream instead of ending it: ending here would surface a silently + // truncated result set as a SUCCESSFUL result (#228 review; same defect + // class as #209/#224) — watchTermination still owns the single PIT close (#202) + Future.failed(ex) } - } - }(system, logger).recoverWith { case ex: Exception => - logger.error(s"PIT search_after failed after retries: ${ex.getMessage}", ex) - // fail the stream instead of ending it: ending here would surface a silently - // truncated result set as a SUCCESSFUL result (#228 review; same defect class - // as #209/#224) — watchTermination still owns the single PIT close (#202) - Future.failed(ex) } - } - .watchTermination() { (_, done) => - // Single owner of the PIT close (#202): completion, failure and downstream - // cancellation all land here. The former in-loop closes made every clean run - // close twice and log a spurious "PIT close reported failure" WARN. - done.onComplete { + + val pages = + if (sliceCount <= 1) Seq(pageSource(None)) + else (0 until sliceCount).map(i => pageSource(Some((i, sliceCount)))) + + // Single owner of the PIT close (#202): completion, failure and downstream cancellation + // all land here, exactly once, whatever the slice count. Page-granular merge, ONE + // mapConcat after it (#238 — no per-row merge traffic). + SliceMerge(pages) { done => + terminated.set(true) + done match { case scala.util.Success(_) => - logger.info(s"PIT search_after completed, closing PIT: ${pitId.take(20)}...") - closePit(pitId) + logger.info( + s"PIT search_after completed ($sliceCount slice(s)), closing PIT: ${pitId.take(20)}..." + ) case scala.util.Failure(ex) => logger.error( s"PIT search_after failed: ${ex.getMessage}, closing PIT: ${pitId.take(20)}..." ) - closePit(pitId) } - NotUsed - } - .mapConcat(identity) + // closePit is a blocking call on the dispatcher: let the pool compensate + scala.concurrent.blocking(closePit(pitId)) + }.mapConcat(identity) + } } } - .mapMaterializedValue(_ => NotUsed) + .watchTermination() { (_, done) => + done.onComplete(_ => outerDone.set(true)) + NotUsed + } } /** Open PIT (ES 7.10+) @@ -2032,6 +2132,10 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel } /** Extract ONLY hits (for search_after optimization) + * + * A parse failure FAILS the page (non-retriable `IllegalStateException`): returning an empty + * page here used to read as "end of stream" and surfaced a silently truncated result as a + * success (#238, same defect class as #228 / #209 / #224). */ private def extractHitsOnly( json: JsonNode, @@ -2046,8 +2150,7 @@ trait RestHighLevelClientScrollApi extends ScrollApi with RestHighLevelClientHel ) match { case Success(rows) => rows case Failure(ex) => - logger.error(s"Failed to parse search after response: ${ex.getMessage}", ex) - Seq.empty + throw new IllegalStateException(s"Failed to parse PIT page: ${ex.getMessage}", ex) } } diff --git a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala index 2ab5d8ca..6d3e56f7 100644 --- a/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala +++ b/es7/rest/src/main/scala/app/softnetwork/elastic/client/rest/RestHighLevelClientCompanion.scala @@ -25,6 +25,7 @@ import app.softnetwork.elastic.client.{ import org.elasticsearch.client.{RequestOptions, RestClient, RestClientBuilder, RestHighLevelClient} import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder import org.apache.http.message.BasicHeader import org.elasticsearch.search.SearchModule import org.elasticsearch.common.settings.Settings @@ -63,7 +64,18 @@ trait RestHighLevelClientCompanion extends ElasticClientCompanion[RestHighLevelC } } - /** Build RestClientBuilder with authentication + /** REST connection pool sized to the slice ceiling (#238 — `ScrollSettings.restPoolPerRoute` / + * `restPoolTotal`): an extraction may hold up to `elastic.scroll.max-slices` page requests in + * flight per route on top of the PIT open / close and `_settings` calls sharing the route. + */ + private def withPoolSizing(httpClient: HttpAsyncClientBuilder): HttpAsyncClientBuilder = + httpClient + .setMaxConnPerRoute(elasticConfig.scroll.restPoolPerRoute) + .setMaxConnTotal(elasticConfig.scroll.restPoolTotal) + + /** Build RestClientBuilder with authentication. ONE `setHttpClientConfigCallback` per builder (a + * second call replaces the first): the auth branch yields a function and the pool sizing is + * composed with it in a single callback. */ private def buildRestClient(): RestClientBuilder = { val httpHost = parseHttpHost(elasticConfig.credentials.url) @@ -77,44 +89,47 @@ trait RestHighLevelClientCompanion extends ElasticClientCompanion[RestHighLevelC } // Authenticate - elasticConfig.credentials.authMethod match { - case Some(BasicAuth) if elasticConfig.credentials.username.nonEmpty => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - val credentialsProvider = new BasicCredentialsProvider() - credentialsProvider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials( - elasticConfig.credentials.username, - elasticConfig.credentials.password - ) - ) - httpClientConfigCallback.setDefaultCredentialsProvider(credentialsProvider) - } - case Some(ApiKeyAuth) if elasticConfig.credentials.encodedApiKey.exists(_.nonEmpty) => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - httpClientConfigCallback.setDefaultHeaders( - Seq( - new BasicHeader( - "Authorization", - ApiKeyAuth.createAuthHeader(elasticConfig.credentials) + val authenticate: HttpAsyncClientBuilder => HttpAsyncClientBuilder = + elasticConfig.credentials.authMethod match { + case Some(BasicAuth) if elasticConfig.credentials.username.nonEmpty => + httpClientConfigCallback => { + val credentialsProvider = new BasicCredentialsProvider() + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + elasticConfig.credentials.username, + elasticConfig.credentials.password ) - ).asJava - ) - } - case Some(BearerTokenAuth) if elasticConfig.credentials.bearerToken.exists(_.nonEmpty) => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - httpClientConfigCallback.setDefaultHeaders( - Seq( - new BasicHeader( - "Authorization", - BearerTokenAuth.createAuthHeader(elasticConfig.credentials) - ) - ).asJava - ) - } - case _ => // No authentication - builder - } + ) + httpClientConfigCallback.setDefaultCredentialsProvider(credentialsProvider) + } + case Some(ApiKeyAuth) if elasticConfig.credentials.encodedApiKey.exists(_.nonEmpty) => + httpClientConfigCallback => + httpClientConfigCallback.setDefaultHeaders( + Seq( + new BasicHeader( + "Authorization", + ApiKeyAuth.createAuthHeader(elasticConfig.credentials) + ) + ).asJava + ) + case Some(BearerTokenAuth) if elasticConfig.credentials.bearerToken.exists(_.nonEmpty) => + httpClientConfigCallback => + httpClientConfigCallback.setDefaultHeaders( + Seq( + new BasicHeader( + "Authorization", + BearerTokenAuth.createAuthHeader(elasticConfig.credentials) + ) + ).asJava + ) + case _ => // No authentication + identity + } + + builder.setHttpClientConfigCallback(httpClientConfigCallback => + withPoolSizing(authenticate(httpClientConfigCallback)) + ) } /** Test connection to Elasticsearch cluster diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSlicedScrollCompletenessSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSlicedScrollCompletenessSpec.scala new file mode 100644 index 00000000..4c094e3d --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientSlicedScrollCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientSlicedScrollCompletenessSpec extends SlicedScrollCompletenessSpec diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index f7537bc7..e5984002 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -54,6 +54,7 @@ import app.softnetwork.elastic.sql.watcher.{ WatcherStatus } import app.softnetwork.elastic.utils.CronIntervalCalculator +import co.elastic.clients.elasticsearch._types.SlicedScroll import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ Conflicts, @@ -121,6 +122,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} +import _root_.java.util.concurrent.atomic.AtomicBoolean import _root_.java.util.{Map => JMap} import scala.collection.immutable.ListMap import scala.jdk.CollectionConverters._ @@ -1367,7 +1369,10 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { * [[ScrollApi]] for scroll operations */ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { - _: JavaClientVersionApi with JavaClientSearchApi with JavaClientCompanion => + _: JavaClientVersionApi + with JavaClientSearchApi + with JavaClientSettingsApi + with JavaClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1525,182 +1530,258 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { ): Source[ListMap[String, Any], NotUsed] = { implicit val ec: ExecutionContext = system.dispatcher - // Step 1: Open PIT - val pitIdFuture: Future[String] = openPit(elasticQuery.indices, config.keepAlive) - + // Parsed ONCE per stream (it used to be once per page) and read-only from here on — shared + // by every slice's page builder. Hoisted ABOVE openPit: nothing between a successful openPit + // and the attachment of watchTermination may throw (#202, the single PIT owner rule). + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject + val hasQueryClause = queryJson.has("query") + val hasJsonSort = queryJson.has("sort") + // Sorts already present: check ONCE whether a tie-breaker exists + val needsTiebreaker = hasSorts && hasJsonSort && !queryJson + .getAsJsonArray("sort") + .asScala + .exists { sortElem => + sortElem.isJsonObject && ( + sortElem.getAsJsonObject.has("_shard_doc") || + sortElem.getAsJsonObject.has("_id") + ) + } + // #238 — core applied the whole slicing policy (strategy, sorts, gate, ceiling, shard count); + // the client applies the resolved count verbatim and never re-derives it (D1). + val sliceCount = math.max(1, config.slices) + + // The PIT is opened LAZILY — at materialization, on first demand — so a source that is + // built but never materialized never opens a PIT. A consumer that cancels WHILE the open is + // in flight is caught by `outerDone` below: the inner source would never be materialized + // (its watchTermination never attached), so the PIT is closed right here instead. The + // residual window is the ordering of two callbacks on the dispatcher — microseconds. + val outerDone = new AtomicBoolean(false) Source - .futureSource { - pitIdFuture.map { pitId => - logger.info(s"Opened PIT: $pitId for indices: ${elasticQuery.indices.mkString(", ")}") - - Source - .unfoldAsync[Option[Seq[Any]], Seq[ListMap[String, Any]]](None) { searchAfterOpt => - retryWithBackoff(config.retryConfig) { - Future { - searchAfterOpt match { - case None => - logger.info(s"Starting PIT search_after (pitId: ${pitId.take(20)}...)") - case Some(values) => - logger.debug( - s"Fetching next PIT search_after batch (after: ${if (values.length > 3) - s"[${values.take(3).mkString(", ")}...]" - else values.mkString(", ")})" - ) - } + .lazyFutureSource { () => + openPit(elasticQuery.indices, config.keepAlive).map { pitId => + if (outerDone.get) { + logger.info( + s"Stream cancelled while opening the PIT; closing PIT: ${pitId.take(20)}..." + ) + closePit(pitId) + Source.empty[ListMap[String, Any]] + } else { + logger.info( + s"Opened PIT: ${pitId.take(20)}... for indices: ${elasticQuery.indices.mkString(", ")} ($sliceCount slice(s))" + ) - // Build search request with PIT - val requestBuilder = new SearchRequest.Builder() - .size(config.scrollSize) - .pit( - PointInTimeReference - .of(p => p.id(pitId).keepAlive(Time.of(t => t.time(config.keepAlive)))) - ) + // Set by onTerminate BEFORE the PIT is closed: every slice's next / in-flight step + // short-circuits to end-of-stream instead of racing the close with retried requests. + val terminated = new AtomicBoolean(false) + + /** One slice (or the whole PIT when `slice` is None) as a source of PAGES. */ + def pageSource(slice: Option[(Int, Int)]): Source[Seq[ListMap[String, Any]], NotUsed] = + Source.unfoldAsync[Option[Seq[Any]], Seq[ListMap[String, Any]]](None) { + searchAfterOpt => + retryWithBackoff(config.retryConfig) { + // By-name: re-evaluated on every retry, so the terminated check stops retries too. + if (terminated.get) Future.successful(None) + else { + // The request is built on a dispatcher thread — NOT on the stream's interpreter + // thread (withJson re-parses the query JSON per page, which would serialise with + // the graph's own processing) — and a synchronous failure stays inside the + // Future chain (retry / stream failure), never an escape out of unfoldAsync. + Future { + searchAfterOpt match { + case None if slice.isEmpty => + logger.info(s"Starting PIT search_after (pitId: ${pitId.take(20)}...)") + case None => + // per slice: DEBUG, so a sliced extraction keeps ONE INFO line (AC 12) + logger.debug( + s"Starting PIT search_after (pitId: ${pitId + .take(20)}...${slice.fold("")(s => s", slice ${s._1}/${s._2}")})" + ) + case Some(values) => + logger.debug( + s"Fetching next PIT search_after batch (after: ${if (values.length > 3) + s"[${values.take(3).mkString(", ")}...]" + else values.mkString(", ")})" + ) + } - // Parse query to add query clause (not indices, they're in PIT) - val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject + // Build search request with PIT + val requestBuilder = new SearchRequest.Builder() + .size(config.scrollSize) + .pit( + PointInTimeReference + .of(p => + p.id(pitId).keepAlive(Time.of(t => t.time(config.keepAlive))) + ) + ) - // Extract query clause if present - if (queryJson.has("query")) { - requestBuilder.withJson(new StringReader(elasticQuery.query)) - } + // Query clause (not indices, they're in the PIT) — a FRESH reader per + // page: a StringReader is single-use + if (hasQueryClause) { + requestBuilder.withJson(new StringReader(elasticQuery.query)) + } - // The paging path never reads hits.total — computing it costs ~30% of the - // ES-side CPU per page (#200). Set after withJson so the query cannot re-enable it. - requestBuilder.trackTotalHits(TrackHits.of(t => t.enabled(false))) - - // Check if sorts already exist in the query - if (!hasSorts && !queryJson.has("sort")) { - // _doc, NOT _shard_doc: from ES 8 / Lucene 9 a primary _shard_doc sort - // defeats the doc-id skip optimisation and every page re-scans the whole - // index (#197). Under a PIT (>= 7.12) ES appends _shard_doc as an automatic - // tiebreaker, so _doc is a total order and row-complete across shards. - logger.debug( - "No sort fields in query for PIT search_after, adding default _doc sort." - ) - requestBuilder.sort( - SortOptions.of { sortBuilder => - sortBuilder.field( - FieldSort.of(fieldSortBuilder => - fieldSortBuilder.field("_doc").order(SortOrder.Asc) + // The paging path never reads hits.total — computing it costs ~30% of the + // ES-side CPU per page (#200). Set after withJson so the query cannot + // re-enable it. + requestBuilder.trackTotalHits(TrackHits.of(t => t.enabled(false))) + + if (!hasSorts && !hasJsonSort) { + // _doc, NOT _shard_doc: from ES 8 / Lucene 9 a primary _shard_doc sort + // defeats the doc-id skip optimisation and every page re-scans the whole + // index (#197). Under a PIT (>= 7.12) ES appends _shard_doc as an + // automatic tiebreaker, so _doc is a total order and row-complete + // across shards. + requestBuilder.sort( + SortOptions.of { sortBuilder => + sortBuilder.field( + FieldSort.of(fieldSortBuilder => + fieldSortBuilder.field("_doc").order(SortOrder.Asc) + ) + ) + } ) - ) - } - ) - } else if (hasSorts && queryJson.has("sort")) { - // Sorts already present, check that a tie-breaker exists - val existingSorts = queryJson.getAsJsonArray("sort") - val hasShardDocSort = existingSorts.asScala.exists { sortElem => - sortElem.isJsonObject && ( - sortElem.getAsJsonObject.has("_shard_doc") || - sortElem.getAsJsonObject.has("_id") - ) - } - if (!hasShardDocSort) { - // Add _id as tie-breaker - logger.debug("Adding _shard_doc as tie-breaker to existing sorts") - requestBuilder.sort( - SortOptions.of { sortBuilder => - sortBuilder.field( - FieldSort.of(fieldSortBuilder => - fieldSortBuilder.field("_shard_doc").order(SortOrder.Asc) - ) + } else if (needsTiebreaker) { + // Add _shard_doc as tie-breaker to the existing sorts + requestBuilder.sort( + SortOptions.of { sortBuilder => + sortBuilder.field( + FieldSort.of(fieldSortBuilder => + fieldSortBuilder.field("_shard_doc").order(SortOrder.Asc) + ) + ) + } ) } - ) - } - } - // Add search_after if available - searchAfterOpt.foreach { searchAfter => - val fieldValues: Seq[FieldValue] = searchAfter.map { - case s: String => FieldValue.of(s) - case i: Int => FieldValue.of(i.toLong) - case l: Long => FieldValue.of(l) - case d: Double => FieldValue.of(d) - case b: Boolean => FieldValue.of(b) - case other => FieldValue.of(other.toString) - } - requestBuilder.searchAfter(fieldValues.asJava) - } - - val response = apply().search( - requestBuilder.build(), - classOf[ObjectNode] - ) - - // Check errors - if ( - response.shards() != null && - response.shards().failed() != null && - response.shards().failed().intValue() > 0 - ) { - val failures = response.shards().failures() - val errorMsg = if (failures != null && !failures.isEmpty) { - failures.asScala.map(_.reason()).mkString("; ") - } else { - "Unknown shard failure" - } - throw new IOException(s"PIT search_after failed: $errorMsg") - } + // Add search_after if available + searchAfterOpt.foreach { searchAfter => + requestBuilder.searchAfter(toFieldValues(searchAfter).asJava) + } - val hits = extractHitsOnly(response, fieldAliases, config.retainDocumentId) + // #238 — after withJson AND trackTotalHits, so the statement JSON can + // never clobber it. No slice object when the count is 1: ES rejects + // `max <= 1`. SlicedScroll.id is a String (ES coerces numeric strings). + slice.foreach { case (id, max) => + requestBuilder.slice(SlicedScroll.of(s => s.id(id.toString).max(max))) + } - if (hits.isEmpty) { - None // end of stream — watchTermination owns the single PIT close (#202) - } else { - val lastHit = response.hits().hits().asScala.lastOption - val nextSearchAfter = lastHit.flatMap { hit => - val sortValues = hit.sort().asScala - if (sortValues.nonEmpty) { - Some(sortValues.map { fieldValue => - if (fieldValue.isString) fieldValue.stringValue() - else if (fieldValue.isDouble) fieldValue.doubleValue() - else if (fieldValue.isLong) fieldValue.longValue() - else if (fieldValue.isBoolean) fieldValue.booleanValue() - else if (fieldValue.isNull) null - else fieldValue.toString - }.toSeq) - } else { - None + requestBuilder.build() } + .flatMap { request => + // Asynchronous page fetch (#238): no dispatcher thread blocks on the + // wire. fromCompletableFuture unwraps CompletionException so a transient + // IOException still reaches retryWithBackoff. NOTE: the typed client + // decodes the page (SearchResponse[ObjectNode]) on the transport's + // completion thread — the RestClient IO reactor — before this future + // completes; only the row extraction below runs on system.dispatcher. + fromCompletableFuture(async().search(request, classOf[ObjectNode])) + } + .map { response => + // Row extraction on system.dispatcher. + // Check errors + if ( + response.shards() != null && + response.shards().failed() != null && + response.shards().failed().intValue() > 0 + ) { + val failures = response.shards().failures() + val errorMsg = if (failures != null && !failures.isEmpty) { + failures.asScala.map(_.reason()).mkString("; ") + } else { + "Unknown shard failure" + } + throw new IOException(s"PIT search_after failed: $errorMsg") + } + + val rawHits = response.hits().hits() + if (rawHits.isEmpty) { + None // end of this slice — watchTermination owns the single PIT close (#202) + } else { + // end-of-slice is decided on the RAW hits above: a page whose hits + // extracted to zero rows must never read as end-of-stream + val hits = + extractHitsOnly(response, fieldAliases, config.retainDocumentId) + val sortValues = rawHits.asScala.last.sort().asScala + if (sortValues.isEmpty) { + // paging on without a cursor would refetch the same page forever + throw new IllegalStateException( + "search_after page returned hits without sort values — cannot continue paging" + ) + } + val nextSearchAfter = Some(sortValues.map(toScalaSortValue).toSeq) + + logger.debug(s"Retrieved ${hits.size} documents, continuing with PIT") + Some((nextSearchAfter, hits)) + } + } } - - logger.debug(s"Retrieved ${hits.size} documents, continuing with PIT") - Some((nextSearchAfter, hits)) + }(system, logger).recoverWith { + case _ if terminated.get => + // a late failure after cancel / close (search_context_missing...): the + // stream is already over, drop it quietly + logger.debug("PIT page failed after the stream terminated; ignoring") + Future.successful(None) + case ex: Exception => + logger.error(s"PIT search_after failed after retries: ${ex.getMessage}", ex) + // fail the stream instead of ending it: ending here would surface a silently + // truncated result set as a SUCCESSFUL result (#228 review; same defect + // class as #209/#224) — watchTermination still owns the single PIT close (#202) + Future.failed(ex) } - } - }(system, logger).recoverWith { case ex: Exception => - logger.error(s"PIT search_after failed after retries: ${ex.getMessage}", ex) - // fail the stream instead of ending it: ending here would surface a silently - // truncated result set as a SUCCESSFUL result (#228 review; same defect class - // as #209/#224) — watchTermination still owns the single PIT close (#202) - Future.failed(ex) } - } - .watchTermination() { (_, done) => - // Single owner of the PIT close (#202): completion, failure and downstream - // cancellation all land here. The former in-loop closes made every clean run - // close twice and log a spurious "PIT close reported failure" WARN. - done.onComplete { + + val pages = + if (sliceCount <= 1) Seq(pageSource(None)) + else (0 until sliceCount).map(i => pageSource(Some((i, sliceCount)))) + + // Single owner of the PIT close (#202): completion, failure and downstream cancellation + // all land here, exactly once, whatever the slice count. Page-granular merge, ONE + // mapConcat after it (#238 — no per-row merge traffic). + SliceMerge(pages) { done => + terminated.set(true) + done match { case scala.util.Success(_) => logger.info( - s"PIT search_after completed successfully, closing PIT: ${pitId.take(20)}..." + s"PIT search_after completed ($sliceCount slice(s)), closing PIT: ${pitId.take(20)}..." ) - closePit(pitId) case scala.util.Failure(ex) => logger.error( s"PIT search_after failed: ${ex.getMessage}, closing PIT: ${pitId.take(20)}..." ) - closePit(pitId) } - NotUsed - } - .mapConcat(identity) + // closePit is a blocking call on the dispatcher: let the pool compensate + scala.concurrent.blocking(closePit(pitId)) + }.mapConcat(identity) + } } } - .mapMaterializedValue(_ => NotUsed) + .watchTermination() { (_, done) => + done.onComplete(_ => outerDone.set(true)) + NotUsed + } } + /** `search_after` cursor values → typed `FieldValue`s for the next page request. */ + private def toFieldValues(searchAfter: Seq[Any]): Seq[FieldValue] = + searchAfter.map { + case null => FieldValue.NULL + case s: String => FieldValue.of(s) + case i: Int => FieldValue.of(i.toLong) + case l: Long => FieldValue.of(l) + case d: Double => FieldValue.of(d) + case b: Boolean => FieldValue.of(b) + case other => FieldValue.of(other.toString) + } + + /** A hit's `sort` value → the Scala value fed back as `search_after`. */ + private def toScalaSortValue(fieldValue: FieldValue): Any = + if (fieldValue.isString) fieldValue.stringValue() + else if (fieldValue.isDouble) fieldValue.doubleValue() + else if (fieldValue.isLong) fieldValue.longValue() + else if (fieldValue.isBoolean) fieldValue.booleanValue() + else if (fieldValue.isNull) null + else fieldValue.toString + /** Open a Point In Time */ private def openPit(indices: Seq[String], keepAlive: String)(implicit @@ -1786,6 +1867,10 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } /** Extract ONLY hits (for search_after optimization) Ignores aggregations for better performance + * + * A parse failure FAILS the page (non-retriable `IllegalStateException`): returning an empty + * page here used to read as "end of stream" and surfaced a silently truncated result as a + * success (#238, same defect class as #228 / #209 / #224). */ private def extractHitsOnly( response: SearchResponse[ObjectNode], @@ -1804,8 +1889,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { logger.debug(s"Parsed ${rows.size} hits from response") rows case Failure(ex) => - logger.error(s"Failed to parse search after response: ${ex.getMessage}", ex) - Seq.empty + throw new IllegalStateException(s"Failed to parse PIT page: ${ex.getMessage}", ex) } } diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala index 382eb4e5..b9c45de6 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala @@ -27,13 +27,15 @@ import co.elastic.clients.json.jackson.JacksonJsonpMapper import co.elastic.clients.transport.rest_client.RestClientTransport import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder import org.apache.http.message.BasicHeader import org.elasticsearch.client.{RestClient, RestClientBuilder} import org.slf4j.{Logger, LoggerFactory} -import java.util.concurrent.CompletableFuture +import java.util.concurrent.{CompletableFuture, CompletionException, ExecutionException} import java.util.concurrent.atomic.AtomicReference import scala.concurrent.{Future, Promise} +import scala.util.Try import scala.jdk.CollectionConverters._ @@ -54,8 +56,11 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { ) c } else { - // Another thread initialized while we were waiting - asyncRef.get().get + // Another thread initialized while we were waiting: release OUR transport (its own + // pool + IO reactor would otherwise leak) and use theirs — re-read rather than `.get`, + // a concurrent close() may have cleared the reference in between + Try(c.close()) + async() } } } @@ -70,7 +75,33 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { } } - /** Build RestClientBuilder with authentication + /** Close the sync client AND the async transport (#238): the async client owns its own RestClient + * pool, and the PIT paging path now runs on it. Idempotent. + */ + override def close(): Unit = { + super.close() + asyncRef.getAndSet(None).foreach { c => + Try { + c.close() + logger.info("Elasticsearch async Client closed successfully") + }.recover { case ex: Exception => + logger.warn(s"Error closing Elasticsearch async Client: ${ex.getMessage}", ex) + } + } + } + + /** REST connection pool sized to the slice ceiling (#238 — `ScrollSettings.restPoolPerRoute` / + * `restPoolTotal`): an extraction may hold up to `elastic.scroll.max-slices` page requests in + * flight per route on top of the PIT open / close and `_settings` calls sharing the route. + */ + private def withPoolSizing(httpClient: HttpAsyncClientBuilder): HttpAsyncClientBuilder = + httpClient + .setMaxConnPerRoute(elasticConfig.scroll.restPoolPerRoute) + .setMaxConnTotal(elasticConfig.scroll.restPoolTotal) + + /** Build RestClientBuilder with authentication. ONE `setHttpClientConfigCallback` per builder (a + * second call replaces the first): the auth branch yields a function and the pool sizing is + * composed with it in a single callback. */ private def buildRestClient(): RestClientBuilder = { val httpHost = parseHttpHost(elasticConfig.credentials.url) @@ -84,44 +115,47 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { } // Authenticate - elasticConfig.credentials.authMethod match { - case Some(BasicAuth) if elasticConfig.credentials.username.nonEmpty => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - val credentialsProvider = new BasicCredentialsProvider() - credentialsProvider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials( - elasticConfig.credentials.username, - elasticConfig.credentials.password - ) - ) - httpClientConfigCallback.setDefaultCredentialsProvider(credentialsProvider) - } - case Some(ApiKeyAuth) if elasticConfig.credentials.encodedApiKey.exists(_.nonEmpty) => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - httpClientConfigCallback.setDefaultHeaders( - Seq( - new BasicHeader( - "Authorization", - ApiKeyAuth.createAuthHeader(elasticConfig.credentials) + val authenticate: HttpAsyncClientBuilder => HttpAsyncClientBuilder = + elasticConfig.credentials.authMethod match { + case Some(BasicAuth) if elasticConfig.credentials.username.nonEmpty => + httpClientConfigCallback => { + val credentialsProvider = new BasicCredentialsProvider() + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + elasticConfig.credentials.username, + elasticConfig.credentials.password ) - ).asJava - ) - } - case Some(BearerTokenAuth) if elasticConfig.credentials.bearerToken.exists(_.nonEmpty) => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - httpClientConfigCallback.setDefaultHeaders( - Seq( - new BasicHeader( - "Authorization", - BearerTokenAuth.createAuthHeader(elasticConfig.credentials) - ) - ).asJava - ) - } - case _ => // No authentication - builder - } + ) + httpClientConfigCallback.setDefaultCredentialsProvider(credentialsProvider) + } + case Some(ApiKeyAuth) if elasticConfig.credentials.encodedApiKey.exists(_.nonEmpty) => + httpClientConfigCallback => + httpClientConfigCallback.setDefaultHeaders( + Seq( + new BasicHeader( + "Authorization", + ApiKeyAuth.createAuthHeader(elasticConfig.credentials) + ) + ).asJava + ) + case Some(BearerTokenAuth) if elasticConfig.credentials.bearerToken.exists(_.nonEmpty) => + httpClientConfigCallback => + httpClientConfigCallback.setDefaultHeaders( + Seq( + new BasicHeader( + "Authorization", + BearerTokenAuth.createAuthHeader(elasticConfig.credentials) + ) + ).asJava + ) + case _ => // No authentication + identity + } + + builder.setHttpClientConfigCallback(httpClientConfigCallback => + withPoolSizing(authenticate(httpClientConfigCallback)) + ) } private def buildTransport(): RestClientTransport = { @@ -159,13 +193,36 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { } } + /** Bridge a Java `CompletableFuture` to a Scala `Future`. + * + * `whenComplete` hands over the failure wrapped in a `CompletionException` when the stage failed + * upstream; that wrapper is unwrapped here (#238) so the cause — e.g. the `IOException` / + * `SocketTimeoutException` that `isRetriableError` matches — reaches `retryWithBackoff`. Without + * it the asynchronous PIT paging path would silently never retry. + */ def fromCompletableFuture[T](cf: CompletableFuture[T]): Future[T] = { val promise = Promise[T]() cf.whenComplete { (result: T, err: Throwable) => - if (err != null) promise.failure(err) + if (err != null) promise.failure(JavaClientCompanion.unwrapCompletion(err)) else promise.success(result) } promise.future } } + +object JavaClientCompanion { + + /** Strip the `CompletionException` / `ExecutionException` layers a failed `CompletableFuture` + * chain adds (bounded — a dependent stage may wrap an already wrapped failure). + */ + @scala.annotation.tailrec + def unwrapCompletion(t: Throwable, depth: Int = 8): Throwable = t match { + case ce: CompletionException if ce.getCause != null && depth > 0 => + unwrapCompletion(ce.getCause, depth - 1) + case ee: ExecutionException if ee.getCause != null && depth > 0 => + unwrapCompletion(ee.getCause, depth - 1) + case other => other + } + +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientCompletionUnwrapSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientCompletionUnwrapSpec.scala new file mode 100644 index 00000000..b78dc723 --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientCompletionUnwrapSpec.scala @@ -0,0 +1,119 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.actor.ActorSystem +import app.softnetwork.elastic.client.java.JavaClientCompanion +import com.typesafe.config.ConfigFactory +import org.scalatest.BeforeAndAfterAll +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import _root_.java.io.IOException +import _root_.java.net.SocketTimeoutException +import _root_.java.util.concurrent.{CompletableFuture, CompletionException} +import _root_.java.util.concurrent.atomic.AtomicInteger +import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.duration._ + +/** #238 — the asynchronous PIT paging path rides `fromCompletableFuture`. A failed + * `CompletableFuture` hands its cause over wrapped in a `CompletionException`, which + * `isRetriableError` does not match: without the unwrap, `retryWithBackoff` would silently stop + * retrying transient `IOException`s on every es8/es9 extraction. No Docker. + */ +class JavaClientCompletionUnwrapSpec extends AnyFlatSpec with Matchers with BeforeAndAfterAll { + + implicit val system: ActorSystem = ActorSystem("completion-unwrap-spec") + implicit val ec: ExecutionContext = system.dispatcher + implicit val logger: Logger = LoggerFactory.getLogger(getClass) + + private val companion: JavaClientCompanion = new JavaClientCompanion { + override def elasticConfig: ElasticConfig = ElasticConfig(ConfigFactory.load()) + } + + override def afterAll(): Unit = { + system.terminate() + super.afterAll() + } + + private def wrappedTimeout(): CompletableFuture[String] = { + val cf = new CompletableFuture[String]() + cf.completeExceptionally( + new CompletionException(new SocketTimeoutException("read timed out")) + ) + cf + } + + "fromCompletableFuture" should "unwrap a CompletionException to its cause" in { + val failure = Await.result(companion.fromCompletableFuture(wrappedTimeout()).failed, 5.seconds) + failure shouldBe a[SocketTimeoutException] + failure.getMessage shouldBe "read timed out" + } + + it should "leave a raw failure untouched" in { + val cf = new CompletableFuture[String]() + cf.completeExceptionally(new IOException("raw")) + val failure = Await.result(companion.fromCompletableFuture(cf).failed, 5.seconds) + failure shouldBe an[IOException] + failure.getMessage shouldBe "raw" + } + + it should "keep a CompletionException without a cause" in { + val cf = new CompletableFuture[String]() + cf.completeExceptionally(new CompletionException("no cause", null)) + Await.result(companion.fromCompletableFuture(cf).failed, 5.seconds) shouldBe a[ + CompletionException + ] + } + + it should "complete normally" in { + val cf = new CompletableFuture[String]() + cf.complete("ok") + Await.result(companion.fromCompletableFuture(cf), 5.seconds) shouldBe "ok" + } + + "retryWithBackoff over fromCompletableFuture" should "still retry a wrapped SocketTimeoutException" in { + val attempts = new AtomicInteger(0) + val result = + retryWithBackoff( + RetryConfig(maxRetries = 2, initialDelay = 10.millis, maxDelay = 20.millis) + ) { + attempts.incrementAndGet() + companion.fromCompletableFuture(wrappedTimeout()) + } + val failure = Await.result(result.failed, 10.seconds) + failure shouldBe a[SocketTimeoutException] + attempts.get() shouldBe 3 // 1 + 2 retries + } + + it should "not retry a wrapped non-IO failure" in { + val attempts = new AtomicInteger(0) + val result = + retryWithBackoff( + RetryConfig(maxRetries = 2, initialDelay = 10.millis, maxDelay = 20.millis) + ) { + attempts.incrementAndGet() + val cf = new CompletableFuture[String]() + cf.completeExceptionally(new CompletionException(new IllegalStateException("permanent"))) + companion.fromCompletableFuture(cf) + } + val failure = Await.result(result.failed, 10.seconds) + failure shouldBe an[IllegalStateException] + attempts.get() shouldBe 1 + } +} diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSlicedScrollCompletenessSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSlicedScrollCompletenessSpec.scala new file mode 100644 index 00000000..381db6be --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSlicedScrollCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientSlicedScrollCompletenessSpec extends SlicedScrollCompletenessSpec diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index 49268597..8aa78e1c 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -54,6 +54,7 @@ import app.softnetwork.elastic.sql.watcher.{ WatcherStatus } import app.softnetwork.elastic.utils.CronIntervalCalculator +import co.elastic.clients.elasticsearch._types.SlicedScroll import co.elastic.clients.elasticsearch._types.mapping.TypeMapping import co.elastic.clients.elasticsearch._types.{ Conflicts, @@ -120,6 +121,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode import com.google.gson.JsonParser import _root_.java.io.{IOException, StringReader} +import _root_.java.util.concurrent.atomic.AtomicBoolean import _root_.java.util.{Map => JMap} import scala.collection.immutable.ListMap import scala.jdk.CollectionConverters._ @@ -1367,7 +1369,10 @@ trait JavaClientBulkApi extends BulkApi with JavaClientHelpers { * [[ScrollApi]] for scroll operations */ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { - _: JavaClientVersionApi with JavaClientSearchApi with JavaClientCompanion => + _: JavaClientVersionApi + with JavaClientSearchApi + with JavaClientSettingsApi + with JavaClientCompanion => /** Classic scroll (works for both hits and aggregations) */ @@ -1525,182 +1530,258 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { ): Source[ListMap[String, Any], NotUsed] = { implicit val ec: ExecutionContext = system.dispatcher - // Step 1: Open PIT - val pitIdFuture: Future[String] = openPit(elasticQuery.indices, config.keepAlive) - + // Parsed ONCE per stream (it used to be once per page) and read-only from here on — shared + // by every slice's page builder. Hoisted ABOVE openPit: nothing between a successful openPit + // and the attachment of watchTermination may throw (#202, the single PIT owner rule). + val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject + val hasQueryClause = queryJson.has("query") + val hasJsonSort = queryJson.has("sort") + // Sorts already present: check ONCE whether a tie-breaker exists + val needsTiebreaker = hasSorts && hasJsonSort && !queryJson + .getAsJsonArray("sort") + .asScala + .exists { sortElem => + sortElem.isJsonObject && ( + sortElem.getAsJsonObject.has("_shard_doc") || + sortElem.getAsJsonObject.has("_id") + ) + } + // #238 — core applied the whole slicing policy (strategy, sorts, gate, ceiling, shard count); + // the client applies the resolved count verbatim and never re-derives it (D1). + val sliceCount = math.max(1, config.slices) + + // The PIT is opened LAZILY — at materialization, on first demand — so a source that is + // built but never materialized never opens a PIT. A consumer that cancels WHILE the open is + // in flight is caught by `outerDone` below: the inner source would never be materialized + // (its watchTermination never attached), so the PIT is closed right here instead. The + // residual window is the ordering of two callbacks on the dispatcher — microseconds. + val outerDone = new AtomicBoolean(false) Source - .futureSource { - pitIdFuture.map { pitId => - logger.info(s"Opened PIT: $pitId for indices: ${elasticQuery.indices.mkString(", ")}") - - Source - .unfoldAsync[Option[Seq[Any]], Seq[ListMap[String, Any]]](None) { searchAfterOpt => - retryWithBackoff(config.retryConfig) { - Future { - searchAfterOpt match { - case None => - logger.info(s"Starting PIT search_after (pitId: ${pitId.take(20)}...)") - case Some(values) => - logger.debug( - s"Fetching next PIT search_after batch (after: ${if (values.length > 3) - s"[${values.take(3).mkString(", ")}...]" - else values.mkString(", ")})" - ) - } + .lazyFutureSource { () => + openPit(elasticQuery.indices, config.keepAlive).map { pitId => + if (outerDone.get) { + logger.info( + s"Stream cancelled while opening the PIT; closing PIT: ${pitId.take(20)}..." + ) + closePit(pitId) + Source.empty[ListMap[String, Any]] + } else { + logger.info( + s"Opened PIT: ${pitId.take(20)}... for indices: ${elasticQuery.indices.mkString(", ")} ($sliceCount slice(s))" + ) - // Build search request with PIT - val requestBuilder = new SearchRequest.Builder() - .size(config.scrollSize) - .pit( - PointInTimeReference - .of(p => p.id(pitId).keepAlive(Time.of(t => t.time(config.keepAlive)))) - ) + // Set by onTerminate BEFORE the PIT is closed: every slice's next / in-flight step + // short-circuits to end-of-stream instead of racing the close with retried requests. + val terminated = new AtomicBoolean(false) + + /** One slice (or the whole PIT when `slice` is None) as a source of PAGES. */ + def pageSource(slice: Option[(Int, Int)]): Source[Seq[ListMap[String, Any]], NotUsed] = + Source.unfoldAsync[Option[Seq[Any]], Seq[ListMap[String, Any]]](None) { + searchAfterOpt => + retryWithBackoff(config.retryConfig) { + // By-name: re-evaluated on every retry, so the terminated check stops retries too. + if (terminated.get) Future.successful(None) + else { + // The request is built on a dispatcher thread — NOT on the stream's interpreter + // thread (withJson re-parses the query JSON per page, which would serialise with + // the graph's own processing) — and a synchronous failure stays inside the + // Future chain (retry / stream failure), never an escape out of unfoldAsync. + Future { + searchAfterOpt match { + case None if slice.isEmpty => + logger.info(s"Starting PIT search_after (pitId: ${pitId.take(20)}...)") + case None => + // per slice: DEBUG, so a sliced extraction keeps ONE INFO line (AC 12) + logger.debug( + s"Starting PIT search_after (pitId: ${pitId + .take(20)}...${slice.fold("")(s => s", slice ${s._1}/${s._2}")})" + ) + case Some(values) => + logger.debug( + s"Fetching next PIT search_after batch (after: ${if (values.length > 3) + s"[${values.take(3).mkString(", ")}...]" + else values.mkString(", ")})" + ) + } - // Parse query to add query clause (not indices, they're in PIT) - val queryJson = JsonParser.parseString(elasticQuery.query).getAsJsonObject + // Build search request with PIT + val requestBuilder = new SearchRequest.Builder() + .size(config.scrollSize) + .pit( + PointInTimeReference + .of(p => + p.id(pitId).keepAlive(Time.of(t => t.time(config.keepAlive))) + ) + ) - // Extract query clause if present - if (queryJson.has("query")) { - requestBuilder.withJson(new StringReader(elasticQuery.query)) - } + // Query clause (not indices, they're in the PIT) — a FRESH reader per + // page: a StringReader is single-use + if (hasQueryClause) { + requestBuilder.withJson(new StringReader(elasticQuery.query)) + } - // The paging path never reads hits.total — computing it costs ~30% of the - // ES-side CPU per page (#200). Set after withJson so the query cannot re-enable it. - requestBuilder.trackTotalHits(TrackHits.of(t => t.enabled(false))) - - // Check if sorts already exist in the query - if (!hasSorts && !queryJson.has("sort")) { - // _doc, NOT _shard_doc: from ES 8 / Lucene 9 a primary _shard_doc sort - // defeats the doc-id skip optimisation and every page re-scans the whole - // index (#197). Under a PIT (>= 7.12) ES appends _shard_doc as an automatic - // tiebreaker, so _doc is a total order and row-complete across shards. - logger.debug( - "No sort fields in query for PIT search_after, adding default _doc sort." - ) - requestBuilder.sort( - SortOptions.of { sortBuilder => - sortBuilder.field( - FieldSort.of(fieldSortBuilder => - fieldSortBuilder.field("_doc").order(SortOrder.Asc) + // The paging path never reads hits.total — computing it costs ~30% of the + // ES-side CPU per page (#200). Set after withJson so the query cannot + // re-enable it. + requestBuilder.trackTotalHits(TrackHits.of(t => t.enabled(false))) + + if (!hasSorts && !hasJsonSort) { + // _doc, NOT _shard_doc: from ES 8 / Lucene 9 a primary _shard_doc sort + // defeats the doc-id skip optimisation and every page re-scans the whole + // index (#197). Under a PIT (>= 7.12) ES appends _shard_doc as an + // automatic tiebreaker, so _doc is a total order and row-complete + // across shards. + requestBuilder.sort( + SortOptions.of { sortBuilder => + sortBuilder.field( + FieldSort.of(fieldSortBuilder => + fieldSortBuilder.field("_doc").order(SortOrder.Asc) + ) + ) + } ) - ) - } - ) - } else if (hasSorts && queryJson.has("sort")) { - // Sorts already present, check that a tie-breaker exists - val existingSorts = queryJson.getAsJsonArray("sort") - val hasShardDocSort = existingSorts.asScala.exists { sortElem => - sortElem.isJsonObject && ( - sortElem.getAsJsonObject.has("_shard_doc") || - sortElem.getAsJsonObject.has("_id") - ) - } - if (!hasShardDocSort) { - // Add _id as tie-breaker - logger.debug("Adding _shard_doc as tie-breaker to existing sorts") - requestBuilder.sort( - SortOptions.of { sortBuilder => - sortBuilder.field( - FieldSort.of(fieldSortBuilder => - fieldSortBuilder.field("_shard_doc").order(SortOrder.Asc) - ) + } else if (needsTiebreaker) { + // Add _shard_doc as tie-breaker to the existing sorts + requestBuilder.sort( + SortOptions.of { sortBuilder => + sortBuilder.field( + FieldSort.of(fieldSortBuilder => + fieldSortBuilder.field("_shard_doc").order(SortOrder.Asc) + ) + ) + } ) } - ) - } - } - // Add search_after if available - searchAfterOpt.foreach { searchAfter => - val fieldValues: Seq[FieldValue] = searchAfter.map { - case s: String => FieldValue.of(s) - case i: Int => FieldValue.of(i.toLong) - case l: Long => FieldValue.of(l) - case d: Double => FieldValue.of(d) - case b: Boolean => FieldValue.of(b) - case other => FieldValue.of(other.toString) - } - requestBuilder.searchAfter(fieldValues.asJava) - } - - val response = apply().search( - requestBuilder.build(), - classOf[ObjectNode] - ) - - // Check errors - if ( - response.shards() != null && - response.shards().failed() != null && - response.shards().failed().intValue() > 0 - ) { - val failures = response.shards().failures() - val errorMsg = if (failures != null && !failures.isEmpty) { - failures.asScala.map(_.reason()).mkString("; ") - } else { - "Unknown shard failure" - } - throw new IOException(s"PIT search_after failed: $errorMsg") - } + // Add search_after if available + searchAfterOpt.foreach { searchAfter => + requestBuilder.searchAfter(toFieldValues(searchAfter).asJava) + } - val hits = extractHitsOnly(response, fieldAliases, config.retainDocumentId) + // #238 — after withJson AND trackTotalHits, so the statement JSON can + // never clobber it. No slice object when the count is 1: ES rejects + // `max <= 1`. SlicedScroll.id is a String (ES coerces numeric strings). + slice.foreach { case (id, max) => + requestBuilder.slice(SlicedScroll.of(s => s.id(id.toString).max(max))) + } - if (hits.isEmpty) { - None // end of stream — watchTermination owns the single PIT close (#202) - } else { - val lastHit = response.hits().hits().asScala.lastOption - val nextSearchAfter = lastHit.flatMap { hit => - val sortValues = hit.sort().asScala - if (sortValues.nonEmpty) { - Some(sortValues.map { fieldValue => - if (fieldValue.isString) fieldValue.stringValue() - else if (fieldValue.isDouble) fieldValue.doubleValue() - else if (fieldValue.isLong) fieldValue.longValue() - else if (fieldValue.isBoolean) fieldValue.booleanValue() - else if (fieldValue.isNull) null - else fieldValue.toString - }.toSeq) - } else { - None + requestBuilder.build() } + .flatMap { request => + // Asynchronous page fetch (#238): no dispatcher thread blocks on the + // wire. fromCompletableFuture unwraps CompletionException so a transient + // IOException still reaches retryWithBackoff. NOTE: the typed client + // decodes the page (SearchResponse[ObjectNode]) on the transport's + // completion thread — the RestClient IO reactor — before this future + // completes; only the row extraction below runs on system.dispatcher. + fromCompletableFuture(async().search(request, classOf[ObjectNode])) + } + .map { response => + // Row extraction on system.dispatcher. + // Check errors + if ( + response.shards() != null && + response.shards().failed() != null && + response.shards().failed().intValue() > 0 + ) { + val failures = response.shards().failures() + val errorMsg = if (failures != null && !failures.isEmpty) { + failures.asScala.map(_.reason()).mkString("; ") + } else { + "Unknown shard failure" + } + throw new IOException(s"PIT search_after failed: $errorMsg") + } + + val rawHits = response.hits().hits() + if (rawHits.isEmpty) { + None // end of this slice — watchTermination owns the single PIT close (#202) + } else { + // end-of-slice is decided on the RAW hits above: a page whose hits + // extracted to zero rows must never read as end-of-stream + val hits = + extractHitsOnly(response, fieldAliases, config.retainDocumentId) + val sortValues = rawHits.asScala.last.sort().asScala + if (sortValues.isEmpty) { + // paging on without a cursor would refetch the same page forever + throw new IllegalStateException( + "search_after page returned hits without sort values — cannot continue paging" + ) + } + val nextSearchAfter = Some(sortValues.map(toScalaSortValue).toSeq) + + logger.debug(s"Retrieved ${hits.size} documents, continuing with PIT") + Some((nextSearchAfter, hits)) + } + } } - - logger.debug(s"Retrieved ${hits.size} documents, continuing with PIT") - Some((nextSearchAfter, hits)) + }(system, logger).recoverWith { + case _ if terminated.get => + // a late failure after cancel / close (search_context_missing...): the + // stream is already over, drop it quietly + logger.debug("PIT page failed after the stream terminated; ignoring") + Future.successful(None) + case ex: Exception => + logger.error(s"PIT search_after failed after retries: ${ex.getMessage}", ex) + // fail the stream instead of ending it: ending here would surface a silently + // truncated result set as a SUCCESSFUL result (#228 review; same defect + // class as #209/#224) — watchTermination still owns the single PIT close (#202) + Future.failed(ex) } - } - }(system, logger).recoverWith { case ex: Exception => - logger.error(s"PIT search_after failed after retries: ${ex.getMessage}", ex) - // fail the stream instead of ending it: ending here would surface a silently - // truncated result set as a SUCCESSFUL result (#228 review; same defect class - // as #209/#224) — watchTermination still owns the single PIT close (#202) - Future.failed(ex) } - } - .watchTermination() { (_, done) => - // Single owner of the PIT close (#202): completion, failure and downstream - // cancellation all land here. The former in-loop closes made every clean run - // close twice and log a spurious "PIT close reported failure" WARN. - done.onComplete { + + val pages = + if (sliceCount <= 1) Seq(pageSource(None)) + else (0 until sliceCount).map(i => pageSource(Some((i, sliceCount)))) + + // Single owner of the PIT close (#202): completion, failure and downstream cancellation + // all land here, exactly once, whatever the slice count. Page-granular merge, ONE + // mapConcat after it (#238 — no per-row merge traffic). + SliceMerge(pages) { done => + terminated.set(true) + done match { case scala.util.Success(_) => logger.info( - s"PIT search_after completed successfully, closing PIT: ${pitId.take(20)}..." + s"PIT search_after completed ($sliceCount slice(s)), closing PIT: ${pitId.take(20)}..." ) - closePit(pitId) case scala.util.Failure(ex) => logger.error( s"PIT search_after failed: ${ex.getMessage}, closing PIT: ${pitId.take(20)}..." ) - closePit(pitId) } - NotUsed - } - .mapConcat(identity) + // closePit is a blocking call on the dispatcher: let the pool compensate + scala.concurrent.blocking(closePit(pitId)) + }.mapConcat(identity) + } } } - .mapMaterializedValue(_ => NotUsed) + .watchTermination() { (_, done) => + done.onComplete(_ => outerDone.set(true)) + NotUsed + } } + /** `search_after` cursor values → typed `FieldValue`s for the next page request. */ + private def toFieldValues(searchAfter: Seq[Any]): Seq[FieldValue] = + searchAfter.map { + case null => FieldValue.NULL + case s: String => FieldValue.of(s) + case i: Int => FieldValue.of(i.toLong) + case l: Long => FieldValue.of(l) + case d: Double => FieldValue.of(d) + case b: Boolean => FieldValue.of(b) + case other => FieldValue.of(other.toString) + } + + /** A hit's `sort` value → the Scala value fed back as `search_after`. */ + private def toScalaSortValue(fieldValue: FieldValue): Any = + if (fieldValue.isString) fieldValue.stringValue() + else if (fieldValue.isDouble) fieldValue.doubleValue() + else if (fieldValue.isLong) fieldValue.longValue() + else if (fieldValue.isBoolean) fieldValue.booleanValue() + else if (fieldValue.isNull) null + else fieldValue.toString + /** Open a Point In Time */ private def openPit(indices: Seq[String], keepAlive: String)(implicit @@ -1786,6 +1867,10 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { } /** Extract ONLY hits (for search_after optimization) Ignores aggregations for better performance + * + * A parse failure FAILS the page (non-retriable `IllegalStateException`): returning an empty + * page here used to read as "end of stream" and surfaced a silently truncated result as a + * success (#238, same defect class as #228 / #209 / #224). */ private def extractHitsOnly( response: SearchResponse[ObjectNode], @@ -1804,8 +1889,7 @@ trait JavaClientScrollApi extends ScrollApi with JavaClientHelpers { logger.debug(s"Parsed ${rows.size} hits from response") rows case Failure(ex) => - logger.error(s"Failed to parse search after response: ${ex.getMessage}", ex) - Seq.empty + throw new IllegalStateException(s"Failed to parse PIT page: ${ex.getMessage}", ex) } } diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala index 382eb4e5..b9c45de6 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientCompanion.scala @@ -27,13 +27,15 @@ import co.elastic.clients.json.jackson.JacksonJsonpMapper import co.elastic.clients.transport.rest_client.RestClientTransport import org.apache.http.auth.{AuthScope, UsernamePasswordCredentials} import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder import org.apache.http.message.BasicHeader import org.elasticsearch.client.{RestClient, RestClientBuilder} import org.slf4j.{Logger, LoggerFactory} -import java.util.concurrent.CompletableFuture +import java.util.concurrent.{CompletableFuture, CompletionException, ExecutionException} import java.util.concurrent.atomic.AtomicReference import scala.concurrent.{Future, Promise} +import scala.util.Try import scala.jdk.CollectionConverters._ @@ -54,8 +56,11 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { ) c } else { - // Another thread initialized while we were waiting - asyncRef.get().get + // Another thread initialized while we were waiting: release OUR transport (its own + // pool + IO reactor would otherwise leak) and use theirs — re-read rather than `.get`, + // a concurrent close() may have cleared the reference in between + Try(c.close()) + async() } } } @@ -70,7 +75,33 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { } } - /** Build RestClientBuilder with authentication + /** Close the sync client AND the async transport (#238): the async client owns its own RestClient + * pool, and the PIT paging path now runs on it. Idempotent. + */ + override def close(): Unit = { + super.close() + asyncRef.getAndSet(None).foreach { c => + Try { + c.close() + logger.info("Elasticsearch async Client closed successfully") + }.recover { case ex: Exception => + logger.warn(s"Error closing Elasticsearch async Client: ${ex.getMessage}", ex) + } + } + } + + /** REST connection pool sized to the slice ceiling (#238 — `ScrollSettings.restPoolPerRoute` / + * `restPoolTotal`): an extraction may hold up to `elastic.scroll.max-slices` page requests in + * flight per route on top of the PIT open / close and `_settings` calls sharing the route. + */ + private def withPoolSizing(httpClient: HttpAsyncClientBuilder): HttpAsyncClientBuilder = + httpClient + .setMaxConnPerRoute(elasticConfig.scroll.restPoolPerRoute) + .setMaxConnTotal(elasticConfig.scroll.restPoolTotal) + + /** Build RestClientBuilder with authentication. ONE `setHttpClientConfigCallback` per builder (a + * second call replaces the first): the auth branch yields a function and the pool sizing is + * composed with it in a single callback. */ private def buildRestClient(): RestClientBuilder = { val httpHost = parseHttpHost(elasticConfig.credentials.url) @@ -84,44 +115,47 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { } // Authenticate - elasticConfig.credentials.authMethod match { - case Some(BasicAuth) if elasticConfig.credentials.username.nonEmpty => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - val credentialsProvider = new BasicCredentialsProvider() - credentialsProvider.setCredentials( - AuthScope.ANY, - new UsernamePasswordCredentials( - elasticConfig.credentials.username, - elasticConfig.credentials.password - ) - ) - httpClientConfigCallback.setDefaultCredentialsProvider(credentialsProvider) - } - case Some(ApiKeyAuth) if elasticConfig.credentials.encodedApiKey.exists(_.nonEmpty) => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - httpClientConfigCallback.setDefaultHeaders( - Seq( - new BasicHeader( - "Authorization", - ApiKeyAuth.createAuthHeader(elasticConfig.credentials) + val authenticate: HttpAsyncClientBuilder => HttpAsyncClientBuilder = + elasticConfig.credentials.authMethod match { + case Some(BasicAuth) if elasticConfig.credentials.username.nonEmpty => + httpClientConfigCallback => { + val credentialsProvider = new BasicCredentialsProvider() + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + elasticConfig.credentials.username, + elasticConfig.credentials.password ) - ).asJava - ) - } - case Some(BearerTokenAuth) if elasticConfig.credentials.bearerToken.exists(_.nonEmpty) => - builder.setHttpClientConfigCallback { httpClientConfigCallback => - httpClientConfigCallback.setDefaultHeaders( - Seq( - new BasicHeader( - "Authorization", - BearerTokenAuth.createAuthHeader(elasticConfig.credentials) - ) - ).asJava - ) - } - case _ => // No authentication - builder - } + ) + httpClientConfigCallback.setDefaultCredentialsProvider(credentialsProvider) + } + case Some(ApiKeyAuth) if elasticConfig.credentials.encodedApiKey.exists(_.nonEmpty) => + httpClientConfigCallback => + httpClientConfigCallback.setDefaultHeaders( + Seq( + new BasicHeader( + "Authorization", + ApiKeyAuth.createAuthHeader(elasticConfig.credentials) + ) + ).asJava + ) + case Some(BearerTokenAuth) if elasticConfig.credentials.bearerToken.exists(_.nonEmpty) => + httpClientConfigCallback => + httpClientConfigCallback.setDefaultHeaders( + Seq( + new BasicHeader( + "Authorization", + BearerTokenAuth.createAuthHeader(elasticConfig.credentials) + ) + ).asJava + ) + case _ => // No authentication + identity + } + + builder.setHttpClientConfigCallback(httpClientConfigCallback => + withPoolSizing(authenticate(httpClientConfigCallback)) + ) } private def buildTransport(): RestClientTransport = { @@ -159,13 +193,36 @@ trait JavaClientCompanion extends ElasticClientCompanion[ElasticsearchClient] { } } + /** Bridge a Java `CompletableFuture` to a Scala `Future`. + * + * `whenComplete` hands over the failure wrapped in a `CompletionException` when the stage failed + * upstream; that wrapper is unwrapped here (#238) so the cause — e.g. the `IOException` / + * `SocketTimeoutException` that `isRetriableError` matches — reaches `retryWithBackoff`. Without + * it the asynchronous PIT paging path would silently never retry. + */ def fromCompletableFuture[T](cf: CompletableFuture[T]): Future[T] = { val promise = Promise[T]() cf.whenComplete { (result: T, err: Throwable) => - if (err != null) promise.failure(err) + if (err != null) promise.failure(JavaClientCompanion.unwrapCompletion(err)) else promise.success(result) } promise.future } } + +object JavaClientCompanion { + + /** Strip the `CompletionException` / `ExecutionException` layers a failed `CompletableFuture` + * chain adds (bounded — a dependent stage may wrap an already wrapped failure). + */ + @scala.annotation.tailrec + def unwrapCompletion(t: Throwable, depth: Int = 8): Throwable = t match { + case ce: CompletionException if ce.getCause != null && depth > 0 => + unwrapCompletion(ce.getCause, depth - 1) + case ee: ExecutionException if ee.getCause != null && depth > 0 => + unwrapCompletion(ee.getCause, depth - 1) + case other => other + } + +} diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSlicedScrollCompletenessSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSlicedScrollCompletenessSpec.scala new file mode 100644 index 00000000..381db6be --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientSlicedScrollCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientSlicedScrollCompletenessSpec extends SlicedScrollCompletenessSpec diff --git a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala index 7d1e445e..986db97a 100644 --- a/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala +++ b/macros/src/main/scala/app/softnetwork/elastic/sql/macros/SQLQueryValidator.scala @@ -156,7 +156,7 @@ trait SQLQueryValidator { | scrollAs[Product](\"\"\"SELECT id, name FROM products\"\"\".stripMargin) | |❌ For dynamic queries, use: - | scrollAsUnchecked[Product](SelectStatement(dynamicSql), ScrollConfig()) + | scrollAsUnchecked[Product](SelectStatement(dynamicSql)) | |""".stripMargin ) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/ScrollCompletenessSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/ScrollCompletenessSpec.scala index 222d2a5f..e23e9424 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/ScrollCompletenessSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/ScrollCompletenessSpec.scala @@ -143,4 +143,19 @@ trait ScrollCompletenessSpec extends AnyFlatSpecLike with ElasticDockerTestKit w rows should have size totalDocs.toLong rows.map(_._1("id").toString).toSet should have size totalDocs.toLong } + + it should "return every row exactly once with slicing disabled (maxSlices = Some(1))" in { + // #238 — the sequential PIT path keeps its own multi-shard completeness guard now that the + // default path slices on ES 7.15+; on ES 6 this is simply the search_after path + val source = client.scroll( + SelectStatement(s"SELECT id FROM $index"), + ScrollConfig(scrollSize = 100, maxSlices = Some(1)) + ) + + val rows = Await.result(source.runWith(Sink.seq), 5.minutes) + + rows should have size totalDocs.toLong + rows.map(_._1("id").toString).toSet should have size totalDocs.toLong + rows.head._2.slices shouldBe 1 + } } diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/SlicedScrollCompletenessSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/SlicedScrollCompletenessSpec.scala new file mode 100644 index 00000000..978e8880 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/SlicedScrollCompletenessSpec.scala @@ -0,0 +1,440 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.{Sink, Source} +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.client.scroll.{ScrollConfig, ScrollMetrics} +import app.softnetwork.elastic.client.spi.{ElasticClientFactory, ElasticClientSpi} +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.sql.query.SelectStatement +import app.softnetwork.persistence.generateUUID +import ch.qos.logback.classic.{Level, Logger => LogbackLogger} +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import com.typesafe.config.ConfigFactory +import org.elasticsearch.client.Request +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import scala.collection.immutable.ListMap +import scala.concurrent.Await +import scala.concurrent.duration._ +import scala.io.{Source => IoSource} +import scala.jdk.CollectionConverters._ +import scala.language.implicitConversions + +/** #238 — sliced PIT row extraction: a no-`ORDER BY` extraction from an N-shard index reads `min(N, + * max-slices)` slices of ONE PIT concurrently and merges them into the single stream the caller + * consumes. On real Elasticsearch, for 1 / 3 / 6 shards (+ a 10-shard wildcard): exact row count + * AND distinct ids on every run, the resolved slice count through `ScrollMetrics.slices`, the + * sequential guarantees (`ORDER BY`, `maxSlices = Some(1)`, the HOCON opt-out), the quota binding + * on the merged total, and the single PIT close (no open search context left behind). + * + * The `slices == N` expectations apply only where PIT slicing exists (ES >= 7.15); completeness + * and `slices == 1` on ES 6 / `maxSlices = Some(1)` are asserted unconditionally. + * + * `ScrollMetrics.slices` is stamped by core's policy, so (m) additionally pins that the CLIENT + * really opened N readers: it captures the client's own log and counts the per-slice `Starting PIT + * search_after … slice i/N` lines (one per reader), with a positive control so a capture bound to + * the wrong logger fails loudly instead of passing vacuously. + */ +trait SlicedScrollCompletenessSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + implicit val context: ConversionContext = NativeContext + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + /** A second client over the same cluster with extra HOCON. Straight from the SPI: + * [[ElasticClientFactory]] caches clients per cluster URL, so a second `create` would return the + * first client regardless of configuration. + */ + private def spiClient(hocon: String): ElasticClientApi = + java.util.ServiceLoader + .load(classOf[ElasticClientSpi]) + .iterator() + .next() + .client(ConfigFactory.parseString(hocon).withFallback(elasticConfig)) + + private val oneShard = "sliced_1" + private val threeShards = "sliced_3" + private val sixShards = "sliced_6" + + private val oneShardDocs = 2000 + private val threeShardDocs = 6000 + private val sixShardDocs = 12000 + + private lazy val pitSlicing: Boolean = client.version match { + case ElasticSuccess(v) => ElasticsearchVersion.supportsPitSlicing(v) + case ElasticFailure(_) => false + } + + /** The slice count core must have resolved: `n` where PIT slicing exists, 1 everywhere else. */ + private def expectedSlices(n: Int): Int = + if (pitSlicing) math.min(n, ScrollConfig.DefaultMaxSlices) else 1 + + private def indexDocs(indexName: String, count: Int, shards: Int): Unit = { + // explicit number_of_shards is mandatory: the testkit pins a 1-shard wildcard template + val settings = s"""{"number_of_shards": $shards, "number_of_replicas": 0}""" + val mapping = + """{ + | "properties": { + | "id": { "type": "keyword" }, + | "value": { "type": "integer" } + | } + |}""".stripMargin + + client.createIndex(indexName, settings = settings).get shouldBe true + client.setMapping(indexName, mapping).get shouldBe true + + val docs = (1 to count).map(i => s"""{"id":"${indexName}_$i","value":$i}""").toList + + implicit val bulkOptions: BulkOptions = BulkOptions( + defaultIndex = indexName, + logEvery = 10000 + ) + + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + + client.bulk[String](docs, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(_) => // ok + case ElasticFailure(error) => + error.cause.foreach(_.printStackTrace()) + fail(s"Bulk indexing failed: ${error.message}") + } + + client.refresh(indexName) + } + + override def beforeAll(): Unit = { + super.beforeAll() + indexDocs(oneShard, oneShardDocs, 1) + indexDocs(threeShards, threeShardDocs, 3) + indexDocs(sixShards, sixShardDocs, 6) + } + + override def afterAll(): Unit = { + client.deleteIndex(oneShard) + client.deleteIndex(threeShards) + client.deleteIndex(sixShards) + system.terminate() + super.afterAll() + } + + /** Run `body` with a second SPI client and close it afterwards (own pools / transports). */ + private def withSpiClient[T](hocon: String)(body: ElasticClientApi => T): T = { + val c = spiClient(hocon) + try body(c) + finally c.close() + } + + private type Row = (ListMap[String, Any], ScrollMetrics) + + private def run( + sql: String, + config: Option[ScrollConfig], + c: ElasticClientApi = client + ): Seq[Row] = { + val source = config match { + case Some(cfg) => c.scroll(SelectStatement(sql), cfg) + case None => c.scroll(SelectStatement(sql)) + } + Await.result(source.runWith(Sink.seq), 5.minutes) + } + + private def assertComplete(rows: Seq[Row], expected: Int, slices: Int, label: String): Unit = { + rows should have size expected.toLong + rows.map(_._1("id").toString).toSet should have size expected.toLong + rows.head._2.slices shouldBe slices + rows.last._2.slices shouldBe slices + log.info(s"✓ $label: ${rows.size} rows, ${rows.size} distinct ids, slices = $slices") + } + + /** Open search contexts across every node — a leaked PIT shows up here. Polled because the close + * is issued from `watchTermination` right after the stream completes. + */ + private def openContexts(): Int = { + val response = restClient.performRequest( + new Request( + "GET", + "/_nodes/stats/indices/search?filter_path=nodes.*.indices.search.open_contexts" + ) + ) + val stream = response.getEntity.getContent + val body = + try IoSource.fromInputStream(stream).mkString + finally stream.close() + // the probe must never pass vacuously: the key has to be there + body should include("open_contexts") + "\"open_contexts\"\\s*:\\s*(\\d+)".r.findAllMatchIn(body).map(_.group(1).toInt).sum + } + + /** Positive control for the probe: a PIT opened by the test IS counted as an open context. */ + private def assertProbeCountsPits(): Unit = { + val open = restClient.performRequest(new Request("POST", s"/$sixShards/_pit?keep_alive=1m")) + val openStream = open.getEntity.getContent + val pitId = + try "\"id\"\\s*:\\s*\"([^\"]+)\"".r + .findFirstMatchIn(IoSource.fromInputStream(openStream).mkString) + .map(_.group(1)) + .getOrElse(fail("no PIT id in the open response")) + finally openStream.close() + try openContexts() should be > 0 + finally { + val close = new Request("DELETE", "/_pit") + close.setJsonEntity(s"""{"id":"$pitId"}""") + restClient.performRequest(close) + } + } + + private def awaitNoOpenContext(): Unit = { + val deadline = System.currentTimeMillis() + 5000 + var open = openContexts() + while (open > 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(100) + open = openContexts() + } + open shouldBe 0 + } + + /** Every client logger (`LoggerFactory getLogger getClass.getName` in the companions: the SPI's + * anonymous subclass, `…client.java`, `…client.rest`, `…client.jest`) lives under this package, + * and a child's events reach the appenders of its ancestors. + */ + private val clientLoggerPackage = "app.softnetwork.elastic.client" + + /** Run `body` while capturing, at DEBUG, everything logged under [[clientLoggerPackage]]; returns + * the result and the formatted messages. The package logger is raised to DEBUG for the window + * (the suites run at INFO, and the per-slice line is DEBUG by design — one INFO line per + * extraction, AC 12) and restored afterwards; additivity is left alone so the client's own + * WARN/ERROR lines still reach the console. `ListAppender` extends the synchronized + * `AppenderBase`, so N slices logging concurrently cannot lose an event; the list is read only + * AFTER the appender is detached and under the appender's lock — the stream's `onTerminate` + * ("closing PIT") still logs from a dispatcher thread after `Await.result` returns, and a plain + * `ArrayList` iterated concurrently with that `add` throws `ConcurrentModificationException` + * (seen once on ES 9.0.3). + */ + private def captureClientLog[A](body: => A): (A, Seq[String]) = { + val lb = LoggerFactory.getLogger(clientLoggerPackage) match { + case l: LogbackLogger => l + case other => + cancel( + s"'$clientLoggerPackage' is not a logback logger (${other.getClass.getName}) — log capture unavailable" + ) + } + val appender = new ListAppender[ILoggingEvent]() + appender.start() + val prevLevel = lb.getLevel + lb.setLevel(Level.DEBUG) + lb.addAppender(appender) + try { + val a = body + lb.detachAppender(appender) + val lines = appender.synchronized(appender.list.asScala.map(_.getFormattedMessage).toVector) + (a, lines) + } finally { + lb.detachAppender(appender) // idempotent + appender.stop() + lb.setLevel(prevLevel) + } + } + + private val startLine = "Starting PIT search_after" + private val sliceSuffix = "slice (\\d+)/(\\d+)\\)".r + + /** The distinct `(slice, of)` pairs among the captured start lines — distinct because the line is + * logged inside the page retry, so a retried first page re-logs its reader's start. + */ + private def readers(lines: Seq[String]): Seq[(Int, Int)] = + lines + .filter(_.startsWith(startLine)) + .flatMap(l => sliceSuffix.findFirstMatchIn(l).map(m => (m.group(1).toInt, m.group(2).toInt))) + .distinct + + // ---- (a) one shard ------------------------------------------------------------------------- + + "a no-ORDER-BY extraction from a 1-shard index" should "be complete and sequential (slices = 1)" in { + val rows = run(s"SELECT id, value FROM $oneShard", Some(ScrollConfig(scrollSize = 500))) + assertComplete(rows, oneShardDocs, 1, "1 shard") + } + + // ---- (b) three shards ----------------------------------------------------------------------- + + "a no-ORDER-BY extraction from a 3-shard index" should "open one slice per primary shard" in { + val rows = run(s"SELECT id, value FROM $threeShards", Some(ScrollConfig(scrollSize = 500))) + assertComplete(rows, threeShardDocs, expectedSlices(3), "3 shards") + } + + // (f) partial last page + it should "be complete with a page size that does not divide the row count" in { + // 6000 % 7 != 0 per slice as well — exercises every slice's partial last page + val rows = run(s"SELECT id FROM $threeShards", Some(ScrollConfig(scrollSize = 7))) + assertComplete(rows, threeShardDocs, expectedSlices(3), "3 shards, page 7") + } + + // (h) ORDER BY + it should "stay sequential and ordered with ORDER BY" in { + val rows = + run( + s"SELECT id, value FROM $threeShards ORDER BY value", + Some(ScrollConfig(scrollSize = 500)) + ) + assertComplete(rows, threeShardDocs, 1, "3 shards, ORDER BY") + val values = rows.map(_._1("value").toString.toInt) + values shouldBe values.sorted + } + + // ---- (c) six shards ------------------------------------------------------------------------- + + "a no-ORDER-BY extraction from a 6-shard index" should "open min(6, max-slices) = 6 slices" in { + val rows = run(s"SELECT id, value FROM $sixShards", Some(ScrollConfig(scrollSize = 500))) + assertComplete(rows, sixShardDocs, expectedSlices(6), "6 shards") + } + + // (d) explicit ceiling below the shard count + it should "honour maxSlices = Some(2) (still a whole-shard split on the Elasticsearch side)" in { + // With max < shards ES assigns whole shards to slices (shardIndex % max) — never a + // per-document filter — so 2 slices over 6 shards is cheap and complete. + val rows = + run(s"SELECT id FROM $sixShards", Some(ScrollConfig(scrollSize = 500, maxSlices = Some(2)))) + assertComplete(rows, sixShardDocs, if (pitSlicing) 2 else 1, "6 shards, maxSlices 2") + } + + // (e) explicit opt-out — asserted UNCONDITIONALLY + it should "page sequentially with maxSlices = Some(1)" in { + val rows = + run(s"SELECT id FROM $sixShards", Some(ScrollConfig(scrollSize = 500, maxSlices = Some(1)))) + assertComplete(rows, sixShardDocs, 1, "6 shards, maxSlices 1") + } + + // (g) quota on the merged total + it should "bind maxDocuments on the MERGED total (exactly 2,500 distinct rows)" in { + val rows = run( + s"SELECT id FROM $sixShards", + Some(ScrollConfig(scrollSize = 500, maxDocuments = Some(2500))) + ) + rows should have size 2500 + rows.map(_._1("id").toString).toSet should have size 2500 + rows.head._2.slices shouldBe expectedSlices(6) + } + + // (j) mechanical AC 5 — the PIT is closed exactly once and never leaked + it should "leave no open search context behind" in { + // positive control first (PIT API exists from 7.10; on ES 6 the path has no PIT to leak) + if (client.version.toOption.exists(ElasticsearchVersion.supportsPit)) assertProbeCountsPits() + awaitNoOpenContext() + val rows = run(s"SELECT id FROM $sixShards", Some(ScrollConfig(scrollSize = 500))) + rows should have size sixShardDocs.toLong + awaitNoOpenContext() + // ...and a cancelled (quota-capped) stream closes its PIT too + val capped = run( + s"SELECT id FROM $sixShards", + Some(ScrollConfig(scrollSize = 500, maxDocuments = Some(700))) + ) + capped should have size 700 + awaitNoOpenContext() + } + + // ---- (i) wildcard: several concrete indices, shard SUM, and the cap --------------------------- + + "a no-ORDER-BY extraction over sliced_* (1 + 3 + 6 = 10 shards)" should "sum the shards and cap at max-slices" in { + val rows = run("SELECT id FROM sliced_*", Some(ScrollConfig(scrollSize = 500))) + assertComplete( + rows, + oneShardDocs + threeShardDocs + sixShardDocs, + expectedSlices(10), + "sliced_* (10 shards)" + ) + if (pitSlicing) rows.head._2.slices shouldBe ScrollConfig.DefaultMaxSlices + } + + // ---- (k) HOCON opt-out reaches an explicit config --------------------------------------------- + + "elastic.scroll.max-slices = 1" should "disable slicing for an explicit ScrollConfig that leaves maxSlices unset" in { + withSpiClient("elastic.scroll.max-slices = 1") { optOut => + val rows = run(s"SELECT id FROM $sixShards", Some(ScrollConfig(scrollSize = 500)), optOut) + assertComplete(rows, sixShardDocs, 1, "6 shards, HOCON max-slices = 1") + } + } + + // ---- (l) HOCON page size on the no-argument path --------------------------------------------- + + "elastic.scroll.size = 250" should "drive the no-argument scroll path" in { + withSpiClient("elastic.scroll.size = 250") { paged => + val rows = run(s"SELECT id FROM $threeShards", None, paged) + assertComplete(rows, threeShardDocs, expectedSlices(3), "3 shards, HOCON size = 250") + // 6,000 rows in 250-row batches on the merged stream + rows.last._2.totalBatches shouldBe (threeShardDocs / 250).toLong + rows.last._2.totalDocuments shouldBe threeShardDocs.toLong + } + } + + // ---- (m) the CLIENT really opens N readers — not only core's `ScrollMetrics.slices` ---------- + + "the client" should "open exactly one PIT reader per resolved slice, and exactly one when sequential" in { + assume( + client.version.toOption.exists(ElasticsearchVersion.supportsPit), + "PIT paging (and its 'Starting PIT search_after' line) needs ES >= 7.12" + ) + val n = expectedSlices(6) + + val (rows, lines) = captureClientLog { + run(s"SELECT id FROM $sixShards", Some(ScrollConfig(scrollSize = 500))) + } + rows should have size sixShardDocs.toLong + rows.head._2.slices shouldBe n + val starts = lines.filter(_.startsWith(startLine)) + // positive control: the capture IS bound to the logger the client writes to + withClue(s"captured ${lines.size} line(s) under '$clientLoggerPackage' — ") { + starts should not be empty + } + val opened = readers(lines) + withClue(s"readers: ${starts.mkString(" | ")} — ") { + if (pitSlicing) { + // one reader per slice: `slice i/N` for every i in 0 until N, nothing else + opened.map(_._2).distinct shouldBe Seq(n) + opened.map(_._1).sorted shouldBe (0 until n) + starts.filterNot(l => sliceSuffix.findFirstIn(l).isDefined) shouldBe empty + } else { + // 7.12–7.14: PIT without slicing — a single reader, no slice suffix + opened shouldBe empty + starts.distinct should have size 1 + } + } + log.info(s"✓ client readers: ${if (pitSlicing) opened.size else 1} (expected $n)") + + // the sequential opt-out opens exactly ONE reader, without a slice suffix + val (seqRows, seqLines) = captureClientLog { + run(s"SELECT id FROM $sixShards", Some(ScrollConfig(scrollSize = 500, maxSlices = Some(1)))) + } + seqRows should have size sixShardDocs.toLong + val seqStarts = seqLines.filter(_.startsWith(startLine)) + withClue(s"sequential readers: ${seqStarts.mkString(" | ")} — ") { + readers(seqLines) shouldBe empty + seqStarts.distinct should have size 1 // one reader (a retried first page re-logs the same) + } + } +}