From 97279e08fa9933a972644b9ceea7a28529053d77 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Wed, 22 Jul 2026 18:43:42 +0300 Subject: [PATCH 01/20] [HSTACK] chore: datafusion 54, buoyant_kernel 0.25.1 main Cargo.toml patch --- Cargo.toml | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dd5011e36..25fde96d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,13 +17,13 @@ repository = "https://github.com/delta-io/delta.rs" [workspace.dependencies] #delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [ -delta_kernel = { package = "buoyant_kernel", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = [ +delta_kernel = { package = "buoyant_kernel", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1", features = [ #delta_kernel = { package = "buoyant_kernel", version = "0.25.0,<0.25.100", features = [ "arrow-58", "internal-api", ] } #delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false } -delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = ["arrow-58", "rustls"], default-features = false } +delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1", features = ["arrow-58", "rustls"], default-features = false } #delta_kernel_default_engine = { package = "buoyant_kernel_engine", version = "0.25.0,<0.25.100", features = ["arrow-58", "rustls"], default-features = false } @@ -160,3 +160,41 @@ inherits = "release" opt-level = 3 codegen-units = 1 lto = "thin" + +[patch.crates-io] +datafusion = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-catalog = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-catalog-listing = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-common-runtime = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-avro = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-csv = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-json = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-parquet = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-execution = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-ffi = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-aggregate = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-aggregate-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-nested = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-table = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-window = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-window-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-expr-adapter = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-pruning = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-plan = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-proto = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-proto-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-session = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-spark = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-sql = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-substrait = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +buoyant_kernel = { git = 'https://github.com/hstack/delta-kernel-rs.git', branch = 'buoyant-v0.25.1' } +#buoyant_kernel = { path = "../delta-kernel-rs-main-024/kernel" } From 4524c07694eab140ed71a3bb5f1b31b5472d07af Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Tue, 7 Jul 2026 15:00:52 +0300 Subject: [PATCH 02/20] [HSTACK] chore: expose extra methods: scan, DeltaScan struct properties, object_store_url, metrics --- .../core/src/delta_datafusion/table_provider.rs | 16 ++++++++-------- crates/core/src/kernel/schema/cast/mod.rs | 8 ++++---- crates/core/src/logstore/mod.rs | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/crates/core/src/delta_datafusion/table_provider.rs b/crates/core/src/delta_datafusion/table_provider.rs index 7159a9998..a26da01c9 100644 --- a/crates/core/src/delta_datafusion/table_provider.rs +++ b/crates/core/src/delta_datafusion/table_provider.rs @@ -524,15 +524,15 @@ pub(crate) fn update_datafusion_session( #[derive(Debug)] pub(super) struct DeltaScan { /// The normalized [Url] of the ObjectStore root - table_url: Url, + pub table_url: Url, /// Column that contains an index that maps to the original metadata Add - config: DeltaScanConfig, + pub config: DeltaScanConfig, /// The parquet scan to wrap - parquet_scan: Arc, + pub parquet_scan: Arc, /// The schema of the table to be used when evaluating expressions - logical_schema: Arc, + pub logical_schema: Arc, /// Metrics for scan reported via DataFusion - metrics: ExecutionPlanMetricsSet, + pub metrics: ExecutionPlanMetricsSet, } impl DeltaScan { @@ -556,9 +556,9 @@ impl DeltaScan { #[derive(Debug, Serialize, Deserialize)] pub(super) struct DeltaScanWire { /// This [Url] should have already been passed through [normalize_table_url] - table_url: Url, - config: DeltaScanConfig, - logical_schema: Arc, + pub(crate) table_url: Url, + pub(crate) config: DeltaScanConfig, + pub(crate) logical_schema: Arc, } impl From<&DeltaScan> for DeltaScanWire { diff --git a/crates/core/src/kernel/schema/cast/mod.rs b/crates/core/src/kernel/schema/cast/mod.rs index 36a10cc26..7956ff7d3 100644 --- a/crates/core/src/kernel/schema/cast/mod.rs +++ b/crates/core/src/kernel/schema/cast/mod.rs @@ -35,7 +35,7 @@ fn should_cast_nanos_timestamps_to_micros() -> bool { CAST_NANOS_TS_TO_MICROS.load(Ordering::Acquire) } -fn cast_struct( +pub fn cast_struct( struct_array: &StructArray, fields: &Fields, cast_options: &CastOptions, @@ -68,7 +68,7 @@ fn cast_struct( ) } -fn cast_list( +pub fn cast_list( array: &GenericListArray, field: &FieldRef, cast_options: &CastOptions, @@ -83,7 +83,7 @@ fn cast_list( ) } -fn cast_map( +pub fn cast_map( array: &MapArray, entries_field: &FieldRef, sorted: bool, @@ -107,7 +107,7 @@ fn cast_map( } } -fn cast_field( +pub fn cast_field( col: &ArrayRef, field: &FieldRef, cast_options: &CastOptions, diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index 327036a83..b1a2f351f 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -605,7 +605,7 @@ pub(crate) fn get_engine(store: Arc) -> Arc { } #[cfg(feature = "datafusion")] -fn object_store_url(location: &Url) -> ObjectStoreUrl { +pub fn object_store_url(location: &Url) -> ObjectStoreUrl { use object_store::path::DELIMITER; // azure storage urls encode the container as user in the url From ea3359c969a9392a88fee9ba60d515348c9a5a57 Mon Sep 17 00:00:00 2001 From: Adrian Tanase Date: Tue, 7 Jul 2026 15:04:50 +0300 Subject: [PATCH 03/20] [HSTACK] fix: optimize scan builder limit pushdown --- crates/core/src/kernel/snapshot/iterators.rs | 18 ++++++++++++++++++ crates/core/src/kernel/snapshot/log_data.rs | 10 ++++++++++ 2 files changed, 28 insertions(+) diff --git a/crates/core/src/kernel/snapshot/iterators.rs b/crates/core/src/kernel/snapshot/iterators.rs index 91023c0f4..9bcdce28a 100644 --- a/crates/core/src/kernel/snapshot/iterators.rs +++ b/crates/core/src/kernel/snapshot/iterators.rs @@ -340,6 +340,24 @@ impl LogicalFileView { pub fn add_action(&self) -> Add { self.to_add() } + + /// Converts this file view into an Add action for log operations. + /// FIXME: Double json > stats conversion TOO EXPENSIVE + pub(crate) fn add_action_no_stats(&self) -> Add { + Add { + path: self.path().to_string(), + partition_values: self.partition_values_map(), + size: self.size(), + modification_time: self.modification_time(), + data_change: true, + stats: None, + tags: None, + deletion_vector: None, + base_row_id: None, + default_row_commit_version: None, + clustering_provider: None, + } + } /// Converts this file view into a Remove action for log operations. pub fn remove_action(&self, data_change: bool) -> Remove { diff --git a/crates/core/src/kernel/snapshot/log_data.rs b/crates/core/src/kernel/snapshot/log_data.rs index 0ffff1f67..876871e3c 100644 --- a/crates/core/src/kernel/snapshot/log_data.rs +++ b/crates/core/src/kernel/snapshot/log_data.rs @@ -88,6 +88,16 @@ impl<'a> LogDataHandler<'a> { self.data.iter().map(|batch| batch.num_rows()).sum() } + pub fn get(&self, mut index: usize) -> Option { + for batch in self.data { + if index < batch.num_rows() { + return Some(LogicalFileView::new(batch.clone(), index)); + } + index -= batch.num_rows(); + } + None + } + /// Iterate over each file as a [`LogicalFileView`] without materializing all of them at once. pub fn iter(&self) -> impl Iterator + '_ { self.data.iter().flat_map(|batch| { From b63e470b27314221ff5141a5dd14035e32f5450d Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Tue, 7 Jul 2026 15:01:50 +0300 Subject: [PATCH 04/20] [HSTACK] fix: REEVALUATE, bring back old table provider --- crates/core/src/delta_datafusion/mod.rs | 1 + .../delta_datafusion/table_provider_old.rs | 649 ++++++++++++++++++ 2 files changed, 650 insertions(+) create mode 100644 crates/core/src/delta_datafusion/table_provider_old.rs diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index 0ca6aae55..d83efcab9 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -102,6 +102,7 @@ pub use session::SessionFallbackPolicy; pub(crate) use session::{SessionResolveContext, resolve_session_state}; mod table_provider; pub(crate) mod utils; +pub mod table_provider_old; impl From for DataFusionError { fn from(err: DeltaTableError) -> Self { diff --git a/crates/core/src/delta_datafusion/table_provider_old.rs b/crates/core/src/delta_datafusion/table_provider_old.rs new file mode 100644 index 000000000..339dbebc5 --- /dev/null +++ b/crates/core/src/delta_datafusion/table_provider_old.rs @@ -0,0 +1,649 @@ +use std::borrow::Cow; +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; +use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef}; +use chrono::{DateTime, TimeZone, Utc}; +use datafusion::catalog::{ScanArgs, ScanResult, Session, TableProvider}; +use datafusion::common::{Column, Result, ScalarValue, Statistics, ToDFSchema}; +use datafusion::common::pruning::PruningStatistics; +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion::config::TableParquetOptions; +use datafusion::datasource::physical_plan::ParquetSource; +use datafusion::datasource::TableType; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::logical_expr::{BinaryExpr, Expr, LogicalPlan, Operator, TableProviderFilterPushDown}; +use datafusion::logical_expr::utils::{conjunction, split_conjunction}; +use datafusion::physical_expr_common::metrics::{ExecutionPlanMetricsSet, MetricBuilder}; +use datafusion::physical_optimizer::pruning::PruningPredicate; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_datasource::{FileExtensions, PartitionedFile, TableSchema}; +use datafusion_datasource::file_groups::FileGroup; +use datafusion_datasource::file_scan_config::FileScanConfigBuilder; +use datafusion_datasource::source::DataSourceExec; +use object_store::ObjectMeta; +use url::Url; +use crate::delta_datafusion::{get_null_of_arrow_type, to_correct_scalar_value, DataFusionMixins, DeltaScanConfig, DeltaScanConfigBuilder}; +use crate::delta_datafusion::table_provider::{simplify_expr, DeltaScan}; +use crate::{DeltaResult, DeltaTable, DeltaTableConfig, DeltaTableError}; +use crate::delta_datafusion::file_id::{file_id_data_type, wrap_file_id_value}; +use crate::kernel::{Add, EagerSnapshot}; +use crate::kernel::transaction::PROTOCOL; +use crate::logstore::LogStoreRef; +use crate::table::state::DeltaTableState; + + +pub(crate) fn get_pushdown_filters( + filter: &[&Expr], + partition_cols: &[String], +) -> Vec { + filter + .iter() + .cloned() + .map(|expr| { + let applicable = expr_is_exact_predicate_for_cols(partition_cols, expr); + if !expr.column_refs().is_empty() && applicable { + TableProviderFilterPushDown::Exact + } else { + TableProviderFilterPushDown::Inexact + } + }) + .collect() +} + +// inspired from datafusion::listing::helpers, but adapted to only stats based pruning +fn expr_is_exact_predicate_for_cols(partition_cols: &[String], expr: &Expr) -> bool { + let mut is_applicable = true; + expr.apply(|expr| match expr { + Expr::Column(Column { name, .. }) => { + is_applicable &= partition_cols.contains(name); + + // TODO: decide if we should constrain this to Utf8 columns (including views, dicts etc) + + if is_applicable { + Ok(TreeNodeRecursion::Jump) + } else { + Ok(TreeNodeRecursion::Stop) + } + } + Expr::BinaryExpr(BinaryExpr { op, .. }) => { + is_applicable &= matches!( + op, + Operator::And + | Operator::Or + | Operator::NotEq + | Operator::Eq + | Operator::Gt + | Operator::GtEq + | Operator::Lt + | Operator::LtEq + ); + if is_applicable { + Ok(TreeNodeRecursion::Continue) + } else { + Ok(TreeNodeRecursion::Stop) + } + } + Expr::Literal(_, _) + | Expr::Not(_) + | Expr::IsNotNull(_) + | Expr::IsNull(_) + | Expr::Between(_) + | Expr::InList(_) => Ok(TreeNodeRecursion::Continue), + _ => { + is_applicable = false; + Ok(TreeNodeRecursion::Stop) + } + }) + .unwrap(); + is_applicable +} + +fn partitioned_file_from_action( + action: &Add, + partition_columns: &[String], + schema: &Schema, +) -> PartitionedFile { + let partition_values = partition_columns + .iter() + .map(|part| { + action + .partition_values + .get(part) + .map(|val| { + schema + .field_with_name(part) + .map(|field| match val { + Some(value) => to_correct_scalar_value( + &serde_json::Value::String(value.to_string()), + field.data_type(), + ) + .unwrap_or(Some(ScalarValue::Null)) + .unwrap_or(ScalarValue::Null), + None => get_null_of_arrow_type(field.data_type()) + .unwrap_or(ScalarValue::Null), + }) + .unwrap_or(ScalarValue::Null) + }) + .unwrap_or(ScalarValue::Null) + }) + .collect::>(); + + let ts_secs = action.modification_time / 1000; + let ts_ns = (action.modification_time % 1000) * 1_000_000; + let last_modified = Utc.from_utc_datetime( + &DateTime::from_timestamp(ts_secs, ts_ns as u32) + .unwrap() + .naive_utc(), + ); + PartitionedFile { + object_meta: ObjectMeta { + last_modified, + ..action.try_into().unwrap() + }, + partition_values, + range: None, + statistics: None, + ordering: None, + extensions: FileExtensions::new(), + metadata_size_hint: None, + table_reference: None, + } +} + +/// The logical schema for a Deltatable is different from the protocol level schema since partition +/// columns must appear at the end of the schema. This is to align with how partition are handled +/// at the physical level +pub(crate) fn df_logical_schema( + snapshot: &EagerSnapshot, + file_column_name: &Option, + schema: Option, +) -> DeltaResult { + let input_schema = match schema { + Some(schema) => schema, + None => snapshot.input_schema(), + }; + let table_partition_cols = snapshot.metadata().partition_columns(); + + let mut fields: Vec> = input_schema + .fields() + .iter() + .filter(|f| !table_partition_cols.contains(f.name())) + .cloned() + .collect(); + + for partition_col in table_partition_cols.iter() { + fields.push(Arc::new( + input_schema + .field_with_name(partition_col)? + .to_owned(), + )); + } + + if let Some(file_column_name) = file_column_name { + fields.push(Arc::new(Field::new(file_column_name, DataType::Utf8, true))); + } + + Ok(Arc::new(Schema::new(fields))) +} + +pub(crate) struct DeltaScanBuilder<'a> { + snapshot: &'a EagerSnapshot, + log_store: LogStoreRef, + filter: Option, + session: &'a dyn Session, + projection: Option<&'a Vec>, + limit: Option, + files: Option<&'a [Add]>, + config: Option, +} + +impl<'a> DeltaScanBuilder<'a> { + pub fn new( + snapshot: &'a EagerSnapshot, + log_store: LogStoreRef, + session: &'a dyn Session, + ) -> Self { + DeltaScanBuilder { + snapshot, + log_store, + filter: None, + session, + projection: None, + limit: None, + files: None, + config: None, + } + } + + pub fn with_filter(mut self, filter: Option) -> Self { + self.filter = filter; + self + } + + pub fn with_files(mut self, files: &'a [Add]) -> Self { + self.files = Some(files); + self + } + + pub fn with_projection(mut self, projection: Option<&'a Vec>) -> Self { + self.projection = projection; + self + } + + pub fn with_limit(mut self, limit: Option) -> Self { + self.limit = limit; + self + } + + pub fn with_scan_config(mut self, config: DeltaScanConfig) -> Self { + self.config = Some(config); + self + } + + pub async fn build(self) -> DeltaResult { + PROTOCOL.can_read_from(self.snapshot)?; + let config = match self.config { + Some(config) => config, + None => DeltaScanConfigBuilder::new().build(self.snapshot)?, + }; + + let schema = match config.schema.clone() { + Some(value) => value, + None => self.snapshot.read_schema(), + }; + + let logical_schema = df_logical_schema( + self.snapshot, + &config.file_column_name, + Some(schema.clone()), + )?; + + let logical_schema = if let Some(used_columns) = self.projection { + let mut fields = Vec::with_capacity(used_columns.len()); + for idx in used_columns { + fields.push(logical_schema.field(*idx).to_owned()); + } + // partition filters with Exact pushdown were removed from projection by DF optimizer, + // we need to add them back for the predicate pruning to work + if let Some(expr) = &self.filter { + // Avoid non-determinism of HashSet (affects the expressions in pushed-down filters) + let column_refs = expr.column_refs().into_iter().collect::>(); + for c in column_refs { + let idx = logical_schema.index_of(c.name.as_str())?; + if !used_columns.contains(&idx) { + fields.push(logical_schema.field(idx).to_owned()); + } + } + } + Arc::new(Schema::new(fields)) + } else { + logical_schema + }; + + let df_schema = Arc::new(logical_schema.clone().to_dfschema()?); + + let logical_filter = self + .filter + .clone() + .map(|expr| simplify_expr(self.session, df_schema.clone(), expr)) + .transpose()?; + // only inexact filters should be pushed down to the data source, doing otherwise + // will make stats inexact and disable datafusion optimizations like AggregateStatistics + let pushdown_filter = self + .filter + .and_then(|expr| { + let predicates = split_conjunction(&expr); + let pushdown_filters = + get_pushdown_filters(&predicates, self.snapshot.metadata().partition_columns()); + + let filtered_predicates = predicates + .into_iter() + .zip(pushdown_filters.into_iter()) + .filter_map(|(filter, pushdown)| { + if pushdown == TableProviderFilterPushDown::Inexact { + Some(filter.clone()) + } else { + None + } + }); + conjunction(filtered_predicates) + }) + .map(|expr| simplify_expr(self.session, df_schema.clone(), expr)) + .transpose()?; + + // Perform Pruning of files to scan + let (files, files_scanned, files_pruned, _) = match self.files { + Some(files) => { + let files = files.to_owned(); + let files_scanned = files.len(); + (files, files_scanned, 0, None) + } + None => { + // early return in case we have no push down filters or limit + if logical_filter.is_none() && self.limit.is_none() { + // let files = self + // .snapshot + // .file_views(&self.log_store, None) + // .map_ok(|f| f.to_add()) + // .try_collect::>() + // .await?; + let files = self + .snapshot + .log_data() + .iter() + .map(|f| f.to_add()) + .collect::>(); + + let files_scanned = files.len(); + (files, files_scanned, 0, None) + } else { + let num_containers = self.snapshot.num_containers(); + + let files_to_prune = if let Some(predicate) = &logical_filter { + let pruning_predicate = + PruningPredicate::try_new(predicate.clone(), logical_schema.clone())?; + pruning_predicate.prune(self.snapshot)? + } else { + vec![true; num_containers] + }; + + // needed to enforce limit and deal with missing statistics + // rust port of https://github.com/delta-io/delta/pull/1495 + let mut pruned_without_stats = Vec::new(); + let mut rows_collected = 0; + let mut files = Vec::with_capacity(num_containers); + + // let file_actions: Vec<_> = self + // .snapshot + // .file_views(&self.log_store, None) + // .map_ok(|f| f.to_add()) + // .try_collect::>() + // .await?; + + use rand::seq::SliceRandom; + + let mut indices = (0..num_containers).collect::>(); + if self.limit.is_some() && std::env::var("DELTA_RS_SHUFFLE_FILES").is_ok() { + let mut rng = rand::rng(); + indices.shuffle(&mut rng); + } + + let log_data_handler = self.snapshot.log_data(); + for i in indices { + let file_view = log_data_handler.get(i).unwrap(); + let keep = files_to_prune[i]; + + // prune file based on predicate pushdown + let action = file_view.add_action_no_stats(); + let num_records = file_view.num_records(); + if keep { + // prune file based on limit pushdown + if let Some(limit) = self.limit { + if let Some(num_records) = num_records { + if rows_collected < limit as i64 { + rows_collected += num_records as i64; + files.push(action.to_owned()); + } else { + break; + } + } else { + // some files are missing stats; skipping but storing them + // in a list in case we can't reach the target limit + pruned_without_stats.push(action.to_owned()); + } + } else { + files.push(action.to_owned()); + } + } + } + + if let Some(limit) = self.limit + && rows_collected < limit as i64 + { + files.extend(pruned_without_stats); + } + + let files_scanned = files.len(); + let files_pruned = num_containers - files_scanned; + (files, files_scanned, files_pruned, Some(files_to_prune)) + } + } + }; + + // TODO we group files together by their partition values. If the table is partitioned + // and partitions are somewhat evenly distributed, probably not the worst choice ... + // However we may want to do some additional balancing in case we are far off from the above. + let mut file_groups: HashMap, Vec> = HashMap::new(); + + let table_partition_cols = &self.snapshot.metadata().partition_columns(); + + for action in files.iter() { + let mut part = partitioned_file_from_action(action, table_partition_cols, &schema); + + if config.file_column_name.is_some() { + let partition_value = if config.wrap_partition_values { + wrap_file_id_value(action.path.clone()) + } else { + ScalarValue::Utf8(Some(action.path.clone())) + }; + part.partition_values.push(partition_value); + } + + file_groups + .entry(part.partition_values.clone()) + .or_default() + .push(part); + } + + let file_schema = Arc::new(Schema::new( + schema + .fields() + .iter() + .filter(|f| !table_partition_cols.contains(f.name())) + .cloned() + .collect::>(), + )); + + let mut table_partition_cols = table_partition_cols + .iter() + .map(|name| schema.field_with_name(name).map(|f| f.to_owned())) + .collect::, ArrowError>>()?; + + if let Some(file_column_name) = &config.file_column_name { + let field_name_datatype = if config.wrap_partition_values { + file_id_data_type() + } else { + DataType::Utf8 + }; + table_partition_cols.push(Field::new( + file_column_name.clone(), + field_name_datatype, + false, + )); + } + + let parquet_options = TableParquetOptions { + global: self.session.config().options().execution.parquet.clone(), + ..Default::default() + }; + + let partition_fields: Vec> = + table_partition_cols.into_iter().map(Arc::new).collect(); + let table_schema = TableSchema::new(file_schema, partition_fields); + + let mut file_source = + ParquetSource::new(table_schema).with_table_parquet_options(parquet_options); + + // Sometimes (i.e Merge) we want to prune files that don't make the + // filter and read the entire contents for files that do match the + // filter + if let Some(predicate) = pushdown_filter + && config.enable_parquet_pushdown + { + file_source = file_source.with_predicate(predicate); + }; + + let file_scan_config = + FileScanConfigBuilder::new(self.log_store.object_store_url(), Arc::new(file_source)) + .with_file_groups( + // If all files were filtered out, we still need to emit at least one partition to + // pass datafusion sanity checks. + // + // See https://github.com/apache/datafusion/issues/11322 + if file_groups.is_empty() { + vec![FileGroup::from(vec![])] + } else { + file_groups.into_values().map(FileGroup::from).collect() + }, + ) + .with_projection_indices(self.projection.cloned())? + .with_limit(self.limit) + .build(); + + let metrics = ExecutionPlanMetricsSet::new(); + MetricBuilder::new(&metrics) + .global_counter("files_scanned") + .add(files_scanned); + MetricBuilder::new(&metrics) + .global_counter("files_pruned") + .add(files_pruned); + + Ok(DeltaScan { + table_url: self.log_store.root_url().clone(), + config, + parquet_scan: DataSourceExec::from_data_source(file_scan_config), + logical_schema, + metrics, + }) + } +} + + + +impl DeltaTable { + pub fn table_provider_old(&self) -> DeltaTableOldProvider { + self.clone().into() + } +} + +// each delta table must register a specific object store, since paths are internally +// handled relative to the table root. +pub(crate) fn register_store(store: LogStoreRef, env: &RuntimeEnv) { + let object_store_url = store.object_store_url(); + let url: &Url = object_store_url.as_ref(); + env.register_object_store(url, store.object_store(None)); +} + +#[derive(Debug, Clone)] +pub struct DeltaTableOldProvider { + /// The state of the table as of the most recent loaded Delta log entry. + pub state: Option, + /// the load options used during load + pub config: DeltaTableConfig, + /// log store + pub(crate) log_store: LogStoreRef, + /// Optional schema override for scanning + pub(crate) schema: Option, +} + +impl DeltaTableOldProvider { + pub fn snapshot(&self) -> DeltaResult<&DeltaTableState> { + self.state.as_ref().ok_or(DeltaTableError::NotInitialized) + } + pub fn log_store(&self) -> LogStoreRef { + self.log_store.clone() + } + pub fn with_schema(mut self, schema: SchemaRef) -> Self { + self.schema = Some(schema); + self + } +} + +impl From for DeltaTableOldProvider { + fn from(value: DeltaTable) -> Self { + Self { + state: value.state.clone(), + config: value.config.clone(), + log_store: value.log_store.clone(), + schema: None, + } + } +} + +#[async_trait::async_trait] +impl TableProvider for DeltaTableOldProvider { + fn schema(&self) -> Arc { + match &self.schema { + Some(s) => df_logical_schema( + self.snapshot().unwrap().snapshot(), + &None, + Some(s.clone()), + ) + .unwrap_or_else(|_| s.clone()), + None => self.snapshot().unwrap().snapshot().read_schema(), + } + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + fn get_table_definition(&self) -> Option<&str> { + None + } + + fn get_logical_plan(&self) -> Option> { + None + } + + async fn scan( + &self, + _session: &dyn Session, + _projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result> { + unimplemented!("scan is not available for this table provider; use scan_with_args") + } + + async fn scan_with_args<'a>(&self, state: &dyn Session, args: ScanArgs<'a>) -> Result { + register_store(self.log_store(), state.runtime_env().as_ref()); + let filters = args.filters().unwrap_or(&[]); + let filter_expr = conjunction(filters.iter().cloned()); + + let config = DeltaScanConfigBuilder { + include_file_column: false, + file_column_name: None, + wrap_partition_values: None, + enable_parquet_pushdown: true, + schema: self.schema.clone(), + }; + + let config = config + .build(self.snapshot()?.snapshot())?; + + let projection = args.projection().map(|p| p.to_vec()); + let scan = DeltaScanBuilder::new(self.snapshot()?.snapshot(), self.log_store(), state) + .with_projection(projection.as_ref()) + .with_limit(args.limit()) + .with_filter(filter_expr) + .with_scan_config(config) + .build() + .await?; + + Ok(ScanResult::new(Arc::new(scan))) + } + + fn supports_filters_pushdown( + &self, + filter: &[&Expr], + ) -> Result> { + let partition_cols = self.snapshot()?.metadata().partition_columns(); + Ok(get_pushdown_filters(filter, partition_cols)) + } + + fn statistics(&self) -> Option { + // HSTACK: removed in upstream as well + None + } +} From 668a05394c6bfef1ffe8f610f2a01cdf7ced78c3 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Fri, 6 Feb 2026 09:27:24 +0200 Subject: [PATCH 05/20] [HSTACK] feat: add delta table udtf and register function --- crates/core/src/delta_datafusion/mod.rs | 1 + crates/core/src/delta_datafusion/udtf.rs | 109 +++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 crates/core/src/delta_datafusion/udtf.rs diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index d83efcab9..4bb96987f 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -101,6 +101,7 @@ mod session; pub use session::SessionFallbackPolicy; pub(crate) use session::{SessionResolveContext, resolve_session_state}; mod table_provider; +pub mod udtf; pub(crate) mod utils; pub mod table_provider_old; diff --git a/crates/core/src/delta_datafusion/udtf.rs b/crates/core/src/delta_datafusion/udtf.rs new file mode 100644 index 000000000..207ee31af --- /dev/null +++ b/crates/core/src/delta_datafusion/udtf.rs @@ -0,0 +1,109 @@ +use async_trait::async_trait; +use datafusion::catalog::{TableFunctionImpl, TableProvider}; +use datafusion::common::{internal_datafusion_err, DataFusionError, Result, ScalarValue}; +use datafusion::logical_expr::Expr; +use datafusion::prelude::SessionContext; +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; +use tokio::runtime::Runtime; +use url::Url; +use crate::open_table_with_storage_options; + +pub fn register_delta_table_udtf(ctx: &SessionContext, name: Option<&str>, settings: Option<&HashMap>) { + let prefix = name + .or_else(|| Some("delta_table")).unwrap(); + + ctx.register_udtf( + prefix, + Arc::new(DeltaTableUdtf { + flavor: DeltaTableUdtfFlavor::Old, + settings: settings.cloned(), + }), + ); + ctx.register_udtf( + format!("{prefix}_next").as_str(), + Arc::new(DeltaTableUdtf { + flavor: DeltaTableUdtfFlavor::Next, + settings: settings.cloned(), + }), + ); +} + +#[derive(Debug, Clone)] +pub enum DeltaTableUdtfFlavor { + Old, + Next +} + +#[derive(Debug)] +pub struct DeltaTableUdtf { + flavor: DeltaTableUdtfFlavor, + settings: Option>, +} + +#[async_trait] +impl TableFunctionImpl for DeltaTableUdtf { + fn call(&self, args: &[Expr]) -> Result> { + if args.len() != 1 { + return Err(DataFusionError::Execution( + "Delta table function expects table_uri as argument".to_string(), + )); + } + + let mut args_string = args + .iter() + .map(|arg| match arg.clone() { + Expr::Literal(ScalarValue::Utf8(Some(path)), _) + | Expr::Literal(ScalarValue::LargeUtf8(Some(path)), _) + | Expr::Literal(ScalarValue::Utf8View(Some(path)), _) => Ok(path), + _ => Err(DataFusionError::Execution(format!( + "Unexpected argument type: {:?}", + arg + ))), + }) + .collect::>>()?; + + let path = args_string + .pop_front() + .expect("DeltaTableUdtf missing path"); + assert_eq!(args_string.len() % 2, 0, "DeltaTableUdtf: Can't build hashmap out of odd-sized args"); + + let settings = if let Some(global_settings) = &self.settings { + global_settings.clone() + } else { + HashMap::new() + }; + + let flavor = self.flavor.clone(); + let table = std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let table_uri = Url::parse(&path) + .expect(&format!("Invalid table uri: {}", path)); + rt.block_on(async { + let delta_table = open_table_with_storage_options(table_uri, settings) + .await + .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) + .unwrap(); + + match flavor { + DeltaTableUdtfFlavor::Old => { + let provider = delta_table.table_provider_old(); + Arc::new(provider) as Arc + } + DeltaTableUdtfFlavor::Next => { + let provider = delta_table + .table_provider() + .build() + .await + .unwrap(); + Arc::new(provider) as Arc + } + } + }) + }) + .join() + .map_err(|e| internal_datafusion_err!("DeltaTableFunc error opening table"))?; + + Ok(Arc::clone(&table)) + } +} From 162a2692a2f23cca9ffe6126c16ebb473fb1fc0f Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Fri, 6 Feb 2026 17:10:48 +0200 Subject: [PATCH 06/20] [HSTACK] test: add support for deep projections TEST --- Cargo.toml | 1 + crates/core/Cargo.toml | 1 + ...4a7d-b866-abb97c953ac9-c000.snappy.parquet | Bin 0 -> 4305 bytes .../deep/_delta_log/00000000000000000000.json | 4 + ...4cbb-8575-1008884b3df5-c000.snappy.parquet | Bin 0 -> 4564 bytes .../_delta_log/00000000000000000000.json | 4 + .../integration_datafusion_deep.rs | 186 ++++++++++++++++++ crates/core/tests/it_datafusion/main.rs | 1 + 8 files changed, 197 insertions(+) create mode 100644 crates/core/tests/data/deep/_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet create mode 100644 crates/core/tests/data/deep/_delta_log/00000000000000000000.json create mode 100644 crates/core/tests/data/hstack_nullable_difference/_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet create mode 100644 crates/core/tests/data/hstack_nullable_difference/_delta_log/00000000000000000000.json create mode 100644 crates/core/tests/it_datafusion/integration_datafusion_deep.rs diff --git a/Cargo.toml b/Cargo.toml index 25fde96d9..11cc9d9d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ datafusion-proto = { version = "54.0.0" } # serde serde = { version = "1.0.194", features = ["derive"] } serde_json = "1" +prost = { version = "0.14.3" } # "stdlib" bytes = { version = "1" } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index adb3fc6c3..a2f22e72e 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -51,6 +51,7 @@ datafusion-proto = { workspace = true, optional = true } # serde serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +prost = { workspace = true } # "stdlib" bytes = { workspace = true } diff --git a/crates/core/tests/data/deep/_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet b/crates/core/tests/data/deep/_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..13737d9a977399a80b198ba6d82685cbcb2059a5 GIT binary patch literal 4305 zcmb_gO>7fa5Pn%ZjscSflwH}C4_sJ*p!g>-#1&P#i~m9ry{W|mIu};48g3l`yL$rv3Phyv^L+9C%`wnQR z&xZq{x_NxZSJR6&o&apz_nO+e zRk602=mzohM!ljN3pGnK(fiD@v9)2;E5^ER@OY=RbZ$d=cgv@m=a$|ATcG1xb(!Qc)h4-Tv$B5?n^Skmt@Z-&L5t3~_Ki3+$%k z64&(DgVN|j|Sw`6T|JD75&m*@4$KSEKt0!rIP$+K>l?A1L9bq z+0Zuw*JT_QOnQY8zda>}dlN_AxV(k(i8V6{O!%w^o$fxCL)TQE4AUkQHSLCD^jxAkE@V9Nd%?ePbv! z7{~+yN%+Ee{E5%rAUA~L{#s@y5Qr>H1OgLlxEaF*pSdwb0c?W4JqNJrBFaf}LsGXE zF@6jEw@@Zve5S92I2-;|EsIK813ZPr9xk7fdDAKn>ZaSCLOVaQC4778;Sg zN_@#avxD|Ev`>=0lW#ghayQ>pW-Jg0u13M0>YX4xEOWCY8=E&EDU`WF#doSuE~c_3 z?x>h*G8-=BBEebsUcq?p=D$Gr-TYGvfOlmD`4JNeiqYaRlG8I+fDg*J3C|+@sNsBW zD^wsJd6dmkHQOwaM~|F>-rvFa6S&#azc9~$RwNzEBfgq}{I-Z69vXbqSdNT%>1m7X zC(wSH>`C9TSLf{dLMc$+BEG;cv5hrR5;4pNH+%Rj+Wwd}>Z$aFr9xR>rM$VE1$muJ z2{b>J-6$=_8#Aco(mN>MLR}(i8Y$J(tEO43rcFGF zi52YVxDspeB15tiKZ#DNoliVQGH$KlAexoRSq50`Hja1;AQ6+fh3-T9HQZwz?B=Uh8= z+*Ql@G4-w>m$f8TikP{{e;|gctw- literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/deep/_delta_log/00000000000000000000.json b/crates/core/tests/data/deep/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..78b334973 --- /dev/null +++ b/crates/core/tests/data/deep/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"3c5a399a-65d7-47ac-a792-e9525322f73e","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_acp_system_metadata\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"acp_sourceBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"commitBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"_id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"productListItems\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"SKU\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"quantity\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"priceTotal\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_experience\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"analytics\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"customDimensions\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"eVars\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"evar1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"evar2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"events\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"event1\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"event2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_DATE\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCHID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_ACP_DATE","_ACP_BATCHID"],"createdTime":1770389266405,"configuration":{}}} +{"add":{"path":"_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"b1","_ACP_DATE":"2026-01-01"},"size":4305,"modificationTime":1770389266454,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"b1\",\"commitBatchId\":\"b1\"},\"_id\":\"id1\"},\"maxValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"b1\",\"commitBatchId\":\"b1\"},\"_id\":\"id1\"},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"commitBatchId\":0},\"_id\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1770389266455,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Overwrite"},"engineInfo":"delta-rs:py-1.3.1","operationMetrics":{"execution_time_ms":52,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.3.1"}} \ No newline at end of file diff --git a/crates/core/tests/data/hstack_nullable_difference/_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet b/crates/core/tests/data/hstack_nullable_difference/_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..386d831cccc0218756971d66ba4ebacbcb6226ab GIT binary patch literal 4564 zcmb_g&u<%55Pn{79LIHw6Y`#Rk%b~eS1lCmI8EE2fY|k~Bu#@I5+{xj(Z+TfJ9ZN6 zO#!4YFJ7^?3!PXK*n%W3?}Jjg>1WZx({w@&ae z#-E)BbjW6C8euiI_s84c7{u{3f94XQCr+QZhBm!GDr541$BWbwO51|XQUH|yekWsN zI%7W=r{8JUfIrN>{P5E|&-wahM#juI%nT}CfLJd6BQ>7?QZukg9g$SUXV;@U;F2oX z#Vb&X>{NQclWA3PhsSMrn5uZ#@mU#5N>%7{T<4nVV^y+XH@uGTm9ZwkE_`C7Q#DCa zH-ST_0y`)fLBkTtJqFxk31%3UP?CEu@jIymkk0Tp7%${aRq$N+hA;p2RQ1GFszpukoXFD7$NZwTG&J$iw2%08fzN5K0exB;oMKL3j9_ z-`#v?sBP7UYd#9Jr68vA&nIVsW*t(v-Y9I9Y9{xyNl6)9Hw$~~jaI`d)z)hz%Pg9f z$;!P(+@HXnunNPD)hTdS3=V;ULu1j3T0ZoOElClI2H=+ErccZ>T4 zYrfpDpe_35rvcogtjJZGz$Js=w^rJ*pk)(k@9Wo>ohT%L=6`G7+_B15i*Y|f->a7k zrKMfVtTK+kWFVw;esunDBzy^sw`2FNmyd7}T-W27bwgi@&&(#;=7pq&)rT4j^PoD( zd?EEhPqP0`GM9uTHbh06*8Jfs>_Yp*J12j5n5oz^A*psxlGwBe+8uz2-$MK;@qK{5 z%1-orG=Z+yyB@9nuiYLF^RRgDOxvy8FW|1~AfIoFP3=(99(ux^hSV7ecERl}o!MR- zOi>SH;`3oH#P?_T3^Bme^UU`T)OX5~fJDV*?VdCZjC&ZgKlBj>mXYUa$N0Bs{4mHb zolLJ?F?xMZI5Pm0>_mR`Dg8Ul!>d$HC__z)aW$mH&f8z8phW$9$ z=Cp4;cO6`UvL-gPUuEr{jIl%a#~Yw$+_Dq*`y&|lSsv}?c(rr(yH7oxbmxrY*#@7o zcw4;Z(H;$GcLs=XyjbXZ>Fqdl1y2;S_vz z-Jp38LDuzIlwivFf`nTWS^VTtnZQkY5?@y`2a(8aGJOMOcnl-kv`6rcp}vW-v6)?7 z$gW1`j9hYI_Vr~OPC7HucYhr9O*Cz6nk!j7L+i=D>o2j~#0UVo6C~k_Mm8gnd=LBa zEz;w1y?6`wL35oc5D=TOonKw7!a&#OmZl<+Yj)@~0k7=AHVr8Z+#H`pe-`>{k*0%h zgN$Pi^a{RGw2ACB2p>${M5mYaMAPOKf}ureTyHphCKDv%)J@D!9>4ZOs{sYdGe*l? zS*YgHQ_=hi?2Ruy5BNLwa7T7_89&>Zs}5ci+m(PujN8SP)P6Btjh1Mqk&ggZDO?5h zV?W6h1x11}i+WO}7UpwVeFmr3R&G}}R&x7AU7ts}Ff(^+Z6;SKCU5T*CZOKXc6zF{ zmd>>n)2V14w&v;Nk8$$FvQ7`tWvN-#Eg^~LcQqv}KW<>NH} M*yA8$8Tfn7UzpKq?*IS* literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/hstack_nullable_difference/_delta_log/00000000000000000000.json b/crates/core/tests/data/hstack_nullable_difference/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..52c979fa6 --- /dev/null +++ b/crates/core/tests/data/hstack_nullable_difference/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz"],"writerFeatures":["timestampNtz"]}} +{"metaData":{"id":"898c9da5-4a43-48cb-ac42-3946be8a86cc","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_acp_system_metadata\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"acp_sourceBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"commitBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ingestTime\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowVersion\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trackingId\",\"type\":\"string\",\"nullable\":false,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"orderData\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"productList\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"SKU\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"quantity\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"priceTotal\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"_id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCHID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_DATE\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCH_ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_ACP_DATE","_ACP_BATCH_ID"],"createdTime":1771242914810,"configuration":{}}} +{"add":{"path":"_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2026-02-16","_ACP_BATCH_ID":"b1"},"size":4564,"modificationTime":1771242914852,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"orderData\":{\"date\":\"2025-01-01T13:00:00Z\"},\"_ACP_BATCHID\":\"batch1\",\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch1\",\"commitBatchId\":\"batch1\",\"isDeleted\":false,\"ingestTime\":1,\"rowId\":\"row1\",\"rowVersion\":1,\"trackingId\":\"t1\"},\"_id\":\"id1\"},\"maxValues\":{\"_ACP_BATCHID\":\"batch1\",\"_acp_system_metadata\":{\"trackingId\":\"t1\",\"isDeleted\":false,\"commitBatchId\":\"batch1\",\"ingestTime\":1,\"rowId\":\"row1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch1\"},\"orderData\":{\"date\":\"2025-01-01T13:00:00Z\"},\"_id\":\"id1\"},\"nullCount\":{\"_id\":0,\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0,\"trackingId\":0,\"isDeleted\":0,\"ingestTime\":0,\"commitBatchId\":0},\"_ACP_BATCHID\":0,\"orderData\":{\"date\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1771242914853,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCH_ID\"]"},"engineInfo":"delta-rs:py-1.3.1","operationMetrics":{"execution_time_ms":44,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.3.1"}} diff --git a/crates/core/tests/it_datafusion/integration_datafusion_deep.rs b/crates/core/tests/it_datafusion/integration_datafusion_deep.rs new file mode 100644 index 000000000..32e223839 --- /dev/null +++ b/crates/core/tests/it_datafusion/integration_datafusion_deep.rs @@ -0,0 +1,186 @@ +use std::ops::Deref; +use std::sync::Arc; +use std::sync::Once; +use arrow_cast::display::FormatOptions; +use arrow_cast::pretty; +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion::datasource::physical_plan::ParquetSource; +use datafusion::datasource::physical_plan::parquet::push_all_projection_hints::PushAllProjectionHints; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::physical_expr::projection::ProjectionExprs; +use datafusion::physical_plan::{collect, displayable, ExecutionPlan}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_datasource::source::DataSourceExec; +use datafusion_proto::physical_plan::{AsExecutionPlan, ComposedPhysicalExtensionCodec, DefaultPhysicalExtensionCodec}; +use datafusion_proto::protobuf::PhysicalPlanNode; +use prost::Message; +use tracing::info; +use deltalake_core::delta_datafusion::DeltaPhysicalCodec; +use deltalake_core::delta_datafusion::udtf::register_delta_table_udtf; + +/// DF 53 stores deep-projection state on `ParquetSource` as +/// `projection_hints: ProjectionExprs` + `projection_hints_indices: Vec`, +/// populated post-planning by the `PushAllProjectionHints` physical optimizer rule. +/// `ProjectionExprs` derives `PartialEq`, sufficient for serde round-trip equality. +fn extract_projection_deep_from_plan( + plan: Arc, +) -> Vec<(ProjectionExprs, Vec)> { + let mut deep_projections: Vec<(ProjectionExprs, Vec)> = vec![]; + let _ = plan.apply(|pp| { + if let Some(dse) = pp.downcast_ref::() { + if let Some((_file_scan_conf, parquet_source)) = + dse.downcast_to_file_source::() + { + deep_projections.push(( + parquet_source.projection_hints.clone(), + parquet_source.projection_hints_indices.clone(), + )); + } + } + Ok(TreeNodeRecursion::Continue) + }); + deep_projections +} + +/// Setting `tracing`'s global default subscriber is a one-shot per process, +/// so multiple tests in the same binary must share a single init. +fn init_tracing() { + static INIT: Once = Once::new(); + INIT.call_once(|| { + let filter = tracing_subscriber::EnvFilter::from_default_env(); + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_env_filter(filter) + .finish(); + let _ = tracing::subscriber::set_global_default(subscriber); + let _ = pretty_env_logger::try_init(); + }); +} + +/// Build a `SessionContext` with the DF 53 `PushAllProjectionHints` physical +/// optimizer rule registered. Without this rule, `ParquetSource.projection_hints` +/// stays empty and the test's serde assertion is trivially true. +fn build_context(config: SessionConfig) -> SessionContext { + let state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(Arc::new(RuntimeEnv::default())) + .with_default_features() + .with_physical_optimizer_rule(Arc::new(PushAllProjectionHints {})) + .build(); + SessionContext::new_with_state(state) +} + +#[tokio::test] +async fn test_hstack_deep_column_pruning() -> datafusion::common::Result<()> { + init_tracing(); + + let config = SessionConfig::new() + .set_bool("datafusion.sql_parser.enable_ident_normalization", false); + + let ctx = build_context(config); + + register_delta_table_udtf(&ctx, None, None); + + let delta_path = format!( + "{}/tests/data/deep", + env!("CARGO_MANIFEST_DIR") + ); + + let query = format!(r#" + select + t1._id, t1.productListItems['SKU'], _ACP_DATE + from + delta_table('file://{}') as t1 + "#, delta_path); + + let plan = ctx.state().create_logical_plan(&query).await.expect("Error creating logical plan"); + let optimized_plan = ctx.state().optimize(&plan).expect("Error optimizing plan"); + let state = ctx.state(); + let query_planner = state.query_planner().clone(); + let physical_plan = query_planner + .create_physical_plan(&optimized_plan, &state) + .await.expect("Error creating physical plan"); + info!( + "Physical plan: {}", + displayable(physical_plan.deref()).set_show_schema(true).indent(true) + ); + let proj1 = extract_projection_deep_from_plan(physical_plan.clone()); + let batches1 = collect(physical_plan.clone(), ctx.state().task_ctx()).await?; + let results1 = pretty::pretty_format_batches_with_options(&batches1, &FormatOptions::default())?.to_string(); + println!("{}", results1); + + // codec + let codec = ComposedPhysicalExtensionCodec::new( + vec![ + Arc::new(DefaultPhysicalExtensionCodec {}), + Arc::new(DeltaPhysicalCodec{}) + ] + ); + let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec) + .unwrap(); + let bytes = proto.encode_to_vec(); + let plan_after_serde = PhysicalPlanNode::try_decode(&bytes) + .expect("Error try_decode") + .try_into_physical_plan(&ctx.task_ctx(), &codec) + .expect("try_into_physical_plan"); + info!( + "Physical plan after serde: {}", + displayable(plan_after_serde.deref()).set_show_schema(true).indent(true) + ); + let proj2 = extract_projection_deep_from_plan(plan_after_serde.clone()); + let batches2 = collect(plan_after_serde.clone(), ctx.state().task_ctx()).await?; + let results2 = pretty::pretty_format_batches_with_options(&batches2, &FormatOptions::default())?.to_string(); + println!("{}", results2); + + assert_eq!(results1, results2, "Batches not equal !"); + println!("proj1: {:?}", proj1); + println!("proj2: {:?}", proj2); + + assert_eq!(proj1, proj2, "Deep Projection not equal !"); + Ok(()) +} + + +#[tokio::test] +async fn test_hstack_nullable_new() -> datafusion::common::Result<()> { + init_tracing(); + + let config = SessionConfig::new() + .set_bool("datafusion.sql_parser.enable_ident_normalization", false) + .set_bool("datafusion.execution.parquet.schema_force_view_types", false); + + let ctx = build_context(config); + + register_delta_table_udtf(&ctx, None, None); + + let delta_path = format!( + "{}/tests/data/hstack_nullable_difference", + env!("CARGO_MANIFEST_DIR") + ); + + let query = format!(r#" + select + * + from + delta_table('file://{}') as t1 + "#, delta_path); + + let plan = ctx.state().create_logical_plan(&query).await.expect("Error creating logical plan"); + let optimized_plan = ctx.state().optimize(&plan).expect("Error optimizing plan"); + let state = ctx.state(); + let query_planner = state.query_planner().clone(); + let physical_plan = query_planner + .create_physical_plan(&optimized_plan, &state) + .await.expect("Error creating physical plan"); + info!( + "Physical plan: {}", + displayable(physical_plan.deref()).set_show_schema(true).indent(true) + ); + let proj1 = extract_projection_deep_from_plan(physical_plan.clone()); + let batches1 = collect(physical_plan.clone(), ctx.state().task_ctx()).await?; + let results1 = pretty::pretty_format_batches_with_options(&batches1, &FormatOptions::default())?.to_string(); + println!("{}", results1); + + Ok(()) +} diff --git a/crates/core/tests/it_datafusion/main.rs b/crates/core/tests/it_datafusion/main.rs index 98953739c..e269d216d 100644 --- a/crates/core/tests/it_datafusion/main.rs +++ b/crates/core/tests/it_datafusion/main.rs @@ -19,3 +19,4 @@ mod integration_datafusion; mod nanosecond_timestamps; mod read_delta_log_test; mod read_delta_partitions_test; +mod integration_datafusion_deep; \ No newline at end of file From f2624f86d1ee4e58e9f1529f25d941a39d0a8548 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Tue, 7 Jul 2026 15:08:55 +0300 Subject: [PATCH 07/20] [HSTACK] fix: schema mapping for record batches with an empty schema (due to schema evolution) --- crates/core/src/kernel/schema/cast/mod.rs | 26 ++++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/crates/core/src/kernel/schema/cast/mod.rs b/crates/core/src/kernel/schema/cast/mod.rs index 7956ff7d3..325f188c4 100644 --- a/crates/core/src/kernel/schema/cast/mod.rs +++ b/crates/core/src/kernel/schema/cast/mod.rs @@ -220,15 +220,25 @@ pub fn cast_record_batch( ..Default::default() }; - // Can be simplified with StructArray::try_new_with_length in arrow 55.1 - let col_arrays = batch.columns().to_owned(); - let s = if col_arrays.is_empty() { - StructArray::new_empty_fields(batch.num_rows(), None) - } else { - StructArray::new(batch.schema().as_ref().to_owned().fields, col_arrays, None) - }; + // // Can be simplified with StructArray::try_new_with_length in arrow 55.1 + // let col_arrays = batch.columns().to_owned(); + // let s = if col_arrays.is_empty() { + // StructArray::new_empty_fields(batch.num_rows(), None) + // } else { + // StructArray::new(batch.schema().as_ref().to_owned().fields, col_arrays, None) + // }; + // + // let struct_array = cast_struct(&s, target_schema.fields(), &cast_options, add_missing)?; + + // @HStack Fix schema mapping for record batches with an empty schema + let mut struct_array = StructArray::try_new_with_length( + batch.schema().as_ref().to_owned().fields, + batch.columns().to_owned(), + None, + batch.num_rows(), + )?; + struct_array = cast_struct(&struct_array, target_schema.fields(), &cast_options, add_missing)?; - let struct_array = cast_struct(&s, target_schema.fields(), &cast_options, add_missing)?; Ok(RecordBatch::try_new_with_options( target_schema, From 6611ae19f4e09b1628fd4f672b2f7e04645db6a4 Mon Sep 17 00:00:00 2001 From: Costi Ciudatu Date: Thu, 7 May 2026 14:50:18 +0300 Subject: [PATCH 08/20] [HSTACK] feat: log segment size limiter --- Cargo.toml | 1 + crates/core/Cargo.toml | 1 + crates/core/src/kernel/snapshot/mod.rs | 8 +- .../core/src/kernel/snapshot/size_limits.rs | 992 ++++++++++++++++++ crates/core/src/table/builder.rs | 21 + 5 files changed, 1022 insertions(+), 1 deletion(-) create mode 100644 crates/core/src/kernel/snapshot/size_limits.rs diff --git a/Cargo.toml b/Cargo.toml index 11cc9d9d4..ed06c3cf6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ datafusion-proto = { version = "54.0.0" } # serde serde = { version = "1.0.194", features = ["derive"] } serde_json = "1" +strum = { version = "0.28", features = ["derive"] } prost = { version = "0.14.3" } # "stdlib" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index a2f22e72e..0001a0ac3 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -51,6 +51,7 @@ datafusion-proto = { workspace = true, optional = true } # serde serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +strum = { workspace = true, features = ["derive"] } prost = { workspace = true } # "stdlib" diff --git a/crates/core/src/kernel/snapshot/mod.rs b/crates/core/src/kernel/snapshot/mod.rs index d9b60144f..89f0351b2 100644 --- a/crates/core/src/kernel/snapshot/mod.rs +++ b/crates/core/src/kernel/snapshot/mod.rs @@ -62,6 +62,7 @@ mod scan; mod serde; mod stats_projection; mod stream; +pub mod size_limits; pub(crate) static SCAN_ROW_ARROW_SCHEMA: LazyLock = LazyLock::new(|| Arc::new(scan_row_schema().as_ref().try_into_arrow().unwrap())); @@ -215,8 +216,10 @@ impl Snapshot { config: DeltaTableConfig, version: Option, ) -> DeltaResult { + let engine2 = engine.clone(); + let table_root2 = table_root.clone(); let snapshot = match spawn_blocking_with_span(move || { - let mut builder = KernelSnapshot::builder_for(table_root); + let mut builder = KernelSnapshot::builder_for(table_root.clone()); if let Some(version) = version { builder = builder.at_version(version); } @@ -236,6 +239,9 @@ impl Snapshot { } }; + let (snapshot, load_metrics) = + size_limits::apply_optional_log_limiter(snapshot, config.log_size_limiter.as_ref(), engine2.as_ref(), &table_root2).await?; + Ok(Self { inner: snapshot, config, diff --git a/crates/core/src/kernel/snapshot/size_limits.rs b/crates/core/src/kernel/snapshot/size_limits.rs new file mode 100644 index 000000000..79db6083f --- /dev/null +++ b/crates/core/src/kernel/snapshot/size_limits.rs @@ -0,0 +1,992 @@ +use crate::logstore::{LogStore, LogStoreExt}; +use crate::{DeltaResult, DeltaTableError}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::convert::identity; +use std::num::{NonZeroU64, NonZeroUsize}; +use std::ops::RangeInclusive; +use std::sync::Arc; +use delta_kernel::log_segment::{LogSegment, LogSegmentFiles}; +use delta_kernel::path::{LogPathFileType, ParsedLogPath}; +use delta_kernel::{Engine, Snapshot}; +use itertools::Itertools; +use strum::Display; +use tracing::{debug, info, trace, warn}; +use url::Url; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SnapshotLoadMetrics { + /// Current version of the snapshot + pub current_version: i64, + + /// Oldest commit version included in the loaded snapshot + /// None if only checkpoint was loaded (no commits) + pub oldest_version_loaded: Option, + + /// Final log segment size in bytes (after truncation if applied) + pub loaded_log_segment_size: u64, + + /// Total number of commit and checkpoint files in the loaded log segment + pub num_log_files: usize, + + /// Number of commit files in the loaded log segment + pub num_commit_files: usize, + + /// Number of checkpoint files in the loaded log segment + pub num_checkpoint_files: usize, + + /// Whether log size limiting was applied during load + pub log_size_limiter_applied: bool, + + /// Original log segment size in bytes (before any truncation) + /// None if no limiter was configured + pub original_log_segment_size: Option, + + /// Number of commits discarded due to truncation + /// None if no truncation occurred + pub num_commits_discarded: Option, +} + +impl Default for SnapshotLoadMetrics { + fn default() -> Self { + Self { + current_version: 0, + oldest_version_loaded: None, + loaded_log_segment_size: 0, + num_log_files: 0, + num_commit_files: 0, + num_checkpoint_files: 0, + log_size_limiter_applied: false, + original_log_segment_size: None, + num_commits_discarded: None, + } + } +} + +pub(crate) fn log_segment_total_byte_size(segment: &LogSegment) -> u64 { + segment + .listed + .checkpoint_parts + .iter() + .chain(segment.listed.ascending_commit_files.iter()) + .map(|p| p.location.size) + .sum() +} + +pub(crate) async fn apply_optional_log_limiter( + snapshot: Arc, + limiter: Option<&LogSizeLimiter>, + engine: &dyn Engine, + root_url: &Url, +) -> DeltaResult<(Arc, SnapshotLoadMetrics)> { + let current_version = snapshot.version() as i64; + + if let Some(limiter) = limiter { + let original_segment = snapshot.log_segment().clone(); + let original_size = log_segment_total_byte_size(&original_segment); + + let (truncated_segment, truncation_info) = + limiter.truncate(original_segment, engine, root_url).await?; + let table_configuration = snapshot.table_configuration().clone(); + + let oldest_version = truncated_segment + .listed + .ascending_commit_files + .first() + .map(|p| p.version as i64); + + let metrics = match truncation_info { + Some(info) => SnapshotLoadMetrics::with_truncation( + current_version, + oldest_version, + info.truncated_size, + info.original_size, + info.commits_discarded, + ), + None => SnapshotLoadMetrics::no_truncation( + current_version, + oldest_version, + original_size, + ), + } + .with_log_segment_file_counts(&truncated_segment); + + Ok(( + Arc::new(Snapshot::new(truncated_segment, table_configuration)), + metrics, + )) + } else { + let metrics = SnapshotLoadMetrics::from_snapshot(snapshot.as_ref()); + Ok((snapshot, metrics)) + } +} + +impl SnapshotLoadMetrics { + pub fn from_snapshot(snapshot: &Snapshot) -> Self { + let log_segment = snapshot.log_segment(); + let log_size = log_segment_total_byte_size(log_segment); + let oldest_version = log_segment + .listed + .ascending_commit_files + .first() + .map(|p| p.version as i64); + + SnapshotLoadMetrics::no_truncation( + snapshot.version() as i64, + oldest_version, + log_size, + ) + .with_log_segment_file_counts(log_segment) + } + + fn with_log_segment_file_counts(mut self, log_segment: &LogSegment) -> Self { + self.num_commit_files = log_segment.listed.ascending_commit_files.len(); + self.num_checkpoint_files = log_segment.listed.checkpoint_parts.len(); + self.num_log_files = self.num_commit_files + self.num_checkpoint_files; + self + } + + pub fn no_truncation( + version: i64, + oldest_version: Option, + log_segment_size: u64, + ) -> Self { + Self { + current_version: version, + oldest_version_loaded: oldest_version, + loaded_log_segment_size: log_segment_size, + num_log_files: 0, + num_commit_files: 0, + num_checkpoint_files: 0, + log_size_limiter_applied: false, + original_log_segment_size: None, + num_commits_discarded: None, + } + } + + pub fn with_truncation( + version: i64, + oldest_version: Option, + truncated_size: u64, + original_size: u64, + commits_discarded: usize, + ) -> Self { + Self { + current_version: version, + oldest_version_loaded: oldest_version, + loaded_log_segment_size: truncated_size, + num_log_files: 0, + num_commit_files: 0, + num_checkpoint_files: 0, + log_size_limiter_applied: true, + original_log_segment_size: Some(original_size), + num_commits_discarded: Some(commits_discarded), + } + } + + pub fn was_truncated(&self) -> bool { + self.log_size_limiter_applied && self.num_commits_discarded.is_some() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TruncationInfo { + /// Original log segment size in bytes (before truncation) + pub original_size: u64, + /// Final log segment size in bytes (after truncation) + pub truncated_size: u64, + /// Number of commits discarded + pub commits_discarded: usize, +} + +#[derive(Debug, Clone, PartialEq, Display, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OversizePolicy { + Reject, + /// Skip checkpoints and only load JSON commits. + UseTruncatedCommitLog(NonZeroUsize), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LogSizeLimiter { + /// Maximum allowed size in bytes for the total log segment (checkpoint + commit files). + size_limit: NonZeroU64, + oversize_policy: OversizePolicy, +} + + +impl LogSizeLimiter { + + pub fn new(size_limit: NonZeroU64, oversize_policy: OversizePolicy) -> Self { + Self { + size_limit, + oversize_policy, + } + } + + pub fn try_new(size_limit: u64, truncated_commit_log_size: Option) -> DeltaResult { + let size_limit = NonZeroU64::new(size_limit) + .ok_or_else(|| DeltaTableError::Generic("max_log_bytes must be nonzero".into()))?; + let oversize_policy = if let Some(num_commits) = truncated_commit_log_size { + let num_commits = NonZeroUsize::new(num_commits) + .ok_or_else(|| DeltaTableError::Generic("pseudo_cdf_lookback_count must be nonzero".into()))?; + OversizePolicy::UseTruncatedCommitLog(num_commits) + } else { + OversizePolicy::Reject + }; + Ok(Self { + size_limit, + oversize_policy, + }) + } + + pub fn from_storage_options(opts: &mut HashMap) -> DeltaResult> { + let prefix = "log_size_limiter"; + let size_limit_key = &format!("{prefix}.size_limit"); + let use_commit_log_key = &format!("{prefix}.use_truncated_commit_log"); + let num_commits_key = &format!("{prefix}.truncated_commit_log_size"); + + let size_limit: Option = opts.remove(size_limit_key) + .map(|opt| opt.parse() + .expect(&format!("{size_limit_key} must be a positive int; got {opt}"))); + let use_commit_log: bool = opts.remove(use_commit_log_key) + .map(|opt| opt.parse() + .expect(&format!("{use_commit_log_key} must be a boolean; got {opt}"))) + .unwrap_or(false); + let num_commits: usize = opts.remove(num_commits_key) + .map(|opt| opt.parse() + .expect(&format!("{num_commits_key} must be a positive int; got {opt}"))) + .unwrap_or(24); // default number of commits to use when commit log is enabled with no size specified + size_limit + .map(|limit| LogSizeLimiter::try_new(limit, use_commit_log.then_some(num_commits))) + .transpose() + } + + pub(super) async fn truncate(&self, log_segment: LogSegment, engine: &dyn Engine, root_url: &Url) -> DeltaResult<(LogSegment, Option)> { + let total_size: u64 = log_segment_total_byte_size(&log_segment); + let total_size = total_size; + let size_limit = self.size_limit.get(); + let original_commit_count = log_segment.listed.ascending_commit_files.len(); + + if total_size > size_limit { + warn!( + "Log segment size in bytes: {} > {}. Applying policy: {:?}", + total_size, size_limit, self.oversize_policy + ); + trace!("Oversized log segment: {:?}", log_segment); + match &self.oversize_policy { + OversizePolicy::Reject => + Err(DeltaTableError::Generic(format!(r#" + Table log segment size ({} bytes) exceeds maximum allowed size ({} bytes). + Consider increasing the size limit or using an oversize policy other than {}. + "#, total_size, self.size_limit, self.oversize_policy))), + OversizePolicy::UseTruncatedCommitLog(num_commits) => { + let (truncated_segment, truncated_size) = + truncated_commit_log(log_segment, engine, root_url, num_commits, size_limit).await?; + let final_commit_count = truncated_segment.listed.ascending_commit_files.len(); + let commits_discarded = original_commit_count.saturating_sub(final_commit_count); + + let truncation_info = TruncationInfo { + original_size: total_size, + truncated_size, + commits_discarded, + }; + Ok((truncated_segment, Some(truncation_info))) + } + } + } else { + debug!("Log segment size ({} bytes) is within the limit of {} bytes", total_size, size_limit); + Ok((log_segment, None)) + } + } +} + +async fn truncated_commit_log(log_segment: LogSegment, engine: &dyn Engine, root_url: &Url, num_commits: &NonZeroUsize, size_limit: u64) -> DeltaResult<(LogSegment, u64)> { + let num_commits = num_commits.get(); + let truncated_log: Vec = if log_segment.listed.ascending_commit_files.len() < num_commits { + let segment_version = log_segment.end_version as usize; + let first_missing_version = segment_version.saturating_sub(num_commits - 1); // start from zero if num_commits > segment_version + let last_missing_version = segment_version - log_segment.listed.ascending_commit_files.len(); // cannot overflow + info!("Extending the segment commit log with versions {}-{}", first_missing_version, last_missing_version); + let missing_versions = first_missing_version..=last_missing_version; + let previous_commits = list_commit_files(engine, root_url, missing_versions).await?; + previous_commits.into_iter() + .chain(log_segment.listed.ascending_commit_files) + .collect() + } else { + info!("Discarding the last {} entries from the segment commit log", log_segment.listed.ascending_commit_files.len() - num_commits); + log_segment.listed.ascending_commit_files[log_segment.listed.ascending_commit_files.len() - num_commits..] + .to_vec() + }; + let mut truncated_log_size = 0_u64; // keep track of the total size to cut it shorter if needed + let latest_commit_file = truncated_log.last().cloned(); + let final_commits: Vec = truncated_log.into_iter() + .rev() + .take_while(|file_meta| { + truncated_log_size += file_meta.location.size; + truncated_log_size <= size_limit + }) + .collect::>() + .into_iter() + .rev() + .collect(); + info!("final_commits: {:?}", &final_commits); + + // Calculate the actual final size + let final_size: u64 = final_commits.iter().map(|f| f.location.size).sum(); + + let segment_files = LogSegmentFiles { + ascending_commit_files: final_commits, + ascending_compaction_files: vec![], + checkpoint_parts: vec![], + latest_crc_file: None, + latest_commit_file, + max_published_version: Some(log_segment.end_version), + }; + + let segment = LogSegment { + end_version: log_segment.end_version, + listed: segment_files, + // ascending_commit_files: final_commits, + // checkpoint_parts: vec![], + // ascending_compaction_files: vec![], + log_root: root_url.clone(), + checkpoint_version: None, + last_checkpoint_metadata: log_segment.last_checkpoint_metadata.clone() + // latest_crc_file: None, + // latest_commit_file, + }; + + Ok((segment, final_size)) +} + +async fn list_commit_files( + engine: &dyn Engine, + root_url: &Url, + version_range: RangeInclusive, +) -> DeltaResult> { + // @HSTack see LogStore::log_root_url + let delta_log_root_url = root_url.join("_delta_log/").unwrap(); + let log_path = delta_log_root_url.clone(); + let lower_bound = log_path.join(&format!("{:020}", version_range.start())) + .map_err(|_| DeltaTableError::InvalidTableLocation(log_path.to_string()))?; + let upper_bound = log_path.join(&format!("{:020}", version_range.end() + 1)) + .map_err(|_| DeltaTableError::InvalidTableLocation(log_path.to_string()))?; + let commit_files = engine.storage_handler() + .list_from(&lower_bound)? + .map(|meta| { + ParsedLogPath::try_from(meta?) + }) + // TODO this filters out .crc files etc which start with "." - how do we want to use these kind of files? + .filter_map_ok(identity) + .take_while(move |path_res| match path_res { + Ok(path) => version_range.end() >= &(path.version as usize), + Err(_) => true, + }) + .filter_ok(|log_file| matches!(log_file.file_type, LogPathFileType::Commit)) + .try_collect()?; + Ok(commit_files) +} + + +#[cfg(test)] +mod tests { + use delta_kernel::Version; + use delta_kernel::metrics::SnapshotLoadMetricContext; + use super::*; + use test_doubles::*; + use crate::DeltaTableBuilder; + + + async fn create_log_segment(log_store: &TestLogStore, version: Option) -> DeltaResult { + let storage = log_store.engine(None).storage_handler(); + let log_root = log_store.log_root_url(); + Ok(LogSegment::for_snapshot(storage.as_ref(), log_root, Vec::new(), version, SnapshotLoadMetricContext::default())?) + } + + #[test] + fn test_serde() -> DeltaResult<()> { + let json = r#"{ + "size_limit": 10055, + "oversize_policy": "reject" + }"#; + assert_eq!( + serde_json::from_str::(json)?, + LogSizeLimiter::new( + NonZeroU64::new(10055).unwrap(), + OversizePolicy::Reject, + ) + ); + + let json = r#"{ + "size_limit": 10055, + "oversize_policy": { + "use_truncated_commit_log": 100 + } + }"#; + assert_eq!( + serde_json::from_str::(json)?, + LogSizeLimiter::new( + NonZeroU64::new(10055).unwrap(), + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(100).unwrap()), + ) + ); + + Ok(()) + } + + #[test] + fn test_from_storage_opts() -> DeltaResult<()> { + assert_eq!( + LogSizeLimiter::from_storage_options(&mut HashMap::new())?, + None + ); + let mut opts = HashMap::from([ + ("log_size_limiter.size_limit".into(), "10".into()), + ("log_size_limiter.use_truncated_commit_log".into(), "false".into()), + ("log_size_limiter.truncated_commit_log_size".into(), "5".into()), // should be ignored + ("test".into(), "1".into()), + ]); + assert_eq!( + LogSizeLimiter::from_storage_options(&mut opts)?, + Some(LogSizeLimiter::new( + NonZeroU64::new(10).unwrap(), + OversizePolicy::Reject + )) + ); + assert_eq!(opts.len(), 1); + assert!(opts.contains_key("test")); + Ok(()) + } + + #[test] + fn test_storage_opts_propagation() -> DeltaResult<()> { + let url = "memory:///".parse().map_err(|e: url::ParseError| DeltaTableError::Generic(e.to_string()))?; + let table = DeltaTableBuilder::from_url(url)? + .with_storage_options(HashMap::from([ + ("log_size_limiter.size_limit".into(), "10".into()), + ("log_size_limiter.use_truncated_commit_log".into(), "true".into()), + ("log_size_limiter.truncated_commit_log_size".into(), "5".into()), + ])).build()?; + assert_eq!( + table.config.log_size_limiter.expect("LogSizeLimiter should be set"), + LogSizeLimiter::new( + NonZeroU64::new(10).unwrap(), + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(5).unwrap()) + ) + ); + + Ok(()) + } + + #[tokio::test] + async fn test_noop_within_limits() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(10), CommitFsize(100), CheckpointFsize(3000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(5000).unwrap(), + OversizePolicy::Reject, + ); + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 90, 10); + // total size < size limit + // let tmp = log_store.engine(None); + let (truncated_segment, truncation_info) = limiter + .truncate( + segment.clone(), + log_store.engine(None).as_ref(), + &log_store.table_root_url() + ).await?; + assert_eq!(truncated_segment, segment); + assert_eq!(truncation_info, None); + + Ok(()) + } + + #[tokio::test] + async fn test_reject_policy() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(10), CommitFsize(100), CheckpointFsize(3000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(2500).unwrap(), + OversizePolicy::Reject, + ); + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 90, 10); + let result = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await; + + assert!(result.is_err()); + let error_msg = result.unwrap_err().to_string(); + assert!(error_msg.contains("exceeds maximum allowed size")); + assert!(error_msg.contains("4000 bytes"), "`{}` does not contain '4000 bytes'", error_msg); + assert!(error_msg.contains("2500 bytes"), "`{}` does not contain '2500 bytes'", error_msg); + + Ok(()) + } + + #[tokio::test] + async fn test_commit_log_truncation_with_regular_delta_log() -> DeltaResult<()> { + let _ = pretty_env_logger::try_init(); + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(5), CommitFsize(10), CheckpointFsize(1000) + ); + info!("LOG STORE ROOT URL: {}", log_store.root_url().to_string()); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(10).unwrap()), + ); + + let segment = create_log_segment(&log_store, Some(25)).await?; + info!("segment 1: {:#?}", &segment); + assert_segment_with_checkpoint(&segment, 25, 0); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 16..=25); + + let segment = create_log_segment(&log_store, Some(7)).await?; + assert_segment_with_checkpoint(&segment, 5, 2); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 0..=7); + + let segment = create_log_segment(&log_store, Some(19)).await?; + assert_segment_with_checkpoint(&segment, 15, 4); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 10..=19); + + Ok(()) + } + + #[tokio::test] + async fn test_commit_log_truncation_with_no_checkpoints_in_log() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(200), CommitFsize(10), CheckpointFsize(1000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(10).unwrap()), + ); + + let segment = create_log_segment(&log_store, Some(30)).await?; + assert_segment_with_commits_only(&segment, 0..=30); + // size limit not exceeded: 31 commits * 10 bytes < 500 bytes, segment not truncated + let (truncated_segment, _truncation_info) = limiter + .truncate(segment.clone(), log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_eq!(truncated_segment, segment); + + let segment = create_log_segment(&log_store, Some(75)).await?; + assert_segment_with_commits_only(&segment, 0..=75); + // size limit exceeded: 75 commits * 10 bytes > 500 bytes; keeps the last 10 commits + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 66..=75); + + Ok(()) + } + + #[tokio::test] + async fn test_commit_log_truncation_with_vacuumed_log() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(30..=150), CheckpointCadence(25), CommitFsize(10), CheckpointFsize(1000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(50).unwrap()), + ); + + let segment = create_log_segment(&log_store, Some(70)).await?; + assert_segment_with_checkpoint(&segment, 50, 20); + // less than 50 commits available in the vacuumed store + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 30..=70); + + Ok(()) + } + + #[tokio::test] + async fn test_truncated_log_gets_cut_off_to_enforce_size_limit() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(30..=150), CheckpointCadence(25), CommitFsize(10), CheckpointFsize(1000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(100).unwrap()), // go back 100 commits + ); + + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 125, 25); + // only loads 50 commits instead of the configured 100 to stay within the size limit + let (truncated_segment, _truncation_info) = limiter.truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()).await?; + assert_segment_with_commits_only(&truncated_segment, 101..=150); + + Ok(()) + } + + #[tokio::test] + async fn test_compacted_json_files_are_ignored() -> DeltaResult<()> { + let extra_files = vec![ + format!("{:020}.{:020}.compacted.json", 15, 19), + ]; + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(10), CommitFsize(10), CheckpointFsize(1000) + ).with_additional_files(extra_files, 200); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(20).unwrap()), // go back 100 commits + ); + + let segment = create_log_segment(&log_store, Some(23)).await?; + assert_segment_with_checkpoint(&segment, 20, 3); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 4..=23 ); + Ok(()) + } + + fn commit_file_name(version: Version) -> String { + format!("{:020}.json", version) + } + + fn checkpoint_file_name(version: Version) -> String { + format!("{:020}.checkpoint.parquet", version) + } + + fn extract_file_names<'a>(stored_objects: impl IntoIterator) -> Vec { + stored_objects.into_iter() + .filter_map(|parsed_path| { + let path_str = parsed_path.location.location.path(); + path_str.split('/').last().map(ToString::to_string) + }) + .collect() + } + + fn assert_segment_with_checkpoint(segment: &LogSegment, checkpoint_version: Version, num_subsequent_commits: u64) { + assert_eq!(segment.end_version, checkpoint_version + num_subsequent_commits); + assert_eq!( + extract_file_names(&segment.listed.checkpoint_parts), + vec![ checkpoint_file_name(checkpoint_version) ], + ); + assert_eq!( + extract_file_names(&segment.listed.ascending_commit_files), + (checkpoint_version + 1 ..= checkpoint_version + num_subsequent_commits) + .map(|v| commit_file_name(v as u64)) + .collect::>(), + ); + } + + fn assert_segment_with_commits_only(log_segment: &LogSegment, versions: RangeInclusive) { + assert_eq!(log_segment.end_version, *versions.end() as u64); + assert_eq!(log_segment.listed.checkpoint_parts, vec![]); + assert_eq!( + extract_file_names(&log_segment.listed.ascending_commit_files), + versions.map(|v| commit_file_name(v as u64)).collect::>(), + ); + } + + + mod test_doubles { + use object_store::PutMultipartOptions; +use super::*; + use crate::DeltaResult; + use crate::kernel::transaction::TransactionError; + use crate::logstore::{object_store_path, CommitOrBytes, LogStore, LogStoreConfig, LogStoreExt}; + use async_trait::async_trait; + use bytes::Bytes; + use futures::stream; + use futures::stream::BoxStream; + use object_store::path::Path; + use object_store::{CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult}; + use std::ops::RangeInclusive; + use std::sync::Arc; + use url::Url; + use uuid::Uuid; + + // substitute for named arguments to make the test code self documenting + pub(super) struct CommitRange(pub(super) RangeInclusive); + pub(super) struct CheckpointCadence(pub(super) usize); + pub(super) struct CommitFsize(pub(super) u64); + pub(super) struct CheckpointFsize(pub(super) u64); + + #[derive(Debug, Clone)] + pub(super) struct TestLogStore { + config: LogStoreConfig, + files: Vec, + } + + impl TestLogStore { + /// Commit files are generated to span the entire `CommitRange`, and checkpoints are + /// created according to the configured `CheckpointCadence`, starting from 0 (exclusive) + /// up to the last version in the store (also exclusive) and only for versions + /// that are also in the `CommitRange`. + /// E.g. commits: 15 up to 100, cadence: 10 => checkpoints at versions 20, 30, ..., 90 + pub(super) fn new( + commit_range: CommitRange, + checkpoint_cadence: CheckpointCadence, + commit_fsize: CommitFsize, + checkpoint_fsize: CheckpointFsize, + ) -> Self { + // get rid of the self-documenting superfluous types + let commit_range = commit_range.0; + let checkpoint_cadence = checkpoint_cadence.0; + let commit_fsize = commit_fsize.0; + let checkpoint_fsize = checkpoint_fsize.0; + + let mut store = TestLogStore { + config: LogStoreConfig::new( + &Url::parse("memory://test/delta_table").unwrap(), + Default::default(), + ), + files: vec![] + }; + let path = object_store_path(&store.log_root_url()).unwrap(); + let commit_files = commit_range.clone() + .map(|v| commit_file_name(v as u64)) + .map(|f| obj_meta(path.clone().join(f), commit_fsize)); + let checkpoint_files = (0..*commit_range.end()) + .skip(checkpoint_cadence) + .step_by(checkpoint_cadence) + .filter(|version| commit_range.contains(version)) + .map(|v| checkpoint_file_name(v as u64)) + .map(|f| obj_meta(path.clone().join(f), checkpoint_fsize)); + let mut files = commit_files + .chain(checkpoint_files) + .collect::>(); + files.sort_unstable_by(|a, b| a.location.cmp(&b.location)); + // files.shuffle(&mut thread_rng()); // no order guarantees for store listing + store.files = files; + store + } + + pub(super) fn with_additional_files(mut self, fnames: Vec, fsize: u64) -> Self { + let log_path = object_store_path(&self.log_root_url()).unwrap(); + let mut files: Vec = fnames + .into_iter() + .map(|fname| obj_meta(log_path.clone().join(fname), fsize)) + .collect(); + self.files.append(&mut files); + self.files.sort_unstable_by(|a, b| a.location.cmp(&b.location)); + self + } + + } + + #[async_trait] + impl LogStore for TestLogStore { + fn name(&self) -> String { + "TestLogStore".to_string() + } + + async fn read_commit_entry(&self, _version: Version) -> DeltaResult> { + unimplemented!("TestLogStore::read_commit_entry not implemented for tests") + } + + async fn write_commit_entry(&self, _version: Version, _commit_or_bytes: CommitOrBytes, _operation_id: Uuid) -> Result<(), TransactionError> { + unimplemented!("TestLogStore::write_commit_entry not implemented for tests") + } + + async fn abort_commit_entry(&self, _version: Version, _commit_or_bytes: CommitOrBytes, _operation_id: Uuid) -> Result<(), TransactionError> { + unimplemented!("TestLogStore::abort_commit_entry not implemented for tests") + } + + async fn get_latest_version(&self, _start_version: Version) -> DeltaResult { + unimplemented!("TestLogStore::get_latest_version not implemented for tests") + } + + fn object_store(&self, operation_id: Option) -> Arc { + self.root_object_store(operation_id) + } + + fn root_object_store(&self, _operation_id: Option) -> Arc { + Arc::new(self.clone()) + } + + fn config(&self) -> &LogStoreConfig { + &self.config + } + } + + impl std::fmt::Display for TestLogStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } + } + + #[async_trait] + impl ObjectStore for TestLogStore { + + async fn put_opts(&self, _location: &Path, _bytes: PutPayload, _options: PutOptions) -> ObjectStoreResult { + unimplemented!("TestLogStore::put_opts not implemented for tests") + } + + async fn put_multipart_opts(&self, _location: &Path, _opts: PutMultipartOptions) -> ObjectStoreResult> { + unimplemented!("TestLogStore::put_multipart_opts not implemented for tests") + } + + async fn get_opts(&self, location: &Path, _options: GetOptions) -> ObjectStoreResult { + self.files.iter().find(|obj_meta| obj_meta.location == *location) + .map(|obj_meta| GetResult { + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(Bytes::new()) }))), + meta: obj_meta.clone(), + range: 0..obj_meta.size, + attributes: Default::default(), + }) + .ok_or_else(|| object_store::Error::NotFound { + path: location.to_string(), + source: Box::new(std::io::Error::new(std::io::ErrorKind::NotFound, "Not found")), + }) + } + + fn delete_stream(&self, _locations: BoxStream<'static, ObjectStoreResult>) -> BoxStream<'static, ObjectStoreResult> { + todo!() + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { + let log_path = object_store_path(&self.log_root_url()); + // Be more permissive - return files if prefix is None or matches the log path + if prefix.is_none() || prefix == log_path.ok().as_ref() { + Box::pin(stream::iter(self.files.clone().into_iter().map(Ok))) + } else { + Box::pin(stream::empty()) + } + } + + async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> ObjectStoreResult { + unimplemented!("TestLogStore::list_with_delimiter not implemented for tests") + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _options: CopyOptions) -> ObjectStoreResult<()> { + todo!() + } + } + + fn obj_meta(path: impl Into, size: u64) -> ObjectMeta { + ObjectMeta { + location: path.into(), + size, + last_modified: "2025-07-18T15:30:00Z".parse().unwrap(), + e_tag: None, + version: None, + } + } + + fn parsed_log_path(path: impl Into, size: u64) -> ParsedLogPath { + let path = path.into(); + let dummy_url = Url::parse("memory://test/").unwrap(); + let file_url = dummy_url.join(path.as_ref()).unwrap(); + let parsed_url_path = ParsedLogPath::try_from(file_url).unwrap().unwrap(); + // Convert to FileMeta-based ParsedLogPath + let file_meta = delta_kernel::FileMeta { + location: parsed_url_path.location, + last_modified: 1752852600000, + size, + }; + ParsedLogPath::try_from(file_meta).unwrap().unwrap() + } + + #[tokio::test] + async fn test_fake_log_store() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(2..=97), CheckpointCadence(10), CommitFsize(128), CheckpointFsize(1024) + ); + + // before the first checkpoint + let segment = create_log_segment(&log_store, Some(5)).await?; + assert_segment_with_commits_only(&segment, 2..=5); + assert_eq!( + segment, + LogSegment { + end_version: 5, + listed: LogSegmentFiles { + ascending_commit_files: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000002.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000003.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000004.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000005.json", 128), + ], + checkpoint_parts: vec![], + ascending_compaction_files: vec![], + latest_crc_file: None, + latest_commit_file: Some(parsed_log_path("delta_table/_delta_log/00000000000000000005.json", 128)), + max_published_version: Some(5), + }, + log_root: log_store.log_root_url(), + checkpoint_version: None, + last_checkpoint_metadata: None, + }, + ); + + // with checkpoint + let segment = create_log_segment(&log_store, Some(32)).await?; + assert_segment_with_checkpoint(&segment, 30, 2); + assert_eq!( + segment, + LogSegment { + end_version: 32, + listed: LogSegmentFiles { + ascending_commit_files: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000031.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000032.json", 128), + ], + checkpoint_parts: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000030.checkpoint.parquet", 1024), + ], + ascending_compaction_files: vec![], + latest_crc_file: None, + latest_commit_file: Some(parsed_log_path("delta_table/_delta_log/00000000000000000032.json", 128)), + max_published_version: Some(32), + }, + log_root: log_store.log_root_url(), + checkpoint_version: Some(30), + last_checkpoint_metadata: None, + } + ); + + // latest version + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 90, 7); + assert_eq!( + segment, + LogSegment { + end_version: 97, + listed: LogSegmentFiles { + ascending_commit_files: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000091.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000092.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000093.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000094.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000095.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000096.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000097.json", 128), + ], + checkpoint_parts: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000090.checkpoint.parquet", 1024), + ], + ascending_compaction_files: vec![], + latest_crc_file: None, + latest_commit_file: Some(parsed_log_path("delta_table/_delta_log/00000000000000000097.json", 128),), + max_published_version: Some(97), + }, + log_root: log_store.log_root_url(), + checkpoint_version: Some(90), + last_checkpoint_metadata: None, + } + ); + + Ok(()) + } + } +} diff --git a/crates/core/src/table/builder.rs b/crates/core/src/table/builder.rs index 1ecd40538..894f7fb4d 100644 --- a/crates/core/src/table/builder.rs +++ b/crates/core/src/table/builder.rs @@ -16,6 +16,7 @@ use crate::kernel::Version; use crate::logstore::storage::IORuntime; use crate::logstore::{LogStoreRef, StorageConfig, object_store_factories}; use crate::{DeltaResult, DeltaTable, DeltaTableError}; +use crate::kernel::size_limits::LogSizeLimiter; /// possible version specifications for loading a delta table #[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] @@ -66,6 +67,10 @@ pub struct DeltaTableConfig { #[delta(skip)] /// When a runtime handler is provided, all IO tasks are spawn in that handle pub io_runtime: Option, + + #[delta(skip)] + pub log_size_limiter: Option, + } impl Default for DeltaTableConfig { @@ -76,6 +81,7 @@ impl Default for DeltaTableConfig { log_batch_size: 1024, skip_stats: false, io_runtime: None, + log_size_limiter: None, } } } @@ -86,6 +92,7 @@ impl PartialEq for DeltaTableConfig { && self.log_buffer_size == other.log_buffer_size && self.log_batch_size == other.log_batch_size && self.skip_stats == other.skip_stats + && self.log_size_limiter == other.log_size_limiter } } @@ -147,6 +154,13 @@ impl DeltaTableBuilder { self } + /// Sets `log_size_limiter` to the builder + pub fn with_log_size_limiter(mut self, limiter: LogSizeLimiter) -> Self { + self.table_config.log_size_limiter = Some(limiter); + self + } + + /// Sets `version` to the builder pub fn with_version(mut self, version: Version) -> Self { self.version = DeltaVersion::Version(version); @@ -213,6 +227,7 @@ impl DeltaTableBuilder { storage_options .clone() .into_iter() + .map(|(k, v)| (k.strip_prefix("deltalake.").map(ToString::to_string).unwrap_or(k), v)) .map(|(k, v)| { let needs_trim = v.starts_with("http://") || v.starts_with("https://") @@ -225,6 +240,12 @@ impl DeltaTableBuilder { }) .collect(), ); + let mut opts = self.storage_options.unwrap().clone(); + self.table_config.log_size_limiter = LogSizeLimiter::from_storage_options(&mut opts) + .expect("Invalid log_size_limiter options"); + + self.storage_options = Some(opts); + self } From 96edf4fbaf2a3c5fbe3e833f48c8c3aad53a1f93 Mon Sep 17 00:00:00 2001 From: Adrian Tanase Date: Fri, 6 Feb 2026 09:10:05 +0200 Subject: [PATCH 09/20] [HSTACK] feat: expose log metadata and total size to support weighted caching --- crates/core/src/kernel/snapshot/mod.rs | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/core/src/kernel/snapshot/mod.rs b/crates/core/src/kernel/snapshot/mod.rs index 89f0351b2..f582d3a2a 100644 --- a/crates/core/src/kernel/snapshot/mod.rs +++ b/crates/core/src/kernel/snapshot/mod.rs @@ -20,6 +20,8 @@ use std::sync::{Arc, LazyLock}; use arrow::array::RecordBatch; use arrow::compute::{filter_record_batch, is_not_null}; use arrow::datatypes::SchemaRef; +use arrow_arith::aggregate::sum_array_checked; +use arrow_array::{Int64Array, StructArray}; use delta_kernel::actions::{Remove, Sidecar}; use delta_kernel::engine::arrow_conversion::TryIntoArrow as _; use delta_kernel::engine::arrow_data::ArrowEngineData; @@ -41,6 +43,8 @@ use object_store::{ObjectStore, ObjectStoreExt as _}; use serde_json::Deserializer; use url::Url; +use crate::kernel::arrow::extract::{self as ex, ProvidesColumnByName}; + use super::{Action, CommitInfo, Metadata, Protocol}; use crate::checkpoints::parse_last_checkpoint_hint; use crate::kernel::arrow::engine_ext::{ExpressionEvaluatorExt, rb_from_scan_meta}; @@ -1301,6 +1305,14 @@ pub(crate) async fn resolve_snapshot( } } +fn read_adds_size(array: &dyn ProvidesColumnByName) -> usize { + if let Some(size) = ex::extract_and_cast_opt::(array, "size") { + sum_array_checked::(size).unwrap().unwrap_or_default() as usize + } else { + 0 + } +} + impl EagerSnapshot { /// Create a new [`EagerSnapshot`] instance pub async fn try_new( @@ -1448,6 +1460,22 @@ impl EagerSnapshot { } } + /// Get the metadata size in the snapshot + pub fn files_metadata_size(&self) -> usize { + self.snapshot + .materialized_files() + .map(|m| m.batches.iter().map(|frb| frb.get_array_memory_size()).sum()) + .unwrap_or(0) + } + + /// Get the total size of files in the snapshot + pub fn files_total_size(&self) -> usize { + self.snapshot + .materialized_files() + .map(|m| m.batches.iter().map(|frb| read_adds_size(frb)).sum()) + .unwrap_or(0) + } + /// Stream the active files in the snapshot /// /// This function returns a stream of [`LogicalFileView`] objects, From 841c803132ada20b904d83a4bf0298a8d61082aa Mon Sep 17 00:00:00 2001 From: Adrian Tanase Date: Fri, 6 Feb 2026 09:11:28 +0200 Subject: [PATCH 10/20] [HSTACK] feat: add support for log_store reconfigure --- crates/core/src/table/mod.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index c286661a5..ae60e75d3 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -153,6 +153,19 @@ impl DeltaTable { } } + /// Update the current [`DeltaTable`] with an updated [`LogStore`] + /// + /// NOTE: This is for advanced users and allows swapping settings like AZURE_PROXY_URL + /// before passing the table to Datafusion, allowing for example to conditionally change or + /// remove proxy usage for metadata and data + pub fn with_new_store(&self, log_store: LogStoreRef) -> Self { + Self { + state: self.state.clone(), + log_store, + config: self.config.clone(), + } + } + /// get a shared reference to the delta object store pub fn object_store(&self) -> ObjectStoreRef { self.log_store.object_store(None) From ebd223fe9fc77e2acbe439e845c102b1be6eb506 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Tue, 7 Jul 2026 15:24:35 +0300 Subject: [PATCH 11/20] [HSTACK] fix: add table constructor that does not remove final `/` --- crates/core/src/table/builder.rs | 53 +++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/crates/core/src/table/builder.rs b/crates/core/src/table/builder.rs index 894f7fb4d..4f100ef0c 100644 --- a/crates/core/src/table/builder.rs +++ b/crates/core/src/table/builder.rs @@ -130,9 +130,10 @@ impl DeltaTableBuilder { })?; debug!("creating table builder with {table_url}"); + let actual_table_url = parse_table_uri(&table_url)?; Ok(Self { - table_url, + table_url: actual_table_url, storage_backend: None, version: DeltaVersion::default(), storage_options: None, @@ -278,6 +279,7 @@ impl DeltaTableBuilder { /// Build a delta storage backend for the given config pub fn build_storage(&self) -> DeltaResult { debug!("build_storage() with {}", self.table_url); + let location = self.table_url.clone(); let mut storage_config = StorageConfig::parse_options(self.storage_options())?; if let Some(io_runtime) = self.table_config.io_runtime.clone() { @@ -286,14 +288,14 @@ impl DeltaTableBuilder { if let Some((store, _url)) = self.storage_backend.as_ref() { debug!("Loading a logstore with a custom store: {store:?}"); - crate::logstore::logstore_with(store.clone(), &self.table_url, storage_config) + crate::logstore::logstore_with(store.clone(), &location, storage_config) } else { // If there has been no backend defined just default to the normal logstore look up debug!( "Loading a logstore based off the location: {:?}", - self.table_url + location ); - crate::logstore::logstore_for(&self.table_url, storage_config) + crate::logstore::logstore_for(&location, storage_config) } } @@ -410,13 +412,24 @@ pub fn parse_table_uri(table_uri: impl AsRef) -> DeltaResult { let mut url = match uri_type { UriType::LocalPath(path) => { - let path = std::fs::canonicalize(&path).map_err(|err| { - let msg = format!( - "Local path \"{}\" does not exist or you don't have access!\nError: {err:?}", - path.display(), - ); - DeltaTableError::InvalidTableLocation(msg) - })?; + // let path = std::fs::canonicalize(&path).map_err(|err| { + // let msg = format!( + // "Local path \"{}\" does not exist or you don't have access!\nError: {err:?}", + // path.display(), + // ); + // DeltaTableError::InvalidTableLocation(msg) + // })?; + // Tolerate non-existent paths: callers like `DeltaTableBuilder::from_url` + // must work for tables that don't exist on disk yet. Existence is + // enforced separately by `open_table` (see `builder_from_valid_url`). + let path = match std::fs::canonicalize(&path) { + Ok(canonical) => canonical, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => path, + Err(err) => { + let msg = format!("Invalid table location: {table_uri}\nError: {err:?}"); + return Err(DeltaTableError::InvalidTableLocation(msg)); + } + }; Url::from_directory_path(path).map_err(|_| { let msg = format!( "Could not construct a URL from the canonical path: {table_uri}.\n\ @@ -592,6 +605,13 @@ mod tests { assert_eq!(expected.as_str(), url.as_str()); } + #[test] + fn test_invalid_uri() { + // Urls should round trips as-is + DeltaTableBuilder::from_url(Url::parse("this://is.nonsense").unwrap()) + .expect_err("this should be an error"); + } + #[test] fn test_writer_storage_opts_url_trim() { let cases = [ @@ -831,9 +851,12 @@ mod tests { ); let builder = builder_result.unwrap(); - assert_eq!( - builder.table_url.as_str(), - Url::from_directory_path(&new_path).unwrap().as_str() - ); + // `from_url` normalizes the path by trimming the trailing `/` so the + // URL is consistent across object-store registrations (see HSTACK + // commit "FIX table constructor"). Compare against the trimmed form. + let mut expected = Url::from_directory_path(&new_path).unwrap(); + let trimmed = expected.path().trim_end_matches('/').to_owned(); + expected.set_path(&trimmed); + assert_eq!(builder.table_url.as_str(), expected.as_str()); } } From c76cb33291b871d6244f95afb8ac3bf45c9ae84e Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Thu, 27 Aug 2026 14:47:13 +0300 Subject: [PATCH 12/20] [HSTACK] feat: implement physical codec for "next" version of ExecutionPlan: DeltaNextPhysicalCodec --- crates/core/src/delta_datafusion/mod.rs | 1 + .../table_provider/next/mod.rs | 2 +- .../table_provider/next/scan/codec.rs | 1185 +++++++++++++++++ .../table_provider/next/scan/exec.rs | 12 +- .../table_provider/next/scan/exec_meta.rs | 12 +- .../table_provider/next/scan/mod.rs | 4 +- .../table_provider/next/scan/plan.rs | 21 +- crates/core/src/kernel/snapshot/mod.rs | 4 +- crates/core/src/test_utils/mod.rs | 3 + .../integration_datafusion_deep.rs | 91 +- 10 files changed, 1317 insertions(+), 18 deletions(-) create mode 100644 crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index 4bb96987f..ff1c5837d 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -77,6 +77,7 @@ pub(crate) use find_files::*; pub(crate) use table_provider::next::normalize_path_as_file_id; pub use table_provider::{ DeltaScanConfig, DeltaScanConfigBuilder, TableProviderBuilder, next::DeltaScanExec, + next::DeltaNextPhysicalCodec, }; pub(crate) use table_provider::{ next::FILE_ID_COLUMN_DEFAULT, resolve_file_column_name, update_datafusion_session, diff --git a/crates/core/src/delta_datafusion/table_provider/next/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/mod.rs index 4dc48a5af..b95360c86 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/mod.rs @@ -44,7 +44,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use uuid::Uuid; -pub use self::scan::DeltaScanExec; +pub use self::scan::{DeltaScanExec, DeltaNextPhysicalCodec}; pub(crate) use self::scan::KernelScanPlan; use self::scan::ProjectedScanContract; use super::data_sink::DeltaDataSink; diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs new file mode 100644 index 000000000..4b2d94e51 --- /dev/null +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs @@ -0,0 +1,1185 @@ +//! Codec for serializing and deserializing [`DeltaScanExec`] physical plans. +//! +//! Provides a [`PhysicalExtensionCodec`] implementation for distributed execution. +//! Expressions are serialized via DataFusion protobuf; kernel `Transform` expressions +//! use a custom wire format since they have no DataFusion equivalent. + +use std::collections::VecDeque; +use std::sync::Arc; + +use arrow::datatypes::SchemaRef; +use dashmap::DashMap; +use datafusion::common::HashMap; +use datafusion::error::DataFusionError; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::Expr; +use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; +use datafusion_datasource::source::DataSourceExec; +use datafusion_proto::bytes::Serializeable; +use datafusion_proto::physical_plan::PhysicalExtensionCodec; +use delta_kernel::expressions::{ColumnName, Expression, ExpressionFieldPatch, ExpressionStructPatch}; +use delta_kernel::schema::{DataType as KernelDataType}; +use serde::{Deserialize, Serialize}; +use super::{DeltaScanExec, DeltaScanMetaExec, ProjectedScanContract, PublicFileIdMap}; +use super::plan::KernelScanPlan; +use crate::delta_datafusion::engine::{to_datafusion_expr, to_delta_expression}; +use crate::delta_datafusion::DeltaScanConfig; +use crate::DeltaTableConfig; +use crate::kernel::size_limits::SnapshotLoadMetrics; +use crate::kernel::Snapshot; + +/// Codec for serializing/deserializing [`DeltaScanExec`] physical plans. +/// +/// This codec enables distributed execution by serializing the inputs needed +/// to reconstruct the execution plan rather than the plan itself. This approach +/// avoids the need for serde support in delta-kernel types. +#[derive(Debug, Clone, Default)] +pub struct DeltaNextPhysicalCodec; + +impl PhysicalExtensionCodec for DeltaNextPhysicalCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &TaskContext, + ) -> datafusion::common::Result> { + let wire: ScanExecWire = serde_json::from_slice(buf).map_err(|e| { + DataFusionError::Internal(format!("Failed to decode Delta scan exec: {e}")) + })?; + + match wire { + ScanExecWire::Scan(wire) => wire.into_exec(inputs, ctx), + ScanExecWire::Meta(wire) => wire.into_exec(inputs, ctx), + } + } + + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + ) -> datafusion::common::Result<()> { + let wire = if let Some(delta_scan) = node.downcast_ref::() { + ScanExecWire::Scan(DeltaScanExecWire::try_from(delta_scan)?) + } else if let Some(meta_scan) = node.downcast_ref::() { + ScanExecWire::Meta(DeltaScanMetaExecWire::try_from(meta_scan)?) + } else { + return Err(DataFusionError::Internal( + "Expected DeltaScanExec or DeltaScanMetaExec for encoding".to_string(), + )); + }; + + serde_json::to_writer(buf, &wire).map_err(|e| { + DataFusionError::Internal(format!("Failed to encode Delta scan exec: {e}")) + })?; + Ok(()) + } +} + +/// Tagged wire wrapper dispatching between the two scan exec variants. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum ScanExecWire { + Scan(DeltaScanExecWire), + Meta(DeltaScanMetaExecWire), +} + +/// Wire format for a kernel FieldTransform. +#[derive(Debug, Serialize, Deserialize)] +struct ExpressionFieldPatchWire { + keep_input: bool, + insertions: Vec>, + optional: bool, +} + +/// Wire format for a kernel StructPatch expression. +/// +/// StructPatch is a sparse schema modification: specifies which fields to modify, +/// with unmentioned fields passing through unchanged. +#[derive(Debug, Serialize, Deserialize)] +struct StructPatchWire { + input_path: Option>, + field_patches: std::collections::HashMap, + prepended_fields: Vec>, + appended_fields: Vec>, +} + +/// Wire format for serializing [`DeltaScanExec`]. +/// +/// Uses `std::collections::HashMap` instead of `DashMap` for serde compatibility. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct DeltaScanExecWire { + scan: ScanPlanWire, + transforms: std::collections::HashMap, + selection_vectors: std::collections::HashMap>, + input_file_id_column: String, + file_id_column: Option, + public_file_ids: std::collections::HashMap, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct ScanPlanWire { + snapshot: Snapshot, + contract: ProjectedScanContractWire, + filters: Vec>, + skipping_predicate: Option>>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct ProjectedScanContractWire { + table_schema: SchemaRef, + provider_schema: SchemaRef, + projection: Option>, + filters: Vec>, + row_index_column: Option, +} + +/// Build the [`ScanPlanWire`] shared by both scan codecs. +fn scan_plan_wire(scan_plan: &KernelScanPlan) -> Result { + let snapshot = { + let exec_scan_plan_scan_snapshot = scan_plan.scan.snapshot().clone(); + // @HSTACK FIXME AT upgrade + // ATM, DeltaTableConfig is ONLY used with defaults + // The only thing that we set in it is the log_size_limiter - + // which has already been used early in the logical / planning phase + // At upgrade, RECHECK usage sites for DeltaTableConfig, we'll need to re-evaluate if + // stuff begins writing to it + let delta_table_config = DeltaTableConfig::default(); + let load_metrics = SnapshotLoadMetrics::from_snapshot(&exec_scan_plan_scan_snapshot); + Snapshot { + inner: exec_scan_plan_scan_snapshot, + config: delta_table_config, + materialized_files: None, + } + }; + + let row_index_column = scan_plan + .contract + .row_index_field + .as_ref() + .map(|f| f.name().clone()); + + let projected_scan_contract_wire = ProjectedScanContractWire { + table_schema: scan_plan.contract.table_schema.clone(), + provider_schema: scan_plan.contract.provider_schema.clone(), + projection: scan_plan.contract.projection.clone(), + filters: scan_plan + .contract + .filters + .iter() + .map(|p| p.to_bytes().map(|b| b.to_vec()).unwrap()) + .collect::>(), + row_index_column, + }; + + Ok(ScanPlanWire { + snapshot, + contract: projected_scan_contract_wire, + filters: scan_plan + .filters + .iter() + .map(|p| p.to_bytes().map(|b| b.to_vec()).unwrap()) + .collect::>(), + skipping_predicate: scan_plan.skipping_predicate.clone().map(|sp| { + sp.iter() + .map(|e| e.to_bytes().map(|b| b.to_vec()).unwrap()) + .collect::>() + }), + }) +} + +impl TryFrom<&DeltaScanExec> for DeltaScanExecWire { + type Error = DataFusionError; + + fn try_from(exec: &DeltaScanExec) -> Result { + let scan_plan_wire = scan_plan_wire(&exec.scan_plan)?; + + let transforms: std::collections::HashMap = exec + .transforms + .iter() + .map(|(file_url, kernel_expr)| { + serialize_transform(kernel_expr.as_ref()) + .map(|wire| (file_url.clone(), wire)) + }) + .collect::>()?; + + let selection_vectors: std::collections::HashMap> = exec + .selection_vectors + .iter() + .map(|entry| (entry.key().clone(), entry.value().clone())) + .collect(); + + let public_file_ids = exec.public_file_ids.as_ref().iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + + Ok(Self { + scan: scan_plan_wire, + transforms, + selection_vectors, + input_file_id_column: exec.input_file_id_column.clone(), + file_id_column: exec.file_id_column.clone(), + public_file_ids, + }) + } +} + +/// Converts kernel Expression -> DataFusion Expr -> protobuf bytes. +/// Does not support Transform expressions - use `serialize_transform` instead. +fn serialize_kernel_expression(expr: &Expression) -> Result, DataFusionError> { + let placeholder_type = KernelDataType::STRING; + let df_expr = to_datafusion_expr(expr, &placeholder_type)?; + let bytes = df_expr.to_bytes()?; + Ok(bytes.to_vec()) +} + +/// Converts protobuf bytes -> DataFusion Expr -> kernel Expression. +fn deserialize_kernel_expression(bytes: &[u8]) -> Result { + let df_expr = Expr::from_bytes(bytes)?; + to_delta_expression(&df_expr) + .map_err(|e| DataFusionError::Internal(format!("Failed to convert to kernel expr: {e}"))) +} + +/// Converts a kernel Transform expression to wire format. +fn serialize_transform(expr: &Expression) -> Result { + match expr { + Expression::StructPatch(struct_patch) => { + let input_path = struct_patch + .input_path + .as_ref() + .map(|p| p.iter().map(|s| s.to_string()).collect()); + + let field_patches = struct_patch + .field_patches + .iter() + .map(|(name, ft)| { + let insertions = ft + .insertions + .iter() + .map(|e| serialize_kernel_expression(e)) + .collect::, _>>()?; + Ok(( + name.clone(), + ExpressionFieldPatchWire { + keep_input: ft.keep_input, + insertions, + optional: ft.optional, + }, + )) + }) + .collect::>()?; + + let prepended_fields = struct_patch + .prepended_fields + .iter() + .map(|e| serialize_kernel_expression(e)) + .collect::, _>>()?; + + let appended_fields = struct_patch + .appended_fields + .iter() + .map(|e| serialize_kernel_expression(e)) + .collect::, _>>()?; + + Ok(StructPatchWire { + input_path, + field_patches, + prepended_fields, + appended_fields + }) + } + _ => Err(DataFusionError::Internal(format!( + "Expected Transform expression, got {:?}", + expr + ))), + } +} + +/// Converts wire format to a kernel Transform expression. +fn deserialize_transform(wire: StructPatchWire) -> Result { + let input_path = wire.input_path.map(ColumnName::new); + + let field_patches = wire + .field_patches + .into_iter() + .map(|(name, ft_wire)| { + let exprs = ft_wire + .insertions + .iter() + .map(|bytes| deserialize_kernel_expression(bytes).map(Arc::new)) + .collect::, _>>()?; + Ok(( + name, + ExpressionFieldPatch { + keep_input: ft_wire.keep_input, + insertions: exprs, + optional: ft_wire.optional, + }, + )) + }) + .collect::, DataFusionError>>()?; + + let prepended_fields = wire + .prepended_fields + .iter() + .map(|bytes| deserialize_kernel_expression(bytes).map(Arc::new)) + .collect::, _>>()?; + + let appended_fields = wire + .appended_fields + .iter() + .map(|bytes| deserialize_kernel_expression(bytes).map(Arc::new)) + .collect::, _>>()?; + + Ok(Expression::StructPatch(ExpressionStructPatch { + input_path, + field_patches, + prepended_fields, + appended_fields, + })) +} + +impl DeltaScanExecWire { + /// Reconstruct a [`DeltaScanExec`] from the wire format. + fn into_exec( + self, + inputs: &[Arc], + task: &TaskContext, + ) -> datafusion::common::Result> { + if inputs.len() != 1 { + return Err(DataFusionError::Internal(format!( + "DeltaScanExec expects exactly 1 input, got {}", + inputs.len() + ))); + } + let input = inputs[0].clone(); + + let mut execution_plan = input.clone(); + #[allow(clippy::collapsible_if)] + if let Some(ds_exec) = execution_plan.downcast_ref::() { + if let Some(file_conf) = ds_exec + .data_source() + .downcast_ref::() + { + let new_file_scan_config = FileScanConfigBuilder::from(file_conf.clone()) + .build(); + + // DataSourceExec::from_data_source(new_file_scan_config) + execution_plan = Arc::new( + ds_exec + .clone() + .with_data_source(Arc::new(new_file_scan_config)), + ); + } + } + + let mut delta_scan_config = DeltaScanConfig::new(); + if let Some(fic) = self.file_id_column { + delta_scan_config = delta_scan_config.with_file_column_name(fic); + } + + let filters = self.scan.contract.filters + .iter() + .map(|b| { + Expr::from_bytes_with_ctx(b, task).unwrap() + }) + .collect::>(); + + let row_index_column = self.scan.contract + .row_index_column.clone(); + + let contract = ProjectedScanContract::try_new( + self.scan.contract.table_schema.clone(), + self.scan.contract.provider_schema.clone(), + &delta_scan_config, + row_index_column.as_deref(), + self.scan.contract.projection.as_ref(), + &filters + )?; + + let skipping_predicate = self.scan.skipping_predicate + .map(|osp| { + osp.iter() + .map(|b| Expr::from_bytes_with_ctx(b, task).unwrap()) + .collect::>() + }); + + let scan_plan = KernelScanPlan::try_new_with_contract( + &self.scan.snapshot, + contract, + &filters, + &delta_scan_config, + skipping_predicate + )?; + + let transforms: HashMap> = self + .transforms + .into_iter() + .map(|(file_url, wire)| { + deserialize_transform(wire).map(|expr| (file_url, Arc::new(expr))) + }) + .collect::>()?; + + let selection_vectors: DashMap> = + self.selection_vectors.into_iter().collect(); + + let mut public_file_ids = PublicFileIdMap::default(); + if scan_plan.contract.retain_file_id { + for (k, v) in self.public_file_ids.iter() { + public_file_ids.insert(k.clone(), v.clone()); + } + } + + let exec = DeltaScanExec::new( + Arc::new(scan_plan), + execution_plan, + Arc::new(transforms), + Arc::new(selection_vectors), + Arc::new(public_file_ids), + Default::default(), + Default::default(), + ); + + Ok(Arc::new(exec)) + } +} + +/// Wire format for serializing [`DeltaScanMetaExec`]. +/// +/// Unlike [`DeltaScanExecWire`], there is no child plan: the per-partition +/// `(file_id, row_count)` inputs are serialized directly. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct DeltaScanMetaExecWire { + scan: ScanPlanWire, + input: Vec>, + transforms: std::collections::HashMap, + selection_vectors: std::collections::HashMap>, + file_id_column: Option, + public_file_ids: std::collections::HashMap, +} + +impl TryFrom<&DeltaScanMetaExec> for DeltaScanMetaExecWire { + type Error = DataFusionError; + + fn try_from(exec: &DeltaScanMetaExec) -> Result { + let scan_plan_wire = scan_plan_wire(&exec.scan_plan)?; + + let transforms: std::collections::HashMap = exec + .transforms + .iter() + .map(|(file_id, kernel_expr)| { + serialize_transform(kernel_expr.as_ref()).map(|wire| (file_id.clone(), wire)) + }) + .collect::>()?; + + let selection_vectors: std::collections::HashMap> = exec + .selection_vectors + .iter() + .map(|entry| (entry.key().clone(), entry.value().clone())) + .collect(); + + let public_file_ids = exec + .public_file_ids + .as_ref() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + + Ok(Self { + scan: scan_plan_wire, + input: exec.input.clone(), + transforms, + selection_vectors, + file_id_column: exec.file_id_field.as_ref().map(|f| f.name().clone()), + public_file_ids, + }) + } +} + +impl DeltaScanMetaExecWire { + /// Reconstruct a [`DeltaScanMetaExec`] from the wire format. + fn into_exec( + self, + inputs: &[Arc], + task: &TaskContext, + ) -> datafusion::common::Result> { + if !inputs.is_empty() { + return Err(DataFusionError::Internal(format!( + "DeltaScanMetaExec expects 0 inputs, got {}", + inputs.len() + ))); + } + + let mut delta_scan_config = DeltaScanConfig::new(); + if let Some(fic) = self.file_id_column { + delta_scan_config = delta_scan_config.with_file_column_name(fic); + } + + let filters = self + .scan + .contract + .filters + .iter() + .map(|b| Expr::from_bytes_with_ctx(b, task).unwrap()) + .collect::>(); + + let row_index_column = self.scan.contract.row_index_column.clone(); + + let contract = ProjectedScanContract::try_new( + self.scan.contract.table_schema.clone(), + self.scan.contract.provider_schema.clone(), + &delta_scan_config, + row_index_column.as_deref(), + self.scan.contract.projection.as_ref(), + &filters, + )?; + + let file_id_field = contract + .retain_file_id + .then(|| contract.file_id_field.clone()); + + let skipping_predicate = self.scan.skipping_predicate.map(|osp| { + osp.iter() + .map(|b| Expr::from_bytes_with_ctx(b, task).unwrap()) + .collect::>() + }); + + let scan_plan = KernelScanPlan::try_new_with_contract( + &self.scan.snapshot, + contract, + &filters, + &delta_scan_config, + skipping_predicate, + )?; + + let transforms: HashMap> = self + .transforms + .into_iter() + .map(|(file_id, wire)| { + deserialize_transform(wire).map(|expr| (file_id, Arc::new(expr))) + }) + .collect::>()?; + + let selection_vectors: DashMap> = + self.selection_vectors.into_iter().collect(); + + let mut public_file_ids = PublicFileIdMap::default(); + if scan_plan.contract.retain_file_id { + for (k, v) in self.public_file_ids.iter() { + public_file_ids.insert(k.clone(), v.clone()); + } + } + + let exec = DeltaScanMetaExec::new( + Arc::new(scan_plan), + self.input, + Arc::new(transforms), + Arc::new(selection_vectors), + Arc::new(public_file_ids), + file_id_field, + ExecutionPlanMetricsSet::new(), + ); + + Ok(Arc::new(exec)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::physical_plan::joins::CrossJoinExec; + use datafusion::physical_plan::ExecutionPlan; + use datafusion::prelude::{col, lit}; + use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec}; + use datafusion_proto::protobuf; + + use crate::delta_datafusion::session::create_session; + use crate::delta_datafusion::table_provider::next::DeltaScan; + use crate::kernel::Snapshot; + use crate::test_utils::{TestResult, TestTables}; + + use super::*; + + async fn create_delta_scan_exec( + filters: &[Expr], + projection: Option<&Vec>, + ) -> TestResult> { + create_delta_scan_exec_from_table(TestTables::Simple, filters, projection).await + } + + async fn create_delta_scan_exec_from_table( + table: TestTables, + filters: &[Expr], + projection: Option<&Vec>, + ) -> TestResult> { + let log_store = table.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let provider = DeltaScan::builder().with_snapshot(snapshot).await?; + + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let plan = provider.scan(&state, projection, filters, None).await?; + Ok(plan) + } + + fn extract_delta_scan_exec(plan: &Arc) -> Option<&DeltaScanExec> { + plan.downcast_ref::() + } + + /// Build a metadata-only scan by projecting no data columns. + async fn create_delta_scan_meta_exec_from_table( + table: TestTables, + filters: &[Expr], + ) -> TestResult> { + let log_store = table.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let provider = DeltaScan::builder().with_snapshot(snapshot).await?; + + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let empty_projection = vec![]; + let plan = provider + .scan(&state, Some(&empty_projection), filters, None) + .await?; + Ok(plan) + } + + fn extract_delta_scan_meta_exec(plan: &Arc) -> Option<&DeltaScanMetaExec> { + plan.downcast_ref::() + } + + #[tokio::test] + async fn test_codec_roundtrip_basic() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + assert!(!buf.is_empty(), "Encoded buffer should not be empty"); + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.scan_plan.contract.result_schema, + decoded_delta_scan.scan_plan.contract.result_schema, + "Result schemas should match" + ); + assert_eq!( + delta_scan.file_id_column, + decoded_delta_scan.file_id_column, + "File ID columns should match" + ); + assert_eq!( + delta_scan.input_file_id_column, + decoded_delta_scan.input_file_id_column, + "Retain file IDs should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_projection() -> TestResult { + let projection = vec![0usize]; + let plan = create_delta_scan_exec(&[], Some(&projection)).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.scan_plan.contract.result_schema.fields().len(), + decoded_delta_scan.scan_plan.contract.result_schema.fields().len(), + "Projected schema field count should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_filter() -> TestResult { + let filters = vec![col("id").gt(lit(5i64))]; + let plan = create_delta_scan_exec(&filters, None).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.scan_plan.contract.result_schema, + decoded_delta_scan.scan_plan.contract.result_schema, + "Result schemas should match with filter" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_wire_format_serialization() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let wire = DeltaScanExecWire::try_from(delta_scan)?; + + let json = serde_json::to_string(&wire)?; + assert!(!json.is_empty(), "JSON should not be empty"); + + let deserialized: DeltaScanExecWire = serde_json::from_str(&json)?; + + assert_eq!( + wire.file_id_column, deserialized.file_id_column, + "File ID column should roundtrip" + ); + assert_eq!( + wire.input_file_id_column, deserialized.input_file_id_column, + "Retain file IDs should roundtrip" + ); + assert_eq!( + wire.scan.contract.table_schema, deserialized.scan.contract.table_schema, + "Result schema should roundtrip" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_decode_wrong_input_count() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let result = codec.try_decode(&buf, &[], &task_ctx); + assert!(result.is_err(), "Should fail with 0 inputs"); + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + let input = delta_scan.children()[0].clone(); + let result = codec.try_decode(&buf, &[input.clone(), input], &task_ctx); + assert!(result.is_err(), "Should fail with 2 inputs"); + + let meta_plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + let mut meta_buf = Vec::new(); + codec.try_encode(meta_plan.clone(), &mut meta_buf)?; + + let result = codec.try_decode(&meta_buf, &[meta_plan], &task_ctx); + assert!(result.is_err(), "DeltaScanMetaExec should fail with 1 input"); + + Ok(()) + } + + #[test] + fn test_kernel_expression_serialization_roundtrip() { + use delta_kernel::expressions::{ColumnName, Expression as KernelExpression, Scalar}; + + let column_expr = KernelExpression::Column(ColumnName::new(["test_column"])); + let serialized = serialize_kernel_expression(&column_expr).unwrap(); + let deserialized = deserialize_kernel_expression(&serialized).unwrap(); + assert_eq!(column_expr, deserialized, "Column expression should roundtrip"); + + let literal_expr = KernelExpression::Literal(Scalar::Integer(42)); + let serialized = serialize_kernel_expression(&literal_expr).unwrap(); + let deserialized = deserialize_kernel_expression(&serialized).unwrap(); + assert_eq!(literal_expr, deserialized, "Literal expression should roundtrip"); + + let string_literal = KernelExpression::Literal(Scalar::String("hello".to_string())); + let serialized = serialize_kernel_expression(&string_literal).unwrap(); + let deserialized = deserialize_kernel_expression(&serialized).unwrap(); + assert_eq!(string_literal, deserialized, "String literal should roundtrip"); + } + + #[tokio::test] + async fn test_wire_format_with_selection_vectors() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let wire = DeltaScanExecWire::try_from(delta_scan)?; + + assert!( + wire.selection_vectors.is_empty() || !wire.selection_vectors.is_empty(), + "Selection vectors should serialize (empty or not)" + ); + + let json = serde_json::to_string(&wire)?; + let deserialized: DeltaScanExecWire = serde_json::from_str(&json)?; + + assert_eq!( + wire.selection_vectors.len(), + deserialized.selection_vectors.len(), + "Selection vectors count should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_preserves_transforms_and_selection_vectors() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.transforms.len(), + decoded_delta_scan.transforms.len(), + "Transforms count should match" + ); + + assert_eq!( + delta_scan.selection_vectors.len(), + decoded_delta_scan.selection_vectors.len(), + "Selection vectors count should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_deletion_vectors() -> TestResult { + let plan = + create_delta_scan_exec_from_table(TestTables::WithDvSmall, &[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + assert!( + !delta_scan.selection_vectors.is_empty(), + "Table with deletion vectors should have non-empty selection_vectors" + ); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.selection_vectors.len(), + decoded_delta_scan.selection_vectors.len(), + "Selection vectors count should match" + ); + + for entry in delta_scan.selection_vectors.iter() { + let key = entry.key(); + let original_vec = entry.value(); + let decoded_vec = decoded_delta_scan + .selection_vectors + .get(key) + .expect("Decoded should have same keys"); + assert_eq!( + original_vec.as_slice(), + decoded_vec.value().as_slice(), + "Selection vector values should match for key {key}" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_column_mapping() -> TestResult { + let plan = + create_delta_scan_exec_from_table(TestTables::WithColumnMapping, &[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + // Column mapping tables have transforms that inject partition values. + // This test verifies full roundtrip serialization of Transform expressions. + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.transforms.len(), + decoded_delta_scan.transforms.len(), + "Transforms count should match" + ); + + // Verify each transform was correctly serialized and deserialized + for key in delta_scan.transforms.keys() { + assert!( + decoded_delta_scan.transforms.contains_key(key), + "Decoded should have transform for key {key}" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_basic() -> TestResult { + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + assert!(!buf.is_empty(), "Encoded buffer should not be empty"); + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.scan_plan.contract.result_schema, + decoded_meta.scan_plan.contract.result_schema, + "Result schemas should match" + ); + assert_eq!( + meta_scan.input, decoded_meta.input, + "Per-file (file_id, row_count) inputs should match" + ); + assert_eq!( + meta_scan.partition_statistics(None)?.num_rows, + decoded_meta.partition_statistics(None)?.num_rows, + "Exact row counts should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_with_filter() -> TestResult { + // Filter on the partition column so pushdown is Exact and the scan stays + // metadata-only (otherwise data must be read, yielding a DeltaScanExec). + let filters = vec![col("Company Very Short").eq(lit("BMS"))]; + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &filters).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.scan_plan.contract.result_schema, + decoded_meta.scan_plan.contract.result_schema, + "Result schemas should match with filter" + ); + assert_eq!( + meta_scan.partition_statistics(None)?.num_rows, + decoded_meta.partition_statistics(None)?.num_rows, + "Exact row counts should match with filter" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_with_deletion_vectors() -> TestResult { + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithDvSmall, &[]).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + assert!( + !meta_scan.selection_vectors.is_empty(), + "Table with deletion vectors should have non-empty selection_vectors" + ); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.selection_vectors.len(), + decoded_meta.selection_vectors.len(), + "Selection vectors count should match" + ); + for entry in meta_scan.selection_vectors.iter() { + let decoded_vec = decoded_meta + .selection_vectors + .get(entry.key()) + .expect("Decoded should have same keys"); + assert_eq!( + entry.value().as_slice(), + decoded_vec.value().as_slice(), + "Selection vector values should match for key {}", + entry.key() + ); + } + assert_eq!( + meta_scan.partition_statistics(None)?.num_rows, + decoded_meta.partition_statistics(None)?.num_rows, + "Exact row counts should match with deletion vectors" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_with_transforms() -> TestResult { + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.transforms.len(), + decoded_meta.transforms.len(), + "Transforms count should match" + ); + for key in meta_scan.transforms.keys() { + assert!( + decoded_meta.transforms.contains_key(key), + "Decoded should have transform for key {key}" + ); + } + + Ok(()) + } + + /// Test with a plan that has both DeltaScanMetaExec and DeltaScanExec + async fn join_roundtrip(meta_first: bool) -> TestResult { + let scan_plan = create_delta_scan_exec_from_table(TestTables::Simple, &[], None).await?; + let meta_plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + + let join: Arc = if meta_first { + Arc::new(CrossJoinExec::new(meta_plan.clone(), scan_plan.clone())) + } else { + Arc::new(CrossJoinExec::new(scan_plan.clone(), meta_plan.clone())) + }; + + let codec = DeltaNextPhysicalCodec; + let proto = protobuf::PhysicalPlanNode::try_from_physical_plan(join.clone(), &codec)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + let decoded = proto.try_into_physical_plan(&task_ctx, &codec)?; + + let decoded_children = decoded.children(); + assert_eq!(decoded_children.len(), 2, "Join should have two children"); + + let (meta_idx, scan_idx) = if meta_first { (0, 1) } else { (1, 0) }; + assert!( + decoded_children[meta_idx] + .downcast_ref::() + .is_some(), + "Child {meta_idx} should decode to DeltaScanMetaExec" + ); + assert!( + decoded_children[scan_idx] + .downcast_ref::() + .is_some(), + "Child {scan_idx} should decode to DeltaScanExec" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_join_meta_first() -> TestResult { + join_roundtrip(true).await + } + + #[tokio::test] + async fn test_codec_roundtrip_join_meta_last() -> TestResult { + join_roundtrip(false).await + } +} diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs index 3bcbad79a..d60b536aa 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs @@ -108,21 +108,21 @@ pub(crate) fn consume_dv_mask( /// 4. Result is cast to the projected scan contract's result schema #[derive(Clone, Debug)] pub struct DeltaScanExec { - scan_plan: Arc, + pub(crate) scan_plan: Arc, /// Execution plan yielding the raw data read from data files. input: Arc, /// Transforms to be applied to data eminating from individual files - transforms: Arc>, + pub(crate) transforms: Arc>, /// Selection vectors to be applied to data read from individual files - selection_vectors: Arc>>, + pub(crate) selection_vectors: Arc>>, /// Public file paths keyed by compact scan file id. - public_file_ids: Arc, + pub(crate) public_file_ids: Arc, /// Execution metrics metrics: ExecutionPlanMetricsSet, /// File id column name carried by the input batches for per file correlation. - input_file_id_column: String, + pub(crate) input_file_id_column: String, /// User-visible file-id column name when projected in the output. - file_id_column: Option, + pub(crate) file_id_column: Option, /// plan properties properties: Arc, /// Aggregated partition column statistics diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs index b67674b89..5aced90e3 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs @@ -62,19 +62,19 @@ use crate::kernel::arrow::engine_ext::ExpressionEvaluatorExt; /// - No deletion vectors affect the results (or they are accounted for) #[derive(Clone, Debug)] pub(crate) struct DeltaScanMetaExec { - scan_plan: Arc, + pub(super) scan_plan: Arc, /// Execution plan yielding the raw data read from data files. - input: Vec>, + pub(super) input: Vec>, /// Transforms to be applied to data eminating from individual files - transforms: Arc>, + pub(super) transforms: Arc>, /// Deletion vectors for the table - selection_vectors: Arc>>, + pub(super) selection_vectors: Arc>>, /// Public file paths keyed by compact scan file id. - public_file_ids: Arc, + pub(super) public_file_ids: Arc, /// Execution metrics metrics: ExecutionPlanMetricsSet, /// Column name for the file id - file_id_field: Option, + pub(super) file_id_field: Option, /// plan properties properties: Arc, } diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs index 2ff3ad39b..1c06954a0 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs @@ -63,8 +63,9 @@ use object_store::{ObjectMeta, path::Path}; use tracing::debug; use url::Url; +pub use self::codec::DeltaNextPhysicalCodec; pub use self::exec::DeltaScanExec; -use self::exec_meta::DeltaScanMetaExec; +pub(crate) use self::exec_meta::DeltaScanMetaExec; use self::expr_adapter::{DeltaPhysicalExprAdapterFactory, relax_schema_nested_nullability}; pub(crate) use self::plan::{KernelScanPlan, ProjectedScanContract, supports_filters_pushdown}; use self::replay::{ScanFileContext, ScanFileStream}; @@ -85,6 +86,7 @@ mod exec_meta; mod expr_adapter; mod plan; mod replay; +mod codec; type ScanMetadataStream = Pin> + Send>>; type PublicFileIdMap = HashMap; diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs index 7de7e8142..1a84ec6d4 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs @@ -46,6 +46,10 @@ use crate::kernel::{Scan, Snapshot}; /// `TableProvider::scan` request so planning and execution do not rederive them independently. #[derive(Clone, Debug)] pub(crate) struct ProjectedScanContract { + pub(crate) table_schema: SchemaRef, + pub(crate) provider_schema: SchemaRef, + pub(crate) projection: Option>, + pub(crate) filters: Vec, /// Logical data schema after removing temporary predicate only columns and internal metadata. pub(crate) result_schema: SchemaRef, /// Logical schema produced by the kernel scan before dropping filter-only columns. @@ -201,6 +205,13 @@ impl ProjectedScanContract { }; Ok(Self { + table_schema, + provider_schema, + projection: projection.cloned(), + filters: filters + .iter() + .map(|x| x.clone()) + .collect::>(), result_schema, scan_schema, output_schema, @@ -221,6 +232,8 @@ impl ProjectedScanContract { /// be pushed to kernel file skipping vs. Parquet readers. #[derive(Clone, Debug)] pub(crate) struct KernelScanPlan { + pub(crate) filters: Vec, + pub(crate) skipping_predicate: Option>, /// Wrapped kernel scan to produce logical file stream pub(crate) scan: Arc, /// Query scoped contract shared across planning and execution. @@ -277,7 +290,7 @@ impl KernelScanPlan { // if some dedicated file skipping predicate is supplied, // we do not push the scan filters into the kernel scan. - let scan_predicate = if let Some(sp) = skipping_predicate { + let scan_predicate = if let Some(sp) = skipping_predicate.clone() { let (scan_predicate, _) = process_filters(&sp, table_config, config)?; if scan_predicate.is_none() { debug!( @@ -315,6 +328,8 @@ impl KernelScanPlan { let parquet_predicate_schema = build_parquet_predicate_schema(&parquet_read_schema, &contract.file_id_field); Ok(Self { + filters: filters.iter().map(|x| x.clone()).collect::>(), + skipping_predicate: skipping_predicate.clone(), scan, contract, parquet_read_schema, @@ -1326,6 +1341,10 @@ mod tests { fn test_retained_row_index_field_ignores_incomplete_internal_contract() { let schema = Arc::new(Schema::empty()); let contract = ProjectedScanContract { + table_schema: Arc::new(Schema::empty()), + provider_schema: Arc::new(Schema::empty()), + projection: None, + filters: vec![], result_schema: Arc::clone(&schema), scan_schema: Arc::clone(&schema), output_schema: schema, diff --git a/crates/core/src/kernel/snapshot/mod.rs b/crates/core/src/kernel/snapshot/mod.rs index f582d3a2a..8a5cc22b4 100644 --- a/crates/core/src/kernel/snapshot/mod.rs +++ b/crates/core/src/kernel/snapshot/mod.rs @@ -204,9 +204,9 @@ pub struct Snapshot { /// Log segment containing all log files in the snapshot pub(crate) inner: Arc, /// Configuration for the current session - config: DeltaTableConfig, + pub(crate) config: DeltaTableConfig, /// Optional materialized replay state owned by this snapshot. - materialized_files: Option>, + pub(crate) materialized_files: Option>, } impl Snapshot { diff --git a/crates/core/src/test_utils/mod.rs b/crates/core/src/test_utils/mod.rs index 3ade2520d..8bb0f5892 100644 --- a/crates/core/src/test_utils/mod.rs +++ b/crates/core/src/test_utils/mod.rs @@ -129,6 +129,7 @@ pub enum TestTables { LatestNotCheckpointed, /// A small table that uses deletion vectors. WithDvSmall, + WithColumnMapping, /// A custom, caller-named table (path resolution is not provided). Custom(String), } @@ -148,6 +149,7 @@ impl TestTables { Self::Checkpoints => data_path.join("checkpoints"), Self::LatestNotCheckpointed => data_path.join("latest_not_checkpointed"), Self::WithDvSmall => data_path.join("table-with-dv-small"), + Self::WithColumnMapping => data_path.join("table_with_column_mapping"), // the data path for upload does not apply to custom tables. Self::Custom(_) => todo!(), } @@ -166,6 +168,7 @@ impl TestTables { Self::Checkpoints => "checkpoints".into(), Self::LatestNotCheckpointed => "latest_not_checkpointed".into(), Self::WithDvSmall => "table-with-dv-small".into(), + Self::WithColumnMapping => "table_with_column_mapping".into(), Self::Custom(name) => name.to_owned(), } } diff --git a/crates/core/tests/it_datafusion/integration_datafusion_deep.rs b/crates/core/tests/it_datafusion/integration_datafusion_deep.rs index 32e223839..c384684ae 100644 --- a/crates/core/tests/it_datafusion/integration_datafusion_deep.rs +++ b/crates/core/tests/it_datafusion/integration_datafusion_deep.rs @@ -1,3 +1,4 @@ +use deltalake_core::delta_datafusion::DeltaNextPhysicalCodec; use std::ops::Deref; use std::sync::Arc; use std::sync::Once; @@ -16,7 +17,7 @@ use datafusion_proto::physical_plan::{AsExecutionPlan, ComposedPhysicalExtension use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; use tracing::info; -use deltalake_core::delta_datafusion::DeltaPhysicalCodec; +use deltalake_core::delta_datafusion::{DeltaPhysicalCodec, DeltaScanExec}; use deltalake_core::delta_datafusion::udtf::register_delta_table_udtf; /// DF 53 stores deep-projection state on `ParquetSource` as @@ -184,3 +185,91 @@ async fn test_hstack_nullable_new() -> datafusion::common::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_hstack_deep_column_pruning_next_codec() -> datafusion::common::Result<()> { + let filter = tracing_subscriber::EnvFilter::from_default_env(); + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_env_filter(filter) + .finish(); + tracing::subscriber::set_global_default(subscriber).ok(); + pretty_env_logger::try_init().ok(); + + let config = SessionConfig::new() + .set_bool("datafusion.sql_parser.enable_ident_normalization", false); + + let ctx = SessionContext::new_with_config(config); + + register_delta_table_udtf(&ctx, None, None); + + let delta_path = format!( + "{}/tests/data/deep", + env!("CARGO_MANIFEST_DIR") + ); + + let query = format!(r#" + select + t1._id, t1.productListItems['SKU'], _ACP_DATE + from + delta_table_next('file://{}') as t1 + "#, delta_path); + + let plan = ctx.state().create_logical_plan(&query).await.expect("Error creating logical plan"); + let optimized_plan = ctx.state().optimize(&plan).expect("Error optimizing plan"); + let state = ctx.state(); + let query_planner = state.query_planner().clone(); + let physical_plan = query_planner + .create_physical_plan(&optimized_plan, &state) + .await.expect("Error creating physical plan"); + info!( + "Physical plan: {}", + displayable(physical_plan.deref()).set_show_schema(true).indent(true) + ); + let proj1 = extract_projection_deep_from_plan(physical_plan.clone()); + let batches1 = collect(physical_plan.clone(), ctx.state().task_ctx()).await?; + let results1 = pretty::pretty_format_batches_with_options(&batches1, &FormatOptions::default())?.to_string(); + println!("{}", results1); + + // codec + let codec = ComposedPhysicalExtensionCodec::new( + vec![ + Arc::new(DefaultPhysicalExtensionCodec {}), + Arc::new(DeltaNextPhysicalCodec{}) + ] + ); + + let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?; + let bytes = proto.encode_to_vec(); + let plan_after_serde = PhysicalPlanNode::try_decode(&bytes) + .expect("Error try_decode") + .try_into_physical_plan(&ctx.task_ctx(), &codec) + .expect("try_into_physical_plan"); + info!( + "Physical plan after serde: {}", + displayable(plan_after_serde.deref()).set_show_schema(true).indent(true) + ); + + let proj2 = extract_projection_deep_from_plan(plan_after_serde.clone()); + let batches2 = collect(plan_after_serde.clone(), ctx.state().task_ctx()).await?; + let results2 = pretty::pretty_format_batches_with_options(&batches2, &FormatOptions::default())?.to_string(); + println!("{}", results2); + + assert_eq!(results1, results2, "Batches not equal !"); + println!("proj1: {:?}", proj1); + println!("proj2: {:?}", proj2); + + assert_eq!(proj1, proj2, "Deep Projection not equal !"); + + Ok(()) +} + +fn find_exec_node(input: &Arc) -> Option<&T> { + if let Some(found) = input.downcast_ref::() { + Some(found) + } else { + input.children().iter() + .find_map(|child| find_exec_node(child)) + } +} + From 5d3e958d1a01fa5d2eb9c0e3b29015e99f891736 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Thu, 13 Aug 2026 14:48:50 +0300 Subject: [PATCH 13/20] [HSTACK] feat: add option to modify the low-level Delta schema to make some fields nullable --- crates/core/Cargo.toml | 1 + crates/core/src/delta_datafusion/mod.rs | 10 +- .../core/src/delta_datafusion/schema_null.rs | 358 ++++++++++++++++++ .../table_provider/next/scan/mod.rs | 6 +- .../table_provider/next/scan/plan.rs | 11 +- 5 files changed, 381 insertions(+), 5 deletions(-) create mode 100644 crates/core/src/delta_datafusion/schema_null.rs diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0001a0ac3..66e50c58f 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -97,6 +97,7 @@ rand = "0.10" sqlparser = { version = "0.61.0" } humantime = { version = "2.1.0", optional = true } validator = { version = "0.19", features = ["derive"] } +ctor = "0.10" [dev-dependencies] arrow = { workspace = true, features = ["prettyprint"]} diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index ff1c5837d..b8e523850 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -101,10 +101,13 @@ pub mod planner; mod session; pub use session::SessionFallbackPolicy; pub(crate) use session::{SessionResolveContext, resolve_session_state}; +use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields; + mod table_provider; pub mod udtf; pub(crate) mod utils; pub mod table_provider_old; +pub mod schema_null; impl From for DataFusionError { fn from(err: DeltaTableError) -> Self { @@ -235,7 +238,7 @@ fn _arrow_schema( partition_columns: &[String], wrap_partitions: bool, ) -> ArrowSchemaRef { - let fields = schema + let mut fields = schema .fields() .into_iter() .filter(|f| !partition_columns.contains(&f.name().to_string())) @@ -264,7 +267,10 @@ fn _arrow_schema( }), ) .collect::>(); - Arc::new(ArrowSchema::new(fields)) + let mut schema = Arc::new(ArrowSchema::new(fields)); + // @Hstack - add the option to have an env var that can nullify fields in the delta schema + schema = rewrite_schema_with_nullable_fields(schema); + schema } pub(crate) fn files_matching_predicate<'a>( diff --git a/crates/core/src/delta_datafusion/schema_null.rs b/crates/core/src/delta_datafusion/schema_null.rs new file mode 100644 index 000000000..324ffafa3 --- /dev/null +++ b/crates/core/src/delta_datafusion/schema_null.rs @@ -0,0 +1,358 @@ +use std::sync::{Arc, Mutex}; +use arrow_schema::{DataType, Field, FieldRef, Fields, SchemaRef, UnionFields}; +use arrow_schema::DataType::{Dictionary, FixedSizeList, LargeList, LargeListView, List, ListView, Map, RunEndEncoded, Struct, Union}; + +static DELTA_FIELD_PATHS_TO_MAKE_NULLABLE: Mutex> = Mutex::new(Vec::new()); + +#[ctor::ctor] +fn init() { + let paths_flag = if let Ok(v) = std::env::var("DELTA_FIELD_PATHS_TO_MAKE_NULLABLE") { + v + } else { + "_acp_system_metadata.rowId,_acp_system_metadata.rowVersion,_acp_system_metadata.ingestTime,_acp_system_metadata.trackingId".to_string() + }; + + let splits = paths_flag + .split(",") + .map(|s| s.to_string()) + .collect::>(); + *DELTA_FIELD_PATHS_TO_MAKE_NULLABLE.lock().unwrap() = splits; +} + +pub fn rewrite_schema_with_nullable_fields(input: SchemaRef) -> SchemaRef { + let paths = DELTA_FIELD_PATHS_TO_MAKE_NULLABLE.lock().unwrap(); + rewrite_schema_with_nullable_fields_inner(input, &paths) +} + +fn rewrite_schema_with_nullable_fields_inner(input: SchemaRef, paths: &[String]) -> SchemaRef { + if paths.is_empty() { + return input; + } + + let new_fields = rewrite_fields(input.fields(), paths, ""); + Arc::new(arrow_schema::Schema::new_with_metadata(new_fields, input.metadata().clone())) +} + +fn rewrite_fields(fields: &Fields, paths: &[String], parent_path: &str) -> Vec { + fields.iter().map(|field| { + let current_path = if parent_path.is_empty() { + field.name().to_string() + } else { + format!("{}.{}", parent_path, field.name()) + }; + + let make_nullable = !field.is_nullable() && paths.iter().any(|p| *p == current_path); + let new_field = rewrite_field_type(field, paths, ¤t_path); + + if make_nullable { + Arc::new(new_field.with_nullable(true)) + } else { + Arc::new(new_field) + } + }).collect() +} + +fn rewrite_field_type(field: &Field, paths: &[String], current_path: &str) -> Field { + let new_dt = rewrite_data_type(field.data_type(), paths, current_path); + Field::new(field.name(), new_dt, field.is_nullable()) + .with_metadata(field.metadata().clone()) +} + +fn rewrite_data_type(dt: &DataType, paths: &[String], current_path: &str) -> DataType { + match dt { + Struct(fields) => { + Struct(Fields::from(rewrite_fields(fields, paths, current_path))) + }, + List(inner) => { + List(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + LargeList(inner) => { + LargeList(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + FixedSizeList(inner, n) => { + FixedSizeList(Arc::new(rewrite_field_type(inner, paths, current_path)), *n) + }, + ListView(inner) => { + ListView(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + LargeListView(inner) => { + LargeListView(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + Map(inner, sorted) => { + Map(Arc::new(rewrite_field_type(inner, paths, current_path)), *sorted) + }, + Dictionary(key, value) => { + Dictionary(key.clone(), Box::new(rewrite_data_type(value, paths, current_path))) + }, + RunEndEncoded(run_ends, values) => { + RunEndEncoded( + Arc::clone(run_ends), + Arc::new(rewrite_field_type(values, paths, current_path)), + ) + }, + Union(union_fields, mode) => { + let type_ids: Vec = union_fields.iter().map(|(id, _)| id).collect(); + let fields: Vec = union_fields.iter().map(|(_, field)| { + let field_path = if current_path.is_empty() { + field.name().to_string() + } else { + format!("{}.{}", current_path, field.name()) + }; + let make_nullable = !field.is_nullable() && paths.iter().any(|p| *p == field_path); + let mut new_field = rewrite_field_type(field, paths, &field_path); + if make_nullable { + new_field = new_field.with_nullable(true); + } + Arc::new(new_field) + }).collect(); + Union(UnionFields::new(type_ids, fields), *mode) + } + _ => dt.clone(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::{DataType, Field, Schema}; + + fn make_schema(fields: Vec) -> SchemaRef { + Arc::new(Schema::new(fields)) + } + + fn paths(strs: &[&str]) -> Vec { + strs.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn test_schema_null_empty_paths_returns_input() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema.clone(), &[]); + assert_eq!(schema, result); + } + + #[test] + fn test_schema_null_top_level_field_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Utf8, false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + assert!(result.field_with_name("a").unwrap().is_nullable()); + assert!(!result.field_with_name("b").unwrap().is_nullable()); + } + + #[test] + fn test_schema_null_already_nullable_unchanged() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, true), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + assert!(result.field_with_name("a").unwrap().is_nullable()); + } + + #[test] + fn test_schema_null_path_not_in_schema_unchanged() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema.clone(), &paths(&["z"])); + assert_eq!(schema, result); + } + + #[test] + fn test_schema_null_nested_struct_field() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + Field::new("c", DataType::Utf8, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b"])); + let a = result.field_with_name("a").unwrap(); + assert!(!a.is_nullable()); + if let Struct(fields) = a.data_type() { + assert!(fields.find("b").unwrap().1.is_nullable()); + assert!(!fields.find("c").unwrap().1.is_nullable()); + } else { + assert_eq!(true, false, "expected struct"); + } + } + + #[test] + fn test_schema_null_deeply_nested_struct() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", Struct(Fields::from(vec![ + Field::new("c", Struct(Fields::from(vec![ + Field::new("d", DataType::Int32, false), + ])), false), + ])), false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b.c.d"])); + let a = result.field_with_name("a").unwrap(); + assert!(!a.is_nullable()); + let b = match a.data_type() { Struct(f) => f.find("b").unwrap().1, _ => panic!() }; + assert!(!b.is_nullable()); + let c = match b.data_type() { Struct(f) => f.find("c").unwrap().1, _ => panic!() }; + assert!(!c.is_nullable()); + let d = match c.data_type() { Struct(f) => f.find("d").unwrap().1, _ => panic!() }; + assert!(d.is_nullable()); + } + + #[test] + fn test_schema_null_struct_itself_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + let a = result.field_with_name("a").unwrap(); + assert!(a.is_nullable()); + if let Struct(fields) = a.data_type() { + assert!(!fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct"); + } + } + + #[test] + fn test_schema_null_struct_and_child_both_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a", "a.b"])); + let a = result.field_with_name("a").unwrap(); + assert!(a.is_nullable()); + if let Struct(fields) = a.data_type() { + assert!(fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct"); + } + } + + #[test] + fn test_schema_null_list_containing_struct() { + let schema = make_schema(vec![ + Field::new("a", List(Arc::new( + Field::new("item", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + )), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b"])); + let a = result.field_with_name("a").unwrap(); + assert!(!a.is_nullable()); + if let List(inner) = a.data_type() { + if let Struct(fields) = inner.data_type() { + assert!(fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct inside list"); + } + } else { + panic!("expected list"); + } + } + + #[test] + fn test_schema_null_list_itself_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", List(Arc::new( + Field::new("item", DataType::Int32, false), + )), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + assert!(result.field_with_name("a").unwrap().is_nullable()); + } + + #[test] + fn test_schema_null_dictionary_with_nested_value() { + let schema = make_schema(vec![ + Field::new("a", Dictionary( + Box::new(DataType::Int32), + Box::new(Struct(Fields::from(vec![ + Field::new("b", DataType::Utf8, false), + ]))), + ), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b"])); + let a = result.field_with_name("a").unwrap(); + if let Dictionary(_, value) = a.data_type() { + if let Struct(fields) = value.as_ref() { + assert!(fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct in dict value"); + } + } else { + panic!("expected dictionary"); + } + } + + #[test] + fn test_schema_null_map_with_nested_value() { + let schema = make_schema(vec![ + Field::new("a", Map(Arc::new( + Field::new("entries", Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + ])), false), + ), false), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.value.b"])); + let a = result.field_with_name("a").unwrap(); + if let Map(inner, _) = a.data_type() { + if let Struct(entries) = inner.data_type() { + let value_field = entries.find("value").unwrap().1; + if let Struct(vf) = value_field.data_type() { + assert!(vf.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct in value"); + } + } else { + panic!("expected struct in map entries"); + } + } else { + panic!("expected map"); + } + } + + #[test] + fn test_schema_null_multiple_paths_across_tree() { + let schema = make_schema(vec![ + Field::new("x", DataType::Int64, false), + Field::new("y", Struct(Fields::from(vec![ + Field::new("z", DataType::Float64, false), + Field::new("w", DataType::Boolean, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["x", "y.z"])); + assert!(result.field_with_name("x").unwrap().is_nullable()); + let y = result.field_with_name("y").unwrap(); + assert!(!y.is_nullable()); + if let Struct(fields) = y.data_type() { + assert!(fields.find("z").unwrap().1.is_nullable()); + assert!(!fields.find("w").unwrap().1.is_nullable()); + } else { + panic!("expected struct"); + } + } + + #[test] + fn test_schema_null_metadata_preserved() { + let mut field = Field::new("a", DataType::Int32, false); + field.set_metadata([("key".to_string(), "val".to_string())].into()); + let schema = make_schema(vec![field]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + let a = result.field_with_name("a").unwrap(); + assert!(a.is_nullable()); + assert_eq!(a.metadata().get("key").unwrap(), "val"); + } +} diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs index 1c06954a0..3eb62e1f1 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs @@ -80,6 +80,7 @@ use crate::{ }, kernel::LogicalFileView, }; +use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields; mod exec; mod exec_meta; @@ -783,9 +784,12 @@ fn finalize_transformed_batch( // The batch adapter targets a nullability-relaxed schema (see // `expr_adapter`). Restamp to the strict logical schema; this // validates actual data nulls rather than declared nullability. + let output_schema = rewrite_schema_with_nullable_fields( + scan_plan.contract.result_schema.clone() + ); crate::kernel::cast_record_batch( &adapted, - scan_plan.contract.result_schema.clone(), + output_schema, false, false, ) diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs index 1a84ec6d4..c7333e2b7 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs @@ -37,6 +37,7 @@ use crate::delta_datafusion::DeltaScanConfig; use crate::delta_datafusion::engine::{ to_datafusion_expr, to_delta_expression, to_delta_predicate, }; +use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields; use crate::delta_datafusion::table_provider::next::FILE_ID_COLUMN_DEFAULT; use crate::kernel::{Scan, Snapshot}; @@ -321,10 +322,16 @@ impl KernelScanPlan { } else { Arc::new(scan_builder.build()?) }; - let parquet_read_schema = config.physical_arrow_schema( + // let parquet_read_schema = config.physical_arrow_schema( + // scan.snapshot().table_configuration(), + // &scan.physical_schema().as_ref().try_into_arrow()?, + // )?; + let mut parquet_read_schema = config.physical_arrow_schema( scan.snapshot().table_configuration(), &scan.physical_schema().as_ref().try_into_arrow()?, - )?; + )?; + parquet_read_schema = rewrite_schema_with_nullable_fields(parquet_read_schema); + let parquet_predicate_schema = build_parquet_predicate_schema(&parquet_read_schema, &contract.file_id_field); Ok(Self { From 6f088b6f13d7010223949e2ba5a6bfaedf79cba4 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Thu, 27 Aug 2026 14:48:47 +0300 Subject: [PATCH 14/20] [HSTACK] feat: expose snapshot load metrics in order to detect if log truncation was applied or not --- .../table_provider/next/scan/codec.rs | 1 + crates/core/src/kernel/snapshot/mod.rs | 20 +++++++++++++++++++ crates/core/src/kernel/snapshot/serde.rs | 4 +++- crates/core/src/table/mod.rs | 13 ++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs index 4b2d94e51..9eb4400f9 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs @@ -150,6 +150,7 @@ fn scan_plan_wire(scan_plan: &KernelScanPlan) -> Result>, + /// Metrics captured during snapshot loading + pub(crate) load_metrics: SnapshotLoadMetrics, } impl Snapshot { @@ -250,6 +253,7 @@ impl Snapshot { inner: snapshot, config, materialized_files: None, + load_metrics, }) } @@ -319,10 +323,13 @@ impl Snapshot { .await .map_err(|e| DeltaTableError::Generic(e.to_string()))??; + let load_metrics = SnapshotLoadMetrics::from_snapshot(snapshot.as_ref()); + let snapshot = Arc::new(Self { inner: snapshot, config: self.config.clone(), materialized_files: None, + load_metrics, }); if snapshot.version() as u64 == current_version { // `target_version` was `None`, but there were no new commits. @@ -553,6 +560,11 @@ impl Snapshot { } } + /// Get the metrics captured during snapshot loading + pub fn load_metrics(&self) -> &SnapshotLoadMetrics { + &self.load_metrics + } + /// Get the table root of the snapshot pub(crate) fn table_root_path(&self) -> DeltaResult { Ok(Path::from_url_path(self.inner.table_root().path())?) @@ -918,6 +930,7 @@ impl Snapshot { inner: self.inner.clone(), config: self.config.clone(), materialized_files, + load_metrics: self.load_metrics.clone(), } } @@ -1426,6 +1439,11 @@ impl EagerSnapshot { self.snapshot.load_config() } + /// Get the metrics captured during snapshot loading + pub fn load_metrics(&self) -> &SnapshotLoadMetrics { + &self.snapshot.load_metrics() + } + /// Well known table configuration pub fn table_properties(&self) -> &TableProperties { self.snapshot.table_properties() @@ -1618,11 +1636,13 @@ mod tests { let engine = log_store.engine(None); let snapshot = KernelSnapshot::builder_for(table_url.clone()).build(engine.as_ref())?; + let load_metrics = SnapshotLoadMetrics::from_snapshot(snapshot.as_ref()); Ok(( Self { inner: snapshot, config: Default::default(), materialized_files: None, + load_metrics, }, log_store, )) diff --git a/crates/core/src/kernel/snapshot/serde.rs b/crates/core/src/kernel/snapshot/serde.rs index fc0b75ba2..698043d30 100644 --- a/crates/core/src/kernel/snapshot/serde.rs +++ b/crates/core/src/kernel/snapshot/serde.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize, ser::SerializeSeq}; use url::Url; use crate::DeltaTableConfig; - +use crate::kernel::size_limits::SnapshotLoadMetrics; use super::{ EagerSnapshot, MaterializedFiles, MaterializedFilesPolicy, MaterializedFilesScope, Snapshot, SnapshotIdentity, SnapshotMaterializationMode, @@ -432,11 +432,13 @@ impl<'de> Visitor<'de> for SnapshotVisitor { .map_err(de::Error::custom)?; let snapshot = KernelSnapshot::new(log_segment, table_configuration); + let load_metrics = SnapshotLoadMetrics::from_snapshot(&snapshot); let snapshot = Snapshot { inner: Arc::new(snapshot), config, materialized_files: None, + load_metrics, }; let materialized_files = materialized_files .map(|value| { diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index ae60e75d3..508e891e4 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -17,6 +17,7 @@ use url::Url; use self::builder::DeltaTableConfig; use self::state::DeltaTableState; +use crate::kernel::size_limits::SnapshotLoadMetrics; use crate::kernel::{CommitInfo, DataCheck, LogicalFileView, Version}; use crate::logstore::{ LogStoreConfig, LogStoreExt, LogStoreRef, ObjectStoreRef, commit_uri_from_version, @@ -201,6 +202,18 @@ impl DeltaTable { self.state.as_ref().map(|s| s.version()) } + /// Returns the metrics captured during snapshot loading. + /// + /// This method provides access to information about how the snapshot was loaded, + /// including whether log size limiting was applied and if truncation occurred. + /// + /// ## Returns + /// + /// A reference to the snapshot load metrics if the table has been loaded, `None` otherwise. + pub fn snapshot_load_metrics(&self) -> Option<&SnapshotLoadMetrics> { + self.state.as_ref().map(|s| s.snapshot().load_metrics()) + } + /// Load DeltaTable with data from latest checkpoint pub async fn load(&mut self) -> Result<(), DeltaTableError> { self.update_incremental(None).await From d92dabe168927ea6413dcf8f2f760e7ea547c761 Mon Sep 17 00:00:00 2001 From: Daniel Vlascenco Date: Tue, 7 Jul 2026 20:18:29 +0300 Subject: [PATCH 15/20] [HSTACK] feat: delta engine injection support --- crates/core/src/delta_datafusion/mod.rs | 1 + .../table_provider/next/mod.rs | 6 ++- crates/core/src/kernel/snapshot/mod.rs | 38 ++++++++++++++----- crates/core/src/table/builder.rs | 22 +++++++++++ 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index b8e523850..17bf06559 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -73,6 +73,7 @@ pub(crate) use column_mapping::ColumnMappingState; pub(crate) use data_validation::{ DataValidationExec, constraints_to_exprs, generated_columns_to_exprs, validation_predicates, }; +pub use engine::DataFusionEngine; pub(crate) use find_files::*; pub(crate) use table_provider::next::normalize_path_as_file_id; pub use table_provider::{ diff --git a/crates/core/src/delta_datafusion/table_provider/next/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/mod.rs index b95360c86..7f1d40338 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/mod.rs @@ -733,7 +733,11 @@ impl TableProvider for DeltaScan { limit: Option, ) -> Result> { self.ensure_read_ready(session)?; - let engine = DataFusionEngine::new_from_session(session); + // let engine = DataFusionEngine::new_from_session(session); + let engine: Arc = self.snapshot.snapshot().config.engine.as_ref() + .map(|e| e.0.clone()) + .unwrap_or_else(|| DataFusionEngine::new_from_session(session)); + let contract = ProjectedScanContract::try_new( self.scan_schema.clone(), self.full_schema.clone(), diff --git a/crates/core/src/kernel/snapshot/mod.rs b/crates/core/src/kernel/snapshot/mod.rs index 73797a4e6..2504c0d14 100644 --- a/crates/core/src/kernel/snapshot/mod.rs +++ b/crates/core/src/kernel/snapshot/mod.rs @@ -257,6 +257,12 @@ impl Snapshot { }) } + /// Returns the engine from the config, or falls back to the log store's default engine. + fn resolve_engine(config: &DeltaTableConfig, log_store: &dyn LogStore) -> Arc { + config.engine.as_ref().map(|e| e.0.clone()) + .unwrap_or_else(|| log_store.engine(None)) + } + /// Create a new [`Snapshot`] instance pub async fn try_new( log_store: &dyn LogStore, @@ -264,7 +270,8 @@ impl Snapshot { version: Option, ) -> DeltaResult { // TODO: bundle operation_id with logstore ... - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = Self::resolve_engine(&config, log_store); // NB: kernel engine uses Url::join to construct paths, // if the path does not end with a slash, the would override the entire path. @@ -277,6 +284,11 @@ impl Snapshot { Self::try_new_with_engine(engine, table_root, config, version).await } + /// Returns the configured engine, falling back to the log store's default engine. + pub(crate) fn engine(&self, log_store: &dyn LogStore) -> Arc { + Self::resolve_engine(&self.config, log_store) + } + /// Create a [`ScanBuilder`] borrowing this snapshot to configure a read of the table. pub fn scan_builder(&self) -> ScanBuilder { ScanBuilder::new(self.inner.clone()) @@ -478,7 +490,8 @@ impl Snapshot { return Ok(self); } - self.materialize_files_with_engine(log_store.engine(None), None) + let engine = self.engine(log_store); + self.materialize_files_with_engine(engine, None) .await } @@ -618,8 +631,11 @@ impl Snapshot { { return cached; } + + let engine = self.engine(log_store); + if predicate.is_some() && self.config.skip_stats { - return self.files_with_engine(log_store.engine(None), predicate); + return self.files_with_engine(engine, predicate); } match self @@ -630,14 +646,14 @@ impl Snapshot { let (existing_version, existing_data, existing_predicate) = materialized_seed.into_parts(); self.files_from( - log_store.engine(None), + engine, predicate, existing_version, Box::new(existing_data), existing_predicate, ) } - None => self.files_with_engine(log_store.engine(None), predicate), + None => self.files_with_engine(engine, predicate), } } @@ -1070,7 +1086,8 @@ impl Snapshot { let tx = builder.tx(); // TODO: bundle operation id with log store ... - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = self.engine(log_store); let remove_data = match self .inner @@ -1117,7 +1134,8 @@ impl Snapshot { app_id: String, ) -> DeltaResult> { // TODO: bundle operation id with log store ... - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = self.engine(log_store); let inner = self.inner.clone(); let version = spawn_blocking_with_span(move || inner.get_app_id_version(&app_id, engine.as_ref())) @@ -1139,7 +1157,8 @@ impl Snapshot { log_store: &dyn LogStore, domain: impl ToString, ) -> DeltaResult> { - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = self.engine(log_store); let inner = self.inner.clone(); let domain = domain.to_string(); let metadata = @@ -1376,9 +1395,10 @@ impl EagerSnapshot { target_version: Option, ) -> DeltaResult<()> { let previous_snapshot = self.snapshot.clone(); + let engine = self.snapshot.engine(log_store); let updated_snapshot = previous_snapshot .clone() - .update(log_store.engine(None), target_version) + .update(engine, target_version) .await?; if Arc::ptr_eq(&updated_snapshot, &previous_snapshot) { return Ok(()); diff --git a/crates/core/src/table/builder.rs b/crates/core/src/table/builder.rs index 4f100ef0c..a3ec2e1ab 100644 --- a/crates/core/src/table/builder.rs +++ b/crates/core/src/table/builder.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use std::sync::Arc; use chrono::{DateTime, FixedOffset, Utc}; +use delta_kernel::Engine; use deltalake_derive::DeltaConfig; use object_store::DynObjectStore; use serde::{Deserialize, Serialize}; @@ -30,6 +31,16 @@ pub enum DeltaVersion { Timestamp(DateTime), } +/// Wrapper around `Arc` that implements `Debug` and `Clone`. +#[derive(Clone)] +pub struct EngineRef(pub Arc); + +impl std::fmt::Debug for EngineRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "EngineRef()") + } +} + /// Configuration options for delta table #[derive(Debug, Serialize, Deserialize, Clone, DeltaConfig)] #[serde(rename_all = "camelCase")] @@ -71,6 +82,11 @@ pub struct DeltaTableConfig { #[delta(skip)] pub log_size_limiter: Option, + /// Custom kernel engine to use for snapshot loading and scan operations. + /// When set, this engine is used instead of the default engine derived from the log store. + #[serde(skip_serializing, skip_deserializing)] + #[delta(skip)] + pub engine: Option, } impl Default for DeltaTableConfig { @@ -82,6 +98,7 @@ impl Default for DeltaTableConfig { skip_stats: false, io_runtime: None, log_size_limiter: None, + engine: None, } } } @@ -161,6 +178,11 @@ impl DeltaTableBuilder { self } + /// Sets a custom kernel `Engine` to use for snapshot loading and scan operations. + pub fn with_engine(mut self, engine: Arc) -> Self { + self.table_config.engine = Some(EngineRef(engine)); + self + } /// Sets `version` to the builder pub fn with_version(mut self, version: Version) -> Self { From f740a799b240f33a6f4ed9673a1a905034428c58 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Tue, 7 Jul 2026 20:49:25 +0300 Subject: [PATCH 16/20] [HSTACK] feat: add ability to construct Snapshot and EagerSnapshot manually --- crates/core/src/hacks.rs | 18 ++++++++++++++++++ crates/core/src/kernel/snapshot/mod.rs | 4 ++-- crates/core/src/lib.rs | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 crates/core/src/hacks.rs diff --git a/crates/core/src/hacks.rs b/crates/core/src/hacks.rs new file mode 100644 index 000000000..2b6ff32a5 --- /dev/null +++ b/crates/core/src/hacks.rs @@ -0,0 +1,18 @@ +use std::sync::Arc; +use crate::kernel::{EagerSnapshot, Snapshot}; +use delta_kernel::snapshot::Snapshot as KernelSnapshot; + +pub fn new_delta_eager_snapshot(snapshot: Snapshot) -> EagerSnapshot { + EagerSnapshot { + snapshot: Arc::new(snapshot), + } +} + +pub fn new_delta_snapshot(input: &Snapshot, kernel_snapshot: KernelSnapshot) -> Snapshot { + Snapshot { + inner: Arc::new(kernel_snapshot), + config: input.config.clone(), + materialized_files: input.materialized_files.clone(), + load_metrics: input.load_metrics.clone(), + } +} \ No newline at end of file diff --git a/crates/core/src/kernel/snapshot/mod.rs b/crates/core/src/kernel/snapshot/mod.rs index 2504c0d14..002eef718 100644 --- a/crates/core/src/kernel/snapshot/mod.rs +++ b/crates/core/src/kernel/snapshot/mod.rs @@ -203,7 +203,7 @@ impl<'a> ConflictReadSet<'a> { #[derive(Debug, Clone, PartialEq)] pub struct Snapshot { /// Log segment containing all log files in the snapshot - pub(crate) inner: Arc, + pub inner: Arc, /// Configuration for the current session pub(crate) config: DeltaTableConfig, /// Optional materialized replay state owned by this snapshot. @@ -1280,7 +1280,7 @@ impl MaterializedFiles { /// A snapshot of a Delta table that has been eagerly loaded into memory. #[derive(Debug, Clone, PartialEq)] pub struct EagerSnapshot { - snapshot: Arc, + pub snapshot: Arc, } /// Read `_last_checkpoint` and return the hinted version when present. diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 5795e78bd..77fd551bb 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -87,6 +87,7 @@ pub(crate) mod parquet_utils; pub mod protocol; pub use kernel::schema; pub mod table; +pub mod hacks; /// Test fixtures and helpers (reference tables, env scoping, batch assertions) exposed for /// integration tests of this crate and downstream crates. From 419198aaeafeef127df39dc67b5cbfb89a2e19f4 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Thu, 23 Jul 2026 10:34:00 +0300 Subject: [PATCH 17/20] [HSTACK] chore: downgrade time dep to overcome incompat with lancedb --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ed06c3cf6..764cefd07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ prost = { version = "0.14.3" } # "stdlib" bytes = { version = "1" } chrono = { version = "0.4.40", default-features = false, features = ["clock"] } -time = { version = "=0.3.53" } +time = { version = "=0.3.47" } tracing = { version = "0.1", features = ["log"] } regex = { version = "1" } thiserror = { version = "2" } From 1be32fe91f1a342b23e854bad75238649955af51 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Thu, 13 Aug 2026 14:50:35 +0300 Subject: [PATCH 18/20] [HSTACK] Delta-rs controls stats_parsed loading in the kernel --- Cargo.toml | 47 +- .../core/src/delta_datafusion/find_files.rs | 2 +- .../table_provider/next/mod.rs | 216 +++ .../table_provider/next/scan/limit.rs | 232 ++++ .../table_provider/next/scan/mod.rs | 60 +- .../table_provider/next/scan/replay.rs | 56 +- crates/core/src/kernel/snapshot/iterators.rs | 4 +- .../src/kernel/snapshot/iterators/scan_row.rs | 272 +++- .../snapshot/iterators/scan_row_cached.rs | 1219 +++++++++++++++++ crates/core/src/kernel/snapshot/mod.rs | 146 +- crates/core/src/kernel/snapshot/scan.rs | 181 ++- .../snapshot/scan/stats_options_tests.rs | 77 ++ crates/core/src/kernel/snapshot/serde.rs | 11 +- .../src/kernel/snapshot/stats_projection.rs | 4 + crates/core/src/table/builder.rs | 14 + .../tests/it/eager_cache_schema_evolution.rs | 158 +++ crates/core/tests/it/main.rs | 1 + crates/core/tests/it_datafusion/main.rs | 1 + .../it_datafusion/stats_parsed_checkpoint.rs | 447 ++++++ 19 files changed, 3045 insertions(+), 103 deletions(-) create mode 100644 crates/core/src/delta_datafusion/table_provider/next/scan/limit.rs create mode 100644 crates/core/src/kernel/snapshot/iterators/scan_row_cached.rs create mode 100644 crates/core/src/kernel/snapshot/scan/stats_options_tests.rs create mode 100644 crates/core/tests/it/eager_cache_schema_evolution.rs create mode 100644 crates/core/tests/it_datafusion/stats_parsed_checkpoint.rs diff --git a/Cargo.toml b/Cargo.toml index 764cefd07..46bc0f82a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,14 +16,14 @@ documentation = "https://docs.rs/deltalake" repository = "https://github.com/delta-io/delta.rs" [workspace.dependencies] -#delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [ -delta_kernel = { package = "buoyant_kernel", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1", features = [ +# delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [ +delta_kernel = { package = "buoyant_kernel", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = [ #delta_kernel = { package = "buoyant_kernel", version = "0.25.0,<0.25.100", features = [ "arrow-58", "internal-api", ] } -#delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false } -delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1", features = ["arrow-58", "rustls"], default-features = false } +# delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false } +delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = ["arrow-58", "rustls"], default-features = false } #delta_kernel_default_engine = { package = "buoyant_kernel_engine", version = "0.25.0,<0.25.100", features = ["arrow-58", "rustls"], default-features = false } @@ -198,5 +198,40 @@ datafusion-session = { git = 'https://github.com/hstack/datafusion.git', branch datafusion-spark = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } datafusion-sql = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } datafusion-substrait = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } -buoyant_kernel = { git = 'https://github.com/hstack/delta-kernel-rs.git', branch = 'buoyant-v0.25.1' } -#buoyant_kernel = { path = "../delta-kernel-rs-main-024/kernel" } +#datafusion = { path = "../datafusion/datafusion/core" } +#datafusion-catalog = { path = "../datafusion/datafusion/catalog" } +#datafusion-catalog-listing = { path = "../datafusion/datafusion/catalog-listing" } +#datafusion-common = { path = "../datafusion/datafusion/common" } +#datafusion-common-runtime = { path = "../datafusion/datafusion/common-runtime" } +#datafusion-datasource = { path = "../datafusion/datafusion/datasource" } +#datafusion-datasource-arrow = { path = "../datafusion/datafusion/datasource-arrow" } +#datafusion-datasource-avro = { path = "../datafusion/datafusion/datasource-avro" } +#datafusion-datasource-csv = { path = "../datafusion/datafusion/datasource-csv" } +#datafusion-datasource-json = { path = "../datafusion/datafusion/datasource-json" } +#datafusion-datasource-parquet = { path = "../datafusion/datafusion/datasource-parquet" } +#datafusion-execution = { path = "../datafusion/datafusion/execution" } +#datafusion-expr = { path = "../datafusion/datafusion/expr" } +#datafusion-expr-common = { path = "../datafusion/datafusion/expr-common" } +#datafusion-ffi = { path = "../datafusion/datafusion/ffi" } +#datafusion-functions = { path = "../datafusion/datafusion/functions" } +#datafusion-functions-aggregate = { path = "../datafusion/datafusion/functions-aggregate" } +#datafusion-functions-aggregate-common = { path = "../datafusion/datafusion/functions-aggregate-common" } +#datafusion-functions-nested = { path = "../datafusion/datafusion/functions-nested" } +#datafusion-functions-table = { path = "../datafusion/datafusion/functions-table" } +#datafusion-functions-window = { path = "../datafusion/datafusion/functions-window" } +#datafusion-functions-window-common = { path = "../datafusion/datafusion/functions-window-common" } +#datafusion-optimizer = { path = "../datafusion/datafusion/optimizer" } +#datafusion-physical-expr = { path = "../datafusion/datafusion/physical-expr" } +#datafusion-physical-expr-adapter = { path = "../datafusion/datafusion/physical-expr-adapter" } +#datafusion-physical-expr-common = { path = "../datafusion/datafusion/physical-expr-common" } +#datafusion-physical-optimizer = { path = "../datafusion/datafusion/physical-optimizer" } +#datafusion-physical-plan = { path = "../datafusion/datafusion/physical-plan" } +#datafusion-proto = { path = "../datafusion/datafusion/proto" } +#datafusion-proto-common = { path = "../datafusion/datafusion/proto-common" } +#datafusion-pruning = { path = "../datafusion/datafusion/pruning" } +#datafusion-session = { path = "../datafusion/datafusion/session" } +#datafusion-spark = { path = "../datafusion/datafusion/spark" } +#datafusion-sql = { path = "../datafusion/datafusion/sql" } + +#buoyant_kernel = { git = 'https://github.com/hstack/delta-kernel-rs.git', branch = 'buoyant-v0.25.1-stats' } +#buoyant_kernel = { path = "../delta-kernel-rs/kernel" } diff --git a/crates/core/src/delta_datafusion/find_files.rs b/crates/core/src/delta_datafusion/find_files.rs index 0a9988d7f..f2c37b172 100644 --- a/crates/core/src/delta_datafusion/find_files.rs +++ b/crates/core/src/delta_datafusion/find_files.rs @@ -215,7 +215,7 @@ impl TreeNodeVisitor<'_> for FindFilesExprProperties { /// rather than materializing every partition column — which avoids coercing values the /// predicate never reads and prevents an unrelated/malformed partition value from /// weakening pruning. -pub(crate) fn extract_partition_only_predicate( +pub fn extract_partition_only_predicate( predicate: Expr, partition_columns: &[String], ) -> DeltaResult)>> { diff --git a/crates/core/src/delta_datafusion/table_provider/next/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/mod.rs index 7f1d40338..ed2fbd2ba 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/mod.rs @@ -857,6 +857,7 @@ pub(crate) fn test_multi_partitioned_override_schema() -> SchemaRef { #[cfg(test)] mod tests { + use tracing::info; use arrow::{ array::{ BooleanArray, Date32Array, Int32Array, Int64Array, StringArray, @@ -972,6 +973,7 @@ mod tests { struct DeltaScanVisitor { num_scanned: Option, total_bytes_scanned: Option, + num_pruned_by_limit: Option, } impl DeltaScanVisitor { @@ -986,6 +988,9 @@ mod tests { self.num_scanned = metrics .sum_by_name("count_files_scanned") .map(|v| v.as_usize()); + self.num_pruned_by_limit = metrics + .sum_by_name("count_files_pruned_by_limit") + .map(|v| v.as_usize()); Ok(true) } @@ -2961,4 +2966,215 @@ mod tests { Ok(()) } + + + /// LIMIT 1 with no filters: pruning fires, only one file reaches the Parquet reader. + #[tokio::test] + async fn test_limit_pushdown_no_filter_trims_files() -> TestResult { + let id_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + ArrowDataType::Int64, + true, + )])); + + // Three single-row files, one per write. + let table = create_in_memory_id_table_with_rows(vec![1]).await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema.clone(), + vec![Arc::new(Int64Array::from(vec![2]))], + )?]) + .await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema, + vec![Arc::new(Int64Array::from(vec![3]))], + )?]) + .await?; + + let provider = DeltaScan::builder() + .with_log_store(table.log_store()) + .build() + .await?; + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let plan = provider.scan(&state, None, &[], Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + // The headline invariant: file-level pruning kicked in, so only one file + // made it into the scan plan even though three exist in the snapshot. + assert_eq!( + visitor.num_scanned, + Some(1), + "LIMIT 1 should prune to exactly one file" + ); + // The pruned-by-limit metric counts files the stream visited and then + // dropped. Kernel emits scan-metadata in batches, and early-break drops + // the stream before later batches arrive, so the exact value depends on + // kernel batching. We only check that it is non-negative. + assert!(visitor.num_pruned_by_limit.is_some()); + + let batches: Vec<_> = collect_partitioned(plan, session.task_ctx()) + .await? + .into_iter() + .flatten() + .collect(); + let row_count: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(row_count, 1, "LIMIT 1 should return exactly one row"); + + Ok(()) + } + + /// LIMIT 1 + Inexact (data-column) filter: defensive gate disables file pruning even when + /// the caller passes `Some(N)`. DataFusion's own planner would not pass `Some(1)` here in + /// production, but the test exercises the local invariant. + #[tokio::test] + async fn test_limit_pushdown_inexact_filter_does_not_trim_files() -> TestResult { + use datafusion::prelude::{col, lit}; + + let id_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + ArrowDataType::Int64, + true, + )])); + + let table = create_in_memory_id_table_with_rows(vec![1]).await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema.clone(), + vec![Arc::new(Int64Array::from(vec![2]))], + )?]) + .await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema, + vec![Arc::new(Int64Array::from(vec![3]))], + )?]) + .await?; + + let provider = DeltaScan::builder() + .with_log_store(table.log_store()) + .build() + .await?; + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let filters = vec![col("id").gt(lit(0_i64))]; + let plan = provider.scan(&state, None, &filters, Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(3), + "Inexact filter should leave all three files in the scan plan" + ); + assert_eq!(visitor.num_pruned_by_limit, Some(0)); + + Ok(()) + } + + /// LIMIT 1 with an explicit FileSelection: pinned files win, file-level pruning is bypassed. + #[tokio::test] + async fn test_limit_pushdown_file_selection_bypasses_pruning() -> TestResult { + let log_store = TestTables::Simple.table_builder()?.build_storage()?; + let snapshot = Arc::new(Snapshot::try_new(&log_store, Default::default(), None).await?); + let table_root = snapshot.scan_builder().build()?.table_root().clone(); + + // Pin to the first two files explicitly. + let selected_paths: Vec = snapshot + .file_views(log_store.as_ref(), None) + .take(2) + .map_ok(|v| table_root.join(v.path_raw()).unwrap().to_string()) + .try_collect() + .await?; + assert_eq!(selected_paths.len(), 2); + let selection = FileSelection::from_file_paths(selected_paths); + + let provider = DeltaScan::builder() + .with_snapshot(snapshot) + .with_file_column(FILE_ID_COLUMN_DEFAULT) + .build() + .await? + .with_file_selection(selection); + + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + let plan = provider.scan(&state, None, &[], Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(2), + "FileSelection should override file-level limit pruning" + ); + assert_eq!(visitor.num_pruned_by_limit, Some(0)); + + Ok(()) + } + + /// LIMIT 1 + partition-only Exact filter on a multi-file partition: pruning trims to one + /// matching file. Partition predicates are classified Exact, so the safety gate allows pruning. + #[tokio::test] + async fn test_limit_pushdown_partition_only_filter_trims_files() -> TestResult { + use datafusion::prelude::{col, lit}; + + let row_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("part", ArrowDataType::Utf8, true), + ArrowField::new("value", ArrowDataType::Int32, true), + ])); + let make_batch = |part: &str, value: i32| -> crate::DeltaResult { + Ok(RecordBatch::try_new( + row_schema.clone(), + vec![ + Arc::new(StringArray::from(vec![part])), + Arc::new(Int32Array::from(vec![value])), + ], + )?) + }; + + // Two partitions × one+two files. Partition A has two files, B has one. + let table = crate::DeltaTable::new_in_memory() + .write(vec![make_batch("A", 1)?]) + .with_partition_columns(vec!["part"]) + .await?; + let table = table.write(vec![make_batch("B", 2)?]).await?; + let table = table.write(vec![make_batch("A", 3)?]).await?; + + + info!("TEST START ---------------------------------------"); + let provider = DeltaScan::builder() + .with_log_store(table.log_store()) + .build() + .await?; + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let filters = vec![col("part").eq(lit("A"))]; + let plan = provider.scan(&state, None, &filters, Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(1), + "Exact partition filter + LIMIT 1 should leave a single matching file" + ); + // Pruned-by-limit count depends on kernel batch shape (see no-filter + // test for the rationale); we assert presence, not value. + assert!(visitor.num_pruned_by_limit.is_some()); + + let batches: Vec<_> = collect_partitioned(plan, session.task_ctx()) + .await? + .into_iter() + .flatten() + .collect(); + let row_count: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(row_count, 1); + + Ok(()) + } } diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/limit.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/limit.rs new file mode 100644 index 000000000..9703b5fd1 --- /dev/null +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/limit.rs @@ -0,0 +1,232 @@ +//! File-level row-count accumulator for `LIMIT` pushdown. +//! +//! This module owns the correctness logic that decides — given a stream of +//! [`ScanFileContext`]s and a row cap — which files to keep and which to drop. +//! It is intentionally pure (no async, no I/O) so it can be unit-tested in +//! isolation from `replay_files` and the kernel scan-metadata stream. +//! +//! Rules: +//! - Files with a deletion vector (`has_dv == true`) go into a set-aside pool, +//! because a DV may delete an arbitrary number of rows and `num_records` +//! would overstate the file's contribution to the limit. +//! - Files missing `num_records` stats go into the same set-aside pool for the +//! same reason. +//! - Files with exact stats and no DV accumulate; once `rows_collected >= +//! limit` the outer loop should break. +//! - At finalize time, the set-aside pool is drained into the kept list iff +//! the accumulator is still short of the limit; otherwise its entries are +//! counted as pruned. + +use datafusion::common::stats::Precision; +use tracing::info; +use super::replay::ScanFileContext; + +/// Per-scan accumulator for file-level limit pruning. See module docs. +#[derive(Debug)] +pub(crate) struct LimitPruneState { + limit: usize, + rows_collected: usize, + files_with_exact_num_records: Vec, + files_with_unknown_num_records: Vec, +} + +impl LimitPruneState { + pub(crate) fn new(limit: usize) -> Self { + Self { + limit, + rows_collected: 0, + files_with_exact_num_records: Vec::new(), + files_with_unknown_num_records: Vec::new(), + } + } + + /// Try to accept one file. Returns `true` when the row cap has been reached + /// and the outer scan-metadata loop should break. + pub(crate) fn accept(&mut self, file: ScanFileContext) -> bool { + if file.has_deletion_vector { + self.files_with_unknown_num_records.push(file); + return false; + } + info!("ACCEPT {} = {:?}", file.file_url.as_str(), file.stats); + match file.stats.num_rows { + Precision::Exact(n) => { + self.rows_collected = self.rows_collected.saturating_add(n); + self.files_with_exact_num_records.push(file); + self.rows_collected >= self.limit + } + _ => { + self.files_with_unknown_num_records.push(file); + false + } + } + } + + /// Consume the state, returning `(kept_files, count_files_pruned_by_limit)`. + /// + /// The set-aside pool is drained into `kept_files` only when the accumulator + /// is still short of the row cap; otherwise its entries are counted as pruned. + pub(crate) fn finalize(mut self) -> (Vec, usize) { + if self.rows_collected < self.limit { + self.files_with_exact_num_records.extend(self.files_with_unknown_num_records.drain(..)); + (self.files_with_exact_num_records, 0) + } else { + let pruned = self.files_with_unknown_num_records.len(); + (self.files_with_exact_num_records, pruned) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use datafusion::common::Statistics; + use url::Url; + + use crate::delta_datafusion::table_provider::next::scan::replay::ScanFileContext; + + fn stub_file(url: &str, num_rows: Precision, has_deletion_vector: bool) -> ScanFileContext { + let stats = Statistics { + num_rows, + total_byte_size: Precision::Absent, + column_statistics: Vec::new(), + }; + ScanFileContext { + file_url: Url::parse(url).unwrap(), + size: 0, + transform: None, + stats, + partitions: None, + has_deletion_vector, + } + } + + #[test] + fn accept_exact_stats_accumulates_until_cap_reached() { + let mut state = LimitPruneState::new(10); + + let stop = state.accept(stub_file("file:///a", Precision::Exact(4), false)); + assert!( + !stop, + "accumulator at 4 rows should not signal stop with cap=10" + ); + + let stop = state.accept(stub_file("file:///b", Precision::Exact(7), false)); + assert!( + stop, + "accumulator at 11 rows should signal stop with cap=10" + ); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 2); + assert_eq!(pruned, 0); + } + + #[test] + fn dv_files_go_to_unknown_pool() { + let mut state = LimitPruneState::new(5); + + let stop = state.accept(stub_file("file:///dv", Precision::Exact(1000), true)); + assert!( + !stop, + "DV files must not contribute to row count regardless of stats" + ); + + let (files, pruned) = state.finalize(); + assert_eq!( + files.len(), + 1, + "DV file should be drained from unknown pool when limit unmet" + ); + assert_eq!(pruned, 0); + } + + #[test] + fn absent_stats_go_to_unknown_pool() { + let mut state = LimitPruneState::new(5); + + let stop = state.accept(stub_file("file:///stats-less", Precision::Absent, false)); + assert!(!stop); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 1); + assert_eq!(pruned, 0); + } + + #[test] + fn unknown_pool_is_dropped_when_cap_already_satisfied() { + let mut state = LimitPruneState::new(3); + + // DV file goes into unknown pool (does not contribute to rows_collected). + let _ = state.accept(stub_file("file:///dv", Precision::Exact(10), true)); + // Exact-stats file saturates the cap. + let stop = state.accept(stub_file("file:///exact", Precision::Exact(5), false)); + assert!(stop, "exact 5 rows should saturate cap=3"); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 1, "only the exact-stats file should be kept"); + assert_eq!(pruned, 1, "the unknown-pool DV file should count as pruned"); + } + + #[test] + fn unknown_pool_is_drained_when_cap_unmet() { + let mut state = LimitPruneState::new(100); + + let _ = state.accept(stub_file("file:///exact", Precision::Exact(5), false)); + let _ = state.accept(stub_file("file:///dv", Precision::Exact(10), true)); + let _ = state.accept(stub_file("file:///no-stats", Precision::Absent, false)); + + let (files, pruned) = state.finalize(); + assert_eq!( + files.len(), + 3, + "all files should be returned when cap is not reached" + ); + assert_eq!(pruned, 0); + } + + #[test] + fn inexact_stats_go_to_unknown_pool() { + let mut state = LimitPruneState::new(5); + + let stop = state.accept(stub_file("file:///inexact", Precision::Inexact(3), false)); + assert!(!stop); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 1); + assert_eq!(pruned, 0); + } + + #[test] + fn saturating_add_handles_pathological_num_records() { + let mut state = LimitPruneState::new(10); + let stop = state.accept(stub_file( + "file:///big", + Precision::Exact(usize::MAX), + false, + )); + assert!( + stop, + "even saturated accumulator must signal stop once cap is reached" + ); + + let (files, _) = state.finalize(); + assert_eq!(files.len(), 1); + } + + #[test] + fn limit_greater_than_total_keeps_everything() { + let mut state = LimitPruneState::new(1_000_000); + + for i in 0..3 { + let url = format!("file:///f{i}"); + let stop = state.accept(stub_file(&url, Precision::Exact(10), false)); + assert!(!stop, "small files should never trip a generous cap"); + } + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 3); + assert_eq!(pruned, 0); + } +} + diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs index 3eb62e1f1..42f415a1b 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs @@ -46,6 +46,7 @@ use datafusion::{ }, prelude::Expr, }; +use datafusion::logical_expr::TableProviderFilterPushDown; use datafusion_datasource::{ PartitionedFile, TableSchema, compute_all_files_statistics, file_groups::FileGroup, file_scan_config::FileScanConfigBuilder, source::DataSourceExec, @@ -81,6 +82,7 @@ use crate::{ kernel::LogicalFileView, }; use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields; +use crate::delta_datafusion::table_provider::next::scan::limit::LimitPruneState; mod exec; mod exec_meta; @@ -88,6 +90,7 @@ mod expr_adapter; mod plan; mod replay; mod codec; +pub mod limit; type ScanMetadataStream = Pin> + Send>>; type PublicFileIdMap = HashMap; @@ -117,7 +120,24 @@ pub(super) async fn execution_plan( ))); } - let replayed = replay_files(engine, &scan_plan, config.clone(), stream, file_selection).await?; + // File-level limit pruning is safe iff: + // - the caller asked for a row cap, + // - no explicit FileSelection is in play (explicit pins win), and + // - every filter is marked Exact (DataFusion's own contract already forbids + // passing `Some(N)` here when an Inexact filter would survive above the + // scan; this check is a defensive duplicate so the invariant is local). + let prune_by_limit: Option = limit.filter(|_| { + file_selection.is_none() && { + let filter_refs: Vec<&Expr> = scan_plan.filters.iter().collect(); + let pushdown = + supports_filters_pushdown(&filter_refs, scan_plan.table_configuration(), config); + pushdown + .iter() + .all(|p| matches!(p, TableProviderFilterPushDown::Exact)) + } + }); + + let replayed = replay_files(engine, &scan_plan, config.clone(), stream, file_selection, prune_by_limit).await?; let file_id_field = scan_plan.contract.file_id_field.clone(); if scan_plan.is_metadata_only() && !scan_plan.contract.retain_row_index { @@ -309,6 +329,7 @@ async fn replay_files( scan_config: DeltaScanConfig, stream: ScanMetadataStream, file_selection: Option<&ResolvedFileSelection>, + prune_by_limit: Option, ) -> Result { let mut stream = ScanFileStream::new( engine, @@ -317,10 +338,34 @@ async fn replay_files( file_selection.map(|selection| &selection.active_file_ids), stream, ); - let mut files = Vec::new(); - while let Some(file) = stream.try_next().await? { - files.extend(file); - } + // let mut files = Vec::new(); + // while let Some(file) = stream.try_next().await? { + // files.extend(file); + // } + + let mut files = if let Some(cap) = prune_by_limit { + let mut state = LimitPruneState::new(cap); + 'outer: while let Some(batch_files) = stream.try_next().await? { + for file in batch_files { + if state.accept(file) { + break 'outer; + } + } + } + // The second tuple element (unknown-pool drops) is a subset of + // `count_files_pruned_by_limit`, which we compute below from the + // stream's total visit count minus the final file set. That formula + // also captures files in the partially-consumed final batch that we + // dropped on early-break without going through the unknown pool. + let (kept, _) = state.finalize(); + kept + } else { + let mut files = Vec::new(); + while let Some(batch_files) = stream.try_next().await? { + files.extend(batch_files); + } + files + }; let mut public_file_ids = PublicFileIdMap::default(); if scan_plan.contract.retain_file_id { @@ -353,6 +398,10 @@ async fn replay_files( MetricBuilder::new(&metrics) .global_counter("count_files_scanned") .add(stream.metrics.num_scanned); + let pruned_by_limit = stream.metrics.num_scanned.saturating_sub(files.len()); + MetricBuilder::new(&metrics) + .global_counter("count_files_pruned_by_limit") + .add(pruned_by_limit); Ok(ReplayedScanFiles { files, @@ -1051,6 +1100,7 @@ mod tests { transform: None, stats: Statistics::new_unknown(&Schema::empty()), partitions: None, + has_deletion_vector: false, } } diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs index 4ee4c1717..b1809c565 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs @@ -372,6 +372,9 @@ pub(crate) struct ScanFileContext { pub stats: Statistics, /// Partition values for the file. pub partitions: Option, + /// Whether this file carries a deletion vector. Cached at construction time so consumers + /// can branch on DV presence without waiting on the asynchronous DV-load task. + pub has_deletion_vector: bool, } impl ScanFileContext { @@ -383,6 +386,7 @@ impl ScanFileContext { transform: inner.transform, stats, partitions, + has_deletion_vector: inner.dv_info.has_vector() } } } @@ -500,10 +504,16 @@ fn visit_scan_file(ctx: &mut ScanContext, scan_file: ScanFile) { mod tests { use std::collections::HashMap; + use arrow_schema::Schema; + use datafusion::common::Statistics; + use delta_kernel::actions::deletion_vector::{ + DeletionVectorDescriptor, DeletionVectorStorageType, + }; + use delta_kernel::scan::state::{DvInfo, ScanFile}; use url::Url; - use super::{ScanContext, visit_scan_file}; + use super::{ScanContext, visit_scan_file, ScanFileContext, ScanFileContextInner}; fn scan_file(path: impl Into) -> ScanFile { ScanFile { @@ -536,4 +546,48 @@ mod tests { "file:///tmp/delta/part-000.parquet" ); } + + #[test] + fn test_scan_file_context_propagates_has_dv_from_inner() { + let url = Url::parse("file:///tmp/f.parquet").unwrap(); + let descriptor = DeletionVectorDescriptor { + storage_type: DeletionVectorStorageType::Inline, + path_or_inline_dv: String::new(), + offset: None, + size_in_bytes: 0, + cardinality: 0, + }; + let inner_with_dv = ScanFileContextInner { + file_url: url.clone(), + size: 0, + transform: None, + num_records: None, + dv_info: DvInfo::from(descriptor), + }; + let inner_without_dv = ScanFileContextInner { + file_url: url, + size: 0, + transform: None, + num_records: None, + dv_info: DvInfo::default(), + }; + + let empty_schema = Schema::empty(); + let ctx_with = + ScanFileContext::new(inner_with_dv, Statistics::new_unknown(&empty_schema), None); + let ctx_without = ScanFileContext::new( + inner_without_dv, + Statistics::new_unknown(&empty_schema), + None, + ); + + assert!( + ctx_with.has_deletion_vector, + "ScanFileContext built from inner with DV should expose has_dv=true" + ); + assert!( + !ctx_without.has_deletion_vector, + "ScanFileContext built from inner without DV should expose has_dv=false" + ); + } } diff --git a/crates/core/src/kernel/snapshot/iterators.rs b/crates/core/src/kernel/snapshot/iterators.rs index 9bcdce28a..a81a84a7b 100644 --- a/crates/core/src/kernel/snapshot/iterators.rs +++ b/crates/core/src/kernel/snapshot/iterators.rs @@ -27,10 +27,12 @@ use crate::kernel::scalars::ScalarExt; use crate::kernel::{Add, DeletionVectorDescriptor, Remove}; use crate::{DeltaResult, DeltaTableError}; -pub(crate) use self::scan_row::{ScanRowOutStream, scan_row_in_eval}; +pub(crate) use self::scan_row::ScanRowOutStream; +pub(crate) use self::scan_row_cached::CachedScanRowEvaluator; mod scan_row; mod tombstones; +mod scan_row_cached; const FIELD_NAME_PATH: &str = "path"; const FIELD_NAME_SIZE: &str = "size"; diff --git a/crates/core/src/kernel/snapshot/iterators/scan_row.rs b/crates/core/src/kernel/snapshot/iterators/scan_row.rs index 6f01f6485..7b561b8e4 100644 --- a/crates/core/src/kernel/snapshot/iterators/scan_row.rs +++ b/crates/core/src/kernel/snapshot/iterators/scan_row.rs @@ -9,6 +9,7 @@ use arrow_schema::{Field, Schema}; use delta_kernel::engine::arrow_conversion::TryIntoArrow as _; use delta_kernel::engine::arrow_conversion::TryIntoKernel; use delta_kernel::engine::arrow_data::ArrowEngineData; +use delta_kernel::engine::arrow_expression::evaluate_expression::to_json; use delta_kernel::engine::parse_json; use delta_kernel::expressions::Scalar; use delta_kernel::scan::scan_row_schema; @@ -142,7 +143,7 @@ pub(crate) fn parse_stats_column_with_schema( ) } -fn parse_stats_column_impl( +pub(crate) fn parse_stats_column_impl( batch: &RecordBatch, stats_schema: KernelSchemaRef, stats_arrow_fields: Option<&Fields>, @@ -157,20 +158,55 @@ fn parse_stats_column_impl( stats_arrow_schema = stats_schema.as_ref().try_into_arrow()?; stats_arrow_schema.fields() }; + let generated_stats = if stats_materialization.preserves_raw_stats() { + batch + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .filter(|parsed| { + project_struct_array(parsed, stats_arrow_fields, StructProjectionMode::Strict) + .is_ok() + }) + .map(|parsed| to_json(parsed)) + .transpose()? + } else { + None + }; let mut columns = Vec::with_capacity(batch.num_columns() + 2); let mut fields = Vec::with_capacity(batch.num_columns() + 2); + let mut emitted_stats = false; for (field, column) in batch.schema().fields().iter().zip(batch.columns()) { if field.name() == FIELD_STATS_PARSED { continue; } - if field.name() == FIELD_STATS && !stats_materialization.preserves_raw_stats() { + if field.name() == FIELD_STATS { + if !stats_materialization.preserves_raw_stats() { + continue; + } + emitted_stats = true; + fields.push(field.clone()); + columns.push(match generated_stats.as_ref() { + Some(generated_stats) => { + generated_stats_json_with_fallback(generated_stats, Some(column))? + } + None => column.clone(), + }); continue; } fields.push(field.clone()); columns.push(column.clone()); } + if !emitted_stats && let Some(generated_stats) = generated_stats { + let generated_stats = generated_stats_json_with_fallback(&generated_stats, None)?; + fields.push(Arc::new(Field::new( + FIELD_STATS, + generated_stats.data_type().clone(), + true, + ))); + columns.push(generated_stats); + } + if let Some(stats_array) = materialize_stats_array( batch, stats_schema, @@ -206,6 +242,40 @@ fn parse_stats_column_impl( )?) } +fn generated_stats_json_with_fallback( + generated_stats: &ArrayRef, + fallback: Option<&ArrayRef>, +) -> DeltaResult { + let generated_stats = generated_stats + .as_any() + .downcast_ref::() + .ok_or_else(|| DeltaTableError::SchemaMismatch { + msg: "generated stats column is not a string".to_string(), + })?; + let fallback = fallback + .map(|fallback| { + fallback + .as_string_opt::() + .ok_or_else(|| DeltaTableError::SchemaMismatch { + msg: "stats column is not a string".to_string(), + }) + }) + .transpose()?; + let mut output = StringBuilder::new(); + + for row in 0..generated_stats.len() { + if generated_stats.is_valid(row) && generated_stats.value(row) != "{}" { + output.append_value(generated_stats.value(row)); + } else if let Some(fallback) = fallback.filter(|fallback| fallback.is_valid(row)) { + output.append_value(fallback.value(row)); + } else { + output.append_null(); + } + } + + Ok(Arc::new(output.finish())) +} + fn materialize_stats_array( batch: &RecordBatch, stats_schema: KernelSchemaRef, @@ -224,7 +294,11 @@ fn materialize_stats_array( .ok_or_else(|| DeltaTableError::SchemaMismatch { msg: "stats_parsed column is not a struct".to_string(), })?; - match project_struct_array(parsed, stats_arrow_fields) { + match project_struct_array( + parsed, + stats_arrow_fields, + StructProjectionMode::AddMissingNullable, + ) { Ok(projected) => return Ok(Some(Arc::new(projected))), Err(err) => { debug!( @@ -244,6 +318,25 @@ fn materialize_stats_array( } } +// fn parse_raw_stats_array( +// batch: &RecordBatch, +// stats_schema: KernelSchemaRef, +// ) -> DeltaResult> { +// let Some((stats_idx, _)) = batch.schema_ref().column_with_name(FIELD_STATS) else { +// return Err(DeltaTableError::SchemaMismatch { +// msg: "stats column not found".to_string(), +// }); +// }; +// +// let stats_batch = batch.project(&[stats_idx])?; +// let stats_data = Box::new(ArrowEngineData::new(stats_batch)); +// +// let parsed = parse_json(stats_data, stats_schema)?; +// let parsed: RecordBatch = ArrowEngineData::try_from_engine_data(parsed)?.into(); +// +// Ok(Arc::new(parsed.into())) +// } + fn parse_raw_stats_array( batch: &RecordBatch, stats_schema: KernelSchemaRef, @@ -253,8 +346,28 @@ fn parse_raw_stats_array( msg: "stats column not found".to_string(), }); }; + parse_raw_stats_array_at(batch, stats_schema, stats_idx) +} - let stats_batch = batch.project(&[stats_idx])?; +pub(crate) fn parse_raw_stats_array_at( + batch: &RecordBatch, + stats_schema: KernelSchemaRef, + stats_idx: usize, +) -> DeltaResult> { + let stats_column = batch.columns().get(stats_idx).ok_or_else(|| { + DeltaTableError::SchemaMismatch { + msg: "stats column not found".to_string(), + } + })?; + let stats_field = Arc::new(Field::new( + FIELD_STATS, + stats_column.data_type().clone(), + true, + )); + let stats_batch = RecordBatch::try_new( + Arc::new(Schema::new([stats_field])), + vec![stats_column.clone()], + )?; let stats_data = Box::new(ArrowEngineData::new(stats_batch)); let parsed = parse_json(stats_data, stats_schema)?; @@ -263,9 +376,16 @@ fn parse_raw_stats_array( Ok(Arc::new(parsed.into())) } +#[derive(Clone, Copy)] +enum StructProjectionMode { + Strict, + AddMissingNullable, +} + fn project_struct_array( array: &StructArray, requested_fields: &Fields, + mode: StructProjectionMode, ) -> DeltaResult { let ArrowDataType::Struct(existing_fields) = array.data_type() else { return Err(DeltaTableError::SchemaMismatch { @@ -275,10 +395,17 @@ fn project_struct_array( let mut columns = Vec::with_capacity(requested_fields.len()); for requested_field in requested_fields { - let existing_idx = existing_fields + let Some(existing_idx) = existing_fields .iter() .position(|field| field.name() == requested_field.name()) - .ok_or_else(|| DeltaTableError::SchemaMismatch { + else { + if matches!(mode, StructProjectionMode::AddMissingNullable) + && requested_field.is_nullable() + { + columns.push(new_null_array(requested_field.data_type(), array.len())); + continue; + } + return Err(DeltaTableError::SchemaMismatch { msg: format!( "stats_parsed field {} not found; existing fields: [{}]", requested_field.name(), @@ -288,7 +415,8 @@ fn project_struct_array( .collect::>() .join(", ") ), - })?; + }); + }; let existing_column = array.column(existing_idx); let column: ArrayRef = match (requested_field.data_type(), existing_column.data_type()) { (ArrowDataType::Struct(requested_child_fields), ArrowDataType::Struct(_)) => { @@ -301,7 +429,7 @@ fn project_struct_array( requested_field.name() ), })?; - Arc::new(project_struct_array(child, requested_child_fields)?) + Arc::new(project_struct_array(child, requested_child_fields, mode)?) } (requested, existing) if requested == existing => existing_column.clone(), (requested, existing) => { @@ -676,7 +804,7 @@ fn collect_map(val: &StructArray) -> Option RecordBatch { + pub(crate) fn scan_row_batch_with_stats(raw_stats: &str) -> RecordBatch { let schema: ArrowSchema = scan_row_schema().as_ref().try_into_arrow().unwrap(); let mut columns: Vec = schema .fields() @@ -712,20 +841,20 @@ mod tests { RecordBatch::try_new(Arc::new(schema), columns).unwrap() } - fn raw_stats_string(batch: RecordBatch, row: usize) -> Option { + pub(crate) fn raw_stats_string(batch: RecordBatch, row: usize) -> Option { batch .column_by_name("stats") .and_then(|col| col.as_string_opt::()) .and_then(|col| col.is_valid(row).then(|| col.value(row).to_string())) } - fn num_records_stats_schema() -> SchemaRef { + pub(crate) fn num_records_stats_schema() -> SchemaRef { Arc::new( StructType::try_new([StructField::nullable("numRecords", DataType::LONG)]).unwrap(), ) } - fn value_stats_schema() -> SchemaRef { + pub(crate) fn value_stats_schema() -> SchemaRef { Arc::new( StructType::try_new([ StructField::nullable("numRecords", DataType::LONG), @@ -748,7 +877,7 @@ mod tests { ) } - fn append_stats_parsed(batch: &RecordBatch, stats_parsed: StructArray) -> RecordBatch { + pub(crate) fn append_stats_parsed(batch: &RecordBatch, stats_parsed: StructArray) -> RecordBatch { let mut fields = batch.schema().fields().to_vec(); let mut columns = batch.columns().to_vec(); fields.push(Arc::new(Field::new( @@ -778,7 +907,7 @@ mod tests { RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() } - fn num_records_stats_parsed(num_records: i64) -> StructArray { + pub(crate) fn num_records_stats_parsed(num_records: i64) -> StructArray { StructArray::from(vec![( Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), Arc::new(Int64Array::from(vec![Some(num_records)])) as ArrayRef, @@ -792,7 +921,7 @@ mod tests { )]) } - fn value_stats_parsed() -> StructArray { + pub(crate) fn value_stats_parsed() -> StructArray { let min_values = StructArray::from(vec![( Arc::new(Field::new("value", ArrowDataType::Int32, true)), Arc::new(Int32Array::from(vec![Some(1)])) as ArrayRef, @@ -838,7 +967,7 @@ mod tests { ]) } - fn stats_parsed_field_names(batch: &RecordBatch) -> Vec { + pub(crate) fn stats_parsed_field_names(batch: &RecordBatch) -> Vec { let schema = batch.schema(); let field = schema.field_with_name("stats_parsed").unwrap(); let ArrowDataType::Struct(fields) = field.data_type() else { @@ -1099,9 +1228,9 @@ mod tests { } #[test] - fn parse_stats_column_impl_falls_back_to_raw_stats_when_existing_parsed_is_partial() + fn parse_stats_column_impl_adds_missing_nullable_fields_without_parsing_raw_stats() -> DeltaResult<()> { - let raw_stats = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#; + let raw_stats = "not valid JSON"; let batch = scan_row_batch_with_stats(raw_stats); let batch = append_stats_parsed(&batch, num_records_stats_parsed(11)); @@ -1118,11 +1247,110 @@ mod tests { stats_parsed_field_names(&projected), vec!["numRecords", "minValues", "maxValues", "nullCount"] ); - assert_eq!(raw_stats_string(projected, 0), Some(raw_stats.to_string())); + assert_eq!( + raw_stats_string(projected.clone(), 0), + Some(raw_stats.to_string()) + ); + + let stats = projected + .column_by_name(FIELD_STATS_PARSED) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + stats + .column_by_name("numRecords") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 11 + ); + for (field_name, value_type) in [ + ("minValues", ArrowDataType::Int32), + ("maxValues", ArrowDataType::Int32), + ("nullCount", ArrowDataType::Int64), + ] { + let field = stats.column_by_name(field_name).unwrap(); + assert!(field.is_null(0)); + let field = field.as_any().downcast_ref::().unwrap(); + let value = field.column_by_name("value").unwrap(); + assert_eq!(value.data_type(), &value_type); + assert!(value.is_null(0)); + } Ok(()) } + #[test] + fn project_struct_array_adds_nested_nullable_fields_and_preserves_null_buffers() + -> DeltaResult<()> { + let keep: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)])); + let nested = StructArray::new( + Fields::from(vec![Field::new("keep", ArrowDataType::Int32, true)]), + vec![keep.clone()], + Some(NullBuffer::from_iter([true, false, true])), + ); + let array = StructArray::new( + Fields::from(vec![Field::new("nested", nested.data_type().clone(), true)]), + vec![Arc::new(nested)], + Some(NullBuffer::from_iter([false, true, true])), + ); + let requested_nested = Fields::from(vec![ + Field::new("added", ArrowDataType::Int64, true), + Field::new("keep", ArrowDataType::Int32, true), + ]); + let requested = Fields::from(vec![Field::new( + "nested", + ArrowDataType::Struct(requested_nested), + true, + )]); + + let projected = + project_struct_array(&array, &requested, StructProjectionMode::AddMissingNullable)?; + + assert_eq!( + projected.nulls().unwrap(), + &NullBuffer::from_iter([false, true, true]) + ); + let nested = projected + .column_by_name("nested") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + nested.nulls().unwrap(), + &NullBuffer::from_iter([true, false, true]) + ); + assert_eq!( + nested.column_by_name("added").unwrap().data_type(), + &ArrowDataType::Int64 + ); + assert_eq!(nested.column_by_name("added").unwrap().null_count(), 3); + assert!(Arc::ptr_eq(nested.column_by_name("keep").unwrap(), &keep)); + + Ok(()) + } + + #[test] + fn project_struct_array_rejects_missing_non_nullable_field() { + let array = num_records_stats_parsed(11); + let requested = Fields::from(vec![Field::new("required", ArrowDataType::Int64, false)]); + + let err = + project_struct_array(&array, &requested, StructProjectionMode::AddMissingNullable) + .expect_err("missing non-nullable field should fail projection"); + + assert!( + err.to_string() + .contains("stats_parsed field required not found"), + "unexpected error: {err}" + ); + } + #[test] fn parse_stats_column_impl_accepts_raw_stats_without_min_max_values() -> DeltaResult<()> { let raw_stats = r#"{"numRecords":11}"#; @@ -1146,6 +1374,8 @@ mod tests { Ok(()) } + // HSTACK - changed behavior + #[ignore = "Changed behavior, superseded by parse_stats_column_impl_regenerates_raw_stats_when_raw_policy_preserves"] #[test] fn parse_stats_column_impl_keeps_raw_stats_when_raw_policy_preserves() -> DeltaResult<()> { let raw_stats = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#; diff --git a/crates/core/src/kernel/snapshot/iterators/scan_row_cached.rs b/crates/core/src/kernel/snapshot/iterators/scan_row_cached.rs new file mode 100644 index 000000000..2760b27ec --- /dev/null +++ b/crates/core/src/kernel/snapshot/iterators/scan_row_cached.rs @@ -0,0 +1,1219 @@ +use std::sync::Arc; + +use arrow_array::{Array, ArrayRef, RecordBatch, StructArray}; +use arrow_cast::CastOptions; +use arrow_schema::{DataType as ArrowDataType, Field, Fields, Schema}; +use delta_kernel::engine::arrow_conversion::TryIntoArrow; +use delta_kernel::expressions::Scalar; +use delta_kernel::scan::scan_row_schema; +use delta_kernel::schema::{DataType, SchemaRef as KernelSchemaRef, StructField, StructType}; +use delta_kernel::{EvaluationHandler, Expression, ExpressionEvaluator}; + +use crate::kernel::arrow::engine_ext::ExpressionEvaluatorExt; +use crate::kernel::schema::cast::cast_struct; +use crate::kernel::snapshot::stats_projection::{FIELD_PARTITION_VALUES_PARSED, FIELD_STATS}; +use crate::kernel::{ARROW_HANDLER, FIELD_STATS_PARSED}; +use crate::{DeltaResult, DeltaTableError}; + +struct CachedStructProjection { + path: String, + input_fields: Fields, + output_fields: Fields, +} + +impl CachedStructProjection { + fn try_new(input_fields: Fields, output_fields: Fields, path: &str) -> DeltaResult { + Self::validate_fields(&input_fields, &output_fields, path)?; + Ok(Self { + path: path.to_string(), + input_fields, + output_fields, + }) + } + + fn validate_fields( + input_fields: &Fields, + output_fields: &Fields, + path: &str, + ) -> DeltaResult<()> { + for output_field in output_fields { + let field_path = format!("{path}.{}", output_field.name()); + let Some(input_field) = input_fields + .iter() + .find(|field| field.name() == output_field.name()) + else { + if output_field.is_nullable() { + continue; + } + return Err(DeltaTableError::SchemaMismatch { + msg: format!("cached {field_path} is not available"), + }); + }; + + match (input_field.data_type(), output_field.data_type()) { + (ArrowDataType::Struct(input_child), ArrowDataType::Struct(output_child)) => { + Self::validate_fields(input_child, output_child, &field_path)?; + } + (input_type, output_type) + if input_type == output_type + || Self::is_legal_primitive_widening(input_type, output_type) => {} + (input_type, output_type) => { + return Err(DeltaTableError::SchemaMismatch { + msg: format!( + "cached {field_path} has type {input_type:?} but requested {output_type:?}" + ), + }); + } + } + } + Ok(()) + } + + fn is_legal_primitive_widening( + input_type: &ArrowDataType, + output_type: &ArrowDataType, + ) -> bool { + matches!( + (input_type, output_type), + ( + ArrowDataType::Int8, + ArrowDataType::Int16 | ArrowDataType::Int32 | ArrowDataType::Int64 + ) | ( + ArrowDataType::Int16, + ArrowDataType::Int32 | ArrowDataType::Int64 + ) | (ArrowDataType::Int32, ArrowDataType::Int64) + | (ArrowDataType::Float32, ArrowDataType::Float64) + ) + } + + /// Reshape a seed `stats_parsed` array to the requested schema with a targeted arrow cast: + /// reorder / drop / add-nullable / widen / nested, preserving null buffers. + fn cast_to_requested(&self, array: &StructArray) -> DeltaResult { + let expected_type = ArrowDataType::Struct(self.input_fields.clone()); + if array.data_type() != &expected_type { + return Err(DeltaTableError::SchemaMismatch { + msg: format!( + "cached {} type {:?} does not match declared type {:?}", + self.path, + array.data_type(), + expected_type + ), + }); + } + + Ok(cast_struct( + array, + &self.output_fields, + &CastOptions { + safe: true, + ..Default::default() + }, + true, + )?) + } +} + +/// How the requested `stats_parsed` column is produced from a materialized-file seed batch. +enum StatsParsedSource { + /// The seed's typed `stats_parsed` column is compatible with the request: reshape it with a + /// targeted arrow cast ([`CachedStructProjection`]). A kernel expression cannot do this + /// because its evaluator never widens primitives, so this one step stays imperative. + Reshape(CachedStructProjection), + /// The seed's typed stats are missing or incompatible: parse the raw `stats` JSON instead. + /// This path is expressed entirely as a kernel `parse_json` in the assembly expression. + ParseJson, +} + +impl StatsParsedSource { + /// The assembly sub-expression that yields `stats_parsed` for this source. The reshape path + /// references the column that [`CachedScanRowEvaluator::reshape_cached_stats`] casts into + /// place; the JSON path parses raw `stats` directly, fully inside the kernel expression. + fn stats_expression(&self, requested_schema: &KernelSchemaRef) -> Expression { + match self { + StatsParsedSource::Reshape(_) => Expression::column([FIELD_STATS_PARSED]), + StatsParsedSource::ParseJson => { + Expression::parse_json(Expression::column([FIELD_STATS]), requested_schema.clone()) + } + } + } +} + +/// The chosen `stats_parsed` production strategy plus the exact schema it emits. +struct StatsPlan { + source: StatsParsedSource, + schema: KernelSchemaRef, +} + +/// One fixed transformation for every batch in a materialized-file seed. +/// +/// Assembly is a single kernel [`ExpressionEvaluator`]: base scan-row columns pass through by +/// name, a missing raw `stats` column becomes a typed NULL, `stats_parsed` is produced per the +/// [`StatsPlan`] (a by-name column reference on the reshape path, or `parse_json` on the raw-JSON +/// path), and `partitionValues_parsed` passes through by name. Columns resolve by name, so the +/// cached batch may carry extra columns or a different field order. +/// +/// The only step the expression cannot perform is the reshape cast (kernel expressions never +/// widen primitives); [`Self::reshape_cached_stats`] applies it as a per-batch pre-pass, and only +/// on the reshape path. `partition_fields`, when set, guards that the seed carries a +/// `partitionValues_parsed` column with exactly the expected fields, surfacing a typed +/// [`DeltaTableError::SchemaMismatch`] rather than a kernel column-not-found error. +pub(crate) struct CachedScanRowEvaluator { + evaluator: Arc, + stats_plan: Option, + partition_fields: Option, +} + +impl CachedScanRowEvaluator { + pub(crate) fn try_new( + available_stats_schema: Option<&KernelSchemaRef>, + raw_stats_available: bool, + effective_replay_stats_schema: Option, + partition_schema: Option<&KernelSchemaRef>, + ) -> DeltaResult { + let base_schema = scan_row_schema(); + + let available_stats_fields = available_stats_schema + .map(|available| { + let available_arrow: Schema = available.as_ref().try_into_arrow()?; + Ok::<_, DeltaTableError>(available_arrow.fields().clone()) + }) + .transpose()?; + let partition_fields = partition_schema + .map(|schema| { + let partition_arrow: Schema = schema.as_ref().try_into_arrow()?; + Ok::<_, DeltaTableError>(partition_arrow.fields().clone()) + }) + .transpose()?; + + // Decide once how `stats_parsed` is produced (reshape cached typed stats, or parse raw + // JSON). The choice is fixed for every batch of this seed. + let stats_plan = Self::plan_stats( + available_stats_fields.as_ref(), + raw_stats_available, + effective_replay_stats_schema.as_ref(), + )?; + + // Assemble the scan-row output expression: base columns (or a typed NULL for a missing + // raw `stats`), then `stats_parsed`, then `partitionValues_parsed`. Column references are + // resolved by name, so a positional reordering of the seed cannot silently mis-map. + let mut expr_fields: Vec = Vec::with_capacity(base_schema.num_fields() + 2); + let mut output_fields: Vec = + Vec::with_capacity(base_schema.num_fields() + 2); + for field in base_schema.fields() { + if field.name().as_str() == FIELD_STATS && !raw_stats_available { + expr_fields.push(Expression::literal(Scalar::Null(field.data_type().clone()))); + } else { + expr_fields.push(Expression::column([field.name().clone()])); + } + output_fields.push(field.clone()); + } + if let Some(plan) = &stats_plan { + expr_fields.push(plan.source.stats_expression(&plan.schema)); + output_fields.push(StructField::nullable( + FIELD_STATS_PARSED, + DataType::Struct(Box::new(plan.schema.as_ref().clone())), + )); + } + if let Some(partition_schema) = partition_schema { + expr_fields.push(Expression::column([FIELD_PARTITION_VALUES_PARSED])); + output_fields.push(StructField::not_null( + FIELD_PARTITION_VALUES_PARSED, + DataType::Struct(Box::new(partition_schema.as_ref().clone())), + )); + } + + let output_type = DataType::Struct(Box::new(StructType::try_new(output_fields)?)); + let expression = Expression::struct_from(expr_fields); + // The evaluator ignores its declared input schema at runtime (it resolves columns by name + // from each batch), so the base scan-row schema is a sufficient placeholder here. + let evaluator = ARROW_HANDLER.new_expression_evaluator( + base_schema.clone(), + Arc::new(expression), + output_type, + )?; + + Ok(Self { + evaluator, + stats_plan, + partition_fields, + }) + } + + /// Chooses the `stats_parsed` strategy: reshape the seed's typed stats when they are present + /// and compatible (cheapest — no JSON parsing), otherwise parse the raw `stats` JSON, and + /// error when neither is available. The decision is made here rather than at evaluation time + /// because a `cast`/parse can silently produce nulls instead of failing, so a runtime failure + /// cannot be used to trigger the fallback. + fn plan_stats( + available_stats_fields: Option<&Fields>, + raw_stats_available: bool, + effective_replay_stats_schema: Option<&KernelSchemaRef>, + ) -> DeltaResult> { + let Some(requested_schema) = effective_replay_stats_schema else { + return Ok(None); + }; + + // Can the seed's typed `stats_parsed` be reshaped to the request? `Ok(None)` means there + // were no typed stats at all; `Err` means they exist but are incompatible. + let reshape = available_stats_fields + .map(|available_fields| { + let requested_arrow: Schema = requested_schema.as_ref().try_into_arrow()?; + CachedStructProjection::try_new( + available_fields.clone(), + requested_arrow.fields().clone(), + FIELD_STATS_PARSED, + ) + }) + .transpose(); + + let source = match reshape { + Ok(Some(projection)) => StatsParsedSource::Reshape(projection), + Ok(None) | Err(_) if raw_stats_available => StatsParsedSource::ParseJson, + Err(err) => return Err(err), + Ok(None) => { + return Err(DeltaTableError::SchemaMismatch { + msg: "cached seed has neither parsed nor raw statistics required by the scan" + .to_string(), + }); + } + }; + + Ok(Some(StatsPlan { + source, + schema: requested_schema.clone(), + })) + } + + /// Exact `stats_parsed` schema emitted by this evaluator. + pub(crate) fn stats_parsed_schema(&self) -> Option { + self.stats_plan.as_ref().map(|plan| plan.schema.clone()) + } + + pub(crate) fn evaluate(&self, batch: RecordBatch) -> DeltaResult { + self.validate_partition_layout(&batch)?; + let batch = self.reshape_cached_stats(batch)?; + Ok(self.evaluator.evaluate_arrow(batch)?) + } + + /// When a partition schema was declared, require the seed to carry a `partitionValues_parsed` + /// column with exactly the expected fields. This turns a missing / mismatched partition column + /// into a typed [`DeltaTableError::SchemaMismatch`] instead of the kernel's generic + /// column-not-found error. Base and stats columns are intentionally not checked here — they are + /// resolved by name by the expression, so extra or reordered columns are harmless. + fn validate_partition_layout(&self, batch: &RecordBatch) -> DeltaResult<()> { + let Some(expected_fields) = &self.partition_fields else { + return Ok(()); + }; + let column = batch.column_by_name(FIELD_PARTITION_VALUES_PARSED).ok_or_else(|| { + DeltaTableError::SchemaMismatch { + msg: format!("cached batch is missing required {FIELD_PARTITION_VALUES_PARSED} column"), + } + })?; + let ArrowDataType::Struct(actual_fields) = column.data_type() else { + return Err(DeltaTableError::SchemaMismatch { + msg: format!("cached {FIELD_PARTITION_VALUES_PARSED} column is not a struct"), + }); + }; + if actual_fields != expected_fields { + return Err(DeltaTableError::SchemaMismatch { + msg: format!( + "cached {FIELD_PARTITION_VALUES_PARSED} has fields {actual_fields:?} but expected {expected_fields:?}" + ), + }); + } + Ok(()) + } + + /// On the reshape path, cast the seed's `stats_parsed` column to the requested schema + /// (reorder / drop / add-nullable / widen / nested, preserving null buffers) so the assembly + /// expression can reference it by name. Every other path (raw-JSON parse, no stats) is handled + /// entirely inside the assembly expression and passes through untouched here. + fn reshape_cached_stats(&self, batch: RecordBatch) -> DeltaResult { + let Some(StatsPlan { + source: StatsParsedSource::Reshape(projection), + .. + }) = &self.stats_plan + else { + return Ok(batch); + }; + let index = batch.schema().index_of(FIELD_STATS_PARSED)?; + let parsed = batch + .column(index) + .as_any() + .downcast_ref::() + .ok_or_else(|| DeltaTableError::SchemaMismatch { + msg: "stats_parsed column is not a struct".to_string(), + })?; + let reshaped: ArrayRef = Arc::new(projection.cast_to_requested(parsed)?); + + let mut fields = batch.schema().fields().to_vec(); + fields[index] = Arc::new(Field::new( + FIELD_STATS_PARSED, + reshaped.data_type().clone(), + true, + )); + let mut columns = batch.columns().to_vec(); + columns[index] = reshaped; + Ok(RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?) + } + +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ArrayRef, new_null_array}; + use arrow::datatypes::DataType as ArrowDataType; + use arrow::datatypes::Schema as ArrowSchema; + use arrow_array::{Int32Array, Int64Array, StringArray}; + use arrow_buffer::NullBuffer; + use arrow_schema::Field; + use delta_kernel::expressions::Scalar; + use delta_kernel::scan::scan_row_schema; + use delta_kernel::schema::{SchemaRef, StructField, StructType}; + use delta_kernel::table_features::ColumnMappingMode; + use pretty_assertions::assert_eq; + + use crate::kernel::arrow::engine_ext::SnapshotExt; + use crate::kernel::snapshot::iterators::scan_row::parse_stats_column_with_schema; + use crate::kernel::scalars::ScalarExt; + use crate::kernel::snapshot::Snapshot; + use crate::kernel::snapshot::iterators::scan_row::parse_stats_column_impl; + use crate::kernel::snapshot::iterators::scan_row::tests::{append_stats_parsed, num_records_stats_parsed, num_records_stats_schema, raw_stats_string, scan_row_batch_with_stats, stats_parsed_field_names, value_stats_parsed, value_stats_schema}; + use crate::kernel::snapshot::stats_projection::{FileStatsMaterialization, StatsProjection}; + use crate::test_utils::TestTables; + + fn empty_num_records_stats_parsed() -> StructArray { + StructArray::from(vec![( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Option::::None])) as ArrayRef, + )]) + } + + fn scan_row_batch_with_optional_stats(raw_stats: Option<&str>) -> RecordBatch { + let schema: ArrowSchema = scan_row_schema().as_ref().try_into_arrow().unwrap(); + let mut columns: Vec = schema + .fields() + .iter() + .map(|field| new_null_array(field.data_type(), 1)) + .collect(); + columns[schema.index_of("path").unwrap()] = + Arc::new(StringArray::from(vec![Some("part-000.parquet")])); + columns[schema.index_of("size").unwrap()] = Arc::new(Int64Array::from(vec![1])); + columns[schema.index_of("modificationTime").unwrap()] = Arc::new(Int64Array::from(vec![1])); + columns[schema.index_of("stats").unwrap()] = Arc::new(StringArray::from(vec![raw_stats])); + + RecordBatch::try_new(Arc::new(schema), columns).unwrap() + } + + fn wide_min_values_stats_parsed( + num_records: i64, + min_value: i32, + min_values_valid: bool, + ) -> StructArray { + let min_values = StructArray::new( + Fields::from(vec![ + Field::new("value", ArrowDataType::Int32, true), + Field::new("other", ArrowDataType::Int32, true), + ]), + vec![ + Arc::new(Int32Array::from(vec![Some(min_value)])) as ArrayRef, + Arc::new(Int32Array::from(vec![Some(99)])) as ArrayRef, + ], + Some(NullBuffer::from_iter([min_values_valid])), + ); + + StructArray::from(vec![ + ( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(num_records)])) as ArrayRef, + ), + ( + Arc::new(Field::new( + "minValues", + min_values.data_type().clone(), + true, + )), + Arc::new(min_values) as ArrayRef, + ), + ]) + } + + fn wide_min_values_stats_schema() -> SchemaRef { + Arc::new( + StructType::try_new([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable( + "minValues", + StructType::try_new([ + StructField::nullable("value", DataType::INTEGER), + StructField::nullable("other", DataType::INTEGER), + ]) + .unwrap(), + ), + ]) + .unwrap(), + ) + } + + fn selected_min_values_stats_schema() -> SchemaRef { + Arc::new( + StructType::try_new([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable( + "minValues", + StructType::try_new([StructField::nullable("value", DataType::INTEGER)]) + .unwrap(), + ), + ]) + .unwrap(), + ) + } + + pub(crate) fn append_partition_values_parsed( + batch: &RecordBatch, + partition_values: StructArray, + ) -> RecordBatch { + let mut fields = batch.schema().fields().to_vec(); + let mut columns = batch.columns().to_vec(); + fields.push(Arc::new(Field::new( + FIELD_PARTITION_VALUES_PARSED, + partition_values.data_type().clone(), + false, + ))); + columns.push(Arc::new(partition_values)); + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() + } + + #[test] + fn parse_stats_column_impl_regenerates_raw_stats_when_raw_policy_preserves() -> DeltaResult<()> + { + let batch = scan_row_batch_with_stats("stale raw stats"); + let batch = append_stats_parsed(&batch, value_stats_parsed()); + let projected = parse_stats_column_impl( + &batch, + value_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::full()), + )?; + + let raw_stats = raw_stats_string(projected, 0) + .expect("full typed stats should generate compatibility JSON"); + let raw_stats: serde_json::Value = serde_json::from_str(&raw_stats)?; + assert_eq!(raw_stats["numRecords"], serde_json::json!(11)); + assert_eq!(raw_stats["minValues"]["value"], serde_json::json!(1)); + assert_eq!(raw_stats["maxValues"]["value"], serde_json::json!(9)); + assert_eq!(raw_stats["nullCount"]["value"], serde_json::json!(0)); + + Ok(()) + } + + #[test] + fn cached_uniform_evaluator_maps_reordered_base_fields_by_name() -> DeltaResult<()> { + // Give `size` and `modificationTime` distinct values so a positional (rather than by-name) + // read would route the wrong value to the wrong output field. + let batch = scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)); + let schema = batch.schema(); + let size_index = schema.index_of("size").unwrap(); + let modification_time_index = schema.index_of("modificationTime").unwrap(); + let mut columns = batch.columns().to_vec(); + columns[size_index] = Arc::new(Int64Array::from(vec![10])); + columns[modification_time_index] = Arc::new(Int64Array::from(vec![20])); + + // Physically swap the two same-typed columns (field + data together). By-name column + // resolution must still route each value back to its correctly named output field. + let mut fields = schema.fields().to_vec(); + fields.swap(size_index, modification_time_index); + columns.swap(size_index, modification_time_index); + let reordered = RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap(); + + let evaluator = CachedScanRowEvaluator::try_new(None, true, None, None)?; + let projected = evaluator.evaluate(reordered)?; + + let size = projected + .column_by_name("size") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("size should be an Int64 column"); + let modification_time = projected + .column_by_name("modificationTime") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("modificationTime should be an Int64 column"); + assert_eq!(size.value(0), 10); + assert_eq!(modification_time.value(0), 20); + Ok(()) + } + + #[test] + fn cached_uniform_evaluator_requires_exact_declared_partition_layout() -> DeltaResult<()> { + let partition_schema = Arc::new(StructType::try_new([StructField::nullable( + "part", + DataType::STRING, + )])?); + let evaluator = CachedScanRowEvaluator::try_new(None, true, None, Some(&partition_schema))?; + let wrong_partition = StructArray::from(vec![( + Arc::new(Field::new("wrong", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("value")])) as ArrayRef, + )]); + + for batch in [ + scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)), + append_partition_values_parsed( + &scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)), + wrong_partition, + ), + ] { + let error = evaluator + .evaluate(batch) + .expect_err("missing or wrong partition layout must be rejected"); + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + } + + let partition_values = StructArray::from(vec![( + Arc::new(Field::new("part", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("value")])) as ArrayRef, + )]); + let projected = evaluator.evaluate(append_partition_values_parsed( + &scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)), + partition_values, + ))?; + assert!( + projected + .schema() + .field_with_name(FIELD_PARTITION_VALUES_PARSED) + .is_ok() + ); + + Ok(()) + } + + #[test] + fn seeded_input_wider_parsed_stats_project_to_selected_nested_columns() -> DeltaResult<()> { + let available_schema = wide_min_values_stats_schema(); + let requested_schema = selected_min_values_stats_schema(); + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_schema), + true, + Some(requested_schema.clone()), + None, + )?; + + assert_eq!(evaluator.stats_parsed_schema(), Some(requested_schema)); + for (num_records, min_value, min_values_valid) in [(11, 1, true), (12, 2, false)] { + let raw_stats = format!( + r#"{{"numRecords":{num_records},"minValues":{{"value":{min_value},"other":99}}}}"# + ); + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(&raw_stats)), + wide_min_values_stats_parsed(num_records, min_value, min_values_valid), + ); + let projected = evaluator.evaluate(batch)?; + + assert_eq!( + stats_parsed_field_names(&projected), + vec!["numRecords", "minValues"] + ); + assert_eq!( + raw_stats_string(projected.clone(), 0), + Some(raw_stats.clone()) + ); + let stats = projected + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("stats_parsed should be a struct"); + let min_values = stats + .column_by_name("minValues") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("minValues should be a struct"); + let ArrowDataType::Struct(fields) = min_values.data_type() else { + panic!("minValues should have a struct type"); + }; + assert_eq!( + fields + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + vec!["value"] + ); + assert_eq!(min_values.is_valid(0), min_values_valid); + if min_values_valid { + assert_eq!( + Scalar::from_array(min_values.column_by_name("value").unwrap().as_ref(), 0), + Some(Scalar::Integer(min_value)) + ); + } + } + + Ok(()) + } + + #[test] + fn seeded_input_stats_schema_change_between_batches_returns_typed_error() { + let available_schema = wide_min_values_stats_schema(); + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_schema), + true, + Some(available_schema.clone()), + None, + ) + .unwrap(); + + let matching_batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some( + r#"{"numRecords":11,"minValues":{"value":1,"other":99}}"#, + )), + wide_min_values_stats_parsed(11, 1, true), + ); + evaluator + .evaluate(matching_batch) + .expect("matching declared stats_parsed schema should be accepted"); + + let changed_batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(r#"{"numRecords":12}"#)), + num_records_stats_parsed(12), + ); + let error = evaluator + .evaluate(changed_batch) + .expect_err("changed stats_parsed schema should return an error"); + + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + } + + #[test] + fn cached_uniform_evaluator_falls_back_to_exact_raw_json() -> DeltaResult<()> { + let raw_stats = r#"{ "minValues": {"value": 1}, "numRecords": 11 }"#; + let available_schema = Arc::new(StructType::try_new([StructField::nullable( + "minValues", + DataType::STRING, + )])?); + let requested_schema = selected_min_values_stats_schema(); + let incompatible_parsed = StructArray::from(vec![( + Arc::new(Field::new("minValues", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("incompatible")])) as ArrayRef, + )]); + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(raw_stats)), + incompatible_parsed, + ); + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_schema), + true, + Some(requested_schema.clone()), + None, + )?; + let projected = evaluator.evaluate(batch)?; + + assert_eq!(evaluator.stats_parsed_schema(), Some(requested_schema)); + assert_eq!( + raw_stats_string(projected.clone(), 0), + Some(raw_stats.to_string()) + ); + assert_eq!( + stats_parsed_field_names(&projected), + vec!["numRecords", "minValues"] + ); + let min_values = projected + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .and_then(|stats| stats.column_by_name("minValues")) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("raw fallback should produce minValues"); + assert_eq!( + Scalar::from_array(min_values.column_by_name("value").unwrap().as_ref(), 0), + Some(Scalar::Integer(1)) + ); + + Ok(()) + } + + #[test] + fn cached_uniform_evaluator_rejects_incompatible_seed_without_raw() { + let available_schema = Arc::new( + StructType::try_new([StructField::nullable("minValues", DataType::STRING)]).unwrap(), + ); + let error = match CachedScanRowEvaluator::try_new( + Some(&available_schema), + false, + Some(selected_min_values_stats_schema()), + None, + ) { + Ok(_) => panic!("incompatible parsed seed should fail without raw statistics"), + Err(error) => error, + }; + + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + assert!( + error.to_string().contains("stats_parsed.minValues"), + "unexpected error: {error}" + ); + } + + #[test] + fn parse_stats_column_impl_generates_full_json_before_narrowing_parsed() -> DeltaResult<()> { + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(None), + value_stats_parsed(), + ); + let projected = parse_stats_column_impl( + &batch, + num_records_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::NumRecordsOnly), + )?; + + let raw_stats = raw_stats_string(projected.clone(), 0) + .expect("full typed stats should generate compatibility JSON"); + let raw_stats: serde_json::Value = serde_json::from_str(&raw_stats)?; + assert_eq!( + raw_stats, + serde_json::json!({ + "numRecords": 11, + "minValues": {"value": 1}, + "maxValues": {"value": 9}, + "nullCount": {"value": 0} + }) + ); + assert_eq!(stats_parsed_field_names(&projected), vec!["numRecords"]); + + Ok(()) + } + + #[test] + fn parse_stats_column_impl_keeps_raw_null_when_typed_stats_are_empty() -> DeltaResult<()> { + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(None), + empty_num_records_stats_parsed(), + ); + let projected = parse_stats_column_impl( + &batch, + num_records_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::NumRecordsOnly), + )?; + + assert_eq!(raw_stats_string(projected, 0), None); + + let raw_stats = r#"{"numRecords":11}"#; + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(raw_stats)), + empty_num_records_stats_parsed(), + ); + let projected = parse_stats_column_impl( + &batch, + num_records_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::NumRecordsOnly), + )?; + assert_eq!(raw_stats_string(projected, 0), Some(raw_stats.to_string())); + + Ok(()) + } + + fn stats_schema(fields: impl IntoIterator) -> SchemaRef { + Arc::new(StructType::try_new(fields).unwrap()) + } + + fn cached_batch( + rows: usize, + raw_stats: Option<&[Option<&str>]>, + stats_parsed: StructArray, + ) -> RecordBatch { + assert_eq!(stats_parsed.len(), rows); + let base_schema: ArrowSchema = scan_row_schema().as_ref().try_into_arrow().unwrap(); + let mut fields = Vec::new(); + let mut columns = Vec::new(); + + for field in base_schema.fields() { + let column: ArrayRef = match field.name().as_str() { + FIELD_STATS if raw_stats.is_none() => continue, + "path" => Arc::new(StringArray::from(vec!["file.parquet"; rows])), + "size" | "modificationTime" => Arc::new(Int64Array::from(vec![1; rows])), + FIELD_STATS => Arc::new(StringArray::from(raw_stats.unwrap().to_vec())), + _ => new_null_array(field.data_type(), rows), + }; + fields.push(field.clone()); + columns.push(column); + } + + fields.push(Arc::new(Field::new( + FIELD_STATS_PARSED, + stats_parsed.data_type().clone(), + true, + ))); + columns.push(Arc::new(stats_parsed)); + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() + } + + fn projected_stats(batch: &RecordBatch) -> &StructArray { + batch + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("stats_parsed should be a struct") + } + + #[tokio::test] + async fn cached_batches_preserve_raw_and_parsed_stats_for_kernel_roundtrip() -> DeltaResult<()> + { + let raw_stats = + r#"{"maxValues":{"id":9},"numRecords":11,"nullCount":{"id":0},"minValues":{"id":1}}"#; + let batch = scan_row_batch_with_optional_stats(Some(raw_stats)); + let log_store = TestTables::Simple.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let projected = parse_stats_column_with_schema( + snapshot.inner.as_ref(), + &batch, + snapshot.inner.stats_schema()?, + )?; + // Keep this guard so cached batches still pass through the uniform evaluator + // without rebuilding raw `stats` from `ToJson`. + let available_stats_schema = snapshot.inner.stats_schema()?; + let partition_schema = snapshot.inner.partitions_schema()?; + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_stats_schema), + true, + Some(available_stats_schema.clone()), + partition_schema.as_ref(), + )?; + let reparsed = evaluator.evaluate(projected)?; + + assert_eq!( + raw_stats_string(reparsed.clone(), 0), + Some(raw_stats.to_string()) + ); + let stats = reparsed + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("stats_parsed should be preserved"); + let num_records = stats + .column_by_name("numRecords") + .expect("numRecords should be preserved"); + let min_values = stats + .column_by_name("minValues") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("minValues should be a struct"); + let max_values = stats + .column_by_name("maxValues") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("maxValues should be a struct"); + let null_count = stats + .column_by_name("nullCount") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("nullCount should be a struct"); + let min_value = min_values + .column_by_name("id") + .expect("min value should be preserved"); + let max_value = max_values + .column_by_name("id") + .expect("max value should be preserved"); + let null_value = null_count + .column_by_name("id") + .expect("null count should be preserved"); + + assert_eq!( + Scalar::from_array(num_records.as_ref(), 0), + Some(Scalar::Long(11)) + ); + assert_eq!( + Scalar::from_array(min_value.as_ref(), 0), + Some(Scalar::Long(1)) + ); + assert_eq!( + Scalar::from_array(max_value.as_ref(), 0), + Some(Scalar::Long(9)) + ); + assert_eq!( + Scalar::from_array(null_value.as_ref(), 0), + Some(Scalar::Long(0)) + ); + + Ok(()) + } + + #[tokio::test] + async fn cached_batches_without_stats_parsed_use_uniform_evaluator() -> DeltaResult<()> { + let raw_stats = r#"{"numRecords":11}"#; + let batch = scan_row_batch_with_optional_stats(Some(raw_stats)); + let evaluator = CachedScanRowEvaluator::try_new(None, true, None, None)?; + let projected = evaluator.evaluate(batch)?; + + assert!(projected.column_by_name(FIELD_STATS_PARSED).is_none()); + assert_eq!(raw_stats_string(projected, 0), Some(raw_stats.to_string())); + + Ok(()) + } + + #[test] + fn cached_stats_add_missing_nullable_field_without_raw() -> DeltaResult<()> { + let available = stats_schema([StructField::nullable("numRecords", DataType::LONG)]); + let requested = stats_schema([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable("added", DataType::INTEGER), + ]); + let parsed = StructArray::from(vec![( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef, + )]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, None, parsed))?; + let stats = projected_stats(&projected); + + assert_eq!( + stats + .column_by_name("numRecords") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 11 + ); + let added = stats.column_by_name("added").unwrap(); + assert_eq!(added.data_type(), &ArrowDataType::Int32); + assert!(added.is_null(0)); + Ok(()) + } + + #[test] + fn cached_stats_drop_extra_field_without_raw() -> DeltaResult<()> { + let available = stats_schema([ + StructField::nullable("keep", DataType::INTEGER), + StructField::nullable("dropped", DataType::STRING), + ]); + let requested = stats_schema([StructField::nullable("keep", DataType::INTEGER)]); + let parsed = StructArray::from(vec![ + ( + Arc::new(Field::new("keep", ArrowDataType::Int32, true)), + Arc::new(Int32Array::from(vec![Some(3)])) as ArrayRef, + ), + ( + Arc::new(Field::new("dropped", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("unused")])) as ArrayRef, + ), + ]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, None, parsed))?; + let stats = projected_stats(&projected); + let ArrowDataType::Struct(fields) = stats.data_type() else { + unreachable!() + }; + assert_eq!( + fields.iter().map(|field| field.name()).collect::>(), + ["keep"] + ); + Ok(()) + } + + #[test] + fn cached_stats_reorder_fields_by_name_without_raw() -> DeltaResult<()> { + let available = stats_schema([ + StructField::nullable("first", DataType::INTEGER), + StructField::nullable("second", DataType::LONG), + ]); + let requested = stats_schema([ + StructField::nullable("second", DataType::LONG), + StructField::nullable("first", DataType::INTEGER), + ]); + let parsed = StructArray::from(vec![ + ( + Arc::new(Field::new("first", ArrowDataType::Int32, true)), + Arc::new(Int32Array::from(vec![Some(1)])) as ArrayRef, + ), + ( + Arc::new(Field::new("second", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(2)])) as ArrayRef, + ), + ]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, None, parsed))?; + let stats = projected_stats(&projected); + let ArrowDataType::Struct(fields) = stats.data_type() else { + unreachable!() + }; + assert_eq!( + fields.iter().map(|field| field.name()).collect::>(), + ["second", "first"] + ); + assert_eq!( + stats + .column_by_name("second") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 2 + ); + assert_eq!( + stats + .column_by_name("first") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1 + ); + Ok(()) + } + + #[test] + fn cached_stats_evolve_nested_fields_without_raw() -> DeltaResult<()> { + let available_nested = StructType::try_new([ + StructField::nullable("dropped", DataType::INTEGER), + StructField::nullable("keep", DataType::INTEGER), + ])?; + let requested_nested = StructType::try_new([ + StructField::nullable("added", DataType::INTEGER), + StructField::nullable("keep", DataType::INTEGER), + ])?; + let available = stats_schema([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable("minValues", available_nested), + ]); + let requested = stats_schema([ + StructField::nullable("minValues", requested_nested), + StructField::nullable("numRecords", DataType::LONG), + ]); + + let min_values = StructArray::new( + Fields::from(vec![ + Field::new("dropped", ArrowDataType::Int32, true), + Field::new("keep", ArrowDataType::Int32, true), + ]), + vec![ + Arc::new(Int32Array::from(vec![Some(9), Some(8), Some(7)])), + Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)])), + ], + Some(NullBuffer::from_iter([true, true, false])), + ); + let parsed = StructArray::new( + Fields::from(vec![ + Field::new("numRecords", ArrowDataType::Int64, true), + Field::new("minValues", min_values.data_type().clone(), true), + ]), + vec![ + Arc::new(Int64Array::from(vec![Some(10), Some(20), Some(30)])), + Arc::new(min_values), + ], + Some(NullBuffer::from_iter([true, false, true])), + ); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(3, None, parsed))?; + let stats = projected_stats(&projected); + assert_eq!( + stats.nulls().unwrap(), + &NullBuffer::from_iter([true, false, true]) + ); + + let min_values = stats + .column_by_name("minValues") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + min_values.nulls().unwrap(), + &NullBuffer::from_iter([true, true, false]) + ); + let ArrowDataType::Struct(fields) = min_values.data_type() else { + unreachable!() + }; + assert_eq!( + fields.iter().map(|field| field.name()).collect::>(), + ["added", "keep"] + ); + assert_eq!(min_values.column_by_name("added").unwrap().null_count(), 3); + assert_eq!( + min_values + .column_by_name("keep") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values(), + &[1, 2, 3] + ); + Ok(()) + } + + #[test] + fn cached_stats_widen_primitive_without_raw() -> DeltaResult<()> { + let available = stats_schema([StructField::nullable("value", DataType::INTEGER)]); + let requested = stats_schema([StructField::nullable("value", DataType::LONG)]); + let parsed = StructArray::from(vec![( + Arc::new(Field::new("value", ArrowDataType::Int32, true)), + Arc::new(Int32Array::from(vec![Some(7), None])) as ArrayRef, + )]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(2, None, parsed))?; + let values = projected_stats(&projected) + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(0), 7); + assert!(values.is_null(1)); + Ok(()) + } + + #[test] + fn cached_stats_normal_evolution_does_not_parse_raw() -> DeltaResult<()> { + let available = stats_schema([StructField::nullable("numRecords", DataType::LONG)]); + let requested = stats_schema([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable("added", DataType::INTEGER), + ]); + let parsed = StructArray::from(vec![( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef, + )]); + let invalid_raw = [Some("not valid JSON")]; + + let evaluator = CachedScanRowEvaluator::try_new(Some(&available), true, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, Some(&invalid_raw), parsed))?; + assert!( + projected_stats(&projected) + .column_by_name("added") + .unwrap() + .is_null(0) + ); + assert_eq!( + raw_stats_string(projected, 0), + Some("not valid JSON".to_string()) + ); + Ok(()) + } + + #[test] + fn cached_stats_reject_truly_incompatible_present_type() { + let available = stats_schema([StructField::nullable("value", DataType::STRING)]); + let requested = stats_schema([StructField::nullable("value", DataType::INTEGER)]); + let error = + match CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None) { + Ok(_) => panic!("string statistics should not cast to integer"), + Err(error) => error, + }; + + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + assert!(error.to_string().contains("Utf8")); + assert!(error.to_string().contains("Int32")); + } +} diff --git a/crates/core/src/kernel/snapshot/mod.rs b/crates/core/src/kernel/snapshot/mod.rs index 002eef718..0fe2ef96c 100644 --- a/crates/core/src/kernel/snapshot/mod.rs +++ b/crates/core/src/kernel/snapshot/mod.rs @@ -47,7 +47,7 @@ use crate::kernel::arrow::extract::{self as ex, ProvidesColumnByName}; use super::{Action, CommitInfo, Metadata, Protocol}; use crate::checkpoints::parse_last_checkpoint_hint; -use crate::kernel::arrow::engine_ext::{ExpressionEvaluatorExt, rb_from_scan_meta}; +use crate::kernel::arrow::engine_ext::{ExpressionEvaluatorExt, SnapshotExt, rb_from_scan_meta}; use crate::kernel::{ARROW_HANDLER, StructType, spawn_blocking_with_span}; use crate::logstore::{LogStore, LogStoreExt}; use crate::{DeltaResult, DeltaTableConfig, DeltaTableError, PartitionFilter, to_kernel_predicate}; @@ -292,11 +292,13 @@ impl Snapshot { /// Create a [`ScanBuilder`] borrowing this snapshot to configure a read of the table. pub fn scan_builder(&self) -> ScanBuilder { ScanBuilder::new(self.inner.clone()) + .with_checkpoint_stats_json_fallback(self.config.checkpoint_stats_json_fallback) } /// Consume this snapshot and create a [`ScanBuilder`] that owns the underlying state. pub fn into_scan_builder(self) -> ScanBuilder { ScanBuilder::new(self.inner) + .with_checkpoint_stats_json_fallback(self.config.checkpoint_stats_json_fallback) } /// Update the snapshot to the given version @@ -351,8 +353,12 @@ impl Snapshot { } else { match self.materialized_files() { Some(materialized_files) => { + // Cached partitionValues_parsed uses the source snapshot's layout. + let materialized_seed = (self.inner.partitions_schema()? + == snapshot.inner.partitions_schema()?) + .then_some(materialized_files); snapshot - .materialize_files_with_engine(engine, Some(materialized_files)) + .materialize_files_with_engine(engine, materialized_seed) .await } None => Ok(snapshot), @@ -471,6 +477,13 @@ impl Snapshot { } } + fn materialized_stats_parsed_schema(&self) -> DeltaResult> { + match self.materialized_files_policy() { + MaterializedFilesPolicy::FullTablePreserveRaw => Ok(Some(self.inner.stats_schema()?)), + MaterializedFilesPolicy::FullTableWithoutStats => Ok(None), + } + } + pub(crate) fn materialized_files(&self) -> Option<&Arc> { self.materialized_files .as_ref() @@ -643,12 +656,19 @@ impl Snapshot { .and_then(|materialized_files| materialized_files.full_table_seed()) { Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); self.files_from( engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, Box::new(existing_data), existing_predicate, ) @@ -802,12 +822,19 @@ impl Snapshot { .and_then(|materialized_files| materialized_files.full_table_seed()) { Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); self.files_from_materialized_with_options( log_store.engine(None), predicate, existing_version, + available_stats_schema, + raw_stats_available, Box::new(existing_data), existing_predicate, FileMaterializationOptions { @@ -830,6 +857,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, ) -> SendableRBStream { @@ -837,6 +866,8 @@ impl Snapshot { engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, existing_data, existing_predicate, FileStatsMode::PreserveRaw, @@ -848,6 +879,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, ) -> SendableRBStream { @@ -855,6 +888,8 @@ impl Snapshot { engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, existing_data, existing_predicate, FileStatsMode::FullPreserveRaw, @@ -866,6 +901,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, stats_mode: FileStatsMode, @@ -874,6 +911,8 @@ impl Snapshot { engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, existing_data, existing_predicate, FileMaterializationOptions { @@ -888,6 +927,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, options: FileMaterializationOptions, @@ -900,7 +941,14 @@ impl Snapshot { let stats_materialization = scan.stats_materialization().clone(); let stream = scan - .scan_metadata_from(engine, existing_version, existing_data, existing_predicate) + .scan_metadata_from( + engine, + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) .map(|d| Ok(rb_from_scan_meta(d?)?)); match ScanRowOutStream::try_new_with_materialization( @@ -959,14 +1007,23 @@ impl Snapshot { return Ok(self); } + // TODO: Compatible parsed checkpoints avoid physical `stats` JSON I/O, but FullTablePreserveRaw still retains both + // synthesized `stats` JSON and full `stats_parsed`; removing duplication requires a narrower or lazy cache representation. let batches = match materialized_seed.and_then(|seed| seed.full_table_seed()) { Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); self.files_from_preserving_raw( engine, None, existing_version, + available_stats_schema, + raw_stats_available, Box::new(existing_data), existing_predicate, ) @@ -979,7 +1036,7 @@ impl Snapshot { .await? } }; - let materialized_files = Arc::new(MaterializedFiles::full(self.as_ref(), batches)); + let materialized_files = Arc::new(MaterializedFiles::full(self.as_ref(), batches)?); Ok(Arc::new( self.with_materialized_files(Some(materialized_files)), )) @@ -1194,6 +1251,7 @@ pub(crate) enum MaterializedFilesScope { pub(crate) struct MaterializedFiles { identity: SnapshotIdentity, policy: MaterializedFilesPolicy, + stats_parsed_schema: Option, pub(crate) scope: MaterializedFilesScope, pub(crate) existing_predicate: Option, pub(crate) batches: Arc<[RecordBatch]>, @@ -1202,14 +1260,26 @@ pub(crate) struct MaterializedFiles { #[derive(Debug, Clone)] struct MaterializedFilesSeed { version: Version, + stats_parsed_schema: Option, + raw_stats_available: bool, existing_predicate: Option, batches: Arc<[RecordBatch]>, } impl MaterializedFilesSeed { - fn into_parts(self) -> (Version, SharedRecordBatchIter, Option) { + fn into_parts( + self, + ) -> ( + Version, + Option, + bool, + SharedRecordBatchIter, + Option, + ) { ( self.version, + self.stats_parsed_schema, + self.raw_stats_available, SharedRecordBatchIter::new(self.batches), self.existing_predicate, ) @@ -1251,15 +1321,16 @@ impl Iterator for SharedRecordBatchIter { } impl MaterializedFiles { - fn full(snapshot: &Snapshot, batches: Vec) -> Self { + fn full(snapshot: &Snapshot, batches: Vec) -> DeltaResult { let identity = snapshot.identity(); - Self { + Ok(Self { identity, policy: snapshot.materialized_files_policy(), + stats_parsed_schema: snapshot.materialized_stats_parsed_schema()?, scope: MaterializedFilesScope::FullTable, existing_predicate: None, batches: batches.into(), - } + }) } fn is_cache_for(&self, snapshot: &Snapshot) -> bool { @@ -1270,6 +1341,8 @@ impl MaterializedFiles { match self.scope { MaterializedFilesScope::FullTable => Some(MaterializedFilesSeed { version: self.identity.version, + stats_parsed_schema: self.stats_parsed_schema.clone(), + raw_stats_available: matches!(self.policy, MaterializedFilesPolicy::FullTablePreserveRaw), existing_predicate: self.existing_predicate.clone(), batches: self.batches.clone(), }), @@ -2158,7 +2231,7 @@ mod tests { let target = Snapshot::try_new(log_store.as_ref(), Default::default(), Some(12)).await?; let expected = active_add_paths(&target, log_store.as_ref()).await?; - let mut mismatched_cache = MaterializedFiles::full(&target, vec![]); + let mut mismatched_cache = MaterializedFiles::full(&target, vec![])?; mismatched_cache.identity.table_root = Url::parse("memory:///other/")?; let cached = Arc::new(target.with_materialized_files(Some(Arc::new(mismatched_cache)))); @@ -2374,6 +2447,7 @@ mod tests { let value = serde_json::to_value(snapshot.as_ref())?; assert_eq!(value[10]["policy"], json!("FullTablePreserveRaw")); + assert!(value[10].get("stats_parsed_schema").is_none()); let bytes = serde_json::to_vec(snapshot.as_ref())?; let actual: Snapshot = serde_json::from_slice(&bytes)?; @@ -2391,6 +2465,10 @@ mod tests { materialized_files.policy, MaterializedFilesPolicy::FullTablePreserveRaw ); + assert_eq!( + materialized_files.stats_parsed_schema, + Some(actual.inner.stats_schema()?) + ); Ok(()) } @@ -2440,6 +2518,10 @@ mod tests { MaterializedFilesPolicy::FullTablePreserveRaw ); assert_eq!(materialized_files.scope, MaterializedFilesScope::FullTable); + assert_eq!( + materialized_files.stats_parsed_schema, + Some(actual.snapshot().inner.stats_schema()?) + ); assert_eq!(actual.try_log_data()?.num_files(), 1); assert_eq!( actual @@ -2504,6 +2586,7 @@ mod tests { materialized_files.policy, MaterializedFilesPolicy::FullTableWithoutStats ); + assert!(materialized_files.stats_parsed_schema.is_none()); assert!( actual .try_log_data()? @@ -3153,12 +3236,43 @@ mod tests { Ok(()) } + #[tokio::test] + async fn cached_seed_carries_exact_schema_for_empty_cache() -> TestResult { + let log_store = TestTables::Simple.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let expected_schema = snapshot.inner.stats_schema()?; + let materialized_files = MaterializedFiles::full(&snapshot, vec![])?; + assert_eq!( + materialized_files.stats_parsed_schema, + Some(expected_schema.clone()) + ); + let seed = materialized_files + .full_table_seed() + .expect("full-table cache must produce a seed"); + assert_eq!(seed.stats_parsed_schema, Some(expected_schema)); + assert!(seed.raw_stats_available); + + let config = DeltaTableConfig { + skip_stats: true, + ..Default::default() + }; + let snapshot = Snapshot::try_new(&log_store, config, None).await?; + let materialized_files = MaterializedFiles::full(&snapshot, vec![])?; + assert!(materialized_files.stats_parsed_schema.is_none()); + let seed = materialized_files + .full_table_seed() + .expect("full-table cache must produce a seed"); + assert!(seed.stats_parsed_schema.is_none()); + assert!(!seed.raw_stats_available); + Ok(()) + } + #[tokio::test] async fn test_materialized_files_full_table_seed_shares_batches() -> TestResult { let log_store = TestTables::Simple.table_builder()?.build_storage()?; let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; let batch = RecordBatch::new_empty(Arc::new(arrow_schema::Schema::empty())); - let materialized_files = MaterializedFiles::full(&snapshot, vec![batch]); + let materialized_files = MaterializedFiles::full(&snapshot, vec![batch])?; let seed = materialized_files .full_table_seed() diff --git a/crates/core/src/kernel/snapshot/scan.rs b/crates/core/src/kernel/snapshot/scan.rs index b1b259fe6..639e67451 100644 --- a/crates/core/src/kernel/snapshot/scan.rs +++ b/crates/core/src/kernel/snapshot/scan.rs @@ -3,7 +3,10 @@ use std::sync::Arc; use arrow_array::RecordBatch; use delta_kernel::engine::arrow_data::ArrowEngineData; -use delta_kernel::scan::{Scan as KernelScan, ScanBuilder as KernelScanBuilder, ScanMetadata}; +use delta_kernel::scan::{ + Scan as KernelScan, ScanBuilder as KernelScanBuilder, ScanMetadata, + StatsOptions, +}; use delta_kernel::schema::SchemaRef; use delta_kernel::snapshot::Snapshot as KernelSnapshot; use delta_kernel::{Engine, EngineData, PredicateRef, SnapshotRef, Version}; @@ -14,9 +17,12 @@ use url::Url; #[cfg(feature = "datafusion")] use super::MaterializedFiles; -use super::stats_projection::{FileStatsMaterialization, StatsProjection}; +use super::stats_projection::{ + FileStatsMaterialization, RawStatsPolicy, StatsProjection, StatsSourcePolicy, +}; use crate::DeltaResult; -use crate::kernel::{ReceiverStreamBuilder, scan_row_in_eval}; +use crate::kernel::arrow::engine_ext::SnapshotExt; +use crate::kernel::{CachedScanRowEvaluator, ReceiverStreamBuilder}; /// A boxed, `Send`able stream of [`ScanMetadata`] results produced while scanning a snapshot. pub type SendableScanMetadataStream = Pin> + Send>>; @@ -28,6 +34,7 @@ pub struct ScanBuilder { schema: Option, predicate: Option, stats_materialization: Option, + checkpoint_stats_json_fallback: bool, } impl ScanBuilder { @@ -38,9 +45,15 @@ impl ScanBuilder { schema: None, predicate: None, stats_materialization: None, + checkpoint_stats_json_fallback: false, } } + pub(super) fn with_checkpoint_stats_json_fallback(mut self, enabled: bool) -> Self { + self.checkpoint_stats_json_fallback = enabled; + self + } + /// Provide [`Schema`] for columns to select from the [`Snapshot`]. /// /// A table with columns `[a, b, c]` could have a scan which reads only the first @@ -108,6 +121,7 @@ impl ScanBuilder { schema, predicate, stats_materialization, + checkpoint_stats_json_fallback, } = self; let stats_materialization = match stats_materialization { @@ -122,7 +136,13 @@ impl ScanBuilder { // Modernization: Use kernel's StatsOptions when available // TODO: Add condition to use kernel StatsOptions API once fully integrated - let inner = build_kernel_scan(snapshot, schema, predicate, Some(&stats_materialization))?; + let inner = build_kernel_scan( + snapshot, + schema, + predicate, + Some(&stats_materialization), + checkpoint_stats_json_fallback, + )?; Ok(Scan::new(Arc::new(inner), stats_materialization)) } @@ -142,41 +162,73 @@ fn build_kernel_scan( schema: Option, predicate: Option, stats_materialization: Option<&FileStatsMaterialization>, + checkpoint_stats_json_fallback: bool, ) -> DeltaResult { let mut builder = KernelScanBuilder::new(snapshot) .with_schema_opt(schema) .with_predicate(predicate); if let Some(stats_materialization) = stats_materialization { - builder = with_kernel_stats_output(builder, stats_materialization); + builder = with_kernel_stats_output( + builder, + stats_materialization, + checkpoint_stats_json_fallback, + ); } Ok(builder.build()?) } +fn kernel_stats_options( + materialization: &FileStatsMaterialization, + checkpoint_stats_json_fallback: bool, +) -> StatsOptions { + match ( + materialization.stats_source_policy(), + materialization.raw_stats_policy(), + materialization.stats_projection(), + ) { + (StatsSourcePolicy::None, _, _) | (_, _, StatsProjection::None) => StatsOptions::none(), + (StatsSourcePolicy::ParsedWithJsonFallback, RawStatsPolicy::Preserve, _) => { + StatsOptions::all_struct() + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback) + } + ( + StatsSourcePolicy::ParsedWithJsonFallback, + RawStatsPolicy::Omit, + StatsProjection::Full, + ) => StatsOptions::all_struct() + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback), + ( + StatsSourcePolicy::ParsedWithJsonFallback, + RawStatsPolicy::Omit, + StatsProjection::PredicateColumns(columns), + ) => StatsOptions::struct_columns(columns.iter().cloned().collect()) + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback), + ( + StatsSourcePolicy::ParsedWithJsonFallback, + RawStatsPolicy::Omit, + StatsProjection::NumRecordsOnly, + ) => StatsOptions::struct_columns(vec![]) + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback), + } +} + fn with_kernel_stats_output( builder: KernelScanBuilder, materialization: &FileStatsMaterialization, + checkpoint_stats_json_fallback: bool, ) -> KernelScanBuilder { - // Modernize to use kernel's StatsOptions API - // Map our legacy StatsProjection to kernel's modern StatsOptions - - let stats_opts = match materialization.stats_projection() { - StatsProjection::None => delta_kernel::scan::StatsOptions::json_only(), - StatsProjection::Full => delta_kernel::scan::StatsOptions::all_struct(), - StatsProjection::PredicateColumns(_columns) => { - // Use kernel's efficient column-based stats - // TODO: Use specific column selection: StructStats::Columns(_columns.iter().cloned().collect()) - delta_kernel::scan::StatsOptions::all_struct() - } - StatsProjection::NumRecordsOnly => delta_kernel::scan::StatsOptions::json_only(), - }; - - // Apply the kernel's native stats options - // This reuses the kernel's efficient StructStats implementation - builder.with_stats(stats_opts) + builder.with_stats(kernel_stats_options( + materialization, + checkpoint_stats_json_fallback, + )) } +#[cfg(test)] +#[path = "scan/stats_options_tests.rs"] +mod stats_options_tests; + #[cfg(test)] mod tests { use delta_kernel::expressions::{ColumnName, Scalar}; @@ -207,6 +259,13 @@ mod tests { super::super::Snapshot::try_new(table.log_store().as_ref(), Default::default(), None).await } + fn assert_stats_options_eq(actual: StatsOptions, expected: StatsOptions) { + assert_eq!( + serde_json::to_value(actual).expect("actual StatsOptions should serialize"), + serde_json::to_value(expected).expect("expected StatsOptions should serialize") + ); + } + #[tokio::test] async fn scan_builder_infers_num_records_only_for_default_query_scan() -> DeltaResult<()> { let snapshot = synthetic_snapshot().await?; @@ -276,6 +335,16 @@ mod tests { Ok(()) } + #[test] + fn scan_builder_compatibility_materialization_maps_to_all_struct_stats() { + let materialization = FileStatsMaterialization::compatibility(StatsProjection::full()); + + assert_stats_options_eq( + kernel_stats_options(&materialization, false), + StatsOptions::all_struct().with_checkpoint_stats_json_fallback(false), + ); + } + #[tokio::test] async fn scan_builder_preserves_explicit_without_stats_materialization() -> DeltaResult<()> { let snapshot = synthetic_snapshot().await?; @@ -434,20 +503,26 @@ impl Scan { engine: Arc, materialized_files: Option<&Arc>, ) -> SendableScanMetadataStream { - match materialized_files.and_then(|materialized_files| materialized_files.full_table_seed()) - { - Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); - self.scan_metadata_from( - engine, - existing_version, - Box::new(existing_data), - existing_predicate, - ) - } - None => self.scan_metadata(engine), - } + let Some(materialized_seed) = + materialized_files.and_then(|materialized_files| materialized_files.full_table_seed()) + else { + return self.scan_metadata(engine); + }; + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); + self.scan_metadata_from( + engine, + existing_version, + available_stats_schema, + raw_stats_available, + Box::new(existing_data), + existing_predicate, + ) } /// Stream [`ScanMetadata`] incrementally starting from a previously observed state. @@ -459,24 +534,33 @@ impl Scan { &self, engine: Arc, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, ) -> SendableScanMetadataStream { let inner = self.inner.clone(); - let snapshot = self.inner.snapshot().clone(); - - // process our stored / cached data to conform to the expected input for log replay - let evaluator = match scan_row_in_eval(&snapshot) { - Ok(scan_row_in_eval) => scan_row_in_eval, + let effective_replay_stats_schema = inner.effective_replay_stats_schema().cloned(); + let partition_schema = match inner.snapshot().partitions_schema() { + Ok(partition_schema) => partition_schema, + Err(err) => return Box::pin(once(ready(Err(err)))), + }; + let evaluator = match CachedScanRowEvaluator::try_new( + available_stats_schema.as_ref(), + raw_stats_available, + effective_replay_stats_schema, + partition_schema.as_ref(), + ) { + Ok(evaluator) => evaluator, Err(err) => return Box::pin(once(ready(Err(err)))), }; - let scan_row_iter = existing_data - .map(|batch| Box::new(ArrowEngineData::new(batch)) as Box) - .map(move |b| { - evaluator - .evaluate(b.as_ref()) - .expect("malformed cached log data") - }); + let stats_parsed_schema = evaluator.stats_parsed_schema(); + let scan_row_iter = existing_data.map(move |batch| { + evaluator + .evaluate(batch) + .map(|batch| Box::new(ArrowEngineData::new(batch)) as Box) + .map_err(|err| delta_kernel::Error::Generic(err.to_string())) + }); // TODO: which capacity to choose? let mut builder = ReceiverStreamBuilder::::new(100); @@ -485,6 +569,7 @@ impl Scan { for res in inner.scan_metadata_from( engine.as_ref(), existing_version, + stats_parsed_schema, Box::new(scan_row_iter), existing_predicate, )? { diff --git a/crates/core/src/kernel/snapshot/scan/stats_options_tests.rs b/crates/core/src/kernel/snapshot/scan/stats_options_tests.rs new file mode 100644 index 000000000..67a072d7c --- /dev/null +++ b/crates/core/src/kernel/snapshot/scan/stats_options_tests.rs @@ -0,0 +1,77 @@ +use delta_kernel::expressions::ColumnName; +use serde_json::{Value, json}; + +use super::*; + +fn serialized(options: StatsOptions) -> Value { + serde_json::to_value(options).expect("StatsOptions should serialize") +} + +fn assert_typed_options(materialization: FileStatsMaterialization, expected: StatsOptions) { + let actual = serialized(kernel_stats_options(&materialization, false)); + assert_eq!(actual, serialized(expected)); + assert_eq!( + actual.get("checkpoint_stats_json_fallback"), + Some(&json!(false)) + ); +} + +#[test] +fn compatibility_full_disables_checkpoint_json_fallback() { + assert_typed_options( + FileStatsMaterialization::compatibility(StatsProjection::Full), + StatsOptions::all_struct().with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn query_full_disables_checkpoint_json_fallback() { + assert_typed_options( + FileStatsMaterialization::query(StatsProjection::Full), + StatsOptions::all_struct().with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn predicate_columns_disable_checkpoint_json_fallback() { + let columns = [ColumnName::new(["value"])].into(); + assert_typed_options( + FileStatsMaterialization::query(StatsProjection::PredicateColumns(columns)), + StatsOptions::struct_columns(vec![ColumnName::new(["value"])]) + .with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn row_count_only_disables_checkpoint_json_fallback() { + assert_typed_options( + FileStatsMaterialization::query(StatsProjection::NumRecordsOnly), + StatsOptions::struct_columns(vec![]).with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn disabled_stats_map_to_none() { + let none = serialized(StatsOptions::none()); + assert_eq!( + serialized(kernel_stats_options( + &FileStatsMaterialization::query(StatsProjection::None), + false, + )), + none + ); + assert_eq!( + serialized(kernel_stats_options( + &FileStatsMaterialization::compatibility(StatsProjection::None), + false, + )), + none + ); + assert_eq!( + serialized(kernel_stats_options( + &FileStatsMaterialization::without_stats(), + false, + )), + none + ); +} diff --git a/crates/core/src/kernel/snapshot/serde.rs b/crates/core/src/kernel/snapshot/serde.rs index 698043d30..3eddf3de7 100644 --- a/crates/core/src/kernel/snapshot/serde.rs +++ b/crates/core/src/kernel/snapshot/serde.rs @@ -162,6 +162,9 @@ impl MaterializedFilesWire { let materialized_files = MaterializedFiles { identity, policy, + stats_parsed_schema: owning_snapshot + .materialized_stats_parsed_schema() + .map_err(|err| err.to_string())?, scope, existing_predicate: None, batches: deserialize_batches(self.batches)?.into(), @@ -189,10 +192,10 @@ fn materialized_files_from_legacy_eager_payload( return Ok(None); } - Ok(Some(Arc::new(MaterializedFiles::full( - snapshot, - deserialize_batches(legacy_payload)?, - )))) + Ok(Some(Arc::new( + MaterializedFiles::full(snapshot, deserialize_batches(legacy_payload)?) + .map_err(|err| err.to_string())?, + ))) } fn serialize_batches(batches: &[arrow_array::RecordBatch]) -> Result, String> { diff --git a/crates/core/src/kernel/snapshot/stats_projection.rs b/crates/core/src/kernel/snapshot/stats_projection.rs index 009335753..02392828a 100644 --- a/crates/core/src/kernel/snapshot/stats_projection.rs +++ b/crates/core/src/kernel/snapshot/stats_projection.rs @@ -117,6 +117,10 @@ impl FileStatsMaterialization { self.stats_source_policy } + pub(crate) fn raw_stats_policy(&self) -> RawStatsPolicy { + self.raw_stats_policy + } + pub(crate) fn preserves_raw_stats(&self) -> bool { self.raw_stats_policy == RawStatsPolicy::Preserve } diff --git a/crates/core/src/table/builder.rs b/crates/core/src/table/builder.rs index a3ec2e1ab..08581b969 100644 --- a/crates/core/src/table/builder.rs +++ b/crates/core/src/table/builder.rs @@ -74,6 +74,12 @@ pub struct DeltaTableConfig { #[serde(default)] pub skip_stats: bool, + /// Allow parsed file statistics to fall back to JSON stats stored in checkpoints. + /// This defaults to `false`. + #[serde(default)] + #[delta(skip)] + pub checkpoint_stats_json_fallback: bool, + #[serde(skip_serializing, skip_deserializing)] #[delta(skip)] /// When a runtime handler is provided, all IO tasks are spawn in that handle @@ -96,6 +102,7 @@ impl Default for DeltaTableConfig { log_buffer_size: num_cpus::get() * 4, log_batch_size: 1024, skip_stats: false, + checkpoint_stats_json_fallback: false, io_runtime: None, log_size_limiter: None, engine: None, @@ -109,6 +116,7 @@ impl PartialEq for DeltaTableConfig { && self.log_buffer_size == other.log_buffer_size && self.log_batch_size == other.log_batch_size && self.skip_stats == other.skip_stats + && self.checkpoint_stats_json_fallback == other.checkpoint_stats_json_fallback && self.log_size_limiter == other.log_size_limiter } } @@ -172,6 +180,12 @@ impl DeltaTableBuilder { self } + /// Sets whether checkpoint stats parsing may fall back to JSON stats. + pub fn with_checkpoint_stats_json_fallback(mut self, enabled: bool) -> Self { + self.table_config.checkpoint_stats_json_fallback = enabled; + self + } + /// Sets `log_size_limiter` to the builder pub fn with_log_size_limiter(mut self, limiter: LogSizeLimiter) -> Self { self.table_config.log_size_limiter = Some(limiter); diff --git a/crates/core/tests/it/eager_cache_schema_evolution.rs b/crates/core/tests/it/eager_cache_schema_evolution.rs new file mode 100644 index 000000000..e6b089291 --- /dev/null +++ b/crates/core/tests/it/eager_cache_schema_evolution.rs @@ -0,0 +1,158 @@ +use std::error::Error; + +use arrow_array::{Array, Int32Array, Int64Array, StringArray}; +use deltalake_core::kernel::transaction::CommitBuilder; +use deltalake_core::kernel::{Action, Add, DataType, MetadataExt, StructField, StructType}; +use deltalake_core::protocol::{DeltaOperation, SaveMode}; +use deltalake_core::{DeltaTable, DeltaTableBuilder}; +use url::Url; + +fn add(path: &str, stats: &str) -> Add { + Add { + path: path.to_string(), + size: 1, + modification_time: 1, + data_change: true, + stats: Some(stats.to_string()), + ..Default::default() + } +} + +fn write_operation() -> DeltaOperation { + DeltaOperation::Write { + mode: SaveMode::Append, + partition_by: None, + predicate: None, + } +} + +#[tokio::test] +async fn eager_cache_schema_evolution() -> Result<(), Box> { + let temp_dir = tempfile::tempdir()?; + let table_url = Url::from_directory_path(temp_dir.path()) + .map_err(|_| "temporary table path is not a directory")?; + let mut table = DeltaTable::try_from_url(table_url.clone()) + .await? + .create() + .with_columns([StructField::nullable("id", DataType::INTEGER)]) + .await?; + + CommitBuilder::default() + .with_actions(vec![Action::Add(add( + "old.parquet", + r#"{"numRecords":3,"nullCount":{"id":0},"minValues":{"id":1,"added":"not-an-integer"},"maxValues":{"id":3}}"#, + ))]) + .build( + Some(table.snapshot()?), + table.log_store(), + write_operation(), + ) + .await?; + table.update_state().await?; + + // Reopen version 1 so its active Add batch is eagerly materialized with the old stats schema. + let mut cached = DeltaTableBuilder::from_url(table_url)?.load().await?; + assert_eq!(cached.version(), Some(1)); + + let evolved_schema = StructType::try_new([ + StructField::nullable("id", DataType::INTEGER), + StructField::nullable("added", DataType::INTEGER), + ])?; + let metadata = cached + .snapshot()? + .metadata() + .clone() + .with_schema(&evolved_schema)?; + CommitBuilder::default() + .with_actions(vec![ + Action::Metadata(metadata), + Action::Add(add( + "new.parquet", + r#"{"numRecords":2,"nullCount":{"id":0,"added":0},"minValues":{"id":10,"added":100},"maxValues":{"id":20,"added":200}}"#, + )), + ]) + .build(Some(cached.snapshot()?), cached.log_store(), write_operation()) + .await?; + + // No checkpoint is created. The update must evolve the cached parsed struct and replay the + // new JSON commit rather than reparsing raw statistics for the old file. + cached.update_state().await?; + assert_eq!(cached.version(), Some(2)); + let actions = cached.snapshot()?.add_actions_table(true)?; + assert_eq!(actions.num_rows(), 2); + + let paths = actions + .column_by_name("path") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let num_records = actions + .column_by_name("num_records") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let null_id = actions + .column_by_name("null_count.id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let min_id = actions + .column_by_name("min.id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let max_id = actions + .column_by_name("max.id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let null_added = actions + .column_by_name("null_count.added") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let min_added = actions + .column_by_name("min.added") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let max_added = actions + .column_by_name("max.added") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + + for row in 0..actions.num_rows() { + match paths.value(row) { + "old.parquet" => { + assert_eq!(num_records.value(row), 3); + assert_eq!(null_id.value(row), 0); + assert_eq!(min_id.value(row), 1); + assert_eq!(max_id.value(row), 3); + assert!(null_added.is_null(row)); + assert!(min_added.is_null(row)); + assert!(max_added.is_null(row)); + } + "new.parquet" => { + assert_eq!(num_records.value(row), 2); + assert_eq!(null_id.value(row), 0); + assert_eq!(min_id.value(row), 10); + assert_eq!(max_id.value(row), 20); + assert_eq!(null_added.value(row), 0); + assert_eq!(min_added.value(row), 100); + assert_eq!(max_added.value(row), 200); + } + path => panic!("unexpected Add path {path}"), + } + } + + Ok(()) +} diff --git a/crates/core/tests/it/main.rs b/crates/core/tests/it/main.rs index a5e8d39d5..5b9041db3 100644 --- a/crates/core/tests/it/main.rs +++ b/crates/core/tests/it/main.rs @@ -1,6 +1,7 @@ mod column_mapping_guardrails; mod command_update_table_metadata; mod dat; +mod eager_cache_schema_evolution; mod eager_snapshot; mod exotic_tables; mod time_travel; diff --git a/crates/core/tests/it_datafusion/main.rs b/crates/core/tests/it_datafusion/main.rs index e269d216d..1c38ec579 100644 --- a/crates/core/tests/it_datafusion/main.rs +++ b/crates/core/tests/it_datafusion/main.rs @@ -19,4 +19,5 @@ mod integration_datafusion; mod nanosecond_timestamps; mod read_delta_log_test; mod read_delta_partitions_test; +mod stats_parsed_checkpoint; mod integration_datafusion_deep; \ No newline at end of file diff --git a/crates/core/tests/it_datafusion/stats_parsed_checkpoint.rs b/crates/core/tests/it_datafusion/stats_parsed_checkpoint.rs new file mode 100644 index 000000000..7dd37b021 --- /dev/null +++ b/crates/core/tests/it_datafusion/stats_parsed_checkpoint.rs @@ -0,0 +1,447 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use arrow_array::{ArrayRef, Int32Array, RecordBatch, StringArray}; +use datafusion::assert_batches_sorted_eq; +use datafusion::common::{ScalarValue, stats::Precision}; +use datafusion::datasource::physical_plan::ParquetSource; +use datafusion::physical_plan::{ExecutionPlan, collect}; +use datafusion::prelude::{SessionContext, col, lit}; +use datafusion_datasource::source::DataSourceExec; +use delta_kernel::actions::STATS_PARSED; +use delta_kernel::schema::{DataType, SchemaRef, StructField}; +use delta_kernel::{ + DeltaResult as KernelResult, DeltaResultIteratorStatic, Engine, EngineData, EvaluationHandler, + FileDataReadResultIterator, FileMeta, JsonHandler, ParquetFooter, ParquetHandler, PredicateRef, + StorageHandler, +}; +use deltalake_core::checkpoints::create_checkpoint; +use deltalake_core::delta_datafusion::DeltaScanExec; +use deltalake_core::operations::write::SchemaMode; +use deltalake_core::protocol::SaveMode; +use deltalake_core::{DeltaTable, DeltaTableBuilder, TableProperty}; +use deltalake_test::TestResult; +use futures::TryStreamExt; +use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStreamBuilder}; +use url::Url; + +struct StatsParsedParquetHandler { + inner: Arc, + checkpoint_add_reads: AtomicUsize, +} + +impl ParquetHandler for StatsParsedParquetHandler { + fn read_parquet_files( + &self, + files: &[FileMeta], + physical_schema: SchemaRef, + predicate: Option, + ) -> KernelResult { + let reads_checkpoint_adds = physical_schema.field("add").is_some() + && files + .iter() + .any(|file| file.location.path().contains(".checkpoint.")); + if reads_checkpoint_adds { + let add = physical_schema + .field("add") + .and_then(|field| match field.data_type() { + DataType::Struct(add) => Some(add), + _ => None, + }) + .expect("projected checkpoint Add action must be a struct"); + assert!( + add.field("stats").is_none(), + "compatibility reads must not project add.stats when add.stats_parsed is available" + ); + assert!( + add.field(STATS_PARSED).is_some(), + "compatibility reads must project add.stats_parsed" + ); + self.checkpoint_add_reads.fetch_add(1, Ordering::SeqCst); + } + + self.inner + .read_parquet_files(files, physical_schema, predicate) + } + + fn write_parquet_file( + &self, + location: Url, + data: DeltaResultIteratorStatic>, + ) -> KernelResult<()> { + self.inner.write_parquet_file(location, data) + } + + fn read_parquet_footer(&self, file: &FileMeta) -> KernelResult { + self.inner.read_parquet_footer(file) + } +} + +struct RecordingEngine { + inner: Arc, + parquet_handler: Arc, +} + +impl Engine for RecordingEngine { + fn evaluation_handler(&self) -> Arc { + self.inner.evaluation_handler() + } + + fn storage_handler(&self) -> Arc { + self.inner.storage_handler() + } + + fn json_handler(&self) -> Arc { + self.inner.json_handler() + } + + fn parquet_handler(&self) -> Arc { + self.parquet_handler.clone() + } +} + +#[tokio::test] +#[ignore = "Missing / broken fixture"] +async fn table_scan_prefers_checkpoint_stats_parsed() -> TestResult { + let fixture = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../delta-kernel-rs/kernel/tests/data/stats-parsed-partitioned"); + assert!(fixture.is_dir(), "missing fixture at {}", fixture.display()); + + let temp_dir = tempfile::tempdir()?; + let copy_options = fs_extra::dir::CopyOptions { + content_only: true, + ..Default::default() + }; + fs_extra::dir::copy(&fixture, temp_dir.path(), ©_options)?; + + let table_url = Url::from_directory_path(temp_dir.path()) + .map_err(|_| "temporary table path is not a directory")?; + + // Checkpoint version 2 so the latest scan combines checkpoint stats with version 3 JSON stats. + let checkpoint_table = DeltaTableBuilder::from_url(table_url.clone())? + .with_version(2) + .load() + .await?; + create_checkpoint(&checkpoint_table, None).await?; + + let base_engine = checkpoint_table.log_store().engine(None); + let checkpoint_url = table_url.join("_delta_log/00000000000000000002.checkpoint.parquet")?; + let checkpoint_meta = base_engine.storage_handler().head(&checkpoint_url)?; + let checkpoint_footer = base_engine + .parquet_handler() + .read_parquet_footer(&checkpoint_meta)?; + let checkpoint_add = checkpoint_footer + .schema + .field("add") + .and_then(|field| match field.data_type() { + DataType::Struct(add) => Some(add), + _ => None, + }) + .expect("checkpoint Add action must be a struct"); + assert!( + checkpoint_add.field("stats").is_some(), + "checkpoint must physically contain stats so the projection assertion is meaningful" + ); + assert!( + checkpoint_add.field(STATS_PARSED).is_some(), + "checkpoint must physically contain stats_parsed" + ); + + let parquet_handler = Arc::new(StatsParsedParquetHandler { + inner: base_engine.parquet_handler(), + checkpoint_add_reads: AtomicUsize::new(0), + }); + let engine: Arc = Arc::new(RecordingEngine { + inner: base_engine, + parquet_handler: parquet_handler.clone(), + }); + + // This is the builder equivalent of open_table, with an injected engine for observing the + // checkpoint projection. The DataFusion scan replays the opened table's materialized files. + let table = DeltaTableBuilder::from_url(table_url)? + .with_engine(engine) + .load() + .await?; + let checkpoint_add_reads_after_load = + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst); + assert_eq!( + checkpoint_add_reads_after_load, 1, + "the eager compatibility-cache read must read checkpoint Add actions exactly once" + ); + assert_eq!(table.version(), Some(3)); + + let context = SessionContext::new(); + let provider = table.table_provider().await?; + + // Keep all fixture files in the plan so their exact statistics remain observable. The + // narrower row predicate below correctly prunes the first file during kernel replay. + let stats_filter = col("user_id") + .gt_eq(lit(1_i64)) + .and(col("user_id").lt_eq(lit(9_i64))); + let stats_scan = provider + .scan(&context.state(), None, &[stats_filter], None) + .await?; + + let delta_scan = stats_scan + .downcast_ref::() + .expect("expected DeltaScanExec"); + let children = delta_scan.children(); + let data_source = children[0] + .downcast_ref::() + .expect("expected DataSourceExec child"); + let (file_scan, _) = data_source + .downcast_to_file_source::() + .expect("expected Parquet file source"); + let user_id_index = file_scan + .file_source + .table_schema() + .table_schema() + .index_of("user_id")?; + let mut parsed_file_stats = file_scan + .file_groups + .iter() + .flat_map(|group| group.iter()) + .map(|file| { + let stats = file + .statistics + .as_ref() + .expect("scan file must have parsed statistics"); + let user_id = &stats.column_statistics[user_id_index]; + let value = |precision: &Precision| match precision { + Precision::Exact(ScalarValue::Int64(Some(value))) => *value, + other => panic!("expected exact Int64 statistic, got {other:?}"), + }; + (value(&user_id.min_value), value(&user_id.max_value)) + }) + .collect::>(); + parsed_file_stats.sort_unstable(); + assert_eq!(parsed_file_stats, vec![(1, 3), (4, 6), (7, 9)]); + + let row_filter = col("user_id") + .gt_eq(lit(4_i64)) + .and(col("user_id").lt_eq(lit(8_i64))); + let row_scan = provider + .scan(&context.state(), None, &[row_filter], None) + .await?; + let batches = collect(row_scan, context.task_ctx()).await?; + + let expected = [ + "+------------+---------------+---------+", + "| _ACP_DATE | _ACP_BATCH_ID | user_id |", + "+------------+---------------+---------+", + "| 2024-01-01 | batch-002 | 4 |", + "| 2024-01-01 | batch-002 | 5 |", + "| 2024-01-01 | batch-002 | 6 |", + "| 2024-01-02 | batch-001 | 7 |", + "| 2024-01-02 | batch-001 | 8 |", + "+------------+---------------+---------+", + ]; + assert_batches_sorted_eq!(&expected, &batches); + assert_eq!( + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst), + checkpoint_add_reads_after_load, + "broad and narrow query scans must replay from the materialized seed instead of re-reading checkpoint Add actions" + ); + + Ok(()) +} + +#[tokio::test] +async fn table_scan_skips_old_physical_schema_file_from_checkpoint_stats_parsed() -> TestResult { + let temp_dir = tempfile::tempdir()?; + let table_url = Url::from_directory_path(temp_dir.path()) + .map_err(|_| "temporary table path is not a directory")?; + + let table = DeltaTable::try_from_url(table_url.clone()) + .await? + .create() + .with_columns([ + StructField::nullable("id", DataType::INTEGER), + StructField::nullable("part", DataType::STRING), + ]) + .with_partition_columns(["part"]) + .with_configuration_property(TableProperty::CheckpointWriteStatsAsStruct, Some("true")) + .with_configuration_property(TableProperty::CheckpointWriteStatsAsJson, Some("true")) + .await?; + + let old_batch = RecordBatch::try_from_iter_with_nullable(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + false, + ), + ( + "part", + Arc::new(StringArray::from(vec!["old", "old"])) as ArrayRef, + false, + ), + ])?; + let table = table + .write(vec![old_batch]) + .with_save_mode(SaveMode::Append) + .await?; + + let new_batch = RecordBatch::try_from_iter_with_nullable(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![100, 101])) as ArrayRef, + false, + ), + ( + "part", + Arc::new(StringArray::from(vec!["new", "new"])) as ArrayRef, + false, + ), + ( + "added", + Arc::new(StringArray::from(vec![Some("present"), None])) as ArrayRef, + true, + ), + ])?; + let table = table + .write(vec![new_batch]) + .with_save_mode(SaveMode::Append) + .with_schema_mode(SchemaMode::Merge) + .await?; + assert_eq!(table.version(), Some(2)); + + let adds = table + .get_active_add_actions_by_partitions(&[]) + .try_collect::>() + .await?; + assert_eq!(adds.len(), 2, "expected one data file per partition"); + + let mut physical_files = Vec::with_capacity(2); + for add in adds { + let path = add.path().into_owned(); + let partition_values = add.partition_values_map(); + assert_eq!(partition_values.len(), 1); + let part = partition_values + .get("part") + .cloned() + .flatten() + .expect("data file must have a non-null part value"); + + let reader = ParquetObjectReader::new( + table.object_store(), + deltalake_core::Path::from(path.as_str()), + ); + let builder = ParquetRecordBatchStreamBuilder::new(reader).await?; + let physical_fields = builder + .schema() + .fields() + .iter() + .map(|field| field.name().to_string()) + .collect::>(); + physical_files.push((part, path, physical_fields)); + } + physical_files.sort_by(|left, right| left.0.cmp(&right.0)); + + let new_file = &physical_files[0]; + assert_eq!(new_file.0, "new"); + assert!(new_file.1.contains("part=new/"), "{:?}", new_file.1); + assert_eq!(new_file.2, vec!["id", "added"]); + + let old_file = &physical_files[1]; + assert_eq!(old_file.0, "old"); + assert!(old_file.1.contains("part=old/"), "{:?}", old_file.1); + assert_eq!(old_file.2, vec!["id"]); + + create_checkpoint(&table, None).await?; + + let base_engine = table.log_store().engine(None); + let checkpoint_url = table_url.join("_delta_log/00000000000000000002.checkpoint.parquet")?; + let checkpoint_meta = base_engine.storage_handler().head(&checkpoint_url)?; + let checkpoint_footer = base_engine + .parquet_handler() + .read_parquet_footer(&checkpoint_meta)?; + let checkpoint_add = checkpoint_footer + .schema + .field("add") + .and_then(|field| match field.data_type() { + DataType::Struct(add) => Some(add), + _ => None, + }) + .expect("checkpoint Add action must be a struct"); + assert!( + checkpoint_add.field("stats").is_some(), + "checkpoint must physically contain JSON stats" + ); + assert!( + checkpoint_add.field(STATS_PARSED).is_some(), + "checkpoint must physically contain stats_parsed" + ); + + let parquet_handler = Arc::new(StatsParsedParquetHandler { + inner: base_engine.parquet_handler(), + checkpoint_add_reads: AtomicUsize::new(0), + }); + let engine: Arc = Arc::new(RecordingEngine { + inner: base_engine, + parquet_handler: parquet_handler.clone(), + }); + let reopened = DeltaTableBuilder::from_url(table_url)? + .with_engine(engine) + .load() + .await?; + let checkpoint_add_reads_after_load = + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst); + assert_eq!(checkpoint_add_reads_after_load, 1); + + let context = SessionContext::new(); + let provider = reopened.table_provider().await?; + let id_filter = col("id").gt(lit(50_i32)); + let scan = provider + .scan(&context.state(), None, &[id_filter], None) + .await?; + + let delta_scan = scan + .downcast_ref::() + .expect("expected DeltaScanExec"); + let children = delta_scan.children(); + let data_source = children[0] + .downcast_ref::() + .expect("expected DataSourceExec child"); + let (file_scan, _) = data_source + .downcast_to_file_source::() + .expect("expected Parquet file source"); + let scanned_paths = file_scan + .file_groups + .iter() + .flat_map(|group| group.iter()) + .map(|file| file.object_meta.location.to_string()) + .collect::>(); + assert_eq!(scanned_paths.len(), 1, "{scanned_paths:?}"); + assert!(scanned_paths[0].ends_with(&new_file.1), "{scanned_paths:?}"); + assert!( + !scanned_paths[0].ends_with(&old_file.1), + "{scanned_paths:?}" + ); + + let batches = collect(scan.clone(), context.task_ctx()).await?; + assert_eq!( + delta_scan + .metrics() + .and_then(|metrics| metrics.sum_by_name("count_files_scanned")) + .map(|value| value.as_usize()), + Some(1) + ); + + let expected = [ + "+-----+------+---------+", + "| id | part | added |", + "+-----+------+---------+", + "| 100 | new | present |", + "| 101 | new | |", + "+-----+------+---------+", + ]; + assert_batches_sorted_eq!(&expected, &batches); + assert_eq!( + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst), + checkpoint_add_reads_after_load, + "query scan must not re-read checkpoint Add actions" + ); + + Ok(()) +} From a5bdcf59566a669d31d2ce236fe50e0a6765a170 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Thu, 20 Aug 2026 21:03:46 +0300 Subject: [PATCH 19/20] [HSTACK] feat: make to_datafusion_expr public --- crates/core/src/delta_datafusion/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index 17bf06559..bf409e0d4 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -84,6 +84,14 @@ pub(crate) use table_provider::{ next::FILE_ID_COLUMN_DEFAULT, resolve_file_column_name, update_datafusion_session, }; +/// Converts a Delta Kernel expression to a DataFusion expression. +pub fn to_datafusion_expr( + expr: &delta_kernel::expressions::Expression, + output_type: &delta_kernel::schema::DataType, +) -> DataFusionResult { + engine::to_datafusion_expr(expr, output_type) +} + pub(crate) const PATH_COLUMN: &str = "__delta_rs_path"; #[doc(hidden)] From 5a25b6eabdd76e75548e3bae4b2cb27388400985 Mon Sep 17 00:00:00 2001 From: Andrei Dragomir Date: Thu, 20 Aug 2026 21:05:24 +0300 Subject: [PATCH 20/20] [HSTACK][TEST] feat: Add predicate test --- crates/core/src/delta_datafusion/udtf.rs | 57 +- .../tests/data/conversations/create_delta.py | 281 +++++++ ...413c-94b6-5625d1dfdfba-c000.snappy.parquet | Bin 0 -> 8027 bytes ...4f20-8b24-9fa3894dfcb6-c000.snappy.parquet | Bin 0 -> 8026 bytes ...4f3d-a838-3254776dad75-c000.snappy.parquet | Bin 0 -> 8054 bytes ...401d-b9d7-9f063567174a-c000.snappy.parquet | Bin 0 -> 8054 bytes ...454a-acdf-f9a8496b4f70-c000.snappy.parquet | Bin 0 -> 8053 bytes ...49ae-8b4d-3df43530c506-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4568-8215-0e3f5881f549-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4e8a-ab1c-d7c71928a942-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4ac9-8af1-cb19b66c0edb-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4aea-a46c-86905984e372-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4a71-b1bb-8a894fa5d7dd-c000.snappy.parquet | Bin 0 -> 8053 bytes ...46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4796-a4cb-323deb17569b-c000.snappy.parquet | Bin 0 -> 8052 bytes ...47b8-8e47-3a70c6a54b19-c000.snappy.parquet | Bin 0 -> 8053 bytes ...43d5-b09e-3317e6b53514-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4972-a250-b641daa5a6a7-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4033-bedc-026d5b184b78-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4884-a7e2-87e358e8e103-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4a7d-aad6-8664717be548-c000.snappy.parquet | Bin 0 -> 8053 bytes ...472a-8d3f-2890d998f099-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4340-807a-719ec6048133-c000.snappy.parquet | Bin 0 -> 8052 bytes ...48ca-907c-fa042d2e83fa-c000.snappy.parquet | Bin 0 -> 8053 bytes ...45b7-a88a-3662756e970b-c000.snappy.parquet | Bin 0 -> 8053 bytes ...489c-bcaf-6406017afe40-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4be9-b318-4b06a02001fd-c000.snappy.parquet | Bin 0 -> 8053 bytes ...45a7-ae4d-7d5ff915786e-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4b72-a482-566f29c82b5c-c000.snappy.parquet | Bin 0 -> 8053 bytes .../_delta_log/00000000000000000000.json | 5 + .../_delta_log/00000000000000000001.json | 3 + .../_delta_log/00000000000000000002.json | 3 + .../_delta_log/00000000000000000003.json | 3 + .../_delta_log/00000000000000000004.json | 3 + .../_delta_log/00000000000000000005.json | 3 + .../_delta_log/00000000000000000006.json | 3 + .../00000000000000000007.checkpoint.parquet | Bin 0 -> 58489 bytes .../_delta_log/00000000000000000007.json | 3 + .../_delta_log/00000000000000000008.json | 3 + .../_delta_log/00000000000000000009.json | 3 + .../_delta_log/00000000000000000010.json | 3 + .../_delta_log/00000000000000000011.json | 3 + .../_delta_log/00000000000000000012.json | 3 + .../_delta_log/00000000000000000013.json | 3 + .../delta/_delta_log/_last_checkpoint | 1 + .../core/tests/data/conversations/schema.sql | 41 + .../create_delta.py | 245 ++++++ ...401b-91cc-cb6b8aeb1388-c000.snappy.parquet | Bin 0 -> 8026 bytes ...43de-8be4-eae27ba5e088-c000.snappy.parquet | Bin 0 -> 8027 bytes ...403e-8734-c049b9c71d62-c000.snappy.parquet | Bin 0 -> 8054 bytes ...44b8-84b6-6b04b73cb9dc-c000.snappy.parquet | Bin 0 -> 8054 bytes ...4a43-899c-e4f62daedd03-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4669-9717-3ac43656b7f5-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4524-a1e3-cb500834e3ca-c000.snappy.parquet | Bin 0 -> 8053 bytes ...46f1-9298-6aeb8c34eab9-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4902-a824-7a704ee3c700-c000.snappy.parquet | Bin 0 -> 8052 bytes ...49e9-a893-592106ba7c38-c000.snappy.parquet | Bin 0 -> 8053 bytes ...41cc-85a4-ff426fdb2954-c000.snappy.parquet | Bin 0 -> 8053 bytes ...48e7-a6df-6a294b350546-c000.snappy.parquet | Bin 0 -> 8052 bytes ...48a6-b23b-0710588c47d1-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4e52-8bb0-4c16d11b3132-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4a6a-9e5d-25db1c63d6ff-c000.snappy.parquet | Bin 0 -> 8053 bytes ...41b2-9dbe-d66dd9ad8ae1-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4bbf-986e-e08dfe777c7b-c000.snappy.parquet | Bin 0 -> 8053 bytes ...40c0-b7a0-4ead10a5517e-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4054-bb47-4206fe646f8b-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4740-87e4-02b1403ab802-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4172-a4f7-01e5d41ef9c2-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4570-9f56-374968008e3a-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4429-b8b2-ca8b3092ac40-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4772-9561-1d3b6e0ef2bd-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4b71-9d11-7ec5928b2545-c000.snappy.parquet | Bin 0 -> 8052 bytes ...4c96-bddf-351439da8c8f-c000.snappy.parquet | Bin 0 -> 8053 bytes ...4950-87e5-9cd130fbdae7-c000.snappy.parquet | Bin 0 -> 8053 bytes .../_delta_log/00000000000000000000.json | 5 + .../_delta_log/00000000000000000001.json | 3 + .../_delta_log/00000000000000000002.json | 3 + .../_delta_log/00000000000000000003.json | 3 + .../_delta_log/00000000000000000004.json | 3 + .../_delta_log/00000000000000000005.json | 3 + .../_delta_log/00000000000000000006.json | 3 + .../00000000000000000007.checkpoint.parquet | Bin 0 -> 38166 bytes .../_delta_log/00000000000000000007.json | 3 + .../_delta_log/00000000000000000008.json | 3 + .../_delta_log/00000000000000000009.json | 3 + .../_delta_log/00000000000000000010.json | 3 + .../_delta_log/00000000000000000011.json | 3 + .../_delta_log/00000000000000000012.json | 3 + .../_delta_log/00000000000000000013.json | 3 + .../delta/_delta_log/_last_checkpoint | 1 + .../integration_datafusion_predicate.rs | 762 ++++++++++++++++++ crates/core/tests/it_datafusion/main.rs | 3 +- 94 files changed, 1467 insertions(+), 12 deletions(-) create mode 100644 crates/core/tests/data/conversations/create_delta.py create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-5af5ddd7-48ea-413c-94b6-5625d1dfdfba-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-68246fa2-b36c-4f20-8b24-9fa3894dfcb6-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-66777adb-10ca-4f3d-a838-3254776dad75-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-e5cb40ab-fba4-401d-b9d7-9f063567174a-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-d4320b85-10f7-454a-acdf-f9a8496b4f70-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f442863b-8e12-49ae-8b4d-3df43530c506-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-82e0525d-9555-4568-8215-0e3f5881f549-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-8ce2f2b0-a7c6-4e8a-ab1c-d7c71928a942-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1b24d95a-1de7-4ac9-8af1-cb19b66c0edb-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-81dd5d9f-3e71-4aea-a46c-86905984e372-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-95c2849a-4d95-4a71-b1bb-8a894fa5d7dd-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-c4450a76-c10c-46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-801b3b87-b9d3-4796-a4cb-323deb17569b-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-bfc1b106-122a-47b8-8e47-3a70c6a54b19-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-59765f2d-40bb-4972-a250-b641daa5a6a7-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-06c535fb-4c81-4033-bedc-026d5b184b78-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-2ffd1456-206d-4884-a7e2-87e358e8e103-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-b95c7946-cd89-4a7d-aad6-8664717be548-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-6c8cf883-cdf4-48ca-907c-fa042d2e83fa-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-e6b6c2fb-201a-45b7-a88a-3662756e970b-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-01aa1a67-ddba-489c-bcaf-6406017afe40-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000000.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000001.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000002.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000003.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000004.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000005.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000006.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.checkpoint.parquet create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000008.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000009.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000010.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000011.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000012.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/00000000000000000013.json create mode 100644 crates/core/tests/data/conversations/delta/_delta_log/_last_checkpoint create mode 100644 crates/core/tests/data/conversations/schema.sql create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/create_delta.py create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-387b2ef3-cd3e-403e-8734-c049b9c71d62-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-bf68d798-f825-4902-a824-7a704ee3c700-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-26aa5864-10c1-48e7-a6df-6a294b350546-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-9abac23b-07d6-48a6-b23b-0710588c47d1-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-92b1f2f5-21dc-4a6a-9e5d-25db1c63d6ff-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-33f334a1-8c2c-41b2-9dbe-d66dd9ad8ae1-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-ecee6b33-3470-4bbf-986e-e08dfe777c7b-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-3e56e624-1e0b-40c0-b7a0-4ead10a5517e-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-65f27845-2a8b-4172-a4f7-01e5d41ef9c2-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-77d00763-9ba2-4429-b8b2-ca8b3092ac40-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-00f3f17d-20f3-4772-9561-1d3b6e0ef2bd-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-9dca9883-8cd9-4950-87e5-9cd130fbdae7-c000.snappy.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000000.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000001.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000002.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000003.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000004.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000005.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000006.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.checkpoint.parquet create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000008.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000009.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000010.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000011.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000012.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000013.json create mode 100644 crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/_last_checkpoint create mode 100644 crates/core/tests/it_datafusion/integration_datafusion_predicate.rs diff --git a/crates/core/src/delta_datafusion/udtf.rs b/crates/core/src/delta_datafusion/udtf.rs index 207ee31af..2baa3fc65 100644 --- a/crates/core/src/delta_datafusion/udtf.rs +++ b/crates/core/src/delta_datafusion/udtf.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use delta_kernel::Engine; use datafusion::catalog::{TableFunctionImpl, TableProvider}; use datafusion::common::{internal_datafusion_err, DataFusionError, Result, ScalarValue}; use datafusion::logical_expr::Expr; @@ -7,9 +8,18 @@ use std::collections::{HashMap, VecDeque}; use std::sync::Arc; use tokio::runtime::Runtime; use url::Url; -use crate::open_table_with_storage_options; +use crate::{builder_from_valid_url, open_table_with_storage_options}; pub fn register_delta_table_udtf(ctx: &SessionContext, name: Option<&str>, settings: Option<&HashMap>) { + register_delta_table_udtf_with_engine(ctx, name, settings, None); +} + +pub fn register_delta_table_udtf_with_engine( + ctx: &SessionContext, + name: Option<&str>, + settings: Option<&HashMap>, + engine: Option>, +) { let prefix = name .or_else(|| Some("delta_table")).unwrap(); @@ -18,6 +28,7 @@ pub fn register_delta_table_udtf(ctx: &SessionContext, name: Option<&str>, setti Arc::new(DeltaTableUdtf { flavor: DeltaTableUdtfFlavor::Old, settings: settings.cloned(), + engine: None, }), ); ctx.register_udtf( @@ -25,6 +36,7 @@ pub fn register_delta_table_udtf(ctx: &SessionContext, name: Option<&str>, setti Arc::new(DeltaTableUdtf { flavor: DeltaTableUdtfFlavor::Next, settings: settings.cloned(), + engine, }), ); } @@ -35,10 +47,20 @@ pub enum DeltaTableUdtfFlavor { Next } -#[derive(Debug)] pub struct DeltaTableUdtf { flavor: DeltaTableUdtfFlavor, settings: Option>, + engine: Option>, +} + +impl std::fmt::Debug for DeltaTableUdtf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeltaTableUdtf") + .field("flavor", &self.flavor) + .field("settings", &self.settings) + .field("has_engine", &self.engine.is_some()) + .finish() + } } #[async_trait] @@ -75,22 +97,35 @@ impl TableFunctionImpl for DeltaTableUdtf { }; let flavor = self.flavor.clone(); + let engine = self.engine.clone(); let table = std::thread::spawn(move || { let rt = Runtime::new().unwrap(); let table_uri = Url::parse(&path) .expect(&format!("Invalid table uri: {}", path)); rt.block_on(async { - let delta_table = open_table_with_storage_options(table_uri, settings) - .await - .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) - .unwrap(); - match flavor { DeltaTableUdtfFlavor::Old => { - let provider = delta_table.table_provider_old(); - Arc::new(provider) as Arc + let delta_table = open_table_with_storage_options(table_uri, settings) + .await + .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) + .unwrap(); + Arc::new(delta_table.table_provider_old()) as Arc } DeltaTableUdtfFlavor::Next => { + let mut builder = builder_from_valid_url(table_uri) + .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) + .unwrap() + .with_storage_options(settings) + .without_files() + .with_checkpoint_stats_json_fallback(true); + if let Some(engine) = engine { + builder = builder.with_engine(engine); + } + let delta_table = builder + .load() + .await + .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) + .unwrap(); let provider = delta_table .table_provider() .build() @@ -102,8 +137,8 @@ impl TableFunctionImpl for DeltaTableUdtf { }) }) .join() - .map_err(|e| internal_datafusion_err!("DeltaTableFunc error opening table"))?; + .map_err(|_e| internal_datafusion_err!("DeltaTableFunc error opening table"))?; - Ok(Arc::clone(&table)) + Ok(table.clone()) } } diff --git a/crates/core/tests/data/conversations/create_delta.py b/crates/core/tests/data/conversations/create_delta.py new file mode 100644 index 000000000..d4b2be336 --- /dev/null +++ b/crates/core/tests/data/conversations/create_delta.py @@ -0,0 +1,281 @@ +from datetime import date, timedelta +from pathlib import Path +import json +import shutil + +import pyarrow as pa +import pyarrow.parquet as pq +from deltalake import DeltaTable, write_deltalake + +TABLE_PATH = Path("delta") +START_DATE = date.fromisoformat("2024-01-01") +CHECKPOINT_DATE = date.fromisoformat("2024-01-04") # Use None for the final version. +DAYS = 7 +BATCHES_PER_DAY = 2 +FILES_PER_PARTITION = 2 +ROWS_PER_FILE = 2 + +SCHEMA = pa.schema( + [ + pa.field( + "_acp_system_metadata", + pa.struct( + [ + pa.field("acp_sourceBatchId", pa.string()), + pa.field("commitBatchId", pa.string()), + pa.field("trackingId", pa.string()), + pa.field("rowId", pa.string()), + pa.field("rowVersion", pa.int64()), + pa.field( + "primaryIdentity", + pa.struct( + [ + pa.field("id", pa.string()), + pa.field( + "namespace", + pa.struct([pa.field("code", pa.string())]), + ), + ] + ), + ), + pa.field("ingestTime", pa.int64()), + pa.field("isDeleted", pa.bool_()), + ] + ), + ), + pa.field( + "agenticExperience", + pa.struct( + [ + pa.field( + "agents", + pa.list_(pa.struct([pa.field("agentID", pa.string())])), + ), + pa.field("version", pa.string()), + ] + ), + ), + pa.field( + "conversation", + pa.struct( + [ + pa.field("conversationID", pa.string()), + pa.field("turnID", pa.string()), + pa.field( + "text", + pa.struct( + [ + pa.field("raw", pa.string()), + pa.field("source", pa.string()), + ] + ), + ), + ] + ), + ), + pa.field( + "identityMap", + pa.map_( + pa.string(), + pa.list_( + pa.struct( + [ + pa.field("id", pa.string()), + pa.field("prim", pa.bool_()), + ] + ) + ), + ), + ), + pa.field("_ACP_DATE", pa.date32()), + pa.field("_ACP_BATCHID", pa.string()), + ] +) + +shutil.rmtree(TABLE_PATH, ignore_errors=True) + +commit = 0 +for day_index in range(DAYS): + current_date = START_DATE + timedelta(days=day_index) + for batch_number in range(1, BATCHES_PER_DAY + 1): + batch_id = f"batch-{batch_number}-{current_date.isoformat()}" + file_batches = [] + + for file_number in range(1, FILES_PER_PARTITION + 1): + file_batch_id = f"{batch_id}--file-{file_number}" + rows = [] + for row_number in range(1, ROWS_PER_FILE + 1): + row_key = ( + f"{current_date.isoformat()}-batch-{batch_number}" + f"-file-{file_number}-row-{row_number}" + ) + identity = f"user-{row_key}@example.com" + rows.append( + { + "_acp_system_metadata": { + "acp_sourceBatchId": batch_id, + "commitBatchId": f"commit-{batch_id}", + "trackingId": f"tracking-{row_key}", + "rowId": f"row-{row_key}", + "rowVersion": 1, + "primaryIdentity": { + "id": identity, + "namespace": {"code": "email"}, + }, + "ingestTime": int( + f"{current_date:%Y%m%d}" + f"{batch_number}{file_number}{row_number}" + ), + "isDeleted": False, + }, + "agenticExperience": { + "agents": [{"agentID": f"agent-{batch_number}"}], + "version": "1.0", + }, + "conversation": { + "conversationID": f"conversation-{current_date}-{file_number}", + "turnID": f"turn-{batch_number}-{file_number}-{row_number}", + "text": { + "raw": f"Conversation text for {row_key}", + "source": "generated", + }, + }, + "identityMap": [ + ("email", [{"id": identity, "prim": True}]) + ], + "_ACP_DATE": current_date, + # A temporary partition forces delta-rs to create this file. + "_ACP_BATCHID": file_batch_id, + } + ) + + file_batches.append(pa.RecordBatch.from_pylist(rows, schema=SCHEMA)) + + write_deltalake( + TABLE_PATH, + pa.RecordBatchReader.from_batches(SCHEMA, file_batches), + mode="overwrite" if commit == 0 else "append", + partition_by=["_ACP_DATE", "_ACP_BATCHID"], + configuration=( + { + "delta.checkpoint.writeStatsAsJson": "true", + "delta.checkpoint.writeStatsAsStruct": "true", + } + if commit == 0 + else None + ), + ) + + # Move both files into the real partition and normalize their add actions. + transaction_path = TABLE_PATH / "_delta_log" / f"{commit:020}.json" + actions = [json.loads(line) for line in transaction_path.read_text().splitlines()] + add_actions = [action["add"] for action in actions if "add" in action] + if len(add_actions) != FILES_PER_PARTITION: + raise RuntimeError( + f"Expected {FILES_PER_PARTITION} files, got {len(add_actions)}" + ) + + destination_directory = ( + TABLE_PATH + / f"_ACP_DATE={current_date.isoformat()}" + / f"_ACP_BATCHID={batch_id}" + ) + destination_directory.mkdir(parents=True, exist_ok=True) + for add in add_actions: + source = TABLE_PATH / add["path"] + destination = destination_directory / source.name + source.rename(destination) + source.parent.rmdir() + add["path"] = destination.relative_to(TABLE_PATH).as_posix() + add["partitionValues"]["_ACP_BATCHID"] = batch_id + + transaction_path.write_text( + "\n".join(json.dumps(action, separators=(",", ":")) for action in actions) + + "\n" + ) + commit += 1 + + if CHECKPOINT_DATE == current_date: + DeltaTable(TABLE_PATH).create_checkpoint() + +if CHECKPOINT_DATE is None: + DeltaTable(TABLE_PATH).create_checkpoint() + +# delta-rs does not currently materialize these configured checkpoint fields. +checkpoint_path = next((TABLE_PATH / "_delta_log").glob("*.checkpoint.parquet")) +checkpoint = pq.read_table(checkpoint_path) +add_index = checkpoint.schema.get_field_index("add") +add_field = checkpoint.schema.field(add_index) +adds = checkpoint.column(add_index).to_pylist() + +stats = [ + json.loads(add["stats"]) if add and add.get("stats") else None + for add in adds +] +for parsed_stats in stats: + if parsed_stats is not None: + parsed_stats["tightBounds"] = True + +inferred_stats_type = pa.array(stats).type +stats_field_order = [ + "numRecords", + "minValues", + "maxValues", + "nullCount", + "tightBounds", +] +stats_type = pa.struct( + [ + inferred_stats_type.field(name) + for name in stats_field_order + if inferred_stats_type.get_field_index(name) != -1 + ] +) +partition_values_type = pa.struct( + [ + pa.field("_ACP_DATE", pa.date32()), + pa.field("_ACP_BATCHID", pa.string()), + ] +) + +for add, parsed_stats in zip(adds, stats): + if add is None: + continue + partition_values = dict(add["partitionValues"]) + add["stats_parsed"] = parsed_stats + add["partitionValues_parsed"] = { + "_ACP_DATE": date.fromisoformat(partition_values["_ACP_DATE"]), + "_ACP_BATCHID": partition_values["_ACP_BATCHID"], + } + +original_add_type = add_field.type +patched_add_type = pa.struct( + [ + field + for field in original_add_type + if field.name not in {"stats_parsed", "partitionValues_parsed"} + ] + + [ + pa.field("stats_parsed", stats_type), + pa.field("partitionValues_parsed", partition_values_type), + ] +) +patched_add_field = pa.field( + "add", + patched_add_type, + nullable=add_field.nullable, + metadata=add_field.metadata, +) +checkpoint = checkpoint.set_column( + add_index, + patched_add_field, + pa.array(adds, type=patched_add_type), +) +pq.write_table(checkpoint, checkpoint_path, compression="snappy") + +last_checkpoint_path = TABLE_PATH / "_delta_log" / "_last_checkpoint" +last_checkpoint = json.loads(last_checkpoint_path.read_text()) +last_checkpoint["sizeInBytes"] = checkpoint_path.stat().st_size +last_checkpoint_path.write_text(json.dumps(last_checkpoint, separators=(",", ":"))) + +print(f"Created {TABLE_PATH} with {commit} versions and a patched checkpoint") diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-5af5ddd7-48ea-413c-94b6-5625d1dfdfba-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-5af5ddd7-48ea-413c-94b6-5625d1dfdfba-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f449af9924fb95b89c98465e62caa4a501ec1b76 GIT binary patch literal 8027 zcmc&(TWA|+7M_vhYn<3|^3Q5gCF``w#36BX@nz-Wmdx0?*s)#Nb!^!zn;lDITef5i zNw#7?ETM!Ygg%r$gc3>!A?(AlEO`haEM>{VvXs(?QbH*EkcTBKC6qq)A%tbmIWwaf z$&#&xT~^qV{{P&*bNSDI#*WF;e&%B5nQxpq7}v>h!VJ7|i)taU(&O(5_yT=BK0o|0 zj&-o3=YhF{HL^yIYxX#R(dFP=u=g%|-&AlsE5d$aBa=xLdVa2+2RvIE2YrYnKHd$A z3OO~gmResnq43U_sdOoo)_MXxxs76vzxxm8IW7PoJDFeu+0vzh?yruNC; zJD?q94Ig=DutG6^g7e=xv%gx1i|Kr`hX<|r+B;s~@az_ELRlH|fxu>7%UM`HslY-` zUevCtnQU6S0MR9p-}iC8fPJ@C0GxUNAUGz{maYacs%6wvx)xLyrM8QmGJ$ps@f*vJ zTp;H#K5N-?pgoQqVx5=_POq3Iw^1>$C9q>Ak^|jMpiy1c)(bsYW**jsLPnV#>LVeJ z-Wv!JXfz_KLSsrzrgYjXDws;6f9(D3@k3 zc^+zk*E3x~_6RdvgK?J|}h)AC^h@gn7JlRl54;T_{L~FC7cyQ2A-i@^D?Z)aUnfj^S zU_FJz=N*PpGW@KQjgSc))$JqfA{vJ{ygnIiQ*+Q*uH+LdaM)Ng*Y9v2y`m~YCN$>I$*#mkQZtc} zS{OF9d|@V)(O`%JJ1wdFh?dsi%0x4VsUf5ip{rRfm(tcjlMDA7X+IBBbmRCCc-ilA zyV9wA0qg+Pm4XnNCiPA-I$~PIcr@WS0-hLqAkt=U8!&KF*bdn@%QZA&gD7pGO&lZ4 zC^&O!(QdCXPnogI9Y`5*n^3dhgkv1$X>_6qq6vQHbw!-pxu$JfQVm;bdJ|%~hts># zKtXk3wm88hdc@bA66axUVn5f|A$C`-sok~29D+;O5kBIj8L8Pbz;#!-*Q?$2hYxe2 zJ|RH%pogrLqe`_ajL69m0>ibhk-* z*?>{3=XNa{x#}9y%I@Q84hc{Av-%Xk2`@3gw?+1{tQ?|sd1bY6YTxAHZ-o`6+gnDaIq@{tSfeFcw-N#I34PMzmLYJR=U|LQ8y@IyaCdVR*E-v%JD!F{GRr;X?<7^JarRlhubAqRAT;_ytFyN0pY}yW_ zdp8jCQY5>-7LwAZ2c#FRXcy;f8n&z2YcpHU4`}G-an@I6sO)B~xKYWJ-pKE@Nnaj7 zW3>l(Y~(fO_o~23jr_(L0QT48m5sa%nQ@`p5Is03ebo$XNA8meh zg2n8?2$q?okpV{qGWy6NJmsb9?b7!L(de5!A-vgxMRix6THWKx8SEavWdAPf9#yLqdV5qiS0q9YSvDbde(Cc;b$k=WLXI9qmwx<;h01=(?UUd-XZ;| z18vsPmaNAhPI!gp{&kX#*v+j}GpEmC)`>OS{sV{bikF^sN_P*V{r_LvuXxJhgj?N^ zz(2mtUg-(cTU}kg(dbCA9TF}|w~q{gV>}PfYnZ%J(>q)K+{OtHyP^BtKgHhXgugSv zB{d7*9yy9nR_-==disO8cl9S5_xJ5ye4U#L%d$KLzxcW&%SCy3SsueJ_>^UZ_F)NY zSsuj|Y?+_1LQ&W$%JLj;ho(>>1ix}Imh*bAq7r!m-{7T$ri)0&U(_ed<1+(ZZ_tNO zqPRvy`fD;spJ-}?^b!B$D)NWU0e^((CVyyRmDa>c_|BpI?U`Qze*#G(M2EL2KY^0* zeS};m{%CB=>-DW7|7B#g_~%Xhut{_>l8BCF7fDw>(bt5758)z~E*wGrOG#gnjDV zpe|2AQ&txP{egw}*v@=>I=dL?Lw0#2)(ienzD#|}$Re%v(k3oSMaaA4^-d6fZU0A+ zUc}N?$QELd!>faUpHgAv^D5SyepIOll?l|Zk$$pg;hCYebv`3Ipzqu~^4HGa@F>#t zdJC^(ePJv~NhIxCn4@xlQvetuIz0hyhtM@+tWP9S$^nBt+V!pR=Q4gE!Mv5x|tx&HNq_@r+(k}hC9uafM`;#VvY2#EY8jDBqCutbAS@m z?NR@+Rj?N%yeh>8yR=e-ycOo-WBnh_LEP4-v&rb1JOyXk=>OydGRiCDKP0Y*^)W8% z^#~;6Ws6@TGuS&z=PSLn2pX11A2F2b)!q*lkzPdKlC(#+oej1isk1$}a5 zK2XRmW^`HKKK6=a4{ICGyT1TEbr{b3sv4pmkJWe?og7(~W5(ed)enVSaXzu}m@hgv zRBFcgSWScncNTkRx90=1W%?=K;lY*pUS%oKo8C!AW()CvKb?$(*8s)Nr`>wl_Al&zC^Cth~Fy7*E#IALdOUF@|T+3`g-rQ5MIwq;AU zkYvmDLkUX>VF@LKu!IuAVoE7}2%#*?QbJk!Q1&5|P)x}~D1BH;2qjN}h7d~6IWwaf z$&#%sEfuz;|3A0yT>kT)v19U_pSjpo=9^*;#&vO=Fb{9ss#-{F^!xh*zQAC=&ksM0 zW1Xz~Dlm7lM%Ku2%^n9Zx*VJf_FiJImyThAMHb#9Qu$$L%7M8awu#l72 zwS6^{O>5U6x+L<(Am&S<9XS?Q!fW)`iL7^onV68x;dv0(;s-a;~olG^*>`cA+23%)@$6$SAX?21$sc z_Xa`)T20v|w-aRj*L)!N{mOHDPs`=iLTYE*3_8~N8D$=1FPX9M^tOWK!ftN6-_JV+ ztp%PMKyz9hKIYlaOg*{Rhd|HRj)kYEpzRlW)^>6|rOKSDRtgg?zLnxSg< zwSXarvX)Z|TC!HaoD|}kg>p)A^iJ?Utn7>Id0Sz01Z{=UsMi~~iyTJ1wfpQGPO-}& zy1UFsTAi)xCUb}acX=2Z97l3u&4wnrYtt&V^i_53?lWxa+frv|N3ORL^# zpt<-(x3xHtqoGqfNv(v4?kAEntlKitQtB_F+xo6MV`Hn5Fbb)4-s&j||9_*7#9_G4 z#68NTnM|ICTHy6eSCBozJR253LWpvrM}SVQuada>z{!F{y9?Fi5k7Gl%eQEGuYq`x za<|E#JS58PWYa0jL>tlC>?l5LHALi48bxteG2fxVvwv ziR{h%LB616Zf3NCnp6ua>*QG5C5QVw(ROxo2~ELaaw2(&oARJKq_*qrveflF3c7uxI~ZmU6;gpSerP+HFk=9Rcq>GEis4S5)Or1yfiO0dxp8bD))M| zyZ-!nPShs^$UlyZ4sKM4>i0Y+{JobwEqr!?Ww|pUl@~CUb%57TJK_F{7e3^rPh8Sport@hxy4-vQ+2d&mC)WpwD13b-P6&!PAgAaCnCr{ z5!R%84bsmWJl33KeG|C8()48QRtw(9Fpd|T@Rv(W5I%LYs6Fe!swbtg3qtoJNWXFj zU-8o2Ch5xtjAA{vYuU(E*N|3rA6Iioc*tMYrvOg)0t5V>$i6HqhiF}1S#6x!H+lF| zVMDszEPd95{^|3DF%MQn(tUXm8XSf94GB)wt!LPE3h(Kewk`)Unick?*IT5A&5&tx zuymKEP0aMk-D+^mOdm;*=|4VWWSSE%ag8-vqID|~5T7t8J!lyP*Le=cShV4R{swn9 zH)<+>fXZ)s*j0NO#ZO*n##Khw7hPQDg;jF-s8#xB3&z~IbWioH=kvLWroUb)`}aIOzDmM zs7-ow4vp0w;IWa{nBS`cD>d>rUIbttKU>+z%a9otx{2r)y!2ta^jRAsvd#atXMyT1 ztYBi`YGW z$o^f{J*s$P3_9e{hLDxM?vQ@pf$msOC$aDIW-)MBC*p3RU z^3?pW*BkU9lqjxIk^Y(q(kGf4Cw;^}vx)qnE5IKiy2&4!-lR3L622>F|6u+{z@I?U zIMLy4)=!{hd><#*i9Z_K^Ll-o$iINB7XPw|A2x|jMiSA<>?(;XMZ|-FwD5<*C3#s{ zM>^Syq_=>H-5&j(vS~q56v%yi*oD`{iS%6@Xik8(Fmnt3hj+id?@}g z@_)-4tIStuW|@SD)rVh+UpWkVy(1IIUpqeXAkq~?5uMpf#wRydz-%IR2=!CuuVVLi z8R@#e2g~t<9;fJBT8om5ymN;97Re)O#RoQbUt0jV=}o{}ng4}SyyYpPzl#1X2S#?2 znZ<)dAna3@Ms#@!nzFeX7z(V!Cl8n7bJ^9vAhOHju>tUh@@48%##d>rmo{-xDni~h zuXmdGYx_Te^dgqFLbect9NruO{FDkSpZi#I`cb7KRHjkCM*7K~g=e1D*7=Mag1#%u z$X`2uV-rZ%>n*&E^@Xt{C6TmmWr@lGP61$u==21*9YWWPwYoGD)}l%}kx8dlGc##~ z8$^7E<^GQ^tYfT^EZz*oIU*A!F;XCL}T?iTssiFC2}j7?gTtJ*`vx(VQGb zdJ#jVWD`C(n~_Dpj`o#hl&3>WP9S$^jN+ke!pR=N4gE!Mv5x|trJ?PW_>6Bck}hC9 zZ<76r{I8H0M`;dVvP)KtS&7c zCL&=vbAS@mol^g?O|Tavyeh>8yR=e-ycL$?lS8j9LEN_IvdQR{JPT*q=>OydGRhm| zKP0Y*^)V&u^#~;6Ws6@TGcvG9=PSLp3L4f(A2F2b)!q+QkzPdKl!MiQ4Msy%l8<6e(T8|^!u+3UkkurrQmOsPw>NCV;mQQ|Em87)OPJb literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-66777adb-10ca-4f3d-a838-3254776dad75-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-66777adb-10ca-4f3d-a838-3254776dad75-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1d840d0c8eba7eaf77b1ef4e5d5a7e8855285e23 GIT binary patch literal 8054 zcmc&(U2GHC6`rx3U&1B??$pB?HtfO>v;_a;hh&wm7{`g7Z15%_#7Vj;b?ixOGEQ9E z`F*I+3RMX4u-XT-LKQ*?^`WXN9uPv+s^X!lR;ztjtq@xEfrkoJ6+-*C4+vE~=iC|3 z*s&8YYE=vDx%d2i=jWb#1}4-gKXb9u%s0*)OlTJbX&SzSg?J&6>I!rPe1V=WpCA4( zfwi$ir-8YR)w6m*X!JOM(d7_au=fG`&{S|FE5m+bHIqpfx_+vj`yyK$2YrZScDxf5 z6>{;!a(ZRSgd#d8Or?uyUF-69+eK1Yif`7 zzX#e;*6>ku_LnIZj|%>qC-+thaWS25^oXDpKfA~4Tan!qO(>~B9|)}FwVZ|Jvob8? zy;isFsPR^=eREl-dq<+yvS-_>0x0 zhc1x&#Q3dd&w=&?_7-c$WC*-s8r(+3z?Q@gn@IL`Hh{+XlD1Ol!VG#?2MQTw_ErxG z3G`h@h(N0;+u(MBtpAJ;ahe7uleT`_}>b-qWL_p?DW79PDBEEm>tAP+_8 zFInwXh|fj#8&gs4N-roHTD1Um6tvAk$KqS}D6^hqr?NVQC*_@7gM(wcl15^}OJ~UF0wdZokjY;gs7Qvb)`k zq}AFgZZd}~38(s?!wDoOSM6xByE?60OJ{jI!=f`0+54I7u9Y=VZsR>$Q>(7N2Cf~N z8_nh>yNi$I)_wLqR?WPtQXAe;0?vu^?LV(ThP zuqeoGQ7Ct(!RSk}+k7GpSpmz7Xu&k$geD}r@w;3FREKr8OPUQ;P8C|YmS(Hf$Zl(~ zBS*cvc9Lodk=<`3XIQsoqN&(lWViLqow2c1NEn6GI&ZZUh5x@%M{*xrX!1Jc(oCi( zLM@0q(`96jG|l=XkdPvR?2(|8^Hma8x1Fq?Xm_BRJkmEVV|jy?59){)DR&wS%44Ej zZ#1+VLgJMsBI4ttCe{NA@$qgm3zBXRu0G#qNVE{G#g5|senWXD(yq1|t0!dY$5w;& zI1-<=8A{3U({>gj6CBm8gX|m{hd8`G7HNs+ps`%aCsJ_OSfkMEa38u9Ph>CUxAFxo zb19=0;>ma+&e{ala>C(0K(y7hTtW-tG&!0)A&h%a9a8u6c3OebY_`Z!fjTg5?YKZi zGwjU3B@S0Of@#g>(wTT}YcvUiM7prWT-=f)Z5jrqrj>X`%V*;WjX5~kl~_$`CNfeB z!={!mOs6v%3~^wmDV+~#x&~J!nmIrXA)OfB%xbx`wgQ@5xL;5Ed6=T>#|Od7UYFaY zr}G7{1JnjkkO5q#Ro+ZSL#AO2NdrzK;Eb^cQZ4q*0W&v6?k)RfsfK#&5ydS~;KE75 z41_Zm->}=O&r@w;j&{e=7xSg-P(mdsw1ltZK@udH@XZJa#(m6TGhH!9yZ zpnrVMFy_LFNDiHvfi6eky{5pay81RdN8#N))7B+HMzhkUa-~Um+z6RA2TXTy>cmX% z+^q!1%=A44GX2-LjZ6#j38B79OSEn$0^*Z;lt)d2;JPTlIEyws(BI$=7Y0q`e@5ju zJnXc+jAnOUY{r#F*O+;MnHN{l>$}a$%O(u9IUg4X#C&oG2v9AnXv5ZXs5Yiz=CiLB~UL3!4u{ILzK z*3g@*cOpT0g$Dn5jD_q5Qc!9dOCzePPQqXOe59%y>d=xphFkEfszKU^C9G9- z6j!ii{=y1HgTo-E&f@m)5=soiUv+&V=k;!)67@WO!6!I;fi#j<(5I^7p+2v---l2l zxJE_%6YZy8G!-I!#2=kTe)WCe4-?(wA3lGX*2GHq2GIUi=q&IjkQ5?1d`Ndoa8}N`;8PV@ipwjr-&ywY5K3KzefBUNDAiWXP4C> zWDOT|ZDt@a7u)pDlV~34+4(To4eLJbqCb^TL;Z-4f>4bAaP%y2a#KS-d`jaN4_XiM zk4yX>fDB`W2&!HZqhp>La(e;Iu zc|EZbO+h@@xjfI$Twq`=nVH#21V()E*uWZvL){+OS&so;>Sgs9oapv?w?oKZR*yFS zhloz~#BBkFrQn=*ap(D%Cji#@m`45Gmr*|k-B$0u9%7|bA*f&;^<|KMvwHs0NZ-Jt z7EH|tpuUDNmbBIv!ZU^WSTE#+^2_NFNF%h4B7a!FJT;rzn4cZX0`Fz2*9r2Ea#=i2 zlm1aGLn<mqz4Z=R2m^RDSwthB&x~LzA5{ex0rSReceFRrJ*6+q3>4;Pr&9A_ xz(JPOp9^j|sp*AyS1#XS{Igs~cB{*OrteGu{-Fx~8S@SP2%}68Cg6YE{{xx4?zaE{ literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-e5cb40ab-fba4-401d-b9d7-9f063567174a-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-e5cb40ab-fba4-401d-b9d7-9f063567174a-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..36483e4750f326b73db87d1cf6ee41247caf15e6 GIT binary patch literal 8054 zcmc&(TWA~U6`qmgYn<3|@=rBcCF|@a6NinXi!ZSkcge`IC3_vM?KqZgN>f=HTZ%`v zD#_P}5|&Ux$iuP^VF@LK5c*I`$wLUCEF}-6EXzJDO9)FJ@=!u4A?)Kmgiw0U`DZjE zS+cdErHVb8`Oob;m;d}{oC$Tx&s^*b^NlkH6WRqqnud$85HBQBU4gEEFVNHF^TQ7l zSQ|Th2AJDeJ*yXlMvns+T@JwoYag!<2@AhN}A(1%E7$2&n$ zAs0_9r&pFtD57)1RJxefHNco#-RSan{{D<01OQ|^lWZWHIygwt=>V<26WOe(J=*_1 zXh&JYN730|rdT{C_-~!sTP?)JbiUCef>!+O9Eu|uT~3ylLr8TVmO|0Cg0!%|gfGYObSLn&Xwq;rYNWwP|Gh2s>uRYQL`u3_+N+ zT)dzqt9k0A7+0;7U9!Vl!T+$dkFob`mC@R7tBiVHaNsU-7zMZAXXkLr?GD-9Zbs5- zZ50=pLzaZoebC_ql9Q`;G}&F9R<5P9yq#gunTYKDOm^4G8Ys8%zOAWMS6>6y4$X~b zbCcc0+j8qZdmpQ2-rllgcP%2NU5;`y0@Z9Jr;hrV?B?Gb(hQJIdZXF50C=%=l_gjd zWVa}kyVGFwCE0D>5r?dRs-w~{lg+cMEq>@Tw0y64W=*eWE9LTc@|T8hH|->4(G4-PbWopNa=Qxu^V zM4ss~vPYU`eG*7W5kdAy(8>8z5})okSwGS4LN$4$Z(PRXO?rG-N4!Y6+h9Uy3(;EaC?4!Lly@TSTC4H&luZ5DYOtO_ z;`25`DH(p&&O&5@qq=>Fok!ykhd0L~E%6*Qmdp7>3N{;S6nY)*!VDo%D^Qxv7FjA#2ga=x7pQ24 zl^Hn1;Rr`Ct=U{U6VGjpCSj0B7q*y-TXLjL!@$(E63=M)Y&@Yc2PeA{t4Ym7MrvW$ z)bfStbVh?A4(v3g^C3;w;K)QX2dN>X6T_QXEtl3-K$8pC>uEg?Q*{0KAb8p9a=Y|& zz5sTB+5ieNfXlSXo5^U%G>jo>z=;H$G1fq;#ojq!=BCIUurHQssK*{rTml6yoFvRZ zICJq0yS@56CC4(iFKvYGLOct8IL2Y5Mn9UCwXMtRaeWP41q_1Q2`iRd9jEQ#`Hpl@ za9x-+L2}6+`Axebcvy?vE7Z5iofT_qXH7AOv~R zHz$}!`uZ?~zOpAk+kPT+pE{&3MdglDx$Qu7HIErJ4)}17D7N;m{wNmf~`f$t;q^BpDBz@;*5qsAAEAEu?ZU|kFAl-LJKZwfR2IXZP zMzNMtwq*3GX;>>cmn%6WJrPgyX+V%(Vt{YU>}5$gL~8QNYUk9($-`esDdk3^@_hsP z$L9=VF06>;@aY-oaunX{3hb(D@38X}-rX~89TH?TD{U%Qo0KPwkZE(kbQh;i%=FIH zN^s0f-&Y{he|^Wuv>=}p>Z`Ow>vSR@KB-4}+%yQTivoD-CM zJRHgHFQlaM`99@EGup+Go96C{_FBxY^9l{!ILdlT43(U)WhX0{;=TM{i}KYzG**3< z$6j7#l&=V^*vqe<0$_hRTHeb`kQoQN4bg-B%J(gZ$o48=do8Hk)jG{!+((;VA7c~t zVDy)mq|pIK1~U4{Aw3n9Ypu$^_M_3ads%q9cZ|xeJhd9glT+A0-evzTX&@E6F$NxT zC?#c;A6u0#ThSfsMaA~cQZZ{Mif!vThVYByELsvm`v_%?O*k%*Rb475&)bwgwxQJ; zdXx1|BuKB(;6IPEklkRavOzwPm26z|0C?b#UW>}JcIECt1o8hn0LtFG1nE{MLEvaL;1mCI}q;+_NXH^}= zCs;DS;R!{9!yu;4;_~nkN({rVx;~NfdN)ytdI2x+2@YQ*jieRysp@#B&+F~?A(RL{ zqayx^_R}|-3XwkIk4_`M`T_8ViEi=_U${cg#7g)E(Ee8F9PlTQ6e2oYCjA6T#cPOM zC;srny4UNQM*d64YVpsR_+gRgWF!$8%Px?3FpqdJkQV;ok)pgBOdy@?h4p1%B3I{Q zoBBd*W-Y0zB$~{O`P`d>HJ8b)R<0pGv5qe#A#XD8_#{dJZ_bsUaWUrSZap)`R?G z6WfqqvTx!EhGN9TQ&r?w6W$50_agC^;1AmIe}eRlIgko&FLcjr%>^J9kx>86d|Wc^ zfqz7&P$x1Y-9|n7=4VrcS^anq`7xqXDJ3H_TMNrNSv>|Py1m}*5b~GRqs{*z zqEkI_TYzCHIOkp5xj*I!fVDrSQGfRp)Q>^8)w{2USSeKqDws!o8RXxrp1(BGH*l*3 zQ}Y3+uVIWOJ?jhMnZkUm7ji=R<@5-o5n4x)KdfJwnoVuY&yHn*_X^eP1o=m~EbgaC z|0tFr6`8sOU(g=_mX13lO#NpSpV6{PN#gxVoq(;qwGE|P9#7l;4iX*upQ^@Zqr&?Tf(z9Lgw zbFpYH9_w8p*(l+!zW*)|J&LGdzZVu_x^FTw;+qSk#%RC6Ne5s6M2{NU;EsXc(!s)CDvd1JOa+MDQ}(idh13iGp5srfMA vAWQ1c1-G2k^g_HVm+vtCEZ33U>hhoMI~#yMRKcGy-{MCYWr8pP|Kt82^wjSw literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-d4320b85-10f7-454a-acdf-f9a8496b4f70-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-d4320b85-10f7-454a-acdf-f9a8496b4f70-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e68796950c95d7a14b0686a5e74b455b275b9146 GIT binary patch literal 8053 zcmc&(U1%Fu7M_vi-#D@3_|9rlCF``##Gz62<(s%Th`nvY{;dun$Wp%bs)Yj5LxZ zTQMvv8for5=brPO-+RWLQfIu(&Mq;}B(pK0LlC4{_!1Ul`FPUhb@@ELK9|Sq^7xp* z+F9o%U~Xp(tU(Z(oHk&z+XOr8eZW361)RyqupeJbr&D>?+sfG!S#lEOA(HvM9uSnz z#^Nif)nyBmg4CsyuDN`!?AoTw+w+G@f`BMHm}EiO+{ICfbv6+DrN}-p#fKlf2jcN; z(a4ID?_W4vFu2CFzsV_rT>KpDvZo^ZTr|NXpL*cHdQQt8hUTia6-#Gy?J{`SbH7jU z_^dnC?62eYgKtcj&0Y1NQA@{CdNrJO3T+qbHc_?@|2@0>tsTz2Hh!z=v!OnLonRdp z3xQ`$qr=D;*pk?36UnijMvxd=)>iW_FYr297alUw>_i_O66m|05P?+lY@?$N&U!C< z;N07#=jw)*&BgMmwN!hC_I*H_q=TnlO44jDQ$8Tn=wdbBRKWc z#$*To<&frpY|@*|wgtcoovSRsq98j&q1>ECqb?khw#i;*CeMEK~>71e65{c$QbCDgzXKsvztwO-arNw%y zB`LhtoA0Q3BoDxSCT~zI&0vZm%l=gn9=b2GP{%WLVIX)Pa1#PTuLF0j_~HpfY#t*vL{njeSB@x*yy(uv}b zdYre@3X~SJMV20@2j#^bJBVn3ojJI};RZ)AZJBH;9m{TyCt!|9<+qugYqF(G#lY0O z8cS=rOf0T38z+n z`dn0Aipo6^5mlcK4j3FN;!BwL-gGmk^rVwPUpW+@rJe}g9h>yMs64Dwes4o`HIErJ z4vjW<05bc-0 zVBc`G_Jh*X(vb-AucamByL#nby|Xwk6`v%br!YkocPlw>XaL6yL3(zcNzzjX3t6K+ zSg}&dyCHNzg7lS5dIiyMRQ_3yUaaMoEg8LP8rDkgx3dtH(a zp_;5JwsUIZWZ`e5q;juGdC`dW@j1ho3o9b&yf_D4j@Tp7TgF9RpHibVz;rE>Ek~NLy_g`$rl}6W?d4ZW1SHbGt7UidAbhSAj7Y5}@ z_DRl?$yt#8f}DlZyqE8`Do>7~vg)%u_VOyDd_`b|UjFF?0QRRd<-NQFnQ^gO5bYgTUbG@2 z%d3FpwV-lW>oJ|Ni#ng2Wm8sX43?;*(E&#WQu@dyy%3eVZOW_TsPt$r3rBm$sBFsf zVgvd90ydCevj3DckP6lq0}mOLlrqYjHsx^}np1pHvAnZXj9Pc0ZT%Hpc>f#=m$=Y6 zLKVj*9G6I|F6EUU+m*d`)LKJtD!vm5(!WvR%X2JXRhX=-kWXYKg=<~_dp7A`qVl{$ z`RXKs`2Sr1W$#^r^t&F2;~(E;Z+IMe)2rP;nk31~!_rmd!Kq=eP88ua4g*=}ffq|( z=LG5N9%zQU-Rwg_dcq{T(j+}S)rl`$j#hPM=Hti*{DsHyQ=1zI81Vud4cKAC|CI)p1>{y35XM2D|wFM(3=J3yure{gEU?e@$f{|#g<@-LY9VUy^jBp#Z`EYfj*4)LHP zi}**!3g=aS9O5GxM^@OU@(L{PImxz$+Y1Kc?ov02PAz0>X*oRL6 zkAK>c6<@3-Rr80nh} zaLT{4*gLnq-~(TT0)zYGal_OH_7R<29Z!$-8u{pn&L;`8`q3fhCx}j|lnBjjFRthi z+rT*Dtsc*jKBNaQe}dtS#nq@DUkxY0pBwx zDO3o`pF??RN5O5e*joIR!ETg&l;{#vr0kY{YssJ(%#;I z)Gha?rM~?{hyJIkvH9>Q)@UL)hc&B4@cb4&krDXoSCizA1tjlVxzs;{`a1Yx zCowjYS;X2NC;1#2@xs@%j~}IcqDLylBT46}n@IQH zTJ+86Ts@lRUtfrL^?0v8Ni_#3(Vpu0ALv7UmVP#cvZxF_QlUq4?&_6^muxx z_j&}Sf`1OQXXFObDPEzO?S)7<8;kU>(%CTKuU>!Gh#p4NQ1AK0i0+wAk9ih+$qA}A zxaj~4fap^Lo7^z4n@kPjp<=%s<5trqJi--#k4Akg^!k_Lz9DZs9o9?bqbUCpVBVbX z4fn@;XY|Fnp?q|HCK(L^4$`FlN^r;tP0z<%*<6?LXSuG-w#$2Y;Ia??Pz8U+Ji>p% MI1_{^_#f{71-z>F-v9sr literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f442863b-8e12-49ae-8b4d-3df43530c506-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f442863b-8e12-49ae-8b4d-3df43530c506-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f63e35f632608e45bf512b04773677dd9d5dc260 GIT binary patch literal 8052 zcmc&(UuawB6~9-O|HiHz$KRc9s$@><+&DCj{`gPq;FVliwq(~)ZO5@}QI^TlwWV0H zHA%K?KZR0431yT}!YF$%Ve~PCQc4J;`l)TiHctQh^>5W{UtM{{Wf`BMHnPfuQ+|5ynbrul&JCVI-h!6bsO%RV~ z3r1FyeCN!;g26S0{Y`ce?Qk-c(9VO0op%NV zr`xg zNM$`Z_0+{=8~^2y7JzKfn~bgn!1Il(B*3B|+eD$%oJPGX$u{GOIAjGhub>7)MIE#t z*@oYxA|N{om0i?oD08aNN~N^$MgzUYCEE&%vKS2>s*lLFL!HyLnj2@TjI@>?LligSeFh`Mw*XV%w{q2LtM9f|9%yBtn|9hPaB zr_qk)HErkGRyeVVrUC}RRtGzUbM>5dfaPW3Am>^!YJy~y?eb5ZieP80@}SVrA@^3y zt@4~=7Rf5@O7DxxywYSJ5_&7#@0ITSqelgqPYRF+IPqD8QOVEmdqMiHhaH#xbd05h z10fYBGQs&!ls*=f7ou`YL`2o6gM9{viuj@izL!1BE`8m_psySV&{R)^?tP2&m8k61 zDW6&pUCm=g%>#+6BKAGe0RPEpc2auE*y9?&^QtG@e;1|CMdhhgc~*zGYnfka8FDJ4 z{V0$2b42^;&)63nt#!ZjG<76`{7Y#?d04O9s<#*BrNWaW^ya6i!fqw!jSS(KAxKY7 zGfDc!#{A}}4_B;|(ryS{kRbiuBK-i-Z&dzKk6x_hmMt2+Y8uvx?&V4bNsq*{d>Rm> z@6o|`WcIhBbnw??RiT|z8z&2YCB>9mP0F)Iw2#jj`dnBMN!Qs0=yK%V4-}}X8?UoV z8njq4~fN9H5ofzrz-AZtbNbe~S=|`{Ykrw3BLPM37Sh$@C zh*KI+9yE`F>7oGREb6dBe}g+*7&U}{iNZg$vvcM&S}ecV^ec_7G2;R=F0Q=QcUzPn zo6*(Ae4HPYE7>R6iza75`Xl=M3p<-LW9co&VqA`-_dO}9+&-ebXhFR=a?{*hQD3Xk zb$&oaH&3vEB1J_vY{|_^s(3HI+p0W1g37AT^4QC(jPezM<$L)DX8_p46Q#Yp2$_Dd z8;E`*D)-uyXRV0H^eSL_EvVepdJSjnq0X;QvKg~8hKp3v=zt>wDScp(o{7rMcIEpv zRC>6Vg~PpLR5se zw)JOp;U}k9pvZ;h5vnjY;kZOv6H-=r-l5#_OrouasApHv!{`C~|nia+>E94Vd zQQ;aFz`Pxp#vX`3sNj$94yP z-ddedRdp8r;pZJy&8cIn>J)Cluc~@zAC|CI)k$2zmhlTKJn9*TW9lMqkFVm1armch z&7>WUZ4{zj#xMAI#;=e>k_z%vb=o`Ra11*UiXYb~h<^gZ^oy#zB#-z5^T@Bh1^hmu z8~o#!uhW`X3Ev3n-|=1m{s@x1M2D|g7lBgo+e@Yszi(#C;c(6)|5aoy@GlwoVUy^j zB;ub+Ez@yN2JxUH3;4$;^5<1g1nH#DmskTPGBp(3PAmr(HlwOaN2AG!Of*QUXI0NE zccMD1hhUYjVIMwuJf2y@epUT7;?E(;lMXGesbk3M%OBR+CM-v8r)3&6=$jXCitj$hnq1IRx$vkUPh{RSS7H%Lr8RC#_i;+Szb zt`L6_evcXdyGYM1!70z~a{t24k{f*C_YRlG?1n4I+C2|*YnXCT8t59^_>IG zPZ6C`De7O?Szb#(Y`v3+w|YFs29WN>{P6|0me)gx$a)|K{@mi{d3=`KBb(9W!cN3J z;S2{yHpw08?nrq)x_PRXcjo}HQm7D=Cxh~m$iH1ZesQGdu+%)UkQ?%A9DPaa#IkQ88ww6WOensb z?uRf!>&Wx_64&PzW4X}cR0?>nQ@+lSeH6=lJ@KO4A4 z%_;?n_bYVCjB{snqyU~fc)H2rgB+2K{h!?(Q-TWx!6FpKX#!%k{FXBBQK4WVRY+Zzp z&KvbnR9`uy=aBBXzU-b$aP?@KXLBj&N<{iSF{(L0iS|^_fA0Y5Gxf6>ltuk8II*K$ zO{`(8!;5o+>bMsqas0f$0ORZ=*$;sOoN%w?czn1MP!i=RzkmEwTTvI)S8CY}Y3YX^ zQ5HW_J@|-D&?Cu_{wqP03jW#Gp0TS)r+E42c9w#HbT~M;PGBXfj!jX$!A%EX0K|am&2huPZZg$}hYJ06f?G|S@Ca7`KI-+c)bCk|xJO)( zWFS#2A4Pdq0CR4!KQI{SpGzz+jATQLbFq*QaF8bT7lKWWYKd&PFP-Vu|18&?+Uawh ZA3E=bKUBe=F%R*dFv$dA2L6Bhe*tZ*_Amed literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-82e0525d-9555-4568-8215-0e3f5881f549-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-82e0525d-9555-4568-8215-0e3f5881f549-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9f0aa513805d0e161a9939c3107dd08ca56b1fbb GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA31jIIKH!*RLMH6GjV7f{rD$#@s^A%Te9n@w&Pf~D4Ug~v88xq zYbE*jDdeGq(1%i%5K13{Da$^ZQc5UG3Hy|%C4@ZWVOdJaLnvk0hkaN=S@xWBXQYuV z*@|IVrIF^|^Y@*fd+wQZTAlSWJG;a@Q_RMM4ndIS;7eGF<>Sd-U$4*O>+kh=dp$lT zuy)pY37Fei18We3CYKEu?KZ&`Vh&&eh(S>cbD-1@$Of zbh4u6`xlN@452aoZ*qyC7e9xG?5W5;6HPG5ryda4$Z6T5&|KBFW9f{pU4{sI?)M8G zpLM62|8?Adh>Z!;(Nzx?wR9|{SHo$i)ON9M6J`6zwcN_Lc907jzt!y7(4N5FVjY+Z zfmckU)2JBOlGtez$%&pu&=_0M*7CiWKo{#mA*0OR>L($AzUv7QXftRiAe;0ivuy$JLgy+=uqen*Q7Ct((P&Gu(|jThSpmz-Xu&j52R%r3;&-_UsE*>v zF6lK?IaO%oTAGWUMs^k#2XZuUtUe+;k0obVw`9UmXfCp|_{^QLuvJJHg|v9yYAFh@ z_2xTn9m#`mpUIn)OEa0G2(=*cOqY>e(i|IwpPtREEO<6blKlWrHTK0jedv=FVuiekUj zP~L;IyKTnmIhlIhX0Uc6@yB*UDH(p=!2)E0qk3?PT}9&%hhNTxT4GsfEH`uUBpf!@ zBn;S`oi}6g%+1_RF0Z9;rnP)55zEI|yTDq`+ngtfw!V>#YknLiClcp{DHp0kYAbK2 z6)4SSi!2qW2jj&ZJE&-eoq4#!;RZ)AZJBH;9n0=aBw&t6<#(8!Te780!@%TNi>0+( zCKlJ2jg#&1^@L_3BegJVYPtMeDy_i)2X-8(TtL$`xG&MnNookG_}F$v%cit7&}7H` z2HMZT5Zy2}0$vW-opwEy%Yz-DHiCjQ;4-c9W-<{l4P!_eaUcO_j6INQv33p^xhZmQ zSvN~HG+>V?Y=Hth&JkuH>awvdtG$LCC8ucaM9K)=wO9uHuuZ{0jea;*wVj)rF?|DF z1q_0-4t9!i^_+Hu=Lgb3!L?)71j#PD= zsN53~QT6HIkinrMzJ!VIbvJWKPdXX&l_LRK&J&@#W0SrYm4|i87dAv!^O#ZdK;o^4 z{h?@rzki;cm7X*9qz3T()D!N1h|<@h^31Nhs6*Vf%rCVJIhE0VT0r{^qW$t$>>G~O zepq>0Iub$tt+cFsSFhZwcNOQQ;*%uw6sD-+Zl&N258{|1NYBnQNqXvJA#2u$DxQ?` zZU|kFAbn+%UP1O7m4DV_6l=LUYv(6N8$ZMfm3zoZFZHydw8acmjoHjNZZP7 zhw``yGHni+&cf7*nLfB%367cRJq0rTm$!{f3-WoPp-M|E-cAI>BlRnf93$YmD8M+2 zHeAr(;0_l?Oy!SI`8^lAWG$nGgBP1|rO`EJUSQ_MRq%SRS^3$4p*H8^!k}EqKgm@x zISbNXG3MX8*sK*x&p|Ba#= zafz(zQeOG7UDE7 zoy0BpRaHOj!xGl2I)N+LGJj!(qW&=uQx|Z1Yy~C8;IFzloprmnQHgpDzu@B^yG|NO zE9g_zslcGyJ>)?sAzY&({s|A!FPaLFKH?A0A;0<_@CS)*@{e7+MQdUue8XseCvXM$ z<46h+9lmC~1WLv40J%>5!Rbx6+cSs!H;}c+zi8rzO`?;LcxW=SMB@G&;=w=`@sEuc z0~dcuL2Xf8jWo0OOg4FgsPHgB0ZiYw33RELZdtnfAL!>549KV$l@ zs=r74TS)R}qYJC*D6$6gx;8)TTa0Xbqa>O`dL|kKyFuNf-S8&kYG4TQQ4k989}8ar zPHt+{gHI{^;z8?2{>kZG$S>J9@%RG~V&bVP@T+n6wA+21_)GBnt@uAe`qm;y`FEH4 z=64o-5Q|V?=wLo>nD)RwqEo2j>G3|J9zD^8BwdI0Mu7~Wi3i|X;Ua1!FV$>n)|7Jb7TiS+zV+&AutMTR#h9O~}y!Fu%ZQZK7V z|8$?*y&FLOvU;@mKT33}CvFQctoRq*%LmU#lmHaZ#~kYKyM_8O=$3l-^b;$k3PJgE zs4tED+tu@zLi!dSHGeYdgZdi7SkhWw3eM-FkpajF<(Jb#kVa@71^%FZYjz>I6ON=f4VN}YzIy|W9YTOLo# z`SueX`k$)C7Q$nAMiaq#JhN&9<+t&PoWNhdnxuFv!f|lZ6Yd%(m-=T=-+)-`CdOwo zOL(>?NI!>0yzn*Smqz4Z=R2m^RT(_y_-KdSOC1Lm!TzVJZ2 zZ&qKLAI?V?W|Pq%;2=xtZv>~D(DZz)H=FA+{#mXov(xLnJb2j$|4;@0jCqXzgb5}H J)9^ps{|jme^EChf literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-8ce2f2b0-a7c6-4e8a-ab1c-d7c71928a942-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-8ce2f2b0-a7c6-4e8a-ab1c-d7c71928a942-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6055c50dfd4f854d822fc0bcb837da3c17115c08 GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA31jIIKH!*RLMH6GjV7f{rD$#@s^A%Te9n@w&Pf~D4Ug~v88xq zYbE*jDdeGq(1%i%5K13{Da$^ZQc5UG3Hy|%C4@ZWVOdJaLnvk0hkaN=S@xWBXQYuV z*@|IV(MWUe`TNe#J@_z4zG%rh+pW8TRAr>2xaJ`=)yKMV6cbeTZaXzXufM zv$6PUYHh^=rJ!{=rE9(3-t78TueaxqmjnS(b}-3;($U3High+n`=!V}Hr0n8ybJ14 zw&-L<&G#=Htr$XM`rqUdK`(v|57|?ZeI}Y%Hs& zxi>4%wM{LXi{(@6Yu1Q+57#rqM$A~a^k%S}-^hYI)S$OybyOX`6xr=U?z2A(fTq#) zBBHLmww>=?}TWXDn1O=g{#j`7!oZ+Yq6r( zZ#9(nAnk6Ov3gFXUbh*n-AMeg-B3z~pLehTnc%1%oMKneIK<(XbD@@478=XVTs#Sf zjWr1aHfQI}SUhtxx0B0j>6>XSA4|mYG1e}ymh(2}NusT9WaFA2hslY=d11X6#X z+i3+#v)Lj`1?s_gamNlSnqg-i?r^xl5lmYqn@Y#BI}-_*BU1StX6Ke{Dbp}8Io4un zEtiSKHD=>vdwe~inaD^j%$iy*KbJ~tFu;KwM=BT4bPeuHG;@*~LMlGCozb!>Z4ETp zale7~b1+0VOpSn-19qofPv!Dp2dIspAPu-otGt;^1Wdyil13azz!_r?q*|<<14eF& z+*{VoQVk8*BMMufz>af-8Hl=UY|Cn|AxFt6nmds)LU%2e0Y7Y0Fi@i(j#X{v=4MRa zKvw~S;H-n4qFghVq%qIn?0~{S~!iW^&_q`zf(9KRuFHW(Ha3rMCLMAwOMd>q9 zc_}LQL_}15Iyhu-sE99N;(OiAT+)+H27Tp7fR^(_=`yZn8wWvI^D=+F0cP;ZvEkjOaw4WBxeuHSg{1y9# zqqQGao|cY8kbf&JE8o>C_v&56d8zm$2|a}=s<>Mzc*BD@W(d-=^GuSSI$6k?^`VL< zrMw$L7bHku*`!yH{YK@V^%%ukZrPI2tEOSCEb0pMl;g3 za@(OiZh}mk1E#Yubz-Iu?pA_hW_nM7O#kI=Bh!L>UTCP&5{tJJ0r5!v$|J`JxGoAX z&Y}$$^f$P}g%MNvBUFCR#V%RPXyM?+W?X4>jhPpid2toI-fLEVc3`N@`M5ABSMpDC zl}yfp^jD1ew=Ono#nN*Si+MSc-H)Y&^7#qnWi#5vk(=i3iuPK}uJaQbx_gH8ml!I! zVask-GR1rOUW@YN1RAS8%VRIEGRjv3R_NuQTmWEyK2zSyOOP2Cy9LpHtMZ}+5m{aZ zEUyKXyIPMKj6JmZ9iy@<&x;M@ z`wQ4We$D<<(m*PBV+=gxP*Tb$uiKQzZRk$%MaA;YQZZ}Yg|_uK4B`EAEL;*o>j+gG zn{ZqrtGbj|er#9v+tF$by{Y(4BuM{8gD=mqfYo5KvOzwPl^k620@$}n{}Pqw9m-cH z5yb!R0w{a$5~M%$Kpy|}4tve>$eUj64$>q^Q67=5Di2PLfOnz@uW^{jLJz!H`Z^~_ zU-v*W-0Nl^2+|WK*_9^g>8Vb9;c~X9v$G#X-sdko&Y#=d_<3(-TvgQ>_=lelRdq`p zT~Q}-3w~AAPy4WhwW?0w3bxE&SfQwY48+t0+#Xv&i81)6Zcb<2?rl_}Uc)c=_{Xl3 zM$!uURCOva=ynfz5K0KwsEB{UL-dQL0;G@l!*j^5z6bn4qMQ6<*KX08SP9=S+TRIW z0sc6W0z`+e883lS@jF1S6Mt}e)9v=mA^#0zE%Gm#_+gXiWF#J%%q)?(KZkfQkVX7s z;{|!uA4fXb3+k)DM6O07+xk*uej}l(B$`N%=MoVzJ)`<(co5YgBLypb4g2sZ;PKCx z{;TTm5&ssF{MqQjsyd3S!Mv``5BnA)+ukUN=8&F=2ElGn_h>h~$+#LALVOg2Lj1?V zSAdh78uj2)3cq;J`jLNfdKdCb_DwwgK!lihstWvS+&%4fUnl+&{C+F`kC48#2vYvt zrM~%{MIXc>6c{>~j~k{v@Q>&e>Ues*&!|UFbRkKY)en!5pCme!QX(|Jv$U#1ZUYmD zw|YKD`;i{N`U!?Nm)4?sd@Y=Wcy4leo}WeE@J1p%zZ3V3dt#B{4GM?4JAAMneZ17m z>d`;l=XUP~kiV=RE&h)Zo$86(0t_quMfdW-^ARNg#q%+T`ulF7ehj*$-aY-qN~uCn z{v7H{BmZ{w{H2h-g-6YwjQXIy#xRz&)|Z0w`DkPSazgp#^bn*GT1SCDsNb4hNNzu7r8Pqo*7Q2b@ z*~}82?FrJ)p%E{9&G@(|)e}9^QB0z}D*?oNM1DqB-PpPaAIY2NqvU*TA$<$!{##4F zS)H3l*Zdoc5w9Na^C#)d0ZR0zdi@9b(VnHBO`|UAhmr9e?S{UJxsEN&4ya=R(8Tfc z{sN3M6XZW64sgP~vc>bmgMgZ-M)~~{n%qoy>3n6Dd{CAl=n)6 KL70aB;r?F_81lgY literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1b24d95a-1de7-4ac9-8af1-cb19b66c0edb-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1b24d95a-1de7-4ac9-8af1-cb19b66c0edb-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..abc87527a2afcea4d5f360fe845790bacbd4d65a GIT binary patch literal 8052 zcmc&(TWA|+7M_viYwX%_{LgAqCF``#M4@qX@g;WfmW(W0vg@d}<5;#So7H7(DVA)l zBwwFGDWQb2lu*J__Q8auk0F#&LRd^bM3kw!OL zF)S+@Y5xD5|D5k!|1<8iI_qXuc8_^vAsYI^#Rps0lS$qoQA(Dmr4iJ>f zgrlp8wG|VTywv4HQUj2g^{rmFpL z%VXZDW`7;GAADoLZ0@cHjan+4NLIsXrO9s^~^$R9UXzO=%*$NFzIeHPRwu-8~8 z#zNp3(`eH(2DT)2+(2@~(FhX5E81GF*A2XO){TesG<&U|4hi&KPl!OOakkM`2WQ>q zU2yKz(sON7%Vfj3#QK`q{&oYm%P22J2+EfdaZv1+E)k&~;Y z%0A)Yz2JY9*eBT=ro3n$GUY`BPdBjDlx}SCoe*3i*pTMb-glADaaCfo9lrP^Ju+QMX8Ek}xb9K~iF zQdtj9J#{hJ#(z1ac_16~CZlTs@OLar4Q0H{*rbsmBn~Q8KJac1A zY!w1VE-lnsElJ@vUwntnBY6<6GkKF@X#`UgAs0j*=@PPCnqz|!oR9*7Y?q*s^HmI2 zdv$DxXnT-NcIl~AUw%Z(hxNoujEy?w5m9b6=~B9oc%zw!`1qii^@BisyxYROB-@Uw z+eh>htwd`zqqx_m3wI#xX1l)nmQ=lL*I9dz_`E|GN{XL$G9RhnsO}tNmryzQ;ipr9 z)^G+I%e8DY4#mcrgaM1K>smOPzLwp|=Cstcl$Hy}!nrW(5LoMJi|r`U);BUy&5N_- zMC`OMWk+#HZR72<0;R=hk;Mne2V#CZga z(f8m~tGRQ)#7&-i&AeGGp#ghDehUOxagHz?QI`pCne{bfDL4gnM-sa0E{D@#hh+-p zX|$txRol6?8BT7XsenPS)xl2TTs@~9V0l?M$hlUGnjl$ayZlq9BG_50JRmf5$c~D+ zRi0DKB3Y$f>3va|Q=069f}_IyUg^$1dQ_14qyTw<6Q4yGkpld_7o_ic*m3F4$5>i8 z5K?g>6Pyo4>0?oOAu6{-L{xn`*k^F4h%aW~d)dS6($`%K`pSU-P4z_R-nU3!iOOD` z@~H*U)jVd@Jdn66V&4-D@SmJ!C#9#1J+1*fuX@7$cTxIWRGwOuXLX3XmieWYA*V9h zkMd|gN3@^*jD5k;TK7v&Q%54mzm%4hhxN*>dV67BDm+PoBR@qIb}Kn=co4@7L3(nU zNzykq7BEMBsA8p*c0=fb1nKt{=?92@qw^kOZyY|-dd)38=_FIO^1dL*9Z(|{m- zj}E>gv%eLkL!c(B3hkWQI9d2BDX!dVQl2%UeSFT)=fa9ey3Wo+mm~Lnpg>jKc%5A$ z_wFC*!X-gU)6%wbqgi>>1d%odOj~~H#7LL#R)S+hdQX8!KYCq{v>=}r8mhF!!tF#r zTvEUCpm_vL7X=t+QHLG+8{FZ-h#~w-6#k)|oinG=Lixp}UukrW85fvwapkSP+oJr~ zjIK83pU~%D*x9TZi=!NiaXFIS_oSF|`-t+Q1@+>{O>=ieeXT~< z`2iK(Ji+>l6cydDB{wUn;=TNCtMd2=Dyu%rV=u2V%2x!I@8uty0bmbLl=ku>WctN! zAo`7{+-p;wwIU+ZtAOdXpmJAp7|z&3onN11(`IK36{)1r0Y?T>`oJPR6P26o%J*%k z^l&c=hkM7UY|6KV2J(k9*g$^H{!`RIDp;crJY-N@N-Hnhm9N^-oWhHW>7Auw)Ozx5 z>o4fSPfoF5kqgZuRAFqwaf!4hrJVA-L%G+1T5IS{g?A!B`WGtv>nY|lD~wlG$S1O* z!Zj{{`xfb+aE*2}^4M%p|MQBz<$N3tzZwt?KOTyP=Exg~#?|yAwZe zt&FRxIs^ak^Ny-+siQ0EByPd4s(NW3mataU30%RJ@e3K%h)>H=<$t>B3<_@{18 zXPnM$6rx_nFZg)Ju8>5M3i4ES$~WkA4!ICY0M{spe}Y5wi>iDikNAUg$gjQy{C=Vv z{9~7|)0$Wb-!SUm@m&D^D3W|chp!nofl~3?N2U|Me|ppDbj>0ERb(yjFBSz1_CN0HT^OKS7Op2g6%dx?%_k)B@igI<5q zrCoK$qpEKR@sShq{vQio08XxI)P+w8{NhgQNB+s_U5GE~H}H6UA!6d8%JZvH=d{y# zh4_o`d(HUYMf%nvobv8Q`sQ~QJ>ZLgZ>T&TR}Fn&AJNIx(bRaKo{z4jg*ahW-#Os? zB+)6AVuAUc$Z8T|>zhEl)#EwZk8~gAk3YB>SzAg**Mf2I=O#bTksL`LARSB;ZD7NH!R z^n|;5CjCpW+syRq@vbOdXAg5+~(#0y_D9)6Vai5{sGr>7a`kAMcVjW+PDcB@ajH2$iS|^_e_uc9Gxf7+ltuk8G`^!< zO|D|B!wa(m>X;8Cas0f$0OQOA*$;sOoN%vf@%V5jpd`vse*Xj}H)C$9uXMx%X&HbX zQ5HW_J@|=E(4(p0zAGV=3jW#Gp3$pFr+5WscNRmzOgJ>KMrVVBzk2;$CVCK2L%ruB zp`>djHSSvU#3!lV;HCpG0HR;@ZE?fEZZg%6hYJ06oLfzs@Ca7`KI-+c*yml2dWPN6 zR4`dAA4PeW0rS>EUvMDWH=B&i59gK^X5&kKz(JbSUkElirX_RX-b}Vz|Fc|odZ*WY Ze(<~p{!j&f#yrG-!UPk9Y54!`{{;r3^|$~4 literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-81dd5d9f-3e71-4aea-a46c-86905984e372-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-81dd5d9f-3e71-4aea-a46c-86905984e372-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5c6615a083205f394f8da1374cb30689b69faf56 GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA31jIIKH!*RLMH6Gf`+9{rD$#@s^A%Te9n@w&Pf~DVvq0v88xq zYbE*jDdeGq(1%i%5SBg!QoAv$IRgJIQQJ=nw>H7QTd~SU#TY@%8w<{=OcsugBZV z1lGhZb%a7hsSfU<*07L<-Ij#8|%f!HrZ_OU5G^59(% zk7tWUR+N1I!qI}kHKzSdE)nG7=Wv%j71?K^2`2f}3kNoGTJ|V3SGDa}I-_fs!NcDB zeS+6--Kl1O9k(BRW5RTF)q_SY9ZTueaM~%fU98(g**^T&&6RKLaPFD$TTP!0^$F}P z)`77Qc*Zn3jf{aUiJdl)oNzaS#Mp|qmhbTauZwlzAtTM+>Z3yfeb*BrkZPW7bk@OH z-(@eHd$aUh+tjkTSU$DBX7xBG&>?aL*|6ygm);DT^BdVUER!Bz$?B*)d?~Wqh1h4m z?*~aE>qSIed2Ku2wY;9~IwCdyK6FTG8Zkf3&YIq8^*TTkI9kib@>-&rt##z(s=0DV ze0VeXUuE`L_Kqbn+6FC&(ZKT!Y_;TDtJlg=CwJInXUD;gRB@BpWJ$O<01Zx{b8^*& zCOfOs%B8q#?PjvGR#tzxj(2NlX|l5ht^<;r%*G}=3yHsXWM?f$N}C+TW(-o< z2u?kXDztJb&Bab5JBy10G3q~7ACaBMI%jCNM8Z*MF0!-u%#E?IRR|cl zv{-MoB!$m<^Bp&j(Pj zdkER&lAhU(<)^g#ww`#2vC*JBCd$1gLrNzSZ##&HkB=Oz4+P@lUNZ}jY!|LRKVh6` zAzF(S#eSNxmZN}<3se0XJuy!Nyhjv3KDSqC;f~10@dT@$eMdjd!pU;I`Vp(V` zH*@hM6dP+2`fbk6o3VK2W^N~!*U~rBT0WMD~&21hV$nQST@%kGRPV2()TcbJ`PvZYML zz~oqqrL|lp7T1`KlkM^Kgk~Zmu`p|Dx%_M@t-$~XdK{@-P}4QIFHy}&Y6z+L=ypcS zrnEJXWXJsm+Rwod-7q-}R`%PSc0HBLgC3wZf`ByOGPUw%G9EM)qe~ibAOT~HJvh~3 z?Hn+2ljq*DZk9@Dz#dW90s(fMBTPrsWn)`beGNGZPEp;7l;OH-u?*N@n}mTH?QpDW zJ2y9D`UaW`7zAe>>=e({bJ`J>9|#9I*N#yWB)jaAf9_BO7i*FGg@$(7T`{)~<`lC@ zc4=3-BPz2>lWRb5SGeCR-T5a^3NoJ*AP;clvkAjenBVt;^nEuwExkC!GQyFNN)wsj z+!dwIMCGNZ+!GN|_37Y{!J#6)go*EUH*-l(IvMnpBLQ0KiO}7#N#BXe!#d^nHbhtR zm{IdU;;V@Lp=g4?f1aI{o-_8O2JpP<3HLul=_^rrW>;R+A?{k{ms*CL%4k0=p#2)r ze(?+THAibdEIln9i6H-4T2{WTSMJrjit|$ONfO+JDXO?z$$3KqIA#dav-3=no;q3B z8uh`7l~Udfp$ihEFKyB*h<>B;&wBJ?Ew^mR=vC9OR&p;_GDvzXUgXn&AiY8dKakmv zCFu~Z$*N*Ir#4O&{zghF_nMR!jc6aAGmN>gB9hLFbI|3;y`LyhRd?QISINDHN4j`P zkkX8_t=x7fkDDOU=78xeOr03%gS(aBn33L7Aku$++laIvpBEacw8Y}=L_oY!pYq5t z45o_$jI*f21^o@~aADXK{s@KNbFoX-G@3tnu^CqyU1R12W?ozctM{6fpB(6Fb3QH% z%9ZSsTqTpUApHe>{*8;xSh2VdVlgjA()+QLP(DAQylh6jIC9h6T~S|)*>!$IMR(7z zz7jN~(A--)m8xoIqvOXL;=9RYv)WzzV(mlM4XsPiM+|c?mM(Vz(gLZ&hBj zAR^1FfaSHIa#wSk&e%hpPtLL_t1|{mRMO~xBLgXYWRqTq%H1~QRVykz-pj)A-Z3hh z^1RqUzPo@8Ll@pZ$08*z zw2n~4u?fc|(yB{&<%f1D=Xv^SxMoV7r?$v`j@CY z?@+!xi6H)e7eLv2mmvMl4RQSAJM1-&BX4@O8%UERd3jj6sysL~4AzMvyvAW53q9~+ z>Fb;zedUH`xYx}-5TqwevMWu}(^H-J!sTpHXJ$T%zRzEHoIkaB@blivn5wGN@DD#9 zs_K?HvZ7Al7W}Gefc9YtYgHY`6>OQmu)?E(Q8=c~gH6|NWg= zPhj*qNhGNtPgN&_10K(y7omi4je__mGDyFuDoFB(KQfE_>U+Q+BD%>xdhHgiiIwmT zq5hrV72uB}DM)nqn)VSW6~BXII`N05Ha#BiEb`w#)*}Cci61tJPD+n_2qUzP0)I%q zH8Y>wTAZKA0Piiz*D11(Vp*uCY2P@eAtjl*0zc3n0G5sw5~BXIj%(DcQjmDRQm3G_ zcXlCl%l&Do?*P%E|EX$hJ~E0mnh4Ed&8ks6zl~311pWrpB>7_j%E3cVxNB4{_0N#L z0lwHxjLl@0u(roZK8HrU@HOq{M=77^kxDU%`mO{K?-B7CS@mG+B7Ah-tdElV+Cusk z(gU}a{4+XNkER7S7NR~q-Wy0#%>hcZr+WSe`%s^ypG~1G>W9&>9qop`im{H(&-AOK zL6F4p^Zo*i)8k}61P*Y*y|TsQ!<~SVC`bAI6Q0;i_^7@zOMXbpF!YE6@w3!Ji0A}8 zo*wGG9!06(pF`~#xq)G zbR@c#SYNe3$!lFn=vuF*H@%VT_4NGlvLJW?We1ZiD4V-DO0mucYQGfO$ENzw!}mZv z$`+igsQJN#!xcklO#hplBIw1>!6ADlvd=^lO#GP}1U55T`Y<%twCzYTrE6Cp!tMwC zg4=7|spfwj_a9p~%;%ue)^kU-z{gb1{nvW<>9ko8<~ zgWTJd=lYhG&P1|_jdg3pJqPO>lO5G* zrCNGw?Pjv0R#tDRjrVG3X|kgRt}@L{W@D2b`NvZ0?zin>vZEG};wDGA8HZ9fl2cD@ zOm^^J4rw09CcVjQTL3)Yxk?f&3bI2KO5JHR+LG)rpNKP`Hosg@*v!2@)qUNOr|J8Er>kRC1j^G#|9;kkb;8jl%SFGRSZ{;>evv`9ziuZ zrRR2I`6(^Gt0!J!Y&0lOh;qNl(9(&-JIzGI$H&dA9~9!_UJLV)ZYQojKW0d@60OyW zV*j|Iya#Fb+Kkl;GWDj-VC_cYkL`w1GW?>0`N;%F_3#wChQ=WdznlxUM$*t&Ze^lz zIBcv*7_d1yZ$+Z1TbZ3qR!iPWYS~CEl8vx-fwi8uIZhI7V>2Dqd^k)_#Lf#-PE?1~ zFxEZ6~RBPd7$F4HP+CKG>BD0eeJ#3l!LKjxYmJmyYDD_8KyjoPxPy2_tmZBPsC1HU$GU`q8|m?cCam z=$q&&U=SR2uv3t$=d?pSFG~jn*N#~eB)jaCf9X&JCu@}lgobvxr($iD*A%l!c4=3- zD=KqJlXFn$sc^qny7Nz-6l6XrKpo)dXcI=HAiwVg>4$E1T6%ekrG!Hv6&EtWxhG1W ziOMTcxi2E3>eImigF{7pF%#dLZswGpb~5NIhXS;mCqj4ECVej|kLr{!Y>2MrF{9>z z#8VObBhdu^;5<7ky{0kM;4O2Hc*#4$sVo}Xuu^vuD6)~pXz zJSnB!5V{~i`pPD~hU_;g|E$L-)^f`hjb1ekYen~RC5NOZ;zd3U2-0f|@I#sXR8$VZ zn!GBsb86${;qRola=%G=*@*t}Im4I>D>7pl;7k`T2{M|J zwv{{0%9AF@v^iip@>3^fx_q}195d5<3S{~(?--dDQ^2&kAUl< z0OKs$a6*5BJ6sqsl|M%1_nqvrwTu?ZFE-;!qif8(z|4y)@AY1b@^dqW+MJK`gK{PR zBxlj&EJ%OFn1AbJvsNrUaqt)Vv+-iZY1-)QjFIp((-j8`_uC$gf0YhD2RHtAoY@}fie z>Lh~r|6Kqj?_GlQhaSk|pWbC}cpiDvtKC7GBq_=x(lzDbsS)r_6yY@v6PfRU7mHu# z1nKJ@Xoh>;>;pl1$|SqeBt1LTi7#A^R&{pvqwxFug~#!8n+rdeR>xIUoq>P&`A}7J z>gcLEiCgfisy^C>C9G9-0#~qQ{=y1HePbY|F5vdqDoTvOKXq$5?Q(6S67@QM!N)gt zgEW#>(5I?X{y~>($c<2fxJE_%6B?plH039K#2=bNe)SUY2Z(O+k6ph_YhopQ!)SlU ze--$nNb(aMzGgfGO2uzKxla6n=`EMbJ%{`^k+s0TXyS)WqLYzma5A+_;=T;x!9W)9 zkB#T$RbLe8WG|qv0Ta2p6yDaC!}FUlRVC3_ay%0Ylj#}NH^YOd4jCy}ZGPCh7~b|Qk!S|#sigqe4d`y|rY9a%{X>Y4f{>5@ zSm-Kna#N#jd`jRK4_ZI+PfqVbe#yRx$L9|d6His1UyZt^U9KC%UxeRh#s3k~bBiG5 z+ggjxOY5cx@>Qz^xQ^E=CHI^@(R?g zy`&y}(|s=2t{?eJ>e1r=DAB2&xGli2>RWWJl%J0!0#G;~bEv=XHtNTqTk752Ppp(G z1m(-1z9jN*SI=Jp={Y=VzW9T3*RNo##MFrQrt4?s>RznmU~G(zjh^9S_XvkURu z(!yj4cyCj^PLqF>%lvto@l0SDQjw{v@B{q;VCi^50@QytaE+E#N)qo^>NFhfon0v1 z(s)|Vw~y%1|5P=y5E{cX8Vk(hnN`Cmzl~4i1pfNeIK^WTj)RMyaMw7w)IS6ICd6Vl zHa?qL#g|8Ve7o~clM>>jew0G5ycn`_X=$Z>#7vUp$^L!MYuN=~INcY`d z_Ri|uJi6xFTnu~kXrC`mXAV%JKh^8s-;eez{cIX_Q9lfi?`SvmHOzHnVRk?r^MfXi zpZ6DFoS7j1A#s2c?v)(R4-W!rq8jD*PjGT8=ArYITJ}O&2BAlk<BEy{1iggew3ajq|bC=Ua(-hdt3`NH5lps(dSeIk(Uk z8i@AI>dW)P*`os^HIe LV1h6W|HJ*ir`_^N literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-c4450a76-c10c-46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-c4450a76-c10c-46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..38a981f2eff4bbab3e975ff1f57d677ebd6b44e7 GIT binary patch literal 8053 zcmc&(T}&HS7M?NYH&6)S&U#WOOIk7^YKVXM2~=#AF$Nn-Tp$Dksy1uv0XvMrYa0wa zMR};As1H@OilV9y5vo@Eh^nfJs#O*3Q=V2)l!rX5R#laUsH#@`un()KTJ1UK&e-EW zaFAB(H1^zk{=V~b&pngQsB<1>XP20Jn%S7pDG1U$d(4Exc|WHOQMds96JB8yLhK18y3&lGnPL&@}*=-puuRdjEJy5WIl0lSvko&D|WOSZ4#ZUyAHwQ+?>cyPzIr z3r<$leE-7HiXk+n|4mL2^y266kUbUIXQBxv{?rWuTNy2V6q>8rP9&MqwaXA;_x%CE z?X~Vy^S_S!53w;}Hh0&9MJ*Xg=+$uADYf0K$3)pNI+$Dg)(&z*#&0!yHnbZCeX>cQOGE>w+2W^pznG@1X@kmMn@gUdM>*` z?#;?`V_QpSBH6^|hBe}7{}4UHY}AZ}Q*Qyw*{w9l!yix4>Zm$=DYD!7+^2sS1WjX` z1w`FhZ718kx|!}iqBQ?La!6?!IX}(Ln$c==H-jYzw3d!!wOBP@>nO@qYvqvq@MiG8 zs_e7u9ZO-f4_gYOf!7=OYN@w2x0Rz#?zG8{&hm~_ag*6(Nw_!!4Nf3AxoSg`9o1>2 zT6$~kX0oGJR&S||cWY>AvZDsBGR;k9W0M{E$5QL=H|=4vqZX0kCP%rMfKoP+Q%`M7 zcJN;gX#vP4y~%7_06gEhN)jvzvO^R~-Dx!1lI$>_h(lJu@+w*|P1HdTk{$S6ssgH` zaI%Yf4OLDRTB(+nLZ^`(g+*D829MQ8WXG}O4C|ImH0PU(>?k~QXDnJR~EoO@>7N`f~g&jMnXn~ytxWnNFM=Gj!0y8nVnm*B}~J>)VvW% zYME3dsxccU+oPK?%|u3OVb;_#+4)3Lg8>fgG$%5CP1oSQL^CI;Ata*XJ1H%l&^ADm z9rqh(KLbN_!}KV4IcRs-^+YBMc7WOl3X*`!w91>wq~A1*A!)>c1e`JUK&sW+Ibh_b z$h~FVEY{F~JtDsa3hX#Xn1QHEM{-ts4H-&K!Q6?25xQ%U6!>AAhJhOWXkOQLZ*E8Q zEp!zy2#z|~Dah4x+7X_YrGtWN$E*pGU3SX9bSi?AwaSA+LxEMvTp(4JRiSKm}b4pLT81$7R0b0%zp}S*~z895;b;=huL|5~eQS(6J zsfhidXoA0go}HDRGxnqg@ch&h?th5V*P`;wuDqy2+_lUvwG26x(SDjo`wgP~@>lE| zj@Euyd0ILWLH@0@s(e?k+^csM=B2`uB=qK|sKRcg;EfF7m?22d&NE4R>R>@@)`u&e zl+tbpU63GsWs_b(_8XOd)?*ZFxn+w+ubPInqILfG$Vj{X~INb?0q%mBM>?rVEz@8BIw$ z%I#+5aT8?P955aEsS`6@zFP^7ndyB6GX0mgjZ6#jd7+_7ODx<@1jH>3D36*)!F5r9 zaTaYjp})Z$E{vMWAEEMlPIk#!MvLVan{lPlHD+F5=EarwdcQ^axfw%k&d2#dxsrd9 zvuJV_q`zX!zjd-XE0*4JEav4%c0ZP4%I7DPmn~=)M{b(CE81%{yUtH&=o286KJgZERVgs$|zqESiYBkashz-`AlgqFG6Nq>=r}^ZOV&QL}Ymt zu)G#j?rObeF!s^rle28b8jRs0lQcTu$UsIP*`ybua<^T1)rLlo_p)%jcZ|xeJTEkm z?=N5j`8E4bQ3I*qjWO_$Lvbmkylz(>x1&3S7ZuApOU119GA^rpf)ks$pW4Zb|b{8oeU$_DvFR&;R93*f*e{YzAycPd|< zL=gYK3!vn^OOXE13wiw0JM1;jBX4@OJ4llxMR`=ZsysL~3f_q#yvAW7^F8ol@#~x* zeccPqaKDFrAV^P`WLKJ`r>DB`h0D>Z&dq%kexJYaIDT$-;pe@z2~|~R;V*tZRMnh1 zwx&+u7W}HJkM>~+YgL`Z6>OQmutHJaIEbl>xIMmx665e!-JVIiTsx>ly@p@#@r_?6 zjieRysp_*^S?2C}-gFydVb?|7C;G=uciasccGbhmcH6OXF?VZ=v4$j5&? zbOkuMsWCS`CGd*}Z21C6My% zt@JPKE_opqLH}@hK5m%yz(1l>sH4e=exn}U%ZqWstbTZe{1nltlw!e!-IaA6a_gT& zyw&qLHh^?L)=wa`y|S^aM>j%oi03w!=lNOkj%>w}3%gP8ggX)**`jc$dn4ub=;fte zQjflwewS;{kNhR|Xz_oH=u}VK7GPNOExA_9&&M(WD4dUZ)Zc##^<&U2_3j=ZR!S9u z@?}t868U$k=P!Zu93C}aeAx^2HIA{QwZ0Nq$S#KmAt#hyP7gvFp>^c>1NyDG#dvOc zaViD8x2Rrc$Un+u{yfckCb0~u$ka9Xf&KunbUYye>OY&fM$0NCiT5jY29Eab9+Ylr zJT2$jM|9|asv21gjpG@O1s3qks$rDh!6$M8e|>74;;{tB!9`EFYn)u_p8yW&T@N91R0-G!}-@R7WEK8ns)4(U0h`);jx z=X7o!UGr@%g*|$--xsGd2Pn~>>hi&LK@-Q% z`wKA6PLltSIKT<_N{;7;2LUxvjq>{^IJF(~(D_QOc%dwV&?Cz7XE_f6q7(FJa-{!y z7_~xt4)tg32GS{C!MWX~a3~!K4{nfbi11fGf7gf}Ley~HvnyfUJ)4|xFL~orbl%{m z126z$K=tRiW8gQr8bF~!zn$P-(`rtwy~go@g?p7wbn=zE!}STkH=F zM*HXVm4%V)^5R^4IRH4wlKLCLA;&a58|h1Dx{ZI9>rU<+&DCj{#bVG;FVliwq(~)ZO5@}QKZR0431yT}!YF$%Ve~NzrIZjx$y3S0C?S*)_E5qIrHnH6Fh&?&UM958pV2G$@5O?C?~S}lSV_Aauw4FM<8GVDj!Q>jF*?^Wg87g>A?QdyPgn%RO4);tq#t* z&O71UtEK1Krk2TubBXmev&T{YK5~ZGMZ*{No)*xY+sLf-xkQVnxWK9bDf^}23z6L@ zgg*0p00fP$7XfwWwC!B?@_MHGfXw`Ec%RJFLw=l{G@RAuYz9r>XDt)XX|Zai){&E| zrpi9y;l1F0mDnfQ8>YNyA2Q`d15Y=w)s${+PBTZH+-Z?*oqHQn#Z6|BCE@HK^f!Ud z$yNKAY^zQym(p8nGm~w#vbxK4yjep>lWjF{?UCGM^flR5cr4fMeAN~v+iE#d+T$oT z7@3VC2$bz15NwF7w5A*gTR4;X0EyDV9bsMGned^V@0uBEhGI2O)@S%<({Pg`t9iMGCxiE3V) zB`0F1g(*9VLuwmurxhqIMvE*xP!GzBJ5~_U0z30?g~J7oVA|7}L@J!gPsCu1NaXU& z$~9RMhGJl9UJIwRY&sm(n1z$A(e;>SAS1CbYHHcsTq32x{04fO6Iq{@)Zo5EHAkr- zB%)*6X)Tk`)W(CI*If>$!4At5 z%+qK`^QxA=wi!-tps9dCu+_m%@mxKp9boyMaFBDY7&Sq%%69puPDQY@R(U{Z=#YCW z=GNYvViw6N?MUy7%AC?<9~62k-0zj{{G&$&nNJFk2RQLrgb^vg?|VV|u7@3${(Ow3 zg##g#CNjbKP?SCvl^3FNOGHG~r-OY4hl==O2ELa)%r1T1#h|Yo2+&kdgzkNd^p&XW z)+wJ_5M9kt0MMY(E$I+X?9Y2%Gl!?!1Jmn+Ce~~9IbV~^fYxOg8WNqS$SBm+^V-1=cVG4B=i=hsN!xV=M4|ym?20{ zPBTgR#>N8Xs1H@Fl=5x}U63IC-Xi?~(Qj1#QIB4%<(4fOy=ofPO77)K21$>^vwRv5 zr0>zecVzarl5_~vWL2@9QyV7>e+r-g3Q64mpfa#(D<1Fg1Lw|!iTo^Hge~H3Bw6k;OG+NkuvFTSDU1P=tW?WnatM9fb zKQ^PQjrq7RC|9yivX@NGg7hc!`4@IJYsS*M7mIN@lHT{Em~#7w@}dRx;>b;NcSU`z zM%VcP72Q0+JSB=sZrHM$l~nOwez#S5d<2zMpXITaR~h9i0xR_L56%FvhbPK=c?mN8 zVmA=|MpW*#DbHFFk?B>y^jc84tMwYr*hQURpJdZ!XAG67q|pIK22%RKB0Uq8o9)W? zZK(8cFAIlz$Ea+|x5WnX$1~VKe$M_=(m*O$qYpe}P+UqYFWZ%`+R>cii;C%;rDD{2 z3T^8z=)zA5WD*Wpy<})jdS60X; zvXa6zE`a+M>7Q_ob}Dy|B8dOr1yJ_hB}lh>A&#HD!T!zT$eUj62GS%+ULKJyDR+*I zfOVn>uW=a2LJvG!`Z^~__j{ok?)I>^1?e%9tV)yg&9N?g;j*==v$OApF7g*1+mG!I z{Jga?uBz$`{KL;Xs=B3)uBel^1;485rF~e!T2&`-1zW~1tnjFJ434P_xIMOlC&u8P zx;dS3IJQxUdKtgq;~l#~5=kn^Q`ITopu;ibL?{7VqagkX4$&{F@{v5^56&UK`WEo} ziEi+ZUA|6hVkLaTs6X$!0Q^xT`G^i*GcE$9;OBa8UQ#tY|FZxrdI&!1ccCNgy?w4IEE<~L%hN=IX<@oX$as%KR1 z40obBq=#UIuVEiP1w7sv!+ur$HR9hwk~g!ou&RzCt3Q|2=7-&jp>5X^9nB&=z2pbI z{-jg8>WW8I-w@&>ClvfY7Q6tQT-B%(pAz`Ro#sLQ$>|-4FX=b%czq#a;-M<=t5L_a z!*PZ9OYnQm_}@kP)*_tp?nL_M^NViqMZh<-Hy&3FePAEa$<@)+c)y;H&ZUJoVOHNc z;QS=fDV1V@`FvzG39dx@qe01|vFUv>obic!~<3s+kd^FiVN_5I6t_v`%co!YZd+TF~02J$E z4(0b>NBQVb|JpCv@n?l-s_aF(_|mTvQSSmt_e&-N-}i?exN@9tRz;5pZd=_ zu2Hi}LE`;NorcoR??CF7`_ok4UZO+)Q`PW7a13iS=AXx!RYQ1w8=uGs{Pn7F^2Z{S zgM*%MSFc>^pZ??q_+lqEKAVnUZBLMV4vl!>YsSrwQa;fmm0}$AUGO2^1L8Bf>cG}T z_~^V*A0_p*h4d|?d#^{_vq`QVP4jLnhFr;Lzc)@b2Pn~=>iO^UpgvPSn?_mG4@2X5 z?P_urV;x?Y9Z<)7Ac^DW{RJ3jCdhsW9N>g|WsAp$I{_t8j`I5_Fu56XQGKN&Zb-`j z^oTw2Gu4Bi=mb5Q8t%UmLaE@NeeD^&igb!sU^c%P3TDEgfi*fCB>dIu?=sPYh#KlW z7YQYuGpTXsqB}lG^#(T`fB_I5)wjhB1G~voKOQRf+i`9+ZNej50r;rb$6~*CIqDvE zMN`3KseBaWT?Wis3;n@?X#Z?7GC!PKT9}P5`2h!MQhy=X-^w(H~gUr{)~Bu|AYx92-EQY+y4uniSp()@@Muz?PS~{J|cfYNiJ&`3RKprBQ+jE1U zd^Q$eNv$qhpcJGorF5;^-<@6C?Do0;a7hsSfU=!Q7L<-oj#8|%f!Hra_K7Jz^58uX zk7tWUR+N1I!r_9!HKzSdE)nG7=U|sT6WQmY2`2f>3kTM7TJ|tBSGBEJI-_fs!NcDB zy@J(?j=38_Vr*Gk&3F5N*Tp*VkdbC5dg+ir-}QtDq?%_Nopo^5 zci9W)-Yz{?H?(XnmQSs%T0M>lbby=zHe~w3r8k4-{Caj3%cR>^vN|jeUy1B?A@dmW$&9Ia(zc`Z@R);jWX)m%9s zKD-(HuQK~Ad)JZ}tpk?CXyEw2C%4;VXZ!w+RB@BpWJ$Q#4-HPBb8^*& zCOfOs%B8q#?PjvGR#tzxj`wP4X|l5hu6>f5%*G}=3yHvYWM?f$N}C+TW)xD{ z2u?kXDztJb&Bab5JBy2bG3q-~ACa9$I%jCNM8Z*MF0!-u%#E?IRR|cl zv{-MoB!$m<^Bpyhp$43s<3j*Nxmt;XsFse03Duy!Hwhc-hfDSpw;f~10@dT@$eMdjd!pU;I`Vp(V` zH*@hM6dP+2`fSdQo3VK2W^Ox|*U~rBT0WMD>=e({bJ`)6?+XVx*N#yWB)jaAe{NR<7i*FGgoZZRT`{-z=M=L^ zc47jsEZI~eqpLjhXqiO}7#N#BXe!#d^nHbhtR zm{IdU;;V@Lp=g4?f1aI{UNH8w2JpP<3HLul>1$DWZdYE`A?{k{ms*CL%4k0;p#28X ze)$Xb4M%G~C_OD5i6H-4T2da@EBESM#d)dtBnj@q6jj`<e%sypwntK{B;BVD{C zNNGmeQf@nxCruD(bHH>KrcR9X{@qG&%t-Gl5b3|XV?HS1XC|?{?UNxg$9Jy)kuBflY>^eW9qPu5U zZ;7Ik8@B9bB~`qa@3tsUkD;>avpn|lDx-WwV1-`(=>-7x=QHKKyabtXv0D)B9amnq zAR^1FfaSHIa#wSk&e%nrPtUSRt1|{lRMO~xBLgXYWRqTs%H3Aw^>I{sw3mgWy<=21 zJ{N*YK7Ym9-13`$BF@YtI)RohAzB+jzvmb zXdR)7V-t=`q*a&l$`5VIUK?typ*I!Zi3I82sPNS}7PKl%R#wO-vXa6zFMvIp^e<6) z(XM=T5<&d`E`YN4EmH;^Vt^74>$Re5k~2&@xDc#XqA7JA^t z($_gb`q~Z6aJP$nC`eD4WLKJ`XQw*wh0EEZPEUUv{eZvlIDcyO;OG72QB_r^;2(ZI zQq@g$cv&6CE%;T{0PVvP)~Y&&E7&rBVTDHnBXCTe!|jn}JTU_Q)Q!ok$Fqe()NA+! zpTNjrE5IK|QjqBIHRU5vDt-sabm9+9Zg@Q28RWl#tVRBL6F+Pcos`7G^gc zbYv0#$Y|lb8i*sE^o8^lU?Ni&qFef6bap+Vs&q7w9?d19qn|!MK%^!7xegQBnkf9;OBXK=KX`~iS+Dt+&}7#MF-c(9qP{D z{(SWFR4>cNz+{idvlB%AvV64IKTLGWC$0-HEC=R2OZ)3%fdCZiV+Q5-+(P;2bW6T_ zdx@1og`fgCl$S>St?KbhA$=1|Es$LBLw=2*FKMkWhGz2%(LRU?#h25=5JqSn1^$qJ zYkDrZxiB}L0p44buajgS#j;ROQ@$}wLrOAr8GfKY04yCVBt-pZ4cDkyr6BQsrA|U= zZ|^|rmiyCE-vOdS|5Mf2Tx0}mG!dG`npLBCehZ(-2>cDGN%F@$l!J$!aM!3@>YpKf z9elBq7@f{6Vr`F+d=8Cx;cLpzk5WF-Bb8zj^<4=f-b3OuyyC&uMfm8vSsx|!wTbji zqz7&-`lofS9!(3Z&qsZFyeE*Pngf(*Pxbr{_M$#ZKbu5Z)DNSh+u99%1!EnXo9^8T|8V~=?nCzP literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..894ff640e25618030dd7f1f850375501f498525c GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA31jIIKH!*RLMH6Gf`+9{rD$#@s^A%Te9n@w&Pf~DVvq0v88xq zYbE)2UqT*A2z@AJ31R6&FlE_CQ%VVCDPf=Tw1kj{JS+p7D9fI6?u;~& zC0j8pt2ENwd;Y%jbI(1KPN_3KW@nd}caqtd&>;xYEPM$|v3xw)jLw?|D-_`yxwDf<8ntzwZV` z`D`q{np#`2Kq+WlPU#we%&u?s_}qWIBnW;$*})_WN=Fw*Dc0FQ?H3~Z*i;{R@GhuF z*`kvbHQ&E*v|3@?;1iknsH{zGg`n2xS`u&AYDDZLs_JEgXZb(<*LhtK?Op~%;%--rFA%VW@2@z;DWgDG!AnUvA z1-UmX&$Uf0n~UXB>uc7Cqy0nl46xk0)`_Lh!Y2^GgJ8MR()$0ID5NItM%WH{hzSdEctJcaP`Qgpr ze^uFM**liPXdAQ?Mgy-m@YPaptzIifo!nuQogD`|QpHVXlO^He05mv(0a@2pUJ|a7hC1+T-WWrHsF0!-u%$>2YRY(|xw0PcX zDGHzU<~wd3$pdho$(xi*Gnt|YwIK3Lmyuo4EE|wOLJAAAOM*tuR|#A_tYd>jdkEF! zlAhU(<)^g#ww`#2vC*JBCd$1gLrW(TZ##&HkB=Oz4;13#UNZ}jZWpdTKVe9;5Us_E zV!zc;?nc_(He>aiOucS1Si6zgCm%>Og5E{Wp~CCFh`{FJIu~4*;1xqU~;U* z(poMPi)+lr$@chqLNk$(T9`GpTz)o{)?k1GJC0N?sOcKqmuTiBHH1`rbUUMEQ`#D6 zvg3XO?dM>KZkQYfFZ=CIyPnGB!46OxK|vaDnO1o-84sFgP+A(W_WS3p?&mD^3Vl8sN(9kZsE7sP*nqoG|F6~Np zL}gZKat#RX3io@ZJOAWKLFSVJ)B%osHepx_^ZQ<&^p&VQvnwy^5O*!}OD#iAWwf6b(0+|*zxW0F znxnNJR-TrQM38?iEi2#FEBESM#d)dtBnj@q6jj`<6uhAU95V#z*?A^OPn|4m&H7-) zlTzLdp$ihEFKyB*$bO^p&w7kvEw^mR=vC9OR&p;_a!7hCUgXn&Aicr>KakmvCFKyV z$*W>Jr#4O={zghF_nMR!jp!erGmN>gB9hLFbI|1|yq_p=s_wkau2Ogp&vfyUAfp*+ zTemO{m^v}j2X`yMF*CiVK&JolwvlN;J})#>X^F+#iGX;eKIM^P7+els zxk@HyLHaAk{2LdWv0`x_#A05KWcOn!p?rQqdD)D1apb1CyP~}ov+MkbhVGtWeI(NpU;%{@)Bgm#cn~g->ST5K}42U z0n2Ma<*w#7gRzG;pPXe=)?f^ln55AGM+P$b$R@oImAh@qt5!66yqAUJy<=2%<$1Az ze0KpG$S>J{N*YK7Z;XM597;+V<#n6#xDDMYzNlE0hGqyhHi& zB!c+=T>xe8U4ry`H{|h8@37ZAkG$#C?jTK)6y;&*s`B8}FnA}5@EV7SEcC#OrLS{> z^pzW$;a)fUK#-m=$*wd>PfvB?3zxG+otgP4`aXZ*asJfi!OwduW2&l7!(aS-sH$7) z$cj3FTkxx@0osQptW|X!SFmON!U{zLqadcvc{|0gq?Ui%`P2Mn(J+8Khq{6(oJcADKmd^*!JZ5#8h;y>^S%#7g*v(Ed*F z3h>8~6eK!)P5TIxir+zUo%lmjn;wsM7Wr==YmtA!#1ET9CnNFjL}rP^13AQlfh^)5 z9V^JIfjH91UPxaBCUSK#x~(ro=Qa|mN}`GMSS}GI)6;5Tng>xGG*Ym_*RT(t0-nIM z>A$M}7V&Q(DUe;9UsXqtHI&!2xgq~TblbN`qB*2z7DHe+q%*PGW9{5Le3Uxd^)@#(GcX2*RnAH!Dke?trl~N)+x3jdWLvDlP zh_`w^NBWQ+#QF(EHkZ~G_4ryO3Gv+I@;pBa{-KRTdTuA~AM?hdLmLzhb$94sJ^Fd6 zm(^ons@LP$4I+P8JzD%9Av)C)w*?qh0t=qygXd$B02I&1Eb8yQh59k*mU{R05i6w% zK?QQCFOB@$)$^A^`W7CwKyuL!^)-sIq_w^jn#(Un`ynTkUrrB08liO*_(S@wnfc__ z;`~GgcyCd?PLY3<%ffk@_Kjm1Qjw`E@B{q;VCi^5LeziOagCN$N)qo^>J%L9on0v1 z@_1U#cYx^7|5Pg|BHp7o~clM>>j0w09+lc#p`>$f^fh7vUp$^L&(?uPvl+Aw6(w z$v>lW^XOV&V{#393U?1AE^s_0{Mg1^3wxiw9S25SI`I&xoGzgkF ze%@bzaeAEmhr|I+xL3A#es~a26V)icf5H=+2_K!W%#t6jS}Aw+b79#0ST zUXP+yh|i(^jNCvvgVqo(Ibc&&U=0-s(Yu?W8MXSa)Qnq z+;jj2K=i4>E$$fjO|FJesMv4ExYx7^k8lOxqj5eKdIQUG|Bx@9j_9TOQB`0WFmKKG zM*8EuGy2lpP=0ZKCb<{_9AruTmEe>Unx2pKWOH4{Kg)Gxc6xl52QK^JAFAM=F^}<| NFwO*F3jT-te*v4J@ly#sx2PoH~$ z39N&4T>|C~*1#GBp~-0jM!QY0!`}Ps15?47v<&;PjZ`X`>w8l@`yxwBgFZyExZewk za+zpsExEpGfl|=AlGL?6PhVyu-{!SR9e?ALxkP; z2L!j*x>L>nI_^Kj#)R42T@MzuR5Ynq!)d3~cC#K6Wyk2L->rUQ2e~fex0*d0+7sAY ztP^t~@QP`47!?Cs5<6`oInmn)8l$V)dae%>=w#g}WR%%k10*EScRe8jt)^_FqYh*} zm)#)uX63oQrDd|wTykUG8gb9zdWPAk84IW00+w@|8IXq>^pvcQs>7EeyIshA=KDd= zG`3Mh)Sc6|bKNT&neHP>^Y0^vl%|pM)9kDntu}WvSb{)nnP^UnSM#-wqFl9B4#^L1 z2LG$dKFi**6h`~7r7#+Jy@9WmdTVoAIqKw2o9yU3*pVu3GMg+37l)w12_z?1ZD_Ki zI;~twZ>`--cGSx1Ew}M*4J}P})WCH>bCcQFWJlq#+`9WsdzkE~MWnRJQEn!nl#S%n zQyY^V{Fg&o0J2GMGTRmaFLbW51dD>~5QTDg8jZFjJIp8IkQK1Jf)-2@b7PblrqmUNQ zTP;Q5vEF>gts{8|?lXCda%m=06rmPGp6N2OQ<`T(5=clvL3T>e$oVRctA}-Lm}n27 znw-)zyRrO~mfzMBFEKV6l*dH5*JNnvLgMXaBI4ttW;Or{@o}$(`AD}DSD&9SBwC5q zYDKZ%W+?AP+TC_z^_)z-ZZ}wakoZG~p_B|i?__>5!BIUp#jc`ph{Mn4g00aEG?ts$ zSON|kYZ3-+j;@>0So&smC!5n!H&a?J8jt3ptV3X}=WULYMBCWR#55lclaulD!n70B zA+?RS(+ZRpvqhE))PwQjjvZ9Az|I2P;c$Z^nD%rgnTlq1CgU(iBy&5=&Mn!JreR=e zUXP}!8Vw`wg_8g(13OdKA1Iv^(s2GMfWCKy3sCDZph~<;`T$ZyLssG~z%4&KP?j)oSe= zFmhAm-m-3%YG}Y7QP=_ncAO*3K-6WTd8@sKEG4IC?nKfE-L+^M{IE^KK#hJhuW36s zx1#zcx(XNsM;+`G%7a2fhumASwhq=5vq^So zSGprA^GcI*Na(F_zgN2RPo5NHJ}E#Q;OJ-*Mx`LX?*-}m9(G!Kaf+pdBO#R*GQqhk zN}q|!OHsKeBBJWk!6Ab~MSO7+-|HUcl%8}k=qpD8w45hGcgH4uCn^u?lt0)IUCm=g z%>#+2BKC)(3I6_hc2;`M*pnK-^HWc_{~=0WiOMs(@}drL*D}A6xuijalmx@o4&|8?Iio2D9H!_4{h9EsV&m`%og9WWw zAFg;(%DW+SL4x$9O?m~{Z&dzSk5R1UmMs~*Y8uu`?&V4jNsq;gd>Rm>R~X<2GW)Tl z9D+4@RczvCqND1X$lk%bw{o`|nF&9=u(sgkGx*Ub~69rDyowwOl3h&{WE?yF3 zG%al_x0{v6O^|7Gz;qO*PR#Vd-AZuGOz$a>>A$>fWLl8V3k_9TV)1q&AZ}?udDJ`# zu8RVUvuMK!{SEGLVboOq2$kP+vP;%7T0D5M8CM!zW99{BUR(vQ_ga*nnlaSod|Vim zEBPlmOD1PQ`YXo#8z-ByV(C4I#k?HJ?#EJG`TT_PvIXtp$W3#1MSHDg*ZC0*-95tw zN(`0Uuw^$Znc}^CuT^<+0*zIl<*}Dn8RaViEA;YDE&#AUpDFLD0hGOW3DWO-A&-B0hrQ-`>w?)9(_1nCKr>`IgL^i&tVa5-Alxw((R@ADTP$4~7p{Jgh1p{nXE{KL&^+?1?*V^+=qCU8wOh0%R>C)e z_ILbOfIo&LKhfc9)t&#Jyz9z=E6NWm&!!#;cp zczm;_|El_1#Ggl!FB4f@Q^$}skkhq=5${rX+Y=$tEYj1F0N4%aZtaFA5mWuch>wC$ zi2r!#3UG2$V{Uv(;ujCv0P;`G>_UFYzKO@@4-*qlRe@iPxn^9h>%?Dz-)F`D5z_NZ zAm!U#?qAqh@*b33@Cw z(tkaSS|L7%`ZIO|>6EYF+|E)sl!=B1*GV=+_^Y44YeWwrYB=w?<*@FaO-;C$yoo6~ zZ*bEA7yvP#`t#f|@S9u>pir^jPH?Yj6CU9Tz(?bJEcN?VV%`x?EEUpA^`k1^3SiDJ z_J;;z{d4;A!bmQ%IG2b700&u8e zuy)pY37Fei18We3CZ`P;?KZ&zGVI6K(&<#*^`>(6M3$Tcd5C0wuLlI> zv$6O}YIWHHr66@FrE4yqE4#Mo^7j1kk|6j1We1ZiD4V-DO0mucV!sgCN2d7jgLgnY zo-G<#QS$u@hYJSRnD#e0MUac1gI)GiWS@#AnB-Fr99Yk3*~8FW)wW{kjILb<4}0$S z2_Bzyr<(nB+^DZy&I$0MUGScis9~~0tyPgn%RP$`3qYln` zFMHtJo2BRKhL+96@~O2|tH&{c4v;g*hD~2M^%l^aU(c>$nYg?qtHbi}rO0jn(U~7YoFvMv$4sJ!ehC1&ztrz*-^`p(k4f-8G}?d zf>TdzOm^^J4rva^CcVjQTL8S!xyk}83bI2K%FSss+LG)rpNK|)j^bipjQWq%M`Xv5&KcS*k!UV77uivK=EhjqDg=yN zTCBHPlEQ1f`Hq@L@&Mdt@&?7y45lbTE{HtRWn`x`%LXJkA%z6lDM2IWs|2nd*0DjN zJ%ns>O3&=Z@)KHqTTi^i*l1856Xjl$A*B z3{1_dv9y-U#NryWak4$Wme5ROBo<~(Etj87r8OAfKu>cj7tnMK?n_j2k{UuPKDw3B zvMFs9B-wGlf%bDSL^n(hgO&Yuhh0zQ@}LK(jUXTmxJ<3QnT!Wa#pseo97wIIs;?nO!6~XcmNHy-EtUa0Y?ClhqaDpF z+V;(jn7)pt0tUfR2Rp@c^_+Hy<@>@x&b4FI1j#Nt<)1qg!O2?XexadV?x~nt`*Vuf zB)haD-4T^prO7!U^i;UtE8Y1gPYN=h6d(_9WR?Zu}R;F%ELP4_claV z^O#ZdK;o^4{h?@rzki;cm7X*9qz3T3>IwHhMCmJ0d1hB$)FJL#=9gNAoXTiFDWLrt z(SGp@_BBUqKPWvd9f=_ST3S-Rtyk{VJB#yD@ktVT3R6^Zx03UQ25`&}q-W=uBt3Po zkTvRq6)UB@8$uT(NMG8dR}lS1<)8KF#aeFJlF_TCVXfp|u4It(SiH!m0YQ3&4t^lB zA4}39RFhT3c1~@aEc}g>RPHq?FB;K4K4%znVMQdJ7w4eMk$XQ@psMb?#jcWj4~}&4 zk|3oSX-m1?tUPXlNSgzuqcC-1r1$Suf@4N{SAj_X#sN{w%yIDyU@8!F#%9CTLtokgEy}ZgOUlCZLmw$W#fc^PQc`q+PW?bwRM0>}T z7p;iM@+x3?EvVepdQ4~RqRuC0*_728gC#0ybik2;ls>XaFGS^ToAT;7Dm~iE!qMI_ zDx31W*g(F!fDPoA>^~(9q=GfZz(WQlrHt~rO?ljg<`iF4EblB8qt;z$TYp0r-ap5} zB`&m%P{pwc$0gFLOL^soc4e;}wbsy^itj{%^lw!7@*E3T6(%byDC;zwkJIYIEb~-Q_V=Rj1(} zem+puO?6~hoxm;lRaHOj!xGl2I*u#YGJj!(NByI4Or6K=(Pca_3jfrNsjS<*g+kP8 z_yr&T=yj4vQbC@oP6h_t?m-Vi3E>(A@lSY=eo<9`(7Wvh8fj>xelYjKuEm{*R z;TuBz+kq>6 z#T6Z58yH8t)#EwRhx7pEPcXc(xEj^ttKlU0bAzAf@mcTFjoWPtY;kARD9`E%hspbGB+EYFM1AVB^($A()7WKo(*tT{OH?0(LK}YG0%c8 zIYIRXHywZh5PfQ3lN$zhlc_;GRP487+-ll{N4Ns;(WsAwUjI_uH{^|{!+NQF6y;w6 z%$xJQ;r@8G_x|o9i zY)z6a+Ycj@5=tnegp!rA2NOmgLnx($FiM_E9!3eFjIf6iMkr;Jv4=6jC}ZcG?_TLj zmTbi^CK~B}-#Onozw`Gy*PT*l+|0_(G1nxsFri%#q*?e9BH>&#-sA4^xIBG5E_aWs zp9!pub({m{HrBuz1fj`c0YwL|TUZ=vpe3$o0Icockh+Pl7x|GJn4t z1m!Z}=t^RB*#xB^bt#e50Ayw@-{bE7?3^HY0A)LqOek%g9Hm%i0kOXq*}I1L;3sc@ zcsyG)vZCZWXZ9Bit}*Oya)=-oKYP3EiO4<@4KVR1E;z8B)iV2`xuk7|Q|Y939z5*2 z(Engwrch`>fb}oAiHSz!qL?Xnse)!)gHHK=_@XD(tXzZ=>kGxd-kXD1D39d_A36Zl!ngmYS~nyGc< z(O5DSq0{e58V-x^s+OLgnCx zpH2l@!Wn2R*Rs(#6dP+2`Yra3YvE}6T6Qa&(^A(`S}q(5=fbQ_U@fOD_M=2wThBx_ zFV2$VvD3n&1H~crFmI<7D9uKTEIm*U%8NTz5YY@fb8v;j1&&}^)0spnoY@+W!5ERq zZ80m?WJwr`fyuTSPHEY6II1xVCtIUyG0i|mVqw(OvbotrN`v_g^w<(vpO)0%zC<-g zsUakyqnl|hlh9T{k`?zGXg>=xbi?E@SlMs2Ta$@w4)g%E5d@?Fm!Xw6lX0J+7+unc z^9UHD@4=}SbLW7Gn>=^Kyjd!t0eeJY3j|nkjxZcimkH<1`WmtnoT9oz3Eg#IwJXMd@=~ z(v#CnlD@ICfH~@e6)UB@8$uT(NPn1*+=C zYwQxackf6SF9}kbmNu0eHsw(hMA{fI?S-imBfWdK5*#DaI|@Yl(QA671^KklP^Bdn zZzlrclKPYfwqY<`6kwc19S-PkaEA-ShVU;@_y-Pl&YVW`yDv8VN~3GcxWJ5yt6=rr zX5~j4y4skJ3xjec`y@xn$HJKCnp7MCE3y z^8H~{da##;gS}%^Hs#x51Nq|_Y#={p|0!u86|B(*9x^B{rInYh%2%yuPVq&>^v+T- zYF&l4^%r#E$ER4Z#D(S&syH^`xI|i$Qcii^rrc{otu^$f;yaNb{R8 z$A-Z=QH0kx3}m4Po-KWy6Qujy&K%n+>O5|bF5`(& z_@{16Wt`4U6rx_nFZg&zuaHEN3i4ES(l_9A4!RIZ0M{spe}aSbi>iDikNAVL$gjQ$ z{C=Vv{G*qz)0$Wb-w^8G@?8M_D3W|chp%Zjfl~3?N2U|Me`>?&bj>3FRb(ylFBte? zljx)*8kk5&=(snFc+inW{G(%q^Qt$BbkgTft^gC6x)|C_MnZGzF;%6bvD8>L79!Qt zs&|?@Q61Dnu*}!651#@a@3djRs{R`B=aJ;iEY7c}BgpE{CAGOB&q8R^y+}v1NKY^N zL9ajQ(yqGWQPnqy_{a$b|BnVQ04G;9;=-o{esQPuA^*hGHpG|o8+g3F5Hayk75LSt zbIR$wLi{E8y=MIHAU(eTr@Y&d-np#>5BMVB8{8d_tA;+XkLcv;Xlkri&qvqde4H?= zZ|`${g6Nb=vB2C`WF-l)^^GIm>hT=uL%I+1#~<8?tS%;_tHC(2}3KfF#W>H=W`8TV_FM;$tmYO%d=z;tiMPJf78S&5M7DN3I z6N)dV2Ox~lItu*$N5O5e*joXtPnr- zpEX>gW|e}(`;|HcrM1>ejSFgXzL=Pfr zsP|kXlyps}##{@Y_ypA(+;jj2K=i4;JU0yNCR6=*sMv4ExYe`?k8lOxqh22iz22p$ zXUH8*1(T)nQIvNHFz4rcgZ<$)vBuU5LMM`ANFAtRjWPc+!=fP z2M*F|oyMMf&);``?zv~u8FkLX?CcVAPcs`6It4+Rhc96zl8wguynSA`cc9Ph>2nV< zfpxI1OTgU08d!rMG&ya+XtxP=*n6LSU@AD1l3_o(nM@|KeQ&DgKxFZ0(1%DC4|+jS zHXVtsCpOkBQ1V(=6S@W<)0??IPwyWu34#|;b~4F=vbme16zgoD_6w1HY^o1Eco)>8 zY{AKjn(tpYS}}yi^uNg|f?oU_99ko8=4 zgWQ{y=f<{{&P1|_%?)eB(f%QNhS{hY3#Z-!ma|)FkcU5>qSaA#_)=uI^SMudKM0z} zHVcTlv)WF!dv!D2eMD*gedLhRG;)5Loi(G?=57W{5NItO$!f7`zSdEctJcaP`Qgpr ze^uFM**liPXdku|Mgy-m@YPapZEh<^o!n`Y9i8PJsp2NH$&zq!2pXI~a&py%COfLr zO11RX+RbD~t*qWs8}HW8(qu;sTxFV@%*G}=@{gs~-EZ2%WJfI`#Z8WKGXbS+B&VL* znC#%c9MS@iO?s2rwg7m(bCo1m6l8}el)BSsv?bYLJ`snkfaO)RV4A3d9wa;PyHo{K zN8w}_^%|<2Dzs89Erm`aI|_@k91R|;kI0T=$r;uynP|>87uivG=FV8yDkO|TS~zdD z6oton^BuR25!VTTi^i*l1856Xjl$p`{Cnx0{KGkB^$!04T)A{TAjU-A-J6e!`GwC0eT$ z#X*~)yccPA+l|$8GWELMVC_NT4;_Y5GW@)g`N;%F_23k{ipC)hKc5S>M$*t&Zf2r! zIBcv*7_>RMZbqW1o0;8AR!iPYYS~CEl8vwqfwi8uIZhI7b1NOyd^k)_#?A}VPE?1~ zHr`GvP+H6uSu9Ww#tS=kP|*TA3vh?S4US;iQ|Ux9lHQ$+!5opu?lL>KWJ{QafvI^T zlGHM(NK|7sPPRukW15MK)WWQ(WwP^$qy_^V*lA8={F<)8eTimHQbR~Y$9Ga%I-zZV zCOhso(0&Gn=!WT0@N&@Zu#C0|_`|?15CPwR6D8 zO_6)cx>>BD0eeJ#3l!LKjxYmJmyYDD_8KyjoPxO%2_tmZA}R30HVp$c`q8|u?cUsu z=v(M2U=SR2uv3t$=d>d{FG~jn*N#~eB)jaCf9_NSCu@}lg@z8fw_H8jbT6%GcrGz6P6&EtWxhqPa ziONe+xhEo`>eIm?gF{7pF%#eG9_EyubTQ~FM*_5*Cqj3}CVeL=59^da*brUKV@Ayb ziKim=hoTAo{&{v*dd}FB8o={YPq_afN?(b}GrRJl4sq8qztl41R7U%09_`nN_KRPz zuQ^)#VdZJ*NCf%U(yH=py>hSKS(uj!Pm<7^pP~x8m4Y`igky#vJv+}N>8XPStyv$g zcv4EcA#_24^rcOD1=(*@{#lPvtmT$18og>7){5@sN)Abn#fy9z5TsWa;0H4Mv8Wt^ zHF;HN=hVi@!{118-7IX5eYZ)z;Uu?#eM%S2mfteRq-s}Aq<)>y0wK*T>2jxos zNzS6lS&;sUG5^NN=B!wH%dwc3Bia2}iYcF;P+qp6T^zY-?yhLB)$BSyqM^HI*g%n? zq8qm4W+hX+m+!YKPfnn*>a#rd@+zZzMPT_}{>cRZ_UALDy}SsSaj{zv9keMgS`m@u zRlxFEP`Ru1n!(sdn@`TN8EY_xi%in!fFlDLePokfh|1k|JX0@!l~iyYjry zK)$XKSd3sf;Yy%Lk`8Il=8Y=dEAcf6kb#;?<^Iw){}2rf5Q;oKgU8vA+(NA zg|P|8C9=Zg<)8zt*3g>@??i(1Z#4Mw9P?WZ#w#1-6Is#0H7|ezoAfVHdETjf zc@jbV|1N-%_bx&DeJ|wkPw%kTJdeET)$Slok`(1p>8kSJ)F^l-itrkTiOl!Fi^Z>V zg7j4{G{gNK_JJThVUk^GlAfOG!WS+_t2#ILQTToS!sGa<-G!g`)+SU{orS;n`A}7J z>e!k(gC9G9-5?8Qg{=y1Hed8deF5>q18cK}AUv+yX?Q-p)67?E>!N)g# zoivhG(5I@?{vnrZ*o{zvxJE_%6B?#pH039K#2=bRe)T=z4-nntAHQ~s*2GHqM$rDQ z{|fL&k>n>je9d|Yl#1Vea-H}CGutkgdmj03AZvkt$;1zvL?{me9<%}EpPJc&{E~eWkIx?_CZ4K1zZ!MTxLntXzX-q2ivJ^|=axXq zx3|*2u)E}iSOopU<@vZ_+5`WHPN9w_C;E+gbT2Q)3A6g)5%N<+r&5Xq7j{?Hb;zxM z67g2g=hy(!{a8PN(Dus4vL4+C#UY;CT%PA=$vd(YOD^n2y%X+8cw~#hq3(^8*Q1x0 zdPzO{X8K*OJwNi7)T71!F``pFaa(|4&9~%QEk7U21fXy}=23tDE!2-ex753PfLJM2 z2+Eg1eM#issh+zTwdq#{$-;0O8xz|!%A1gQUP;uQXqJ9{j*wt?6>zM1v;@qG*?gvdA zKkqNVI6F!HL*f7@+$%YrA07nML^aCqpWxJX%tPlZwc>@c3__16%b(>u1c*-1qsfu} z>tWOi@j29=u^ULId{rp`cdI(X&dC#tdb@yy?!oB2;PtkdU zn-0JLhym4~td5r&r MNhSz0@IT!D3p91{%>V!Z literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9227e2a1b48a43d7a032540c98555998ab516769 GIT binary patch literal 8053 zcmc&(TWA|+7M_viYn<9~{LgAqCF``#M4@qX@g;WgmW(W0vg@d}<5;#So0X-prFdj( zCHeZWgglfG`cO&4hW*%jDwWLjys4gjktHTUA0nCGcY&f@ zCK_8!uB}+0;RP(=%`wy`(VK#TwgGDVBP3qNf+9|bNtlLD{KK$?Bt$b?-x!1;THG4L+C$P6z z2j)WH71QW2Dh9SBcFII@%+&}Qqbu54t_Kt7WL+p^l-XN-BqY#xJs|?Erfj354rINT zJRtXG<+--0WwOy+a(&Gjaqq!;2HCI~3#Z-!mU9~!kcS%d7Of7e!xtjEmCt?VhknpB zvR**cmD9F!UCZm4u0u-mZ$k%^rjheg?2H+$ zbreo^QLmxOsX{B&(o*O&vZJsl%TfQ4`iSf}lAK}Pl8NSgbCDf|XYP!JtwO>mq=oZV zOHp{OH{Vg~NFIRuOx~nin#mMJs0EQ{x`gbMX4!xQ5>iNzof0&1zKY}OK^+?;+5@O2 zr}WfrEI*~?clE?ejEx565mD|o8Cp7#c&nL+`1r7y^?^cs+-qTe((T06=f@0*R-(09 zQS2W#l)I32r_ESBBU7*14AyQW{@89PCBx4;Sb$7$RQFG^D`*_z@cL}1HJX9Oaw8i{ zz+q!eLch(?c_SK2-^lJ{b6V;~O3Ov#(Oi_Z3#|2=&2fTg>l>Mv=Eq@jJbq4?bfP+> z9_Q_}0;R=lk;MY_V7#zn2Nf-_GY5A#+~5eNEuBfGqM4oXILr~r+zzvIOSYtG7?_&Z zqA4w#j>a@*<79hmJ+7I^NG;5oS~fSEOldH{ft}`LHlXPm+?QzP1T}V4!fSr=D-e68$m$|aG6$lGZ_zCQiKLXi2S0Cj+)qfHo=Lj1lLq#wH3De3u1mKF|$R9wge=Z+|S zCMqvP<*tZ`s!s<83=S3X#Z7##x|vgY+{vJ?9175Ko(SD-oAkY?Jg8HCZ$or7j~O)& zB;JbHABZOSd*|30=^0~>YXHwrJ>mYlD19v|PwmR{I>cSe{8G!1QyJ~2d9>dk+An{> zzTs%?2bHI#BN60ZOUuf4^~&9PXJK9{JV}BpKSdRGD+O<80LKhLdU}pY(h~;@S+hP^ z@uZY?L+FA8=_{M`60+Z@{G%SDSj#P2GHb$WNV^>GIu5aLi2aDUj(uzinh%kk1JXRa#=JPfXj z0*te0!wLNj?r>q)RQ?c^-*vK!)-swezu1f`jjl2C0y8hJyw`gz%FoRhYI8o$56YGN zlbl79vmpHiWB#p^%~-Lx%CVT2Bia2}iYuQVQ(m;7T^zY-?yhLB)$BSyp`kmcSznQ% zq8qm4W+hX+m+!SIkB_0T>a#rd@+zZzMPT_}{>ga&_NUXOy}SsSaj{zv?H^a3w<03T ztAORTpmJApnZejYn~%@1DQhqWi%in!fFlDLeQ1-Oi^`og<>hfSdbF2?qrGEPcI8>2 zfqZ`+8^|x&e~KDN1#gUjha5^sY2{U$@~92nDZHpy-dQSUtvlbg{)!>Icb0{VLTDYK z3S$$FOJr4-a>|eG%6>aqt)Vv+-iZY1Uuf{fSr)JwOjI_=C$gf0YhD2RHtCI8!L|6Kqj?_GlQg$wfd$9LE(o=4vFYIl$(Ns98YbVa#;au~c5MR<+FMCNMbK~c|l`&OSr{N!dK2+5$ zb!0`Iz%BSyRX^>+64t6Zjw{$Qe_@59{!tK9=W%;<1tmt|pSn4fal5xsiFy^k;Nu^? zMjA;g=u_3nz<}F5=s_qUT%#iX2@ldQnhKCU;t$Uvzxp2V2Z?U-k6yh=YhopQLuh{| za2fbxND2@gzNWndO2zL0xla7SsZF=rGmHG!k+s0TVB&{OqLYzWXd=Bt;{Gh+!9W)9 zkB;T#ReucWWG|?%0u#Bq7}?gBB6AyYRVC4QYAhR%km+gFKh1-v4jL&~;cM83Pacnd z+Vo#le~b9HkmS!S&abK?$QsP)+T4(DA+qgVB+)F=(~Ci{8`M48b#Ef31_luy1tA~* z(eP#9{yi9<)B>pP1T({E~eWk3SG0CZ4K1zZ!E-x!u=@zX-qIivJ^|Z!Lh7 ze|M>OZfC&X6&O zIO46I&yhZ)2e5vE;mxJBMLo6_PCz_2xjfI$f^TReo|@Z<`Nlla$j}CbL){%JuSXv* z^^$t@PxZRpy8+}csYi?dBSfcq;lFD%xy+xZY413eAr+ar0zc3n0G5s?BuM>d9oJ}Cr6lowrB1=o-r0rH zEsdw;eEW$G{ZCb+^WjlEqw(Mzo>?`5^4s`CPT;R!O;9`*;5fMH33rW?OZ_vbZ$K<| z<6|@FB|O{Xq@P10Uig~!aZ#!#dZeS6Kzo-1i1(2EjI6q`brC+2H_u1W`PxGI7SjDU zmwYoiH;=CQHx?pZJ=W__(3u02=uh?f5A>luOFx@JUDOXFV>{Y)eHC*ZouBDfM+2aV z|(Cc50`G&l)R9G+8kE;C3fO%`a zH{2iVoza)(hH{JZGl|6@;2=xtuLOr2*YsSpCzI_m{w&v(-s$mP8o1 literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d80a4dfdeae984724f1f795c7278f6d5d4c3dcb9 GIT binary patch literal 8053 zcmc&(U1%Fu7M_vi-`KI^_|9rlHS4s_M4@r?=iW?m6H2y=UBMb=J%5>=N@#F&h&)1wopFFJU>FizRx!y*`hxzt`jK^$aqB zb+E2Wz}&$aSc4!mIc>mbw+VLGd!Kz^3V1s$!+vZdl}hG%-&D?l$P!Z^50NY!xIj=Y z6OFAU*H3KBnUo0*~uge%I0p4QmnIq*e^u(u_-?M;9U@p zXA4GFlzjif(SpG>ru|J$5#-|MaF;z5*=M2&Ch^n*2R5@><|s5*wVh}xt!tOT!=C&7 zg2!jwsb+s2w;z0C!ffuY2aQ@Pn$)Y|v{PuiS&xabW8`;tR==^sxqHTMHGMYJC$N*O z6JsIpjA?Wj83S7qJ7Xd_;c5hl(N%3d*XspdC+o&TMw*@Mr$YjL*ApU;YMyO$)WKQr zWe=Qtv-Diw(lXg-F1fL8^*AQbA##S;i0KQb-U6C)o0)YilU{Gp>Zm+?DYD!7*k`^U z07;`81w`FBZ70{gvXSXNA~pX$d`M~&VMhbLEiu z@MiG8%Ivf39ZO=g4_OkUf#)08YRR`YkCmfN?zG8{&hm~_ag*6(Nw_!&4NjnQa@B?= zJF3%4rMPPCX0oGJR$r-(cWY>AvZDsBGRaM5W0M{E$5QQ{H|=4vqn0DZO^#wS4ykMe zr=HrF?BKr~(maq&dXw3<0C>K0l>}H6WQQn}n$u{sCD~y<5r?dR<`vXns;Gk=Bs=iC zR0L#4p|Xp54P{OhTB($lLZ^`(g+*D629DK7WXG}28QLw8XwEkm*-?1r##q=Y1dLo- zsJB{@!fUKlg#_6tK_lm@IIbSnu_2;8 zgluw3&+NwXQ(AsoPrSs~Xiy##~1!vrEaFQTr?idMOlZyTF=`Yr--((nTctB9405?=Y=UJ zibHA}Z>JR~EoO@>K2Q(J3p;iY(E>a3aEHSUj$qo;nPe)O*`0{P9FffJGCS8~OPY#- zsd+t`(z5AjOk*}qw#PQ&nu(0W!mO!fb92d*1_K=EX-;MXny$fpiE2(!LrBKPcG6lV zsjY(~JMK5oeinx4hN%&-a=`Ae>&a{m^Z>OH1f&3$sg*aAiGZmXUDAjH2^eGS!Kqej z=YWx$Ja^K%SuCLedqjQ<1lVzoFdb2siEdl0H(_n{f3I=Mlqj^o+ zy}1?DH_=qUAUNt^r*N*G(~hvbEF9!qJ4Q{A?6On-xl<9GtW_Qm8aiZG#oQ{-DQ1)G z(w=ljROXZ>=b+%KaKBf&^G}@;WIicC9^lAl6Go&EzwZU<`yO^idU2Yig(D#qCo;jg zD@vb<%1cqXCnBQi)4?HwLq&XX6W{9|=9HdvG3YBt0<_cO&PPrL-GD7bHku+N4(y{YK@V_2|V~ZrP&ItEOSC=w7a5kn~u*$fp58dW8;tAhREf z(jionRfTp=ZJaFpjg(OCH7PF|(LO$B7;|AoBwZKhq05nbKT)8n?!3jWl6w!2bm5X9 zrDS@-MytwjK@3$yFHKVJ|`8Yo) zSF%rX7ER8A^jGxxH%>Nd#o{W*VqT7<_hTupe11ZC*@Aj;->m{?!L|X zixd^zuq8Jusp7qSzg2m10+m&t<*}Dn8RaVi%lGn6E&#AUzg^nPi;x)?y9LofoARO+ z5m{aZEUyKXyPC^%#y;wNa+Xb7oiS9Tl12v{8A$0PoAg3d?zSth+ED57UKWn`j#1f^ z=YKWBP!=0!O86jBZYuJZR z9*=*<-b73d3?V*p zLf-#l;VZz&RgHS^DT!a)Y5mAQIlTw*CH*EIe;`6kJXCppHRhgnyRQ>}5q`fF|3^sQ zUW8Nrz2(07-9;bxA`}=ZkH-yDAJ|89a&;^<-e=^aXK5ipnAH!DI6p~rN~L&ces_6I zhu8)t5O4K(j`kxxfcX;)Z!NDc>9O^20{pqf&-3^!`i3{-srlWQZ`>1&3~!P<)V<;I zeDv{DFUd#$bf4S37eM}!e6-jd01EXnhw}Swp?q|@CEq>$ z#7dz;Q2s2+OCkSG_4p-`zKx~kPb~Q$zsAs)wAPn{^SPzS0K|mi%jqEqBeae@e^9?Q zyO7vkT9`}&?=8yLX|j)EnXjiA?*yhHC7HSkKhPflmW~w?r2eykYt*b#ka)jRr=hfW z_aJpk{b{LhKhdH8scLi~JcczI56)xFsu4WDgHL1x{`%Df`C}2v!A(!NYg8`v&!D~u zzSxV8&!(5LwkJqFheo{cHRIz)DWB+(N-=@@t^^S85%C#ab7Siwe01KdkD~h8M*23= z{kN8VvpQFgrujD)BVIk$=TA`00ZO!|dj1FcQJ}HxAAAW?(}Z2_wwLnAN-*T{)~By O|AYx92-EOC-2V$7>-GNt literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..aafb639f06e5508ff2c80e215b4f3b2f8053a391 GIT binary patch literal 8052 zcmc&(UuawB6~9-O|HyG-$MJWkn<|;pIyVZ9qd)!=J9s5mmMz(JRNHYZ+mvOpbZse? zY)z6a+b;{Dgc8aqp@dQPV8ZBQ2&I$|M#)pj!zdw?5%y5R2&IfN_Ao{mW$c{u-78(m zlC2oVL?hkrJLfy+cm96ox|8a(hgsP<=AK{{CbSEJGy`A4Vk8%hcYC_MZf|e5+tcmt zX98EyQ{q8Lg zk7tWUR+N1A%>IJGHHQ674iV(yXK$B171_t40Ve*`4F}eyIb-)j0Ss83)= zSUbi-;2G0s*E0sTBzD|Da>&^T5+lpnYOdP@ybjihhx9Z%(o2T~`mQHLAk{eAXs?5_ zp7U-v_j>8Mx}jyVkz8VJ)$DQ9zlWRwcG2*KqpKM-=hic;-5$}>TU=n(fRw$`@TJIZ z6hfbQ(FcNt*NT8TbJ}LEb7?Kpxld+(H@HV;>LEYQP8!ZS?6!d>@UxbQ`hZ%v<{f^qJgIy*lJ3*!)`N2o!oAb?d`i8QpHVXktN}5KlC?& z&dF8#nQX64E0^M|wVBEGT3Nm2I^L?GqsjIfxOPcyGWwcqFFclOcfW26lkK$}DeZ9- zn^8z*JvjB$#bi7G<&b89Y|xvGt_8pgjjJrcq9EHvq1>ECy(`If@4z-{m48JBpQE(rPGks?f@%H1kFSy~QKji;G<`>N`*$k?jXMr)xJw!d7T5vc34s zjWMxR2pGAvSZ}o?g~xpH9W;;Rez?x$4T_}^Oi_ef5P77_$PQ_S^-FL<3JS7Af=13) zFqp(KsGs~XI6dr5iK9p6E86~>XgSsx!t5o=|JKQ8xisGp^f!|KzzK{%zPx< zfvY=*^b;*aYcZp^e^?jpMB1%Zef2G=dey44b|LXan=X_TKWk@xQo&K(J;pAfa`3}X zr-Ch!3^bN&*=QV!jWr2<7JJ9FNHl#dyOqspscR`M7l}o35!NQKmeUsdQKGG_XQG-9 zXUVbHX<@>F;*ffnx6=xgW}`)x9;gT9#T_e%Xoj6xxWeHAM=-7FOd=J@Y>mZWj7a3R zn3Zd?Bn-vCWLu4-v}`&O)tH5otBL2OKNamqMD=B z5E9Xm&9s(DXsaN}iu(<;pM@E^VPXiZ?6caf$wW2>dVtyp0#bm>(8`<1nBP#0E@{Mh z1dP%5;8csbbHKz+o;zaRES1oJJ)*D$0<1Vk7>=mRMDk{R4Ot3KQQe`0?z+p7G}vL8 zfO#72u&roY*ES-_bu<+)2=+SIDW0q6w0$h!6%KN)6{99dR@ou{)UF5))*|-_4Q;Zs zVs7ouDQ1zZ(zbL{RA!VWN59~#aKBf&^N$`CWIicC9^k}h5r(87zwZU~I9lso>1pan1o@ZJlJcltxn1ul&P&B7NpKdXsN!xV=MDDbm?20{ zPcupS#?FG~s1H=El=5x}U63IC!6N+t(Qj1#QIB4%<(4fOy=ofPO77)K21$>_vwRv5 zr0>zecV+gsl5_~xWL2@9QyV7>ex zOXS|YBVD{CNNHNyRBqUm$4wAvW5Bc*rcR9X?%hgoj7aY&5b4Kn=#duW(?Ua)mRP)< z2#8ziRUX=gz;sc7aTawrpufQ#E({sMzeM35I@mdL8qMv#*z_xnt}){RGcK-z)%Ti} zA8qJrV?Hhn%9ZSs93_*pApHq_{)K~0o3S`|V=*pA()+#?Q|=s6UN)m%9Jy)kuBflY z=sG{3qFX0eZ;7Ik8@B9bB~`qa-)m8x971K)XL;=9RYv)WzzV(mgEIi^(TVb2UV==& z*bPL#6_xvkmFF#p$n+{;dM&8j)trVic2MWnC)uRg83QFMX>`Dmfs{V9NY6#(R;%*; zVN`msmxY79V^lWf+hPOx;~8uqKWG0bX&@D>(FYzfC@!UySFOrdt!Pg1MaA^aQZZ^> zg|_t<`JqmHsQELT9Z;vdC{iaZ$qs$^rqrFks$pG75?=U^P3gMD=Xv^ zSxMm<7r+CH^iQ}(+m(Aq5yb!R0w{a$5~Mp$h~sB(vVZe9@}^h2fiy{ymxrWF%H3l_ zV4Wz!Ya9l$&;!qwzRn5K11B`Yoi6r{AU$D{RcVsGIo5$MT=o`qdiuTaMgGEL|FPAD zpSPDsRaKpWfB1PC)k`nUWSfIo^PKhfc9%0r-3{PvUS#2=X4aJk$w$bS`Ci~REje%K^B zDTxNh(~ET6mqk42$Rhrc(ZYGv7ezYh3nW*7iA-GxZzdPRv+FTcrK7RbXf_rm)l;f( ziaSvq&_l4y*RT(t0v_L#VZW;W8u90mKM$vT+lxK3Tk~GSg)po9uD9w*AOomX9X;hlx)4#B~9NW#7DOX?J}r5P)KR z%%J?9>nI0mZSv`IUr022JeDMV@!U(OSz#m9npPq~77v{#(zWafR)4w2~hu8 z!!>GFDM-9usgqFJTicMj<^D9)w~y%1|5P}0)KsKocu8l z<=~_MvUU6aT zB7Ah-sE?BR$|F6Gbl>$w?{t!@N7H=k^I=ai+T)8;%>hcZr+WVTdr_aMpG~4H>WAUc zE$wP@1!EnVo9+udS zfwi;FOTgUD8d!rMG`VcRXtxP=*n6LSU@AD1kzqf+o=&IoJ#VUKUu4Ni(1%Fo_uZf< zpN+*=Q)??0Cw#dVKCbToMF7pzL6h1*M~lqZI3Gp!N%qeQc_aJa`w> zqioU1ikk0VI9f4;#`M3*C4yf393HZ#BKu4 z9heJ&S4^YRs2JFi*l82V33nrCjIC&E`5sK5i*=!pQD$%Tk&r;&^@IqtnzD_~I*|2U z_JZ7-mFL=~md(ZTsr5B$#C?bB8Dzs|EL?gsSk7-`K^|(*SF$>)4quAwb|LrK@B2a1 z$a)b`S6rWqjkY8^%_riJ6|lUF7EBX$(1TL{MYj>LarASaODSOC}tJ<{~?b&)gXcTZM#CNQ>vK zmZI=kZ@%N!kvstRnY>B4G?OWcPzxf@bQ#$t&9VUrB&4t)yCi7je3ih}!#Xxdw1-el zF6o)wSbj>&Z|jMd7#j`BW1`$^GPHCe@wS7A`1r`d`amH*?lrRj>2~4j^Amt5hSU$$u1=e!j<~&KX^^I&?3*az0o;WW|x=#C0|_`|?15B^wR6D8 zO_6)cx>>5B0eeJY3l!LKjxYmJmyK;%?KR{mIYo0PQby>m#WLWBZ4w4*^uw{L?cCgq z=^N-OU=W;juv3()=d>d{KadUzt{t-`NOsvJ|JHBVWT6%GcWrQOkl@>C=xhqPa ziONe+xhEo`>eIm?gF{7p2@~JzZswAnbTa5GM*_5*Cqj3}CVeL=59^fQ+YnvNV@Ayb ziLWB|hoTAo{&{v*dd}FB8o={YPq_afN?(b}GrRJl4sq8qztl41R7U%00qxg__KRPz zuQ^)#VdZJ*NCf%U(z5bxy>hSKRh*ZKPmIm4I>D?(!#@Jtsk2{M|I zww2os<#7{a+8i*Qg{c!WeQ>uD95d5<3S|1vZyT8wQf#$hQW1F zfN>UWxS+qm9WD%;${(TfdoFg#T1N8+FE-;!qif8(z|4!Q;Pqa!@{Xz1=4)>mSv zo207DQxu z6|lS(RPJhSGZ=el^T}B@Wevt)iAfqAaAY8(k8IKlQMucuylO?G$9q{g-aAHRSDqIe z$afd8f&7yFr=)>Y@WvQ;$f2Z^QC_zxkK53l;){yqouy*dx(jXVuNcDn=UAj9gw_$N zI5y$9L{@bvul&%i?6;%U8hTUlok)=WjRs$yV?nFIWMzYVA}cw#<^`~Cll~02(NLN$U+aiSo%6A zNME_38SZtn4+QB6lk7^9^z>9GzHm8P)R~!&qVMw;9_LSO9{jwwGN!8PH2lNQhpM`z zj;yE?xCOte8lZhx!dg|waRpoEFRV~BFbZPoJZ_J!pu{NrQ#Yrw9?v!^QLo__d;+7_ zNh4_meX2Sc9PoGsy$B_YYgEKPkwN-JQ$f;4{E=DYSKkBv5YbKk(QCJ8O{|1(2<`6# zuK<4>NkO8+*R+p7srVfv*NHzgwdwJAXOaH~vKILlO#HA(bTSeTPh^%zJdi^?7|0_2 z(XoQO8i*sE?1l7IU?Nu+qucsYbZ#S|swA37kL40kGCi#Zrg;$6K_dk#d=2~XDc}iA zoBpfnZxR0%k^3kLNn^m*5Xr@qdK$tp$(@ z>@M}r?JW2q7UAIF!F=2>?SX$pr%=b!W4%T_dKc%DgjxOY2>A)3Qz<3Fb303`I^;Gu zj(Dr*bEFUHL9CxpWOHe4QID@hk`T{LF3$Rp`Q`L5q!C(2fj^|*nwd{- zEzVD5fcF;F>lFD%xh$NgY2P@OAr+ar0zc3n0G5s?Bt-pZ9oJ}Cr6lowrB1=o-r0rH zEsv+=d?`D^4s`CPT+4qO;S7-;5c~b33rW?OZ_vXZ$K<| z6Js-(B|O{Xq@P10Uig~!b5W`%dZeS6M0;0)i1jI4UFbrC+2H_u1O`PxGI7SaQ^ zmi#k1H;=9bHWs2jJ>DBg(wPI4=uh?f5B8xwOFx@JUDOYwV>{XneHC*Zo1f`dM}we= zum&&Um=Q@+A8I}6cBHWuw)BiRVyuYUfn5j}#a;k@UUqPllFJ?35TCnxB< z!A%EX07Rb}+~SUb-{fitg^K-ljC)O+@Ca7`J{sp^p*OG`_Ye8v>4;vcA5{gG0rS>; zZ=^roJEJen4doZ-XOfE{z(JPOUkOe*q3QWpPd3+O{8_Fmv(w|dJaE|$f2e{#V;uc7Cqx}Q)46zY27EZkdEax^dAP;{$C9A{g@TJIZ7jmEZegHI$ zt``w?<+SZw*YbL%>yXm?`|ts!Y2^GgJ8MSkxVsrFL7=rvG^fR@`C3O&u39Sxf{ca?C99vkt%L7n=ArWqjkY8^%qQZI6|lUF7EBX$(1TL{MLaq_NOFdCOD38N%|&(;pSd#@wh9TOkQUEd zEk)t6-h4-`BY6<+GkKG8X(m$?p%z4*=`ylYnqz|!NJv3Jc1qC5`6`a9hjna-Xb+*9 zoYFJ9vHXOV-_{c^F*X{M$3(f;WN7I`;_YT4;^U)c)(;Btakqu}NVgMLpB*zKT8Y+b zMX`6>P~L;IyKTnmIhlIhX0Uc6@rQOpDH(p=!Te-`qk3?PT}9&%hhNSGTca6hEH|^U z1ROTjBn;Rboj0Sg^v!HOo6}M^Q(7(>kLIGRU0|)}ZH|*fTi?jUG#?I=6Y=xHloQn< z^*C>*6(}uci!2qW2jj&ZJE&-Zoq4#!;RZ)AZRt!h70u));xI=fb9rXxmTXDWFfcW* zMN?Wf9gS(s#>w{BdR#M+ky@BFwQO!KnbKf@13S&htY6bLxG&MnNookm*w}Vj%Otfm z&}7H`2HMZU5Zy2}0$vW-9dIwJHqV$!hJhLk=>JWD=^Ghv5PGz*86wrQ+XutRs z`n_#_EEg(<4ITPb+MgE(dg(zEkSlAbzP(3V1jo$ut^%3<%UedK1^K+tP^BdnZzlrcmim=P%_HEt zD8M+2Hk{Dk;0_l?Oy!SI`8_AQWG$nG{TG{YrO`EJUSQ_MRq%SZMfs^2Lv7B-g+aNJ zf0DCgau%e&V$8pBvRNyZp8Z(N%aQDUB*m4_jwvr&&@PVLGnz9O(fFaP)g0Q>Wq@?Kto%(&Psi1v;vFIo|i zw8GU4vUWm%wHs#fEGS=RCeWg zv4MPd0UOA#*?&qJNCj_w*H18ynl{`NECGZLhN#ud4cJAC|CI)d^g|miY@S6!ndPn7V-5V=E{z27lGf>5R*@jY`yO_yr%| z*mcrKT0x(xPWcC2t|2!<3E~ZWlYi{mEm{*R;TuN# zdH)sQk0HrVboiR_5GWPD{p33F2c|b&F83Vr-$2$P|DuT>Hi=F~V!_Gu5{dh=hzA2% z#6LD(kXL;%q?5gXz6wm_Y9zd^FNNnf;;Krb@zi)W9wyT>s&9q|Q5`Z;u)^1{51#@a z-;C+Ms{S7FZz0K-i7c$DqsSV_>Dv6TcQL%}iI8X(>FG!S>;`nVcEgj1ss16vM?omW ze=KwbIJv1&H$EltiwCVA`6s7$Airea#N+daiHWDGz^}$!(=OL_;xEDPv*P~{>065+ z<=a{6o6j$LAr?XZ(EfbfFztbVM5j>4QsaF_J-Qx*zK&5ZYW?i|DbnPy*t)$>n)|7QMq8@zi`i<{fuO!^0aC4s~aEe?5A6 zsh8EGZ@SOr+VLZQSv^|(A0;}~6SoBzR(y-D<^AU)LI8^AV-EHA-9r5sbW6Rv`-zoO zg`j*{)R#j3?dthUB7F;wnlBOYLVb;4ENQJT1?F>+@Brk5^2_N#NF%h40)IfiHM@}5 ziY!c~f%g{G>ooaCxh$Ng8P5ckAr+ar0zc3n0G5s?BtZRV9oJ}Cr6lowrB1`q&hJ3! zmdDd_zI{Z8{->(Zh0qwD(Rg4U&#W3o`E7h6C-B#&CMX_@a2#CpguBMcrT!VvHy{=} z@$uR85}xe|($Aq0FMQ2-xhT~WJ|~M&Wpg)2Db`s)?C(YPo*_Q)>6;)P z&lZiWDEZErg9U?Y4EvkxBFM$h{w{kWvX4arO#F!x4s2w#%t2@_X1_mw;T3Hy*XIIWJL|?ndYZj9K!*hSt|vqw)i~Q|tAn$y z^G-PTYU#PQsb#X^Tw;CA>~Yk;kDOt4(eQ=6rv)_UHZp5{F3~bjTwv9Jl>O51g~)Cc zLZ5j)2!h7ei-5Xw+E%W6c|Fs8KxTeFvQK8}AwSMe8qR8SHiIVcvz7_xv{*G$>&VGf zQ)Qp<@Lur0O6-&D4O3pU51aC$fu|eTYD%{@r5G2-mIac$+jA}_DF6r`kHJjJeF&BzG@4TZM7UJ?Qs;F z2}or@J0i@#UTk)A2 zV`8fiFmh?J-fBq-m-*s5Y#zx&aGlAU6iXwRq6oPl@<^AF?b19OlHi2o7i7BxjhwGy zxZ15_!$jMKY_dyFt@`pKT0X2NUSe$2DUXP9t4WvAg~S`pM8wAj&1?V!;^W;G<{{a3 zT-`pRpJ*jos~N?;HeGlx(r&ivtM5qF%XXc$2Z_%+bfKj9X(#iN3XbZ|F?I=+gCBlA zne?QBjBB9Yr> zR<6mCFcbq*^IABiWz*rP#w?s{jjqQu0~v{hQB%w2<`XFm<~PvOoXC2$qz3mTsyRvx zArT$lN^6;fwg!@{xZgngS(u?4rbofbL95N0Ok{JQ2dIr8AO*M#t-P5`dJVwasvH15E`Cf~^jAis$M%?EuU7goB)G#i$9ARkq7NcPfINwaSA+LxS6thTHX-9frROXc?`;gFE;eM}l=N~;P$b3?OJiv+1B8*CYe%}ky_dV>m^ygzN zEgT4`G?59;hobbcsJsxBTOuN=J{{~cI8?+JGw{9aVRq@8E(U$&K!B!tB6ROtq_0I~ zw@&%Yg6L`%gV!g2SB%!x3MHP1|Id5bL#|%Mw za+*ofw>IWCM}4?rrIdF==z;|44;JZ1h<>B;k9zcCEw^mR=vC9OR&p;_GDvzPp5@bk zApL+2z9X~0m865eCaa3=oZ2{9_!}v%+-g#uHKKid&d}$=ib%T7E;vVd}(4@7=8g$B6W<0+D|7x*lmkJ}opi(KTjVV8+E&u=;L` z@>4Uq+L(_EgK{PNBzwu^EJ%MspMPm*b7m~Pd$AaoBk6rliYd2`C@)%2FOJ+acURQc zYIL0+QPIs4Y@kF@$qid}vyv*_%kQ=-kB^|T>a#rd@+zZzMPP+q{=pdl_V7e`FE2r+ zU+e~=--*h-Hsx6>A~L-Sm|hDiceP%_8M~J{N*YK7YxIGK42nx>6_bvv3) zPoZu71zq^bDHbSkp?QQVj!if&k=CS?Q=WGy_c~B(4ZW%OP9#YGLWO@l#k^*P@yZJM zL{?I`#szTSBK;Gt(N5*=Q3Ub-y8z1Gy9DWWFU0ZlH`u><9C_2L-9VZo$;+eCCFRbs zQLs)F;WZ8eS?GahOJC;%>3%OX!`&YCwje!bl2vJvzCG53FI=`(b#Ct6;6?tzWBaMy zfuFZlCRA0Og@5>YM^*Fc*or!ZTkxx@9@>W`tW|XqSFmOL!U~Ui#^IQ{h}+{Ucw!v> zshcwyhhqzcsF(2zKA!O_B$1?oJXM|c4mljdPK4seH45UNz%c!yDlf?+{=hu)t8W3n zkLU*f_~q-gCRV~Xg8H|;7l1#CBrnn7Yt}`eRQ&dm>BR4w*>pIZ^T>Y{S&RHj27cHi zIw^_zr_vER?#Ut^bYv0#_(b8n>WLzq^!bvjz(l5of?LT*aA6~+s&q7#n#jh2qWW8I?=a#cClvfY9=HIUT-BHppAz`Roi>2{Q!_gdU(#>j@pyy8#6wl!SEG&@ zhvN$Im*Dr9@xP1o{1Tk<>_qw(wwK)C3%_@GZ#=FV`oKP-ldGetiGDpFouS1zVOHNc z;QSQPDV1XWh3&{{5@PF}M7-7GIW~ZFFXoRguo+nkC8KMBIQVmupXc#ea*u4pQVZKr z_k=SX9N8dus5>Kj^U=*yy(}L+GyM+7ju-jM^3i1f7||)8xGuo3;#qPm@2!sz0VvkT zJj(CCj`GpzrhIn}5G#cWL3y$$FNOSD)#H~ydLB#76A!r|zsAv*v`$8R3%O8m5Mn~{ z<#a!U5n4xq-*wJCM5N{xsFMhv?A%R5iR97{?lo`4+Hd)gYeV!Y48Ue?4lP{ILY( z;Giem)hn0!r!TnyzSxOP%%vk(+mj@pLnB`JnsxJ|luz_Xr5Hzj7rcn~fcT89I8v(uRUW|kxudQ&uuRS1DSAeaE;Ce2!HkZyG--|qK10U zMS@A^Y-+-}dkY*z-}_thlh&&c7j_?oA3x%06yyVvDEKbj=D!& z(NrK=Dj!98mH~5qu|F^v?Vn3V7DjTR#kqLM2RKNR`YXXE$FyWF+?UCA>wlK(PH*?Q Z&JUe;!yl^P&zOh!PncwaFa!U;{l98l^Vk3Y literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..18e63d0befb8cc3a099da6ec202e1a130c021514 GIT binary patch literal 8053 zcmc&(U1%Fu7M_vi-#D@3_|9rlCF``#M4@r?2umM=Da$^ZQc5UG3Hy|%C4@ZWVOdJaLnvk0hkaN=S@xWBXQYuV z*@|IV(MWUeIrp6J{N6L}lse;Oc6Nz*CYg;19fBau!k4fV&BYQu-X5RF*Vp6m_IR!^ zfwi;FOTgUD8d!rMG&ya+XtxP=*!zHeXbL!!mSI1(o=PQiJ#Q;#Ph^QnkcUX-_go+- zmx;z!lWQv$Ct!dc3YbUJ?W!pzL6h1!Z#=M=93XK6M;)B? zUiQGbw@c5pO)Zm+=924cR*z!>9Uy0r4V%7j>Mfu-w~<-HGU@S_tPab=S0cMzh<)aV zevmY>UPRQD)3$S6%j=o0LsIkaLkFa$5%bgRtm&=eo@USlj@B~KoEEQUYaMyHYOWj* zAKnc9SDAg5y=zH~wn0l`H1K=_TP^u^++*dalRIp(qho(Zs<_E)vLsv_fCeYfIk{>> zlO5G*}vGX|+_ z1gD(P_vC*JBA=^Ix>+ir0eeJY3k2A4jxZfjmx*p!^)+NEI7M~Gl7{Q9Mblu1Z4w4*w4-@d z%ir9L>KkY(U=SR2uv0u&&uNEPzAqf)TsuZhknFNk{-r|^oUB#u7aH1SSH;}gpHs{x z*`*!nj;PEkP0j(qRpEZGbmyNuDad?MfIPsF&n65@A%5Qr(huG2wDj^6OACiWDotd9 zb5E2$7nN6{a$iJ5)u)3428W9H;wHW~-OMRH?PSnb4h3kbCqj3}CVej|kLr{^*brUK zV@AybiMJy5N1_S-!FhI8dcoM!8o=|aC*1!KrLRThxm|f#hq!B*Uuqe0Dx>|Zfc6_i z`{l3LHyo|~p!BqKB!c`~X<7NMUb$cIEY3^CCrNM>rl{g>CFczd;Fuvu&(AYSdgfpu zYt#oTR!Vs{gf2*szOqTLA^MHVKkLzpwcN5LqgPGCTFJd!$sp;8c#%&7g7g|4{7_~; zm83(cCaa3=oZ2{9_&X_~+;37|Hllre&M@Y}iby&y&Ow(W_uf^Ys_wkQu9ABXj&$*o zAf;((Te;n=JZXYRn**k!Fm+<2_wQDMV@7&cfk^-59V60$d|qg%(h`fe69Mr^eahqJ zVK7}3V4OuAPUvrNhYQ1|@W&|pzLQnl-Ia>JJ0tfY$f^4(VD=`mDReU`^wUS*W82&~Y{?_L04e?C**%S(_M7rO<~-f`t+ zD5N^}`SdKCvN~h1L?w+5I5LpZ$2RGusN8E)ULQxLM|)W~+B-&N zQ(hDs$oChpf&7~Nr=)>Yu*Mj8$e@IjR^GHJPukF&;){yqouy*bx(jXVZ|K4Y=UBMJ zh1LWT6LM zEPb65q_17j40pTPhl2E!Np_`4dUmQ4U$`8t>def?kq`I_kK^YyH-6q<8B-LrW1c~YSZoZ%p(5{WG(V9nD}9n=%geTnn*9vaeo%^ zpd*X;N5=~1ReucWq%Wwi0u!0K7}?gBB6AyYRi&fx)L1qiA=T5Wf0{c{9W+9)!q>15 zp8_8Lv}wPpev0_FkmS!S&abK?$QsP)+T4(DA+qgVq@!7+rx$~uH>i8G8{R}r4Gbba zazerXqv0#S$yJSb@F|I3+-ZHtKQXlf@g@Bx9)BQ0OgvNtel_Nva=Wh+e+hoS75~Rb z-&%lE{+*@Xx%`3;d=Ux^?vKX}QyDwwhBuei7WLR#I063L+nF2qUzP0)J4y zH8Y>sTAZIq1Me-$*D11(Vp*uCY413uAtjl*0zc3n0G5sw5~TjKj%(DcQjmDRQm3G_ z^E;5b<^HtPx1Z?H|5P00#+%+nf`e#tz z0AK9H$7a$?SliHb?wz8RgXN7MWp3lXm#>-8t7<^UzyQ$7C!eW=gU&!$io^~1nzyuSeB^f=iMfdiayuWa%7a3`Q7%29s*geErQUaGJ3k`K}{1U+J3{4DhlBsxKl zrG|R1M^GyG=RkW#ZXliF6`IK}M8cV9q<@Xhh6#W5`nyK-FrtQf&n-oC&va_cv*1fi zP`$xT2Vek1pBmWWhJoE=Y7h?<`|TLFnl|APt^j;A>SLkTzZ~-od1I-tUMe3&`IiCn z)_iZcKh`^=FU<|*7UyRYi$TCan$+J24mqyrxoA%&+hzP&t}C7I@m?Od?1Mj4!JjdY P@SiZw1YrvPhx>m4xUlty literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d9cb6225602db7228b5fcb60adf43c660ae15a7c GIT binary patch literal 8053 zcmc&(TWA|+7M_viYn<9~{LgAqCF``##G!F?@g;WgmW(W0vg@d}<5;#Ro0X-prFdj( zCHeXk@(@DkLn%uLr4PZBWgkr`C6uLveG2Tdgph|kEK4bQ2&F9hun$Wp%bs)o8EJI0 z6~nTkk>>x;?K_wM{Ab*0b=Je|>>_hdF&h&)1VNgEFJUQ~izRx!yQbzf!z zYiFGofw`SEum(YBa@v5=ZWHXV_dff;RB$>i!+vZ%l}hG%-&D`O$P!ba50NbF_kf~Y zCK_8!uB}+0!6ADpvQI@5Oya2<1U9l-<}fr@wC!jrt!tMc!tVS1 zg4=7|spfwj_a9+U!0VX~tZk>VytxfzF2Hj-0M zZA^CXUk+&=$R@qXY+C?4-?>T>EDEwi6iVG`G}@BvFrSD+R>1NyS};x2K@XB0_+6?3 zs-tkSi+T-JP8C|ImX<=NksXCaS&jyd)JJ5;k>m{PmP|C~n~UryJacC(Y!wnlAuXJ@ zT8hGBz4?w>NAe)tXYwZH(oCi(LM@0q(NA=(&yMo3c4nLm_wnj71SZ-uv z2{>%5Nf@v>I&VZ{=^NRdY)(tvNNKrfJerHLc7e5?vpG%>ZG9sX(|kBgPQ=d%Q%+Qe z)Z@IJR-m+)EwWgk9*h@u?4Y6rcIM#@hZ`Khw52o2R5Y_Q5r;V4%=$H5gZmQAoS=q~jE!xlwMg zk((m-mUXjOLj(4R{1zy%;~ZfIqAnBNvf69NQgRCBjwOxIU5%!}58D(B)aXa^s9(lMDNW8np{K(AUg^$1aYB&!qyTk*qoYk2k%Iib7o_jI*(vG8NtPB4g;ZR~1m~_O zeJUz1MdhA|h^kKq2Mi7s@x@Jiue+I3deX_DuN(@{a-Im?ZJYF+s64DwKDQydn#YWq z2NF+3><>i~{QYz6jP#tbCpCcQr=D>CLzKP}m1lP4MIGXlRVn55$zYh zU|(~z_JhjP(vb-Aucc+>+j`|*y|XYc6`mxaCqG3Mb}I#Mco4@7L3(zMNzzjX3tF>2 zRPm&gc0=fb1nEng^a`@ysQj}Yqgcx=TQqvrG^`cf%at6G9*Y8^TIrIISTI`1y0rNx7ig6@4=ZaToPn7 zEp02enw7^*kZE(kbmXT_%yju~B{*iL_Y}zVpWZeyEy(ADhAJ(wa61tYx74pZY90aC zMFGZHwBdyQ26wnHVk&=x%I`VZMQa%?lwWMdl}6W?d4ZW1SKjNr7UidA47E8Q=Lh9V z{z=ZF$yt#8iZTDj$!4usddjhwmm}HzSc)s39aCPmpj{lfY3{CQuhr~2Kcb<#r&)iI zp`shM7~vFfut_VOyDd_`dSUVi610Q=+V(q3MK%(&Psi1v>wFIo|i z|uxV>BhKfwm=zt>w8GU4vUWm%wHs#fEGS=RCeWg zp@DpN9vjFn*?)=}NCj_TTCJfs72b&i>ECGZLhN#ud4cJAC|CI)d^g|miY@S6!ndPn7V-5V=E{z2LIH}>5R*@jY`z3_yr%| z*fr8fT0x(xPWcC2t|2!<3E~-xV1=(?A3k|J zz8TYhRsAjE-$Ifv6IobQN0Bv<)3y0w?_zk{6Cu$o($kRu*bV4z?YbusQ~g7TkAjeo z|5)fUaB@?lZhT7O7Y|xL@=s3hLVn4!(zg~t z%D21JH@~y!g;)grL*@CnZrTI?h)$u7rN;Y=dUQt?5`3*!AKxlJmEuzQPLJ5fHCYR^=S@aHX#8dM-G4HrL8Xn%DaHzY(<@M<0 zrCw5xzUe-fYuAtbCG}|Wf0XD{Puvz@Sn(~omdnpaga8!I#~kYKyNUWS=$3kS_Y*6n z3PJg@s4s>5+tu@zMEVvUHD4m)h58!9SkhWw3e4vs;Q`1A<(JcgkVa@7dH#TYb9N!I z6ON=f4VN}YzIy|W9Y zTN+Qx`SuYV`k$&s7eZrrM&p5bJhN&T<+t&PoWNh7nxJ?r!f|lX6Yd%(m-=Tw-+)-` z#>Z#VOL(>?NI!>0yzn*S<)Tzi^hifBf%Y!@5$_@S8C`W@>mqz4Z=R2$^Ruk&**ieQ@(<;JB#5^CK?`CBiRt)uYUfn5E zUuYoKH>)qr59cBavx!ImaF8YSSAs*1YkDr)o5^+=f0pY?@AP^u4PNrXAFANbm`C_e Mm|%i14gbUazf0KoVE_OC literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9c2f12f2a3ac152c71f92e4e4bb10595098a5ec5 GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA309#IKH!*RLMH6GjV7f{rD$#@s^A%Te9n@w&Pf~D4Ug~v88xq zYbE*jDdZu9(1%i%5K13{Da$^ZQc5UG3Hy|%C4}sfhh-@x522K0ANFAhW!ZDiosmYe zWGjYcl}4I-&);``?zv~uDRsun?Cc`*Ofnl2+66(Hg)d<-mX9ZUd_6voudm1B?eSb@ z0&8O(7lFBrHLwOjXmZ(r(QXs$u=hUuz*KNLBg1}tEuBu~d)`#fp2(7upbwGE?{$Ns zd^Q$eNv$qhpcJ$&rF0EIX4f`*yxqUQC3@?;1iknsH{zGg`n2yeRu&AYDDZLs_JEgXhb(tvJhW~PB`5QaP{nhxbX3vK91ojqd z$6N@!Vj7)B#lV)tPMJuKbvJ^>*s`{o@4*DRSSJb@W%gDd2?_LFPl!OPDck6*16l7S z56Hb)d9H3~*<382T3fY79PJ;VXOIn>v2f|lU^%~@1$p@6Em<8_hc88TtC0Kb_x+%0 zWUYv(Gp}vsJD1k7orjd>--iw;O(W;0*cmfg$2|_P1cBDFvAmY3=4%~AxoWK(kRRR* z{#TWKhP`7cjMhO*VKnf1179umcHCp-sFT}mva@}EN2<8VY_cR=7=Q*Rkepn#p~=qb zv~n%owRSVvSu3lr+{U{#v^3dS1J^#yO=e@0orTA8>z+65VY0Ipk1NSS};x2K@XCh_+737 zs-t+aOL`4eP8C|YmgZuok)6fGz8v)*sgKCcBgq-oEtzl>nv3i#K67U*Y!wnlAuXP_ zT8hGJz4?w>NAdvNXYvN+(oCi(LM@0q(`96rG|L7gkdQ)x?2@37^Hl;@59`<<(H=rI zxuj=yWBCa!zpW=;Vr(=hkBM@x$jQ=OxZBM9q}zq7&yE=qEktXv zqS!lbDDOtv-Bx4uoJ_rLHCVfl_(Pkalng&_X8|(7Q9U@xuAp&*6)4SSi!2qW2jj&ZJE&-eojJI};RZ)At(j~p9m{TyCt!|9<+qugTe780!@%TN zjit3*CKlJ2jg#&1wS;COBegJVYPtMuDy_i)2X-8(TtL$`xG&Mn32F$b_~=$f%citd z&}7H`2HMZT5Zy323|{uzopwEy%Yz-DHiCjQ;4-c9W-=Zy4P!_eaUcO_j6INQv33p^ zxhZmQSvN~HG+>V?Y=Hth&JkuH>awv-tG$LCC8ucaSjq_9)mR4nuuZ~1jea;*wCx)k zF?}6f1q_0-4t9!i^_+Hy=ljw@!L?)71j#PDNurO7oQbXU0FE8Y1gP6#re6rc`p}!tJeo%Q@Iub$twX~#sTd&-!cNOQQ;*%tF7pAD`6 zP)%MH+c~vy^6)oOQn}Zpyl6!K_?%(Pg%y!>T$qC{N8!Drz^S_ZHoHRMJvh_FOM;AM zq%Gx^LwVc;nKlPZXJP8ZOz+>V1jo$ut^%3<)7wU-1^JxNP^BdnZzlrck@}QJj$v?J z6kwc18!qT?aEA-Srt(Lq{GN+lw3gBQ{)^4H(&!p9FEI1sDtNuyto-D_P@D5{VNkB* zpX4f;oCWF681rvjY{rVEdp{QQawNMSO9|z(W6H~Bw2LD*&D|C4wU}M!M>KTzH0vud zRC2?X-K=Da_wwBq<;gKLR(+PoUS4ICuL!Kr%kP{AV1GPa-pfmn85g?+(cW?8MGGRb zyb4%e3o3WDZZjCWX!FS#Hf0UQV2Mc@9dKkIqmOLT3sJe-s=PXmMvwNgaI|-f%C0;w zHjwYmV*~jm`%g&&so;$<@Q_1EDWkk@RUWsZJH;0j%R5WOtaTOI*55FM_s_C$NeHbY zRB>#=afz(zQeOF?P1$Qht2Okd;yaNb{TmIwJj()BgUQMU`9xN7aLo%~&nEp#RGzmh zU!Fh^|Gx{M?7d5ne%B3o{KGr!HP0h&dbK-9lO#oXSh}J-I5`a7i6XqlVIm7X@IvY9 zoFIMG4b5=3i+vzSPncv^nxv;EJMe|e*`m(Od=z<~zwkJJYIWo1z2z}gRj1)Eem+#y zO?6~hoxm;lRaHOj!xGl2I*u#YGJj!(qW)13Q|EDebQvW^;jg+em36zfP>FgKzu@B^ zy+#^IE9g_z$-scyJ?KFwAzY&({s|A#FPaLFKH?A0BER|`@CS)*@{eA5!Kn?m+cS&&*O9fzzhL5rO`?;LcxWQCNaFq+;=w=` z@sExb0~dcuK*Ld8jWn}i;=nYgsPHgB0ZK%M9B2C>YwI8R0oX|Eb}$&!>549 zKW+N2s=r12n@I9!qw_232(kwAx;8iDTZn9Vqa>O`dL|kKyFuNfUH2yAYG4rYQ4k98 z9}QmyPHt+%gHI{^;z8>}{)wp_$S>J9@%RG~V&bVP@T+n6l-qrc_)GBnt@uAe`sM;i z`F9q3=e8Go5Q|V?aDP6ooA$szqEo2j>9Jm;9zD_dBwCvFQcEc+MSOZ(49lmHaZ$1Ljay@~oU=$3l-^bsqi z3PJgEs4tEDTh;TILi#2iHGeYdgZdi9SkhWw49?}Fk$%Vt<(Jb#kVa@71^%FZb7nrd z8J(ZV0Pjtz*D3Oka#=V})826`Ln<mqz4Z=R2m^RVcHb2v^js`#z z$ItrF}=I^|bHKQEKd&)c^-=l4y~dA~5{=hQJ}|MR@(oa7`q=>>7y z|9?bD&w0=LKDYOIp7(j5_dRb`IK5`AQoHGV?I&4v+SytrpJ9$o``S8&VZQ3DT=7-6 zb7j>M&cs;^CbPk0S)|e~b~#s8)-H1|3HXPC9R|)&JbQY-e_+U9BIJajj4QL5Z53ui zs4Q$XSjs|HL%?JY8_LSenF_4Kzq;}Fq>=!GspQ2 zmhwP_p(1Q4HTh|C*m1HcvLP{Q< zOSlZA21GC%(#;{=6~Qpq&IL>ce|gYmu!Jk1Nx&R5gvx{EW;<8mw_7+A3V^?n3!AK* zHDs_`tw6Tb2J+*~R)Z;A)^4q+Ft=MRcI<)OmnyxVFiVC20Z2V$DdS9m3ai0vY6tqQ z7C;aTwHw;)evpAJU}-Nm8BTqI2zItxEL?@H3?vaYbI`~Bu%RLVLM#ilTgt3url1vi zZ~qn0pGdzM!P{)*<>mfR0Kf-9pzUQLgTJDz0+h{aDKEE${GoEI(5ddQH5jm%`~gFI zz;6LPGKV(50{w4qH`&UpwsLd1#ed*^LTLxVUt{&RTSK7`5M2R+w3y3+2D>F-GgxgP z1aqi8)E)rA1g>DNu~l%;<9^N%C-B zHcgv0)25uiPS%iUVWKHX42?G_Fcg#FOs%3o4zqcY3}&+-irKtKig{w`_~q;|FAw}j zzFP_3Gl?QgsmkS6j@)?UzfXg!acGy;!5?%q#r+vlKa(Zi2x0+Qa#~xd)*X{&MpTvs zSyDbI!oshm0R+u~0|&OTD!8g(6*p&+3}5TM9{|Fq723U|PFp}Rzez%wP$SB<1*l;} zht{A2F2V;gRjM0KgeL`Rv5rnTU#67Ol)G5$H-(B}l}yQYK);D- zFRg=qz;3%}_Wj>F@6SIb408`q@C*2N5dPf?|MmejKN6m86aFSsBepJ3i*=Bib^i?jPS+M7dki(P0z%+_073uKMwxR(7XJ1(ApYjRvDzb*;_&wL1LU5+fPQ?v zq<6T-8xHmjga%6-Txm&9XRps684knU*GqzZy(8g)LH|%^UvG&cm&sb@hDvjox|pe0 z3|vWR$xwLR5UGj`_6-jN!zGTAHR0ayfPW|)f@cH%wNO%7{557aG3Cx~?;Dt1T-zb) zMSEu?4CCiO-&%vYWCK7N9>~~CGP0nI%a~F3Re!MmtApzYhr&Hy?FkS0L;fK@`DQ_| zucxPT$VKXv%R;JOl3ZKs~6E@V21C2M#vJW#wqyTtfI zX5~fUb^f0INZ1GS9&fYbuOY^dezue|$rgsjy!;$cuo9Z0x{>Wgsv;>}p2ZvU6 z_JBA@u)t(CSvbzDI+EoBfmUg!n*oJ?&79Kt!K!ur;epO@Z!kQ^>a?p$vP#TGlj=|E z9{;-b)$E+pr2O?l);V-R2^V3N8%nwT?ABkhuGK8V5qH~a)*Lw4N5}!+WiZaoyG&KL zbp}gnPQSqJegSG6eu3S48(TfZDhK+WUCqwy9gajQ`-Xd~Dw*Jqe9lW6S2q?jrtRNg zOg~!2n5y+GIe+8@mXx#=usts_ra+^TF?nah*XE)nGvM-+f-%{Cq)?hlxvN>#6PsWD z-U9N$l9nvfh7B9Gex@4vl<9nqDGe~B_~Gr#ja$Gc1ZI$t4i#g5Tyb~@>+Mn~&1Rd0 zSe=0ds_d^H{I;rINz~h#nEn5*aQvsT;Z^ozPx@;;d4F_7n1KsbbBdQ<2yI{94SbJ$ zrZOH@Yz;7{o7GF3)fw;Xd%gN7qnh!NMzQ~QWZ&t(tX56ilAV_K08{E2Qusav?!Bv& zrJ5`RWNOhUYb6484_p6L=D64N# zZ9k6`>iaVutQ`r0D2($I{{J!NGJY%h&!YR(MAM_sUy%V4_}ypBhSQ&^?*GjAO@`dS z4Kp{rr~z+SsbNYAfA*t?4{lZF%*Zfjs4|oW^Tyd(p4-{34>h(s7;Y!){dM|HaNW!> zB{|otxkuCMA1-D%){}(<^kJgd-8iAyZlu-ARpr| zh;gT~(VC;y zW&0(=-NA1Bt?J|zitrUcSxw&3qb$hXxfj?m=8!p|#`{zmIU|}3T~V6)a&pq9+GjKW zX6(6urLq)h3UvIbR`aV>wH5joaBre*4TV`tXc!5%-41R;}24 z(MKS}d8GckOj&-84oJEE35NT@KBjErQ<;uZ#xO$E#Z~{|cjOi{{3XYDCE5B;UZ929 z^#tp?kU97SyURVp_MfCG^Y#<-{||0j-|c*FgdtWIp;@RKhJ*U1WW#HZ&2a*n5}+Z<$F}YaH^xep>&TeefQsPjqPl! zO>^keiC6dPmohR-{*={vRP*G*G~?6c!D$OV`;y~$+Ubw;dOyzlkDc=>aw2)$6`AvV zNA@Un)6^&TD7Wv)H_qC9h!~FjCj8**`A^S!@P1<_A%$7s9m{r97dTXk!rwW5rl?;4 zQdzIPHotvV;gV;6t<1jgz>R6Ww`Mz@Q7-LP=KMH=`#7)dDzenhcvt5u%eb7=nQwo@ z+>THk8v0!@dzOmd!JJ|nnBxF&2(P3>K3kv_54<4Ojm7w8aOS!IJ}|ycM5XP z$zk=fD@pYo>lWOQ zZ8JUdiKqU&Sp&u1rrEw>hPzXx8!OtfB(3m}V@cZXtxC_2Ga8SoGW@4I!|%P?`1k@# z#&gRRc}E=J<1)A+h4E)(uUGlphcnt%D~xZl&fffu>YUqVK6xp-c~8%`!6si~xEbmT zW@j6RJx6X=d2VBKmSj3!(R;6B?`}-1UZ;3_Gh5%i?}8)EYP-#IE9>blgdsGrbejt0 zvX_jVcAU~|J&cy{;Q8o@{y6rtYA#K6nyc4z{a|{1D)%G;7K60GL2Txou6> zj(08yT%php%qp6fVcfv@^qQmBsTi(jwCFc8baQfC*Qt!HWP2LJ?f6EXuZ2{w- zo@ogMj7<9q#e(#_&5H&WrE{gs>!&pJKY#qo43{&g$-k`7cFT0`E!Ox7Lkx#sQv|f6 z;WdSSK8^)@6>ZjLq5JFvjHcYYr|Tt{(DQdYf6X-^Y@heZK*JzU}Fv(>qzu zPIk*swpY!P$9BD$20n&co8k54&3YoEYBl%a4CBA6#{R3(YW^7fec+E8z1!0JyykvF z8op)m4y%k$5Z_-_oh8`&uqwxcwdH z`*&V;*L!z+_9{F(*vq@Twrt<+gnIzm!Em&6uLAlZXF~>dP!l1KUH;VTbu6j&)v?t_ z*#Rqi@F`-PPOnz4nn+wjZb; z{E2Z#_R^hMBloIE-`0070Iu)&tgVN%_pl4@neN;*@Gx8Shx3-19%LFS75S(0c3-D* zH8PHOS%-$r&h7C!jxd+$+Ur$w{_Obew9B8eT?;cXl~>Q+ah3B=#zm^CQezH_7QdM} z{oxGVFQzr#1+JdU={SEWcs(my?Ij0m*x|#zJC!$WBew4=N4AmI{sfjo1AQ^@ zN3yvl4+rL}ax-+s$CxAksx10zUhBUqw|vNKygkjKN;8(Q+U#n7M*Udfp860B`9|jL z_mrNyG+SR(X5Vt4BhA|bc6a#1`Oy7;D)xL!QDi7}tV{psC+WuZidk1_@|}yG{D^G+ z{{D|hw(;aGX^-4;!SRE)r0p#MqgQOc8i+XliGqA^@)JepC#3p%h(cdZ8=Gl-nrS$} z+~1csFm`-aY5sR!AQ!iFr5XJ!S#0~CMaFja*^adOzI<&KXHj2&OV;%t&KRj78|zgA zKUOSwQ#bZx`-ckf)YS%+aSiGD6=Tl-c!sA~b@bU>#$5C#P0{^%o&7e4D(B9k1kc!=fxz#1DF^7Y1JAf6wbm!UhjJ3a7Q=5mwqhv#ix z4`bLz;D+lIjCuE#1?1y{H?aG!12(H~WKVyCZTkjm`=iRYCmk?nEqwAuHh*^3(qa|F zc;tT?jh^?(?t$6fO4hZ7>FP^!{*v}1hAM~KPB8oG9JQL>i>zJX$FCr&rltR?+I2ol z4(2?Yp;B`j@{J0`<83+L$Qk?8=AY$hRY7OA%_CJgW8c533^ZimR$}DX*J?JN(m0A) z{a^Du{VZsj_4~$Ll}E4LT?K9-XF5Jj+i|_p{*@!|DvjGS zjg{=5FY(^%-1R*hC@r|Cjm>J~Y8;lIAAcgQD2_nCKI;8Mu@pShjZBU%Bk$E^1p06S0TILtG?v#g;ot!tsu^BZ#O8?!cQTGBHLR?V&c zp~iEJ?EBbjRp&cw#s#eEUhYN$(Rew*F8+W$n4|pQZDmnMp6|6qpjYd3JJzs;HJ-ak zQBdRfwqnTxJ&(5TBdzIK+q<(z4@6YqcKyxjvBJiqr)$#xT$8=4^V!14z_4o0RhKV> z5ahAK-9zc%I*sqCPQS`@y~^yn*BD`rzsjiGr!@+{#&{R77N59vY`KaB zALnH}%#@Jx@<-JBz6EnrWC!*Ibh~<7D z|M(ZpR5Bf3$23GQCOhZ_<{}_FQ5u&Ip--g=l2}dzGKo+bb0U~T5%M3BRLX38L=z#g zOqfA7AyQf*lTyL#gj`+(kqrr%iDeKOw?RHrGj}N%pip@ z9~4Wg(0mR}6S;8dgwiR}6q5#s5ArYcA!deR4^oLqNC-ti^7ql&F#tSdOv2EdmN`CC zgY{rC0_!IDfaY{CiNz;%XvPK;OOlig@`x!i3JeoBGwY`kwp3`d->YlUD`geDbOQ9V#_oWIWmKd62O@=-gUksvo*L)U~`jKtvKI5zB-~eO;SD~rloE&en5jFadZ&P4~l5yN=JuO zdLiv=8SfkIEO;j4lMuz{9x`G<8i12VJ?Q)CLg;z5xDY-Q7Q&ghMR-Qb*1=Q(S99Sz ze!18-7YcJ1v+zBaq0Nb61yqAI`}kHRTD@eVfvxf`;#=XOHZ1((U2-#F@mAO$fz2}K zj)&%4{m&_%TLE_!8#E4t+)kpK{x?D*4+!@?ZyV4mEAMG;u%vpvmRzbCGA z4>q9{uYa1!vt;ji(!zM2Ml*>M59Cu!Q>CgPp#|&&q+2KBV>KxsQqiU@aC{@u11^gchGuuR^o~}4HjUl|$_Pl=B{@Z8)xU7@N?FZF`u!uVfGJ z{t8nd%mQSD{pCr}QV?HBUc6F~Qr{raRwcLviHn4dP4tr}(E_#jBzA?Skz6X3HAP0K ztT$=2*Tz&_b!a}I5yOay1U`R9DXR81^pFRdne5U5*g{cu;1B~#>DM6O@oRvb9QaoCwh2bfoW24xJl*M}PNDzpD*(sHfSkH*Uf{B_< ztUJlR5L}+9JJ}}OO&Z;PC!;|b(T4@GOQ|+d_=MVOzt&Z!RnwrU!h(B6acazG!;5oZ zdno+lW-_^SiVUdYHe7L9Vf<7I7r(;z=~=jNWBk+r7jI+ybOJ6e#rUZm;wM&!pFT+$ zKZ!UB)M6bRB$7+1F8sser?{E^KTZ4;N7ba_r??hU(I)$hvmHO3lEhEfOd@_Fv%ZMv z35To1P&Ga$wW__RG|M4Mo-)q6Ftd^TqY(;A$OUYvLoOG!ao;133JBJ5j}BV zRP-dLArpl`Db!`6W)tg9vM2lkqbGPHFXO^pwuuLnlCm>s za@IKi|Bn7IPBQ-}$Sbrf#q8buJ*ZF><}%LKULH0CD&XV>OS|1-s4&^WhH^NOp~A%Z zP2oTov{vvf!V+@H{JE29S1_qs)PHE^D}k6o@FU7WGd~?r z!#ntRDuxWJ>B6G|@nSLr#|PLbXzBWXDzYcp>Z3W@t!k0{zvjNbbS;F`u$^#yKW={= z92yu74#94e_ReqwU$kBgyOH6|V|XOHv!b*l!@k4Gs4g#&hed+l_7{qFcQA`o zB(Gnj(lQ-EPo*>{DT_u&L3UGD8uW<#edWh*!Y1Q4~^~+vpS(H11ufJz#wT4%+6(_B#vD>xx9IQ}!YZR*A zGL>)p6|OqBoI< zj`u3P0P$>*nY@Qil-Xp2Oj{2|8H_eTueKhFMYl$cFZy(xv48@E?WrOWQ|LIUtK`Ro ziFz~EStG*x3ul0K+>VH00O}A4e~yTVW-=0yHk`U(Vgv|VdPUTe6s)8@?%NexPlj;Z zIw{tlGB#lg<`mmcXr2>Ri1FJg9H0W@W0Z|3+dZ@phRc-+!wMZa`NC-BGd70L<22D( zN9it^g5QueBg$5#Bo>*ffkgv`7)=m2plh6@{YX!FiouwyTQ4{lc}PZ)$w@6{3g;q^ zsK!N&dYm!rT;!23F_T76F|9fmdEn&tT;w5{!OJTOeJw+b1SVsE=OPbbwtFt}fb5cZ zd~MW+m;5~=581Tbl+M1Sn|bu&*_S2r$36RUdL3pgXF~!K7Z)I-m&p(X3rhuY%a%>C zGZ_1B{U~Ehk^a$S)ReZUdR|FGW+F804SIOISA=Kf#M+$Yrz7tkpkii6MdIvqr|$md?%^+N4FTDM3Di<+w>SDg5+i zVr2@_2Ep8W{MsDG$`&a6p^&x+gftJdv)@0|F^{R62}%m}rojpZoKOnKM(_tkfWfeJ z@LB_7RTgBx)cUJ*(y0Z5!o~_H5!%8%TcDy(#S$fgAM6|rgZ05>ZcksRv%ND&vn_Z4 zDHseCXM*GRS9bW}6?+50z-#{+H)HKNuN?Gi5O?x?-Sj1kxP? zF+pTe!F&~ZXfPZ?9uR(J?3`Te+KlKqx!N?mpDwqKqs`^9Wwc>|0>L>HW zApl1naHKu`GQ$u0vgu&r;oeX28Ps z_%Vr*3=;gHfT}NbshUuDFgVcJkE6NR`w-s1>Fqs!!8OBX#r}lBlPO5=qfMq1NH7Tj z4JuX^A4tJ02~0XhKyFl(EBhkDJ-vu0t!^fz=LJ%MOOYkn1x0)xtJ}lsA1>4%F95{JKQn@F zG$fZo{$zZ_OyRsVhPM&P{E}j9)wStgFIt@R!z6_>L2bZ8GK=F_keVAmGo?_r9#ZIj zGNixxtNMq(guOG-S#Hu9h5UJb=g#rOUojJp+)QHhdr!x++W13^QdsS|b39SLQ!%Clu_hd8IIaPo zJI6EWXVa!?3TLAG=ry8?-2b%-$@A{g@Lk|2zJ7{ zTNM1U>xq8v^^*h7^7C6c(uK9VDA=2CAo?vg$b*GBpj6c3K$R{6MnP}-F44b|8ni?l z!W3DKUg=_D6!Hmx{Lpt(5}gd0CW>OKkWoO#z6V16-c$jlTE2{ey?GZ1b=PFTN@vzG z>z+}Fo4yaSOaW2HP)my{msFZ@%cYWKQVRGXqB~CX9{}iwzE2?YfO|9W6T8C9Lvm?~ z%mZdvW%rSxW{?ij1zznkc<{Q-zoJ`p5;-fFTFIi>`8 zwc~D@&rNRpG8ZNFpi1{HtN&;Z(LeD+O6r*&8J0}=&1p%53w=FhIcLU3eoFPD3f+gS ze#^~}>bnV5A^9*>RE6Zy6siy}T|N;tWg274gOtQJT|j@^7=ff!!cnbpNvn9wf_ZgJ zRSgqY?_mv+(r|Q|?$=8Fv7ZqAj=i*V6An(5JsHO=JvenzM}@_;kQ1?PQx>Sg2-V?4 zC0_S=nT_pApewAD@#~2WjPJWY1uwLpGM;#HtBkf1AaN^CVj_)G)&+inmy+_fMt7}J zzxx2ufBI8O$|TMZGf(CWF+2t{nT-?bvuAX%=$mr#Fws2$26r3LUvnE89YL1_pV$=| zU2>@$oiJ9*s{WKEEmW?D}^ph)J)9L z#iteIG+lRdHuQ?T?;Fq2Co?H*(x7}EAUaTrJO2;xd4%#AJAXC)C{`Jn<4i}Mp7As_ zj-E7riwGqJ)}c-i{kHpw{?7JCq4v163fe^e|sF5xL+u}5HN_%|FDk`HFZVIjFx&RCL$h19XwxHQVeVvnA+ zSZrL>6U1VVLoD_f?cB-4VsVGCPAV3QYau7%sm5YcVs@NZ3_^!j{+%*@_G7VeX^fA> zu2JfDJq5AY6O@!m#A1^^&o$Y(O)VB1mx)Z-SnSSci2llF{+_Ye_(aCiymVyXW3f+v zP4rJcOT|6eSd5!=EQV)TCJTOQv6#ryq++q#eoOSHpQGHI#aK)vA#RXO9gE%hI~dKL zr$qdtVli2YSUZAv26jaH^V%(V>b1$zLQ;fw@GnpKV9fBa}qn9Y3Usx9_9D21S72Yz%aoK@ES7lkYqHhI#A2!QHPKzyQ;Eg+8RXr6Ao{I;K(EEmA8`gL zUSX&pUdKx1j65k+I8$0>0x_P>Ew2&%v#muZQR6FTstAoI_Vu@j{_eNV0WI|BIOL79e?EqybJPIoTV zeNBJxFPLRWIGZ_c#iv@b>?mELPVr(@YzI5VaSzcQ10Q+{vIwVuQ^`5a5IITiQbbHr z4KP1RR;&V7L3Vrva(_ zP7Bz>J{KcLT+S5+1QAa1dt{Us1bXG)z>lT?3K1qf-I<3eY)=e6V{zeuf zadDf{qcp|0mwjW6QiYIC$v~2jBJ^CXK+A-$Ma2<&tgx*v+2+oEK8S z;rAo(BuVxHh|*2ir6as)B8c>xAkuHjlLufY5NQJBL^)mwa=da}*l}_c%~?>2iM9Zt zRBpNoqIEqAo{KHw1Z|;OEh_Wz)5?7 z5c+Lbv-%xZr-UxCMZwin$WyJ6oGlPl>aEv+2b~IVQK=Jl1RL0yUo+g&P5*QB9BQ2x8gg2Q}h#47|5gnMAQdNPLJbWQlBD)v5}PDp&CU!Cjx5%nGB)82%PF>>us+p4a)4V>QcSPgqtKmX_1j*7WZcV?`LlPX zb`q_+Ye=H7gf#2K1VAj^}Ak!ev+KSopdU95>n`wSkhJ&U)&jY5_~Vd&g$=e9r0nWrZ}~Ge0))< zibw7x9G{nH1N>XxAHD$=@CHa!el3zN{P4L^Y5KS}D#V17_T-VAcoTB_Z%!4-d_0Sf zKZB1-3Um2Syv6DdzLgS^+|nsRiF*|Z;p~U^fUbF49uD>*U38Hyz##$M1^5C0yzd=((r7k6MqCPr@%A4Dd?&mfyX)Oa0Tz~c6DSqW zCZrH?b@%;u*>Tp1Me4+34C5EUfAt=#Kk(k9s1X;zML@C(U3hUy+1DS=u)fh z=r7ZBbM;!IrtMs3$FDp!cT=W>~uOC;U9i3b~@KOU2B}xPUjkD zge&@!@U5k9E$()N@HL?nP( zg;(^x*4xoqM#*>Dm)9;@w0^V`nTAh84SrYRmq0U8#Zxcap3-05CA6=qrd2^$(N=in z#HKE;rD>hH6-lc?d<&}tdYvvW)}fVn`sgjsu%Gi;owaVP-zV00cF+pl_|5Z2?JC!$Ol9v}%@OA`JMM#aSm+$YFJ4}hsumOjhlRD2e=1LEinXrwfimw1^aX9>tQ!UT?Tt}*JXaEXpj=V; zx)D@uXuZGL8wqx|Mp~M^=3tL+6#8SW6;$Wq<;{XVr_f)Ne~oVtbSDz%ZFJUPecPxY z&#HRrk18qs6e;aW7dCP($1lJIy6Ri+g}>?2QspW9|<`1fC_5^rq#QR1Q6~YRNmCA>zdL<4U4QruWD+ggXarUFx*HSg8M>fn@pZA}%KJad8 zS%rGci+4G~FLU<@`s*D>{;BP^W)#{l?6&gbh12b&`qoELQ+g4#)c=_9zAM04d;Coy z&>!BnV%2`6$Byvm_%WV6#Qc9ftzTso{PBeTe+BXj`ohhK>Y4(`FZa{ae?f|#W9B8tF z-N2`Mv=``El|cUl`ddlqABp1U?TzA-=r47GwGCbV8sAWhufY`Ti43<=dMizcuRPk` zMDa=5Q&&N&3SJILZSAqI4>VOn2fDkleXS@DR~6-7;y)?Zyp&M1u909Z<$cuYa`!KN#m1P`zqSrBDNR$J5gQ~b*QI$Fa-Su`1`PapGdEhmlsbT zwI_;?jt@0Mk#J*WJ=R~BP+!6ytxttS{b_4I`XyRe*M$}`5 zv6o%cA@KEaSa8M0k_L>ODp#RwIE*(hZUk5^3;LT;^f%OLx~TkldE%u@h!22_Wh;@N z7Ml=H3ic-P50t)U{04dSv^4t$L)D&7707Q$q|cdxJ`{aTLr1U&Yy)%^`~$?NUGel?+$GTG6yghNmAnH- zdg#y(@q2TB<{*CseYJa2BOzi?Ig z3f{j6_LSJ46d#>$Q1nhqIq#2znUT9nY%k33kHMrK*jqQ6y z`8av`O2$9xZ>Z|={Y66;(2w;1N3a$2%@8|!DF15(`b2-3$WMYl=jmS*#XpfhclRN^ z?aRxA{>0f1os7dbFCe_9NNRi*i1UZR7j=7kB7?1~%n^`%H^c!QSjSoE9s+;cBFfXn zmnP~n6$PDFt!n6NX|nd>ym2MsUx@9||5WkzERA;&K6S;eZj4oVnPcqX!~P$lNIdRg zzP+IN^&&iH3jIO;F#hmTpX;on^jDRG>fr-76&)X+@%{kgGiM`4v<*nhiZuYB(t--r zi505X;{94T=tcPr*0;pIUu<@BYw4E3=gf^(ru5T1QhkcTi{Qb@x57va%C-PPW>s-dyjI+rDxY=aw z20sx*dIp!n7gldHQPq*yCmqm+5lq!uB$L`36@87>9H?364wT_M2|m?`e^FnQKQG0P zNQD>yS)n7LkaVHXDStTMhaAEBppZYoHcpQl`RT40-+$F$Rh;7mx+6fOdznyQ%7ae$T!Hu(R*CIX_)m&I$k}0j(ty6buOiNWR+SMCfS-%Eo15E=JUynr|)O8otPd=2S_w6lB4Qrdr8mtf#L;^nhkTg#P z^{Wsc7v(=ie1$k`z3Tzi0K_A`m!MrObFYu<59g?GFZg3=e1Y^i(QRVRB-9km;XwYc z0B;uzCsB6dS zLVrTM9g2W%t@88ps75>Qj|Bfx8LR-O2bIPCq(bBUIRzr-LxuuP5mO`NFj}|{etuEa zWr~wW!^(Jh$dC6OEoH8D(ci?uTedPDp35!Vi}_wk8qXiH>Y;9V5bdjml1j%f(3OrR znls0o8R{?acgc$Q{AnGnPy37dFKVBBYKz(@Ev$igSudTF#d5e11R%VQD7+N$GLl-0 zIYZu`)nI+wXp|pHuT)>LKF?Xqzt(yq;aWT-d@2Ke!bknLhswV%p*~V3jDNJja{Nvl?`WJ+kL~R(VtY5go`OzAXipiW@T!n4 zn8Sc?hVi_iI$Z0DfSpGI{21h1gz~bw&BFMRf?w<^S1jLLxf1Fxv_$dC`{#P}&m0Q- zVk8TkAkZ?5L@q<6a#8tA!9Q0ZzN!^MepHx0@&1eNI|NPFzY_iP#a+l>g`M}WqWw|* zlh_{}>d>G1Xneu1C0Ip#V&+J^A9wzSxrPWDiPMGktzDcTUt#@unqIY<{`+BR6;z|<*(xV z>k`CoPi&vSuMq#F^slwFedEFvLj2Chw-WoL{*uNkeCST~fge9%Ey_o8jJO5@@jcRG z>Js=9{WEr;;KlH=9@|Uv;aEuX;gWa{V?c=i5uRnG2rq@bQ~zdyHML>tZ(N9f;b4p( zN-zE5?T7Y1g)bW)S7G}_L9xA%??EI|e^-O|;&=s5)?$6v3P@#3^fd+_Yr_Yh)xz4} z+R9Z}|Kf!D68oUl`TSbKc*D;Js<3@iwEgFZe^-p6yc!pRr~zrJ{6j+g>+Ht*!!i8G znUhboQT?N=xG~;spz-b+H2WZ`0i7RL()p1O8}ab~HQ;8hx+_?19}PD*m|%^povzn3 z#nx-u5#KtuE#AHbfA;U)|;B!1kzbtL#I1SFC{46z!jJ?6nsBp*fawbuM2i z;1le%HW5EnEeU>Pg~rQ5z7^XqY*`5Y0w2i7w~$YU&qVPprQaUKKjru~K>2sWR95OA z6Y0nPLwgkBTMy;GZ=nzm3id$zOEMl&uSN6gbzQ)(y*^sM13BY2aY*V@|KV?LsN(ZH zu&xXrE7DiiCDNB7-%I;9F@9aMi1%M&{5lxL&+mtj*gs-&ju*cIez5;|e);i{Hl<$_ zsxU_z0pHWrx(e6y8~XV9jjICNSFI59r{m}w<{Nyl&c@l-x7LDL)OA59DVa}HtrYIX z{7#~Or~a%9abDk~75J*qELU4MWaKeS)70#uj!>lE`T)Gx{ULp9bnjSBRN`QQY5 zpe86TeBov=Fn5DuM7f^MKGv>@Yi6T8Bo)4zA@&s#_g}z z{)mfLBB&sKA*U(EFFLiS`{U|yJ5~kYX%yrWJULSdRQ2j#CJZZ3AB<@e+ zo6zS=n1b2x6YG}`hldQo{(ggh0AA^vExdX(o8ycoqX~W>dnUu2`~Qq3z%06kVOBW3 G=Kl-rS!jv? literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.json new file mode 100644 index 000000000..626184002 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126205,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-2-2024-01-04"},"size":8053,"modificationTime":1786523126205,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-04-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-1\"},\"turnID\":\"turn-2-2-1\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-04-batch-2-file-2-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-1\",\"isDeleted\":false,\"ingestTime\":20240104221,\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-2-2-2\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-2\",\"ingestTime\":20240104222,\"rowId\":\"row-2024-01-04-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"isDeleted\":false,\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"isDeleted\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"trackingId\":0,\"commitBatchId\":0,\"rowVersion\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-59765f2d-40bb-4972-a250-b641daa5a6a7-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-04","_ACP_DATE":"2024-01-04"},"size":8053,"modificationTime":1786523126205,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240104211,\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"rowId\":\"row-2024-01-04-batch-2-file-1-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-04\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-04-1\",\"turnID\":\"turn-2-1-1\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-04-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-2\"},\"turnID\":\"turn-2-1-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"rowVersion\":1,\"rowId\":\"row-2024-01-04-batch-2-file-1-row-2\",\"isDeleted\":false,\"ingestTime\":20240104212,\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-1-row-2@example.com\"}}},\"nullCount\":{\"_acp_system_metadata\":{\"commitBatchId\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowVersion\":0,\"ingestTime\":0,\"rowId\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000008.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000008.json new file mode 100644 index 000000000..272d2a269 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000008.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126223,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-2ffd1456-206d-4884-a7e2-87e358e8e103-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-1-2024-01-05"},"size":8052,"modificationTime":1786523126223,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-1-row-1@example.com\"},\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-1\",\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-1-row-1\",\"ingestTime\":20240105111},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"rowVersion\":1,\"ingestTime\":20240105112,\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-1-1-2\",\"conversationID\":\"conversation-2024-01-05-1\"}},\"nullCount\":{\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"ingestTime\":0,\"rowId\":0,\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-06c535fb-4c81-4033-bedc-026d5b184b78-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-05","_ACP_DATE":"2024-01-05"},"size":8053,"modificationTime":1786523126223,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"rowId\":\"row-2024-01-05-batch-1-file-2-row-1\",\"rowVersion\":1,\"ingestTime\":20240105121,\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-1\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-1-2-1\",\"conversationID\":\"conversation-2024-01-05-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-1-2-2\",\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-2\",\"ingestTime\":20240105122,\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-05-batch-1-file-2-row-2\",\"isDeleted\":false,\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-05\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0,\"trackingId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000009.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000009.json new file mode 100644 index 000000000..82174c9b7 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000009.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126230,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-05","_ACP_DATE":"2024-01-05"},"size":8053,"modificationTime":1786523126230,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-2-1-1\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-05-batch-2-file-1-row-1\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"rowVersion\":1,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"ingestTime\":20240105211,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-1-row-1@example.com\"}}},\"maxValues\":{\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-2\"},\"turnID\":\"turn-2-1-2\",\"conversationID\":\"conversation-2024-01-05-1\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"ingestTime\":20240105212,\"rowId\":\"row-2024-01-05-batch-2-file-1-row-2\",\"isDeleted\":false,\"rowVersion\":1,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-1-row-2@example.com\"},\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowId\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"isDeleted\":0,\"ingestTime\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-b95c7946-cd89-4a7d-aad6-8664717be548-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-2-2024-01-05"},"size":8053,"modificationTime":1786523126230,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-1\"},\"turnID\":\"turn-2-2-1\"},\"_acp_system_metadata\":{\"ingestTime\":20240105221,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-2-row-1@example.com\"},\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-1\",\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-2-file-2-row-1\",\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-2\",\"turnID\":\"turn-2-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"isDeleted\":false,\"rowVersion\":1,\"ingestTime\":20240105222,\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-2\",\"rowId\":\"row-2024-01-05-batch-2-file-2-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-2-row-2@example.com\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000010.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000010.json new file mode 100644 index 000000000..7de9d78f9 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000010.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126237,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-06","_ACP_DATE":"2024-01-06"},"size":8053,"modificationTime":1786523126237,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"ingestTime\":20240106121,\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-2-row-1@example.com\"},\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-1-file-2-row-1\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-1-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-1\"}}},\"maxValues\":{\"_acp_system_metadata\":{\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-1-file-2-row-2\",\"ingestTime\":20240106122,\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-2\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-2-row-2@example.com\"}},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-1-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-2\",\"source\":\"generated\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"commitBatchId\":0,\"isDeleted\":0,\"ingestTime\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-1-2024-01-06"},"size":8052,"modificationTime":1786523126237,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-1\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"ingestTime\":20240106111,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-1-file-1-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-06-batch-1-file-1-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240106112,\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-2\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-2\",\"source\":\"generated\"}}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowVersion\":0,\"isDeleted\":0,\"rowId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"commitBatchId\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"ingestTime\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000011.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000011.json new file mode 100644 index 000000000..0041d5186 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000011.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126244,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-e6b6c2fb-201a-45b7-a88a-3662756e970b-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-2-2024-01-06"},"size":8053,"modificationTime":1786523126244,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-06-batch-2-file-2-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-1\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-2-row-1@example.com\"},\"rowVersion\":1,\"ingestTime\":20240106221,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"commitBatchId\":\"commit-batch-2-2024-01-06\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-1\"},\"turnID\":\"turn-2-2-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-2\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"ingestTime\":20240106222,\"rowVersion\":1,\"rowId\":\"row-2024-01-06-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-06\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-2-2-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"isDeleted\":0,\"rowId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"ingestTime\":0,\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-6c8cf883-cdf4-48ca-907c-fa042d2e83fa-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-06","_ACP_DATE":"2024-01-06"},"size":8053,"modificationTime":1786523126244,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-1-row-1@example.com\"},\"ingestTime\":20240106211,\"isDeleted\":false,\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-2-file-1-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-06\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-2-1-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240106212,\"rowId\":\"row-2024-01-06-batch-2-file-1-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-2\",\"isDeleted\":false}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"rowVersion\":0,\"ingestTime\":0,\"trackingId\":0,\"rowId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000012.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000012.json new file mode 100644 index 000000000..5d01a5536 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000012.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126251,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-01aa1a67-ddba-489c-bcaf-6406017afe40-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-07","_ACP_DATE":"2024-01-07"},"size":8052,"modificationTime":1786523126251,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-1\",\"rowVersion\":1,\"ingestTime\":20240107111,\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"rowId\":\"row-2024-01-07-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"ingestTime\":20240107112,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-2\",\"rowId\":\"row-2024-01-07-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0,\"rowId\":0,\"isDeleted\":0,\"trackingId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"rowVersion\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-1-2024-01-07"},"size":8053,"modificationTime":1786523126251,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"rowId\":\"row-2024-01-07-batch-1-file-2-row-1\",\"rowVersion\":1,\"isDeleted\":false,\"ingestTime\":20240107121,\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-07\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-1-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-1\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"rowId\":\"row-2024-01-07-batch-1-file-2-row-2\",\"rowVersion\":1,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"ingestTime\":20240107122,\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}}},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-1-2-2\"}},\"nullCount\":{\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"isDeleted\":0,\"rowId\":0,\"trackingId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"ingestTime\":0,\"rowVersion\":0,\"acp_sourceBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000013.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000013.json new file mode 100644 index 000000000..b350d8521 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000013.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126258,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-07","_ACP_DATE":"2024-01-07"},"size":8053,"modificationTime":1786523126258,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowId\":\"row-2024-01-07-batch-2-file-2-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-2-row-1@example.com\"},\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"rowVersion\":1,\"ingestTime\":20240107221},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-1\",\"source\":\"generated\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-2-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-2\",\"ingestTime\":20240107222,\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-2-row-2@example.com\"},\"rowVersion\":1,\"rowId\":\"row-2024-01-07-batch-2-file-2-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-07\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"acp_sourceBatchId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"trackingId\":0,\"rowId\":0,\"rowVersion\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-2-2024-01-07"},"size":8053,"modificationTime":1786523126258,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"ingestTime\":20240107211,\"isDeleted\":false,\"rowVersion\":1,\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-1-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-1\",\"rowId\":\"row-2024-01-07-batch-2-file-1-row-1\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-07-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-2-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-2\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-1-row-2@example.com\"},\"rowId\":\"row-2024-01-07-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"rowVersion\":1,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-2\",\"ingestTime\":20240107212},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowVersion\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"rowId\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0},\"turnID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/_last_checkpoint b/crates/core/tests/data/conversations/delta/_delta_log/_last_checkpoint new file mode 100644 index 000000000..68e3317f3 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":7,"size":18,"sizeInBytes":58489,"numOfAddFiles":16} \ No newline at end of file diff --git a/crates/core/tests/data/conversations/schema.sql b/crates/core/tests/data/conversations/schema.sql new file mode 100644 index 000000000..85fd1d1df --- /dev/null +++ b/crates/core/tests/data/conversations/schema.sql @@ -0,0 +1,41 @@ +CREATE OR REPLACE TABLE conversations( + _acp_system_metadata STRUCT( -- 1 + acp_sourceBatchId VARCHAR, -- 2 + commitBatchId VARCHAR, --3 + trackingId VARCHAR, + rowId VARCHAR, + rowVersion INT8, + primaryIdentity STRUCT( + id VARCHAR, + namespace STRUCT( + code VARCHAR + ) + ), + ingestTime INT8, -- 4 + isDeleted BOOL -- 5 + ), + agenticExperience STRUCT( + agents STRUCT( + agentID VARCHAR + )[], + version VARCHAR + ), + conversation STRUCT( + conversationID VARCHAR, + turnID VARCHAR, + text STRUCT( + raw VARCHAR, + source VARCHAR + ) + ), + identityMap MAP( + VARCHAR, + STRUCT( + id VARCHAR, + prim BOOLEAN + )[] + ), + _ACP_DATE DATE, + _ACP_BATCHID VARCHAR, +); + diff --git a/crates/core/tests/data/conversations_no_stats_parsed/create_delta.py b/crates/core/tests/data/conversations_no_stats_parsed/create_delta.py new file mode 100644 index 000000000..52018358f --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/create_delta.py @@ -0,0 +1,245 @@ +from datetime import date, timedelta +from pathlib import Path +import json +import shutil + +import pyarrow as pa +import pyarrow.parquet as pq +from deltalake import DeltaTable, write_deltalake + +TABLE_PATH = Path("delta") +START_DATE = date.fromisoformat("2024-01-01") +CHECKPOINT_DATE = date.fromisoformat("2024-01-04") # Use None for the final version. +DAYS = 7 +BATCHES_PER_DAY = 2 +FILES_PER_PARTITION = 2 +ROWS_PER_FILE = 2 + +SCHEMA = pa.schema( + [ + pa.field( + "_acp_system_metadata", + pa.struct( + [ + pa.field("acp_sourceBatchId", pa.string()), + pa.field("commitBatchId", pa.string()), + pa.field("trackingId", pa.string()), + pa.field("rowId", pa.string()), + pa.field("rowVersion", pa.int64()), + pa.field( + "primaryIdentity", + pa.struct( + [ + pa.field("id", pa.string()), + pa.field( + "namespace", + pa.struct([pa.field("code", pa.string())]), + ), + ] + ), + ), + pa.field("ingestTime", pa.int64()), + pa.field("isDeleted", pa.bool_()), + ] + ), + ), + pa.field( + "agenticExperience", + pa.struct( + [ + pa.field( + "agents", + pa.list_(pa.struct([pa.field("agentID", pa.string())])), + ), + pa.field("version", pa.string()), + ] + ), + ), + pa.field( + "conversation", + pa.struct( + [ + pa.field("conversationID", pa.string()), + pa.field("turnID", pa.string()), + pa.field( + "text", + pa.struct( + [ + pa.field("raw", pa.string()), + pa.field("source", pa.string()), + ] + ), + ), + ] + ), + ), + pa.field( + "identityMap", + pa.map_( + pa.string(), + pa.list_( + pa.struct( + [ + pa.field("id", pa.string()), + pa.field("prim", pa.bool_()), + ] + ) + ), + ), + ), + pa.field("_ACP_DATE", pa.date32()), + pa.field("_ACP_BATCHID", pa.string()), + ] +) + +shutil.rmtree(TABLE_PATH, ignore_errors=True) + +commit = 0 +for day_index in range(DAYS): + current_date = START_DATE + timedelta(days=day_index) + for batch_number in range(1, BATCHES_PER_DAY + 1): + batch_id = f"batch-{batch_number}-{current_date.isoformat()}" + file_batches = [] + + for file_number in range(1, FILES_PER_PARTITION + 1): + file_batch_id = f"{batch_id}--file-{file_number}" + rows = [] + for row_number in range(1, ROWS_PER_FILE + 1): + row_key = ( + f"{current_date.isoformat()}-batch-{batch_number}" + f"-file-{file_number}-row-{row_number}" + ) + identity = f"user-{row_key}@example.com" + rows.append( + { + "_acp_system_metadata": { + "acp_sourceBatchId": batch_id, + "commitBatchId": f"commit-{batch_id}", + "trackingId": f"tracking-{row_key}", + "rowId": f"row-{row_key}", + "rowVersion": 1, + "primaryIdentity": { + "id": identity, + "namespace": {"code": "email"}, + }, + "ingestTime": int( + f"{current_date:%Y%m%d}" + f"{batch_number}{file_number}{row_number}" + ), + "isDeleted": False, + }, + "agenticExperience": { + "agents": [{"agentID": f"agent-{batch_number}"}], + "version": "1.0", + }, + "conversation": { + "conversationID": f"conversation-{current_date}-{file_number}", + "turnID": f"turn-{batch_number}-{file_number}-{row_number}", + "text": { + "raw": f"Conversation text for {row_key}", + "source": "generated", + }, + }, + "identityMap": [ + ("email", [{"id": identity, "prim": True}]) + ], + "_ACP_DATE": current_date, + # A temporary partition forces delta-rs to create this file. + "_ACP_BATCHID": file_batch_id, + } + ) + + file_batches.append(pa.RecordBatch.from_pylist(rows, schema=SCHEMA)) + + write_deltalake( + TABLE_PATH, + pa.RecordBatchReader.from_batches(SCHEMA, file_batches), + mode="overwrite" if commit == 0 else "append", + partition_by=["_ACP_DATE", "_ACP_BATCHID"], + configuration=( + { + "delta.checkpoint.writeStatsAsJson": "true", + "delta.checkpoint.writeStatsAsStruct": "true", + } + if commit == 0 + else None + ), + ) + + # Move both files into the real partition and normalize their add actions. + transaction_path = TABLE_PATH / "_delta_log" / f"{commit:020}.json" + actions = [json.loads(line) for line in transaction_path.read_text().splitlines()] + add_actions = [action["add"] for action in actions if "add" in action] + if len(add_actions) != FILES_PER_PARTITION: + raise RuntimeError( + f"Expected {FILES_PER_PARTITION} files, got {len(add_actions)}" + ) + + destination_directory = ( + TABLE_PATH + / f"_ACP_DATE={current_date.isoformat()}" + / f"_ACP_BATCHID={batch_id}" + ) + destination_directory.mkdir(parents=True, exist_ok=True) + for add in add_actions: + source = TABLE_PATH / add["path"] + destination = destination_directory / source.name + source.rename(destination) + source.parent.rmdir() + add["path"] = destination.relative_to(TABLE_PATH).as_posix() + add["partitionValues"]["_ACP_BATCHID"] = batch_id + + transaction_path.write_text( + "\n".join(json.dumps(action, separators=(",", ":")) for action in actions) + + "\n" + ) + commit += 1 + + if CHECKPOINT_DATE == current_date: + DeltaTable(TABLE_PATH).create_checkpoint() + +if CHECKPOINT_DATE is None: + DeltaTable(TABLE_PATH).create_checkpoint() + +# Ensure this checkpoint has only raw stats and raw partition values. +checkpoint_path = next((TABLE_PATH / "_delta_log").glob("*.checkpoint.parquet")) +checkpoint = pq.read_table(checkpoint_path) +add_index = checkpoint.schema.get_field_index("add") +add_field = checkpoint.schema.field(add_index) +adds = checkpoint.column(add_index).to_pylist() + +for add in adds: + if add is None: + continue + add.pop("stats_parsed", None) + add.pop("partitionValues_parsed", None) + +original_add_type = add_field.type +patched_add_type = pa.struct( + [ + field + for field in original_add_type + if field.name not in {"stats_parsed", "partitionValues_parsed"} + ] +) +patched_add_field = pa.field( + "add", + patched_add_type, + nullable=add_field.nullable, + metadata=add_field.metadata, +) +checkpoint = checkpoint.set_column( + add_index, + patched_add_field, + pa.array(adds, type=patched_add_type), +) +pq.write_table(checkpoint, checkpoint_path, compression="snappy") + +last_checkpoint_path = TABLE_PATH / "_delta_log" / "_last_checkpoint" +last_checkpoint = json.loads(last_checkpoint_path.read_text()) +last_checkpoint["sizeInBytes"] = checkpoint_path.stat().st_size +last_checkpoint_path.write_text(json.dumps(last_checkpoint, separators=(",", ":"))) + +print( + f"Created {TABLE_PATH} with {commit} versions and a raw-only checkpoint" +) diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5c2b6adcea9000a4dea7c62a2b95ec4abc6306df GIT binary patch literal 8026 zcmc&(TWBNa6`qmg>)Nr`>wl_Al&zC^Cth~Fy7*E#IALdOUF@|T+3`g-rQ5MIwq;AU zkYvmDLkUX>VF@LKu!IuAVoE7}2%#*?QbJk!Q1&5|P)x}~D1BH;2qjN}h7d~6IWwaf z$&#%sEfuz;|3A0yT>kT)v19U_pSjpo=9^*;#&vO=Fb{9ss#-{F^!xh*zQAC=&ksM0 zW1Xz~Dlm7lM%Ku2%^n9Zx*VJf_FiJImyThAMHb#9Qu$$L%7M8awu#l72 zwS6^{O>5U6x+L<(Am&S<9XS?Q!fW)`iL7^onV68x;dv0(;s-a;~olG^*>`cA+23%)@$6$SAX?21$sc z_Xa`)T20v|w-aRj*L)!N{mOHDPs`=iLTYE*3_8~N8D$=1FPX9M^tOWK!ftN6-_JV+ ztp%PMKyz9hKIYlaOg*{Rhd|HRj)kYEpzRlW)^>6|rOKSDRtgg?zLnxSg< zwSXarvX)Z|TC!HaoD|}kg>p)A^iJ?Utn7>Id0Sz01Z{=UsMi~~iyTJ1wfpQGPO-}& zy1UFsTAi)xCUb}acX=2Z97l3u&4wnrYtt&V^i_53?lWxa+frv|N3ORL^# zpt<-(x3xHtqoGqfNv(v4?kAEntlKitQtB_F+xo6MV`Hn5Fbb)4-s&j||9_*7#9_G4 z#68NTnM|ICTHy6eSCBozJR253LWpvrM}SVQuada>z{!F{y9?Fi5k7Gl%eQEGuYq`x za<|E#JS58PWYa0jL>tlC>?l5LHALi48bxteG2fxVvwv ziR{h%LB616Zf3NCnp6ua>*QG5C5QVw(ROxo2~ELaaw2(&oARJKq_*qrveflF3c7uxI~ZmU6;gpSerP+HFk=9Rcq>GEis4S5)Or1yfiO0dxp8bD))M| zyZ-!nPShs^$UlyZ4sKM4>i0Y+{JobwEqr!?Ww|pUl@~CUb%57TJK_F{7e3^rPh8Sport@hxy4-vQ+2d&mC)WpwD13b-P6&!PAgAaCnCr{ z5!R%84bsmWJl33KeG|C8()48QRtw(9Fpd|T@Rv(W5I%LYs6Fe!swbtg3qtoJNWXFj zU-8o2Ch5xtjAA{vYuU(E*N|3rA6Iioc*tMYrvOg)0t5V>$i6HqhiF}1S#6x!H+lF| zVMDszEPd95{^|3DF%MQn(tUXm8XSf94GB)wt!LPE3h(Kewk`)Unick?*IT5A&5&tx zuymKEP0aMk-D+^mOdm;*=|4VWWSSE%ag8-vqID|~5T7t8J!lyP*Le=cShV4R{swn9 zH)<+>fXZ)s*j0NO#ZO*n##Khw7hPQDg;jF-s8#xB3&z~IbWioH=kvLWroUb)`}aIOzDmM zs7-ow4vp0w;IWa{nBS`cD>d>rUIbttKU>+z%a9otx{2r)y!2ta^jRAsvd#atXMyT1 ztYBi`YGW z$o^f{J*s$P3_9e{hLDxM?vQ@pf$msOC$aDIW-)MBC*p3RU z^3?pW*BkU9lqjxIk^Y(q(kGf4Cw;^}vx)qnE5IKiy2&4!-lR3L622>F|6u+{z@I?U zIMLy4)=!{hd><#*i9Z_K^Ll-o$iINB7XPw|A2x|jMiSA<>?(;XMZ|-FwD5<*C3#s{ zM>^Syq_=>H-5&j(vS~q56v%yi*oD`{iS%6@Xik8(Fmnt3hj+id?@}g z@_)-4tIStuW|@SD)rVh+UpWkVy(1IIUpqeXAkq~?5uMpf#wRydz-%IR2=!CuuVVLi z8R@#e2g~t<9;fJBT8om5ymN;97Re)O#RoQbUt0jV=}o{}ng4}SyyYpPzl#1X2S#?2 znZ<)dAna3@Ms#@!nzFeX7z(V!Cl8n7bJ^9vAhOHju>tUh@@48%##d>rmo{-xDni~h zuXmdGYx_Te^dgqFLbect9NruO{FDkSpZi#I`cb7KRHjkCM*7K~g=e1D*7=Mag1#%u z$X`2uV-rZ%>n*&E^@Xt{C6TmmWr@lGP61$u==21*9YWWPwYoGD)}l%}kx8dlGc##~ z8$^7E<^GQ^tYfT^EZz*oIU*A!F;XCL}T?iTssiFC2}j7?gTtJ*`vx(VQGb zdJ#jVWD`C(n~_Dpj`o#hl&3>WP9S$^jN+ke!pR=N4gE!Mv5x|trJ?PW_>6Bck}hC9 zZ<76r{I8H0M`;dVvP)KtS&7c zCL&=vbAS@mol^g?O|Tavyeh>8yR=e-ycL$?lS8j9LEN_IvdQR{JPT*q=>OydGRhm| zKP0Y*^)V&u^#~;6Ws6@TGcvG9=PSLp3L4f(A2F2b)!q+QkzPdKl!MiQ4Msy%l8<6e(T8|^!u+3UkkurrQmOsPw>NCV;mQQ|Em87)OPJb literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f449af9924fb95b89c98465e62caa4a501ec1b76 GIT binary patch literal 8027 zcmc&(TWA|+7M_vhYn<3|^3Q5gCF``w#36BX@nz-Wmdx0?*s)#Nb!^!zn;lDITef5i zNw#7?ETM!Ygg%r$gc3>!A?(AlEO`haEM>{VvXs(?QbH*EkcTBKC6qq)A%tbmIWwaf z$&#&xT~^qV{{P&*bNSDI#*WF;e&%B5nQxpq7}v>h!VJ7|i)taU(&O(5_yT=BK0o|0 zj&-o3=YhF{HL^yIYxX#R(dFP=u=g%|-&AlsE5d$aBa=xLdVa2+2RvIE2YrYnKHd$A z3OO~gmResnq43U_sdOoo)_MXxxs76vzxxm8IW7PoJDFeu+0vzh?yruNC; zJD?q94Ig=DutG6^g7e=xv%gx1i|Kr`hX<|r+B;s~@az_ELRlH|fxu>7%UM`HslY-` zUevCtnQU6S0MR9p-}iC8fPJ@C0GxUNAUGz{maYacs%6wvx)xLyrM8QmGJ$ps@f*vJ zTp;H#K5N-?pgoQqVx5=_POq3Iw^1>$C9q>Ak^|jMpiy1c)(bsYW**jsLPnV#>LVeJ z-Wv!JXfz_KLSsrzrgYjXDws;6f9(D3@k3 zc^+zk*E3x~_6RdvgK?J|}h)AC^h@gn7JlRl54;T_{L~FC7cyQ2A-i@^D?Z)aUnfj^S zU_FJz=N*PpGW@KQjgSc))$JqfA{vJ{ygnIiQ*+Q*uH+LdaM)Ng*Y9v2y`m~YCN$>I$*#mkQZtc} zS{OF9d|@V)(O`%JJ1wdFh?dsi%0x4VsUf5ip{rRfm(tcjlMDA7X+IBBbmRCCc-ilA zyV9wA0qg+Pm4XnNCiPA-I$~PIcr@WS0-hLqAkt=U8!&KF*bdn@%QZA&gD7pGO&lZ4 zC^&O!(QdCXPnogI9Y`5*n^3dhgkv1$X>_6qq6vQHbw!-pxu$JfQVm;bdJ|%~hts># zKtXk3wm88hdc@bA66axUVn5f|A$C`-sok~29D+;O5kBIj8L8Pbz;#!-*Q?$2hYxe2 zJ|RH%pogrLqe`_ajL69m0>ibhk-* z*?>{3=XNa{x#}9y%I@Q84hc{Av-%Xk2`@3gw?+1{tQ?|sd1bY6YTxAHZ-o`6+gnDaIq@{tSfeFcw-N#I34PMzmLYJR=U|LQ8y@IyaCdVR*E-v%JD!F{GRr;X?<7^JarRlhubAqRAT;_ytFyN0pY}yW_ zdp8jCQY5>-7LwAZ2c#FRXcy;f8n&z2YcpHU4`}G-an@I6sO)B~xKYWJ-pKE@Nnaj7 zW3>l(Y~(fO_o~23jr_(L0QT48m5sa%nQ@`p5Is03ebo$XNA8meh zg2n8?2$q?okpV{qGWy6NJmsb9?b7!L(de5!A-vgxMRix6THWKx8SEavWdAPf9#yLqdV5qiS0q9YSvDbde(Cc;b$k=WLXI9qmwx<;h01=(?UUd-XZ;| z18vsPmaNAhPI!gp{&kX#*v+j}GpEmC)`>OS{sV{bikF^sN_P*V{r_LvuXxJhgj?N^ zz(2mtUg-(cTU}kg(dbCA9TF}|w~q{gV>}PfYnZ%J(>q)K+{OtHyP^BtKgHhXgugSv zB{d7*9yy9nR_-==disO8cl9S5_xJ5ye4U#L%d$KLzxcW&%SCy3SsueJ_>^UZ_F)NY zSsuj|Y?+_1LQ&W$%JLj;ho(>>1ix}Imh*bAq7r!m-{7T$ri)0&U(_ed<1+(ZZ_tNO zqPRvy`fD;spJ-}?^b!B$D)NWU0e^((CVyyRmDa>c_|BpI?U`Qze*#G(M2EL2KY^0* zeS};m{%CB=>-DW7|7B#g_~%Xhut{_>l8BCF7fDw>(bt5758)z~E*wGrOG#gnjDV zpe|2AQ&txP{egw}*v@=>I=dL?Lw0#2)(ienzD#|}$Re%v(k3oSMaaA4^-d6fZU0A+ zUc}N?$QELd!>faUpHgAv^D5SyepIOll?l|Zk$$pg;hCYebv`3Ipzqu~^4HGa@F>#t zdJC^(ePJv~NhIxCn4@xlQvetuIz0hyhtM@+tWP9S$^nBt+V!pR=Q4gE!Mv5x|tx&HNq_@r+(k}hC9uafM`;#VvY2#EY8jDBqCutbAS@m z?NR@+Rj?N%yeh>8yR=e-ycOo-WBnh_LEP4-v&rb1JOyXk=>OydGRiCDKP0Y*^)W8% z^#~;6Ws6@TGuS&z=PSLn2pX11A2F2b)!q*lkzPdKlC(#+oej1isk1$}a5 zK2XRmW^`HKKK6=a4{ICGyT1TEbr{b3sv4pmkJWe?og7(~W5(ed)enVSaXzu}m@hgv zRBFcgSWScncNTkRx90=1W%?=K;lY*pUS%oKo8C!AW()CvKb?$(*8sv;_a;hh&wm7{`g7Z15%_#7Vj;b?ixOGEQ9E z`F*I+3RMX4u-XT-LKQ*?^`WXN9uPv+s^X!lR;ztjtq@xEfrkoJ6+-*C4+vE~=iC|3 z*s&8YYE=vDx%d2i=jWb#1}4-gKXb9u%s0*)OlTJbX&SzSg?J&6>I!rPe1V=WpCA4( zfwi$ir-8YR)w6m*X!JOM(d7_au=fG`&{S|FE5m+bHIqpfx_+vj`yyK$2YrZScDxf5 z6>{;!a(ZRSgd#d8Or?uyUF-69+eK1Yif`7 zzX#e;*6>ku_LnIZj|%>qC-+thaWS25^oXDpKfA~4Tan!qO(>~B9|)}FwVZ|Jvob8? zy;isFsPR^=eREl-dq<+yvS-_>0x0 zhc1x&#Q3dd&w=&?_7-c$WC*-s8r(+3z?Q@gn@IL`Hh{+XlD1Ol!VG#?2MQTw_ErxG z3G`h@h(N0;+u(MBtpAJ;ahe7uleT`_}>b-qWL_p?DW79PDBEEm>tAP+_8 zFInwXh|fj#8&gs4N-roHTD1Um6tvAk$KqS}D6^hqr?NVQC*_@7gM(wcl15^}OJ~UF0wdZokjY;gs7Qvb)`k zq}AFgZZd}~38(s?!wDoOSM6xByE?60OJ{jI!=f`0+54I7u9Y=VZsR>$Q>(7N2Cf~N z8_nh>yNi$I)_wLqR?WPtQXAe;0?vu^?LV(ThP zuqeoGQ7Ct(!RSk}+k7GpSpmz7Xu&k$geD}r@w;3FREKr8OPUQ;P8C|YmS(Hf$Zl(~ zBS*cvc9Lodk=<`3XIQsoqN&(lWViLqow2c1NEn6GI&ZZUh5x@%M{*xrX!1Jc(oCi( zLM@0q(`96jG|l=XkdPvR?2(|8^Hma8x1Fq?Xm_BRJkmEVV|jy?59){)DR&wS%44Ej zZ#1+VLgJMsBI4ttCe{NA@$qgm3zBXRu0G#qNVE{G#g5|senWXD(yq1|t0!dY$5w;& zI1-<=8A{3U({>gj6CBm8gX|m{hd8`G7HNs+ps`%aCsJ_OSfkMEa38u9Ph>CUxAFxo zb19=0;>ma+&e{ala>C(0K(y7hTtW-tG&!0)A&h%a9a8u6c3OebY_`Z!fjTg5?YKZi zGwjU3B@S0Of@#g>(wTT}YcvUiM7prWT-=f)Z5jrqrj>X`%V*;WjX5~kl~_$`CNfeB z!={!mOs6v%3~^wmDV+~#x&~J!nmIrXA)OfB%xbx`wgQ@5xL;5Ed6=T>#|Od7UYFaY zr}G7{1JnjkkO5q#Ro+ZSL#AO2NdrzK;Eb^cQZ4q*0W&v6?k)RfsfK#&5ydS~;KE75 z41_Zm->}=O&r@w;j&{e=7xSg-P(mdsw1ltZK@udH@XZJa#(m6TGhH!9yZ zpnrVMFy_LFNDiHvfi6eky{5pay81RdN8#N))7B+HMzhkUa-~Um+z6RA2TXTy>cmX% z+^q!1%=A44GX2-LjZ6#j38B79OSEn$0^*Z;lt)d2;JPTlIEyws(BI$=7Y0q`e@5ju zJnXc+jAnOUY{r#F*O+;MnHN{l>$}a$%O(u9IUg4X#C&oG2v9AnXv5ZXs5Yiz=CiLB~UL3!4u{ILzK z*3g@*cOpT0g$Dn5jD_q5Qc!9dOCzePPQqXOe59%y>d=xphFkEfszKU^C9G9- z6j!ii{=y1HgTo-E&f@m)5=soiUv+&V=k;!)67@WO!6!I;fi#j<(5I^7p+2v---l2l zxJE_%6YZy8G!-I!#2=kTe)WCe4-?(wA3lGX*2GHq2GIUi=q&IjkQ5?1d`Ndoa8}N`;8PV@ipwjr-&ywY5K3KzefBUNDAiWXP4C> zWDOT|ZDt@a7u)pDlV~34+4(To4eLJbqCb^TL;Z-4f>4bAaP%y2a#KS-d`jaN4_XiM zk4yX>fDB`W2&!HZqhp>La(e;Iu zc|EZbO+h@@xjfI$Twq`=nVH#21V()E*uWZvL){+OS&so;>Sgs9oapv?w?oKZR*yFS zhloz~#BBkFrQn=*ap(D%Cji#@m`45Gmr*|k-B$0u9%7|bA*f&;^<|KMvwHs0NZ-Jt z7EH|tpuUDNmbBIv!ZU^WSTE#+^2_NFNF%h4B7a!FJT;rzn4cZX0`Fz2*9r2Ea#=i2 zlm1aGLn<mqz4Z=R2m^RDSwthB&x~LzA5{ex0rSReceFRrJ*6+q3>4;Pr&9A_ xz(JPOp9^j|sp*AyS1#XS{Igs~cB{*OrteGu{-Fx~8S@SP2%}68Cg6YE{{xx4?zaE{ literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..36483e4750f326b73db87d1cf6ee41247caf15e6 GIT binary patch literal 8054 zcmc&(TWA~U6`qmgYn<3|@=rBcCF|@a6NinXi!ZSkcge`IC3_vM?KqZgN>f=HTZ%`v zD#_P}5|&Ux$iuP^VF@LK5c*I`$wLUCEF}-6EXzJDO9)FJ@=!u4A?)Kmgiw0U`DZjE zS+cdErHVb8`Oob;m;d}{oC$Tx&s^*b^NlkH6WRqqnud$85HBQBU4gEEFVNHF^TQ7l zSQ|Th2AJDeJ*yXlMvns+T@JwoYag!<2@AhN}A(1%E7$2&n$ zAs0_9r&pFtD57)1RJxefHNco#-RSan{{D<01OQ|^lWZWHIygwt=>V<26WOe(J=*_1 zXh&JYN730|rdT{C_-~!sTP?)JbiUCef>!+O9Eu|uT~3ylLr8TVmO|0Cg0!%|gfGYObSLn&Xwq;rYNWwP|Gh2s>uRYQL`u3_+N+ zT)dzqt9k0A7+0;7U9!Vl!T+$dkFob`mC@R7tBiVHaNsU-7zMZAXXkLr?GD-9Zbs5- zZ50=pLzaZoebC_ql9Q`;G}&F9R<5P9yq#gunTYKDOm^4G8Ys8%zOAWMS6>6y4$X~b zbCcc0+j8qZdmpQ2-rllgcP%2NU5;`y0@Z9Jr;hrV?B?Gb(hQJIdZXF50C=%=l_gjd zWVa}kyVGFwCE0D>5r?dRs-w~{lg+cMEq>@Tw0y64W=*eWE9LTc@|T8hH|->4(G4-PbWopNa=Qxu^V zM4ss~vPYU`eG*7W5kdAy(8>8z5})okSwGS4LN$4$Z(PRXO?rG-N4!Y6+h9Uy3(;EaC?4!Lly@TSTC4H&luZ5DYOtO_ z;`25`DH(p&&O&5@qq=>Fok!ykhd0L~E%6*Qmdp7>3N{;S6nY)*!VDo%D^Qxv7FjA#2ga=x7pQ24 zl^Hn1;Rr`Ct=U{U6VGjpCSj0B7q*y-TXLjL!@$(E63=M)Y&@Yc2PeA{t4Ym7MrvW$ z)bfStbVh?A4(v3g^C3;w;K)QX2dN>X6T_QXEtl3-K$8pC>uEg?Q*{0KAb8p9a=Y|& zz5sTB+5ieNfXlSXo5^U%G>jo>z=;H$G1fq;#ojq!=BCIUurHQssK*{rTml6yoFvRZ zICJq0yS@56CC4(iFKvYGLOct8IL2Y5Mn9UCwXMtRaeWP41q_1Q2`iRd9jEQ#`Hpl@ za9x-+L2}6+`Axebcvy?vE7Z5iofT_qXH7AOv~R zHz$}!`uZ?~zOpAk+kPT+pE{&3MdglDx$Qu7HIErJ4)}17D7N;m{wNmf~`f$t;q^BpDBz@;*5qsAAEAEu?ZU|kFAl-LJKZwfR2IXZP zMzNMtwq*3GX;>>cmn%6WJrPgyX+V%(Vt{YU>}5$gL~8QNYUk9($-`esDdk3^@_hsP z$L9=VF06>;@aY-oaunX{3hb(D@38X}-rX~89TH?TD{U%Qo0KPwkZE(kbQh;i%=FIH zN^s0f-&Y{he|^Wuv>=}p>Z`Ow>vSR@KB-4}+%yQTivoD-CM zJRHgHFQlaM`99@EGup+Go96C{_FBxY^9l{!ILdlT43(U)WhX0{;=TM{i}KYzG**3< z$6j7#l&=V^*vqe<0$_hRTHeb`kQoQN4bg-B%J(gZ$o48=do8Hk)jG{!+((;VA7c~t zVDy)mq|pIK1~U4{Aw3n9Ypu$^_M_3ads%q9cZ|xeJhd9glT+A0-evzTX&@E6F$NxT zC?#c;A6u0#ThSfsMaA~cQZZ{Mif!vThVYByELsvm`v_%?O*k%*Rb475&)bwgwxQJ; zdXx1|BuKB(;6IPEklkRavOzwPm26z|0C?b#UW>}JcIECt1o8hn0LtFG1nE{MLEvaL;1mCI}q;+_NXH^}= zCs;DS;R!{9!yu;4;_~nkN({rVx;~NfdN)ytdI2x+2@YQ*jieRysp@#B&+F~?A(RL{ zqayx^_R}|-3XwkIk4_`M`T_8ViEi=_U${cg#7g)E(Ee8F9PlTQ6e2oYCjA6T#cPOM zC;srny4UNQM*d64YVpsR_+gRgWF!$8%Px?3FpqdJkQV;ok)pgBOdy@?h4p1%B3I{Q zoBBd*W-Y0zB$~{O`P`d>HJ8b)R<0pGv5qe#A#XD8_#{dJZ_bsUaWUrSZap)`R?G z6WfqqvTx!EhGN9TQ&r?w6W$50_agC^;1AmIe}eRlIgko&FLcjr%>^J9kx>86d|Wc^ zfqz7&P$x1Y-9|n7=4VrcS^anq`7xqXDJ3H_TMNrNSv>|Py1m}*5b~GRqs{*z zqEkI_TYzCHIOkp5xj*I!fVDrSQGfRp)Q>^8)w{2USSeKqDws!o8RXxrp1(BGH*l*3 zQ}Y3+uVIWOJ?jhMnZkUm7ji=R<@5-o5n4x)KdfJwnoVuY&yHn*_X^eP1o=m~EbgaC z|0tFr6`8sOU(g=_mX13lO#NpSpV6{PN#gxVoq(;qwGE|P9#7l;4iX*upQ^@Zqr&?Tf(z9Lgw zbFpYH9_w8p*(l+!zW*)|J&LGdzZVu_x^FTw;+qSk#%RC6Ne5s6M2{NU;EsXc(!s)CDvd1JOa+MDQ}(idh13iGp5srfMA vAWQ1c1-G2k^g_HVm+vtCEZ33U>hhoMI~#yMRKcGy-{MCYWr8pP|Kt82^wjSw literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e68796950c95d7a14b0686a5e74b455b275b9146 GIT binary patch literal 8053 zcmc&(U1%Fu7M_vi-#D@3_|9rlCF``##Gz62<(s%Th`nvY{;dun$Wp%bs)Yj5LxZ zTQMvv8for5=brPO-+RWLQfIu(&Mq;}B(pK0LlC4{_!1Ul`FPUhb@@ELK9|Sq^7xp* z+F9o%U~Xp(tU(Z(oHk&z+XOr8eZW361)RyqupeJbr&D>?+sfG!S#lEOA(HvM9uSnz z#^Nif)nyBmg4CsyuDN`!?AoTw+w+G@f`BMHm}EiO+{ICfbv6+DrN}-p#fKlf2jcN; z(a4ID?_W4vFu2CFzsV_rT>KpDvZo^ZTr|NXpL*cHdQQt8hUTia6-#Gy?J{`SbH7jU z_^dnC?62eYgKtcj&0Y1NQA@{CdNrJO3T+qbHc_?@|2@0>tsTz2Hh!z=v!OnLonRdp z3xQ`$qr=D;*pk?36UnijMvxd=)>iW_FYr297alUw>_i_O66m|05P?+lY@?$N&U!C< z;N07#=jw)*&BgMmwN!hC_I*H_q=TnlO44jDQ$8Tn=wdbBRKWc z#$*To<&frpY|@*|wgtcoovSRsq98j&q1>ECqb?khw#i;*CeMEK~>71e65{c$QbCDgzXKsvztwO-arNw%y zB`LhtoA0Q3BoDxSCT~zI&0vZm%l=gn9=b2GP{%WLVIX)Pa1#PTuLF0j_~HpfY#t*vL{njeSB@x*yy(uv}b zdYre@3X~SJMV20@2j#^bJBVn3ojJI};RZ)AZJBH;9m{TyCt!|9<+qugYqF(G#lY0O z8cS=rOf0T38z+n z`dn0Aipo6^5mlcK4j3FN;!BwL-gGmk^rVwPUpW+@rJe}g9h>yMs64Dwes4o`HIErJ z4vjW<05bc-0 zVBc`G_Jh*X(vb-AucamByL#nby|Xwk6`v%br!YkocPlw>XaL6yL3(zcNzzjX3t6K+ zSg}&dyCHNzg7lS5dIiyMRQ_3yUaaMoEg8LP8rDkgx3dtH(a zp_;5JwsUIZWZ`e5q;juGdC`dW@j1ho3o9b&yf_D4j@Tp7TgF9RpHibVz;rE>Ek~NLy_g`$rl}6W?d4ZW1SHbGt7UidAbhSAj7Y5}@ z_DRl?$yt#8f}DlZyqE8`Do>7~vg)%u_VOyDd_`b|UjFF?0QRRd<-NQFnQ^gO5bYgTUbG@2 z%d3FpwV-lW>oJ|Ni#ng2Wm8sX43?;*(E&#WQu@dyy%3eVZOW_TsPt$r3rBm$sBFsf zVgvd90ydCevj3DckP6lq0}mOLlrqYjHsx^}np1pHvAnZXj9Pc0ZT%Hpc>f#=m$=Y6 zLKVj*9G6I|F6EUU+m*d`)LKJtD!vm5(!WvR%X2JXRhX=-kWXYKg=<~_dp7A`qVl{$ z`RXKs`2Sr1W$#^r^t&F2;~(E;Z+IMe)2rP;nk31~!_rmd!Kq=eP88ua4g*=}ffq|( z=LG5N9%zQU-Rwg_dcq{T(j+}S)rl`$j#hPM=Hti*{DsHyQ=1zI81Vud4cKAC|CI)p1>{y35XM2D|wFM(3=J3yure{gEU?e@$f{|#g<@-LY9VUy^jBp#Z`EYfj*4)LHP zi}**!3g=aS9O5GxM^@OU@(L{PImxz$+Y1Kc?ov02PAz0>X*oRL6 zkAK>c6<@3-Rr80nh} zaLT{4*gLnq-~(TT0)zYGal_OH_7R<29Z!$-8u{pn&L;`8`q3fhCx}j|lnBjjFRthi z+rT*Dtsc*jKBNaQe}dtS#nq@DUkxY0pBwx zDO3o`pF??RN5O5e*joIR!ETg&l;{#vr0kY{YssJ(%#;I z)Gha?rM~?{hyJIkvH9>Q)@UL)hc&B4@cb4&krDXoSCizA1tjlVxzs;{`a1Yx zCowjYS;X2NC;1#2@xs@%j~}IcqDLylBT46}n@IQH zTJ+86Ts@lRUtfrL^?0v8Ni_#3(Vpu0ALv7UmVP#cvZxF_QlUq4?&_6^muxx z_j&}Sf`1OQXXFObDPEzO?S)7<8;kU>(%CTKuU>!Gh#p4NQ1AK0i0+wAk9ih+$qA}A zxaj~4fap^Lo7^z4n@kPjp<=%s<5trqJi--#k4Akg^!k_Lz9DZs9o9?bqbUCpVBVbX z4fn@;XY|Fnp?q|HCK(L^4$`FlN^r;tP0z<%*<6?LXSuG-w#$2Y;Ia??Pz8U+Ji>p% MI1_{^_#f{71-z>F-v9sr literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f63e35f632608e45bf512b04773677dd9d5dc260 GIT binary patch literal 8052 zcmc&(UuawB6~9-O|HiHz$KRc9s$@><+&DCj{`gPq;FVliwq(~)ZO5@}QI^TlwWV0H zHA%K?KZR0431yT}!YF$%Ve~PCQc4J;`l)TiHctQh^>5W{UtM{{Wf`BMHnPfuQ+|5ynbrul&JCVI-h!6bsO%RV~ z3r1FyeCN!;g26S0{Y`ce?Qk-c(9VO0op%NV zr`xg zNM$`Z_0+{=8~^2y7JzKfn~bgn!1Il(B*3B|+eD$%oJPGX$u{GOIAjGhub>7)MIE#t z*@oYxA|N{om0i?oD08aNN~N^$MgzUYCEE&%vKS2>s*lLFL!HyLnj2@TjI@>?LligSeFh`Mw*XV%w{q2LtM9f|9%yBtn|9hPaB zr_qk)HErkGRyeVVrUC}RRtGzUbM>5dfaPW3Am>^!YJy~y?eb5ZieP80@}SVrA@^3y zt@4~=7Rf5@O7DxxywYSJ5_&7#@0ITSqelgqPYRF+IPqD8QOVEmdqMiHhaH#xbd05h z10fYBGQs&!ls*=f7ou`YL`2o6gM9{viuj@izL!1BE`8m_psySV&{R)^?tP2&m8k61 zDW6&pUCm=g%>#+6BKAGe0RPEpc2auE*y9?&^QtG@e;1|CMdhhgc~*zGYnfka8FDJ4 z{V0$2b42^;&)63nt#!ZjG<76`{7Y#?d04O9s<#*BrNWaW^ya6i!fqw!jSS(KAxKY7 zGfDc!#{A}}4_B;|(ryS{kRbiuBK-i-Z&dzKk6x_hmMt2+Y8uvx?&V4bNsq*{d>Rm> z@6o|`WcIhBbnw??RiT|z8z&2YCB>9mP0F)Iw2#jj`dnBMN!Qs0=yK%V4-}}X8?UoV z8njq4~fN9H5ofzrz-AZtbNbe~S=|`{Ykrw3BLPM37Sh$@C zh*KI+9yE`F>7oGREb6dBe}g+*7&U}{iNZg$vvcM&S}ecV^ec_7G2;R=F0Q=QcUzPn zo6*(Ae4HPYE7>R6iza75`Xl=M3p<-LW9co&VqA`-_dO}9+&-ebXhFR=a?{*hQD3Xk zb$&oaH&3vEB1J_vY{|_^s(3HI+p0W1g37AT^4QC(jPezM<$L)DX8_p46Q#Yp2$_Dd z8;E`*D)-uyXRV0H^eSL_EvVepdJSjnq0X;QvKg~8hKp3v=zt>wDScp(o{7rMcIEpv zRC>6Vg~PpLR5se zw)JOp;U}k9pvZ;h5vnjY;kZOv6H-=r-l5#_OrouasApHv!{`C~|nia+>E94Vd zQQ;aFz`Pxp#vX`3sNj$94yP z-ddedRdp8r;pZJy&8cIn>J)Cluc~@zAC|CI)k$2zmhlTKJn9*TW9lMqkFVm1armch z&7>WUZ4{zj#xMAI#;=e>k_z%vb=o`Ra11*UiXYb~h<^gZ^oy#zB#-z5^T@Bh1^hmu z8~o#!uhW`X3Ev3n-|=1m{s@x1M2D|g7lBgo+e@Yszi(#C;c(6)|5aoy@GlwoVUy^j zB;ub+Ez@yN2JxUH3;4$;^5<1g1nH#DmskTPGBp(3PAmr(HlwOaN2AG!Of*QUXI0NE zccMD1hhUYjVIMwuJf2y@epUT7;?E(;lMXGesbk3M%OBR+CM-v8r)3&6=$jXCitj$hnq1IRx$vkUPh{RSS7H%Lr8RC#_i;+Szb zt`L6_evcXdyGYM1!70z~a{t24k{f*C_YRlG?1n4I+C2|*YnXCT8t59^_>IG zPZ6C`De7O?Szb#(Y`v3+w|YFs29WN>{P6|0me)gx$a)|K{@mi{d3=`KBb(9W!cN3J z;S2{yHpw08?nrq)x_PRXcjo}HQm7D=Cxh~m$iH1ZesQGdu+%)UkQ?%A9DPaa#IkQ88ww6WOensb z?uRf!>&Wx_64&PzW4X}cR0?>nQ@+lSeH6=lJ@KO4A4 z%_;?n_bYVCjB{snqyU~fc)H2rgB+2K{h!?(Q-TWx!6FpKX#!%k{FXBBQK4WVRY+Zzp z&KvbnR9`uy=aBBXzU-b$aP?@KXLBj&N<{iSF{(L0iS|^_fA0Y5Gxf6>ltuk8II*K$ zO{`(8!;5o+>bMsqas0f$0ORZ=*$;sOoN%w?czn1MP!i=RzkmEwTTvI)S8CY}Y3YX^ zQ5HW_J@|-D&?Cu_{wqP03jW#Gp0TS)r+E42c9w#HbT~M;PGBXfj!jX$!A%EX0K|am&2huPZZg$}hYJ06f?G|S@Ca7`KI-+c)bCk|xJO)( zWFS#2A4Pdq0CR4!KQI{SpGzz+jATQLbFq*QaF8bT7lKWWYKd&PFP-Vu|18&?+Uawh ZA3E=bKUBe=F%R*dFv$dA2L6Bhe*tZ*_Amed literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6055c50dfd4f854d822fc0bcb837da3c17115c08 GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA31jIIKH!*RLMH6GjV7f{rD$#@s^A%Te9n@w&Pf~D4Ug~v88xq zYbE*jDdeGq(1%i%5K13{Da$^ZQc5UG3Hy|%C4@ZWVOdJaLnvk0hkaN=S@xWBXQYuV z*@|IV(MWUe`TNe#J@_z4zG%rh+pW8TRAr>2xaJ`=)yKMV6cbeTZaXzXufM zv$6PUYHh^=rJ!{=rE9(3-t78TueaxqmjnS(b}-3;($U3High+n`=!V}Hr0n8ybJ14 zw&-L<&G#=Htr$XM`rqUdK`(v|57|?ZeI}Y%Hs& zxi>4%wM{LXi{(@6Yu1Q+57#rqM$A~a^k%S}-^hYI)S$OybyOX`6xr=U?z2A(fTq#) zBBHLmww>=?}TWXDn1O=g{#j`7!oZ+Yq6r( zZ#9(nAnk6Ov3gFXUbh*n-AMeg-B3z~pLehTnc%1%oMKneIK<(XbD@@478=XVTs#Sf zjWr1aHfQI}SUhtxx0B0j>6>XSA4|mYG1e}ymh(2}NusT9WaFA2hslY=d11X6#X z+i3+#v)Lj`1?s_gamNlSnqg-i?r^xl5lmYqn@Y#BI}-_*BU1StX6Ke{Dbp}8Io4un zEtiSKHD=>vdwe~inaD^j%$iy*KbJ~tFu;KwM=BT4bPeuHG;@*~LMlGCozb!>Z4ETp zale7~b1+0VOpSn-19qofPv!Dp2dIspAPu-otGt;^1Wdyil13azz!_r?q*|<<14eF& z+*{VoQVk8*BMMufz>af-8Hl=UY|Cn|AxFt6nmds)LU%2e0Y7Y0Fi@i(j#X{v=4MRa zKvw~S;H-n4qFghVq%qIn?0~{S~!iW^&_q`zf(9KRuFHW(Ha3rMCLMAwOMd>q9 zc_}LQL_}15Iyhu-sE99N;(OiAT+)+H27Tp7fR^(_=`yZn8wWvI^D=+F0cP;ZvEkjOaw4WBxeuHSg{1y9# zqqQGao|cY8kbf&JE8o>C_v&56d8zm$2|a}=s<>Mzc*BD@W(d-=^GuSSI$6k?^`VL< zrMw$L7bHku*`!yH{YK@V^%%ukZrPI2tEOSCEb0pMl;g3 za@(OiZh}mk1E#Yubz-Iu?pA_hW_nM7O#kI=Bh!L>UTCP&5{tJJ0r5!v$|J`JxGoAX z&Y}$$^f$P}g%MNvBUFCR#V%RPXyM?+W?X4>jhPpid2toI-fLEVc3`N@`M5ABSMpDC zl}yfp^jD1ew=Ono#nN*Si+MSc-H)Y&^7#qnWi#5vk(=i3iuPK}uJaQbx_gH8ml!I! zVask-GR1rOUW@YN1RAS8%VRIEGRjv3R_NuQTmWEyK2zSyOOP2Cy9LpHtMZ}+5m{aZ zEUyKXyIPMKj6JmZ9iy@<&x;M@ z`wQ4We$D<<(m*PBV+=gxP*Tb$uiKQzZRk$%MaA;YQZZ}Yg|_uK4B`EAEL;*o>j+gG zn{ZqrtGbj|er#9v+tF$by{Y(4BuM{8gD=mqfYo5KvOzwPl^k620@$}n{}Pqw9m-cH z5yb!R0w{a$5~M%$Kpy|}4tve>$eUj64$>q^Q67=5Di2PLfOnz@uW^{jLJz!H`Z^~_ zU-v*W-0Nl^2+|WK*_9^g>8Vb9;c~X9v$G#X-sdko&Y#=d_<3(-TvgQ>_=lelRdq`p zT~Q}-3w~AAPy4WhwW?0w3bxE&SfQwY48+t0+#Xv&i81)6Zcb<2?rl_}Uc)c=_{Xl3 zM$!uURCOva=ynfz5K0KwsEB{UL-dQL0;G@l!*j^5z6bn4qMQ6<*KX08SP9=S+TRIW z0sc6W0z`+e883lS@jF1S6Mt}e)9v=mA^#0zE%Gm#_+gXiWF#J%%q)?(KZkfQkVX7s z;{|!uA4fXb3+k)DM6O07+xk*uej}l(B$`N%=MoVzJ)`<(co5YgBLypb4g2sZ;PKCx z{;TTm5&ssF{MqQjsyd3S!Mv``5BnA)+ukUN=8&F=2ElGn_h>h~$+#LALVOg2Lj1?V zSAdh78uj2)3cq;J`jLNfdKdCb_DwwgK!lihstWvS+&%4fUnl+&{C+F`kC48#2vYvt zrM~%{MIXc>6c{>~j~k{v@Q>&e>Ues*&!|UFbRkKY)en!5pCme!QX(|Jv$U#1ZUYmD zw|YKD`;i{N`U!?Nm)4?sd@Y=Wcy4leo}WeE@J1p%zZ3V3dt#B{4GM?4JAAMneZ17m z>d`;l=XUP~kiV=RE&h)Zo$86(0t_quMfdW-^ARNg#q%+T`ulF7ehj*$-aY-qN~uCn z{v7H{BmZ{w{H2h-g-6YwjQXIy#xRz&)|Z0w`DkPSazgp#^bn*GT1SCDsNb4hNNzu7r8Pqo*7Q2b@ z*~}82?FrJ)p%E{9&G@(|)e}9^QB0z}D*?oNM1DqB-PpPaAIY2NqvU*TA$<$!{##4F zS)H3l*Zdoc5w9Na^C#)d0ZR0zdi@9b(VnHBO`|UAhmr9e?S{UJxsEN&4ya=R(8Tfc z{sN3M6XZW64sgP~vc>bmgMgZ-M)~~{n%qoy>3n6Dd{CAl=n)6 KL70aB;r?F_81lgY literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9f0aa513805d0e161a9939c3107dd08ca56b1fbb GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA31jIIKH!*RLMH6GjV7f{rD$#@s^A%Te9n@w&Pf~D4Ug~v88xq zYbE*jDdeGq(1%i%5K13{Da$^ZQc5UG3Hy|%C4@ZWVOdJaLnvk0hkaN=S@xWBXQYuV z*@|IVrIF^|^Y@*fd+wQZTAlSWJG;a@Q_RMM4ndIS;7eGF<>Sd-U$4*O>+kh=dp$lT zuy)pY37Fei18We3CYKEu?KZ&`Vh&&eh(S>cbD-1@$Of zbh4u6`xlN@452aoZ*qyC7e9xG?5W5;6HPG5ryda4$Z6T5&|KBFW9f{pU4{sI?)M8G zpLM62|8?Adh>Z!;(Nzx?wR9|{SHo$i)ON9M6J`6zwcN_Lc907jzt!y7(4N5FVjY+Z zfmckU)2JBOlGtez$%&pu&=_0M*7CiWKo{#mA*0OR>L($AzUv7QXftRiAe;0ivuy$JLgy+=uqen*Q7Ct((P&Gu(|jThSpmz-Xu&j52R%r3;&-_UsE*>v zF6lK?IaO%oTAGWUMs^k#2XZuUtUe+;k0obVw`9UmXfCp|_{^QLuvJJHg|v9yYAFh@ z_2xTn9m#`mpUIn)OEa0G2(=*cOqY>e(i|IwpPtREEO<6blKlWrHTK0jedv=FVuiekUj zP~L;IyKTnmIhlIhX0Uc6@yB*UDH(p=!2)E0qk3?PT}9&%hhNTxT4GsfEH`uUBpf!@ zBn;S`oi}6g%+1_RF0Z9;rnP)55zEI|yTDq`+ngtfw!V>#YknLiClcp{DHp0kYAbK2 z6)4SSi!2qW2jj&ZJE&-eoq4#!;RZ)AZJBH;9n0=aBw&t6<#(8!Te780!@%TNi>0+( zCKlJ2jg#&1^@L_3BegJVYPtMeDy_i)2X-8(TtL$`xG&MnNookG_}F$v%cit7&}7H` z2HMZT5Zy2}0$vW-opwEy%Yz-DHiCjQ;4-c9W-<{l4P!_eaUcO_j6INQv33p^xhZmQ zSvN~HG+>V?Y=Hth&JkuH>awvdtG$LCC8ucaM9K)=wO9uHuuZ{0jea;*wVj)rF?|DF z1q_0-4t9!i^_+Hu=Lgb3!L?)71j#PD= zsN53~QT6HIkinrMzJ!VIbvJWKPdXX&l_LRK&J&@#W0SrYm4|i87dAv!^O#ZdK;o^4 z{h?@rzki;cm7X*9qz3T()D!N1h|<@h^31Nhs6*Vf%rCVJIhE0VT0r{^qW$t$>>G~O zepq>0Iub$tt+cFsSFhZwcNOQQ;*%uw6sD-+Zl&N258{|1NYBnQNqXvJA#2u$DxQ?` zZU|kFAbn+%UP1O7m4DV_6l=LUYv(6N8$ZMfm3zoZFZHydw8acmjoHjNZZP7 zhw``yGHni+&cf7*nLfB%367cRJq0rTm$!{f3-WoPp-M|E-cAI>BlRnf93$YmD8M+2 zHeAr(;0_l?Oy!SI`8^lAWG$nGgBP1|rO`EJUSQ_MRq%SRS^3$4p*H8^!k}EqKgm@x zISbNXG3MX8*sK*x&p|Ba#= zafz(zQeOG7UDE7 zoy0BpRaHOj!xGl2I)N+LGJj!(qW&=uQx|Z1Yy~C8;IFzloprmnQHgpDzu@B^yG|NO zE9g_zslcGyJ>)?sAzY&({s|A!FPaLFKH?A0A;0<_@CS)*@{e7+MQdUue8XseCvXM$ z<46h+9lmC~1WLv40J%>5!Rbx6+cSs!H;}c+zi8rzO`?;LcxW=SMB@G&;=w=`@sEuc z0~dcuL2Xf8jWo0OOg4FgsPHgB0ZiYw33RELZdtnfAL!>549KV$l@ zs=r74TS)R}qYJC*D6$6gx;8)TTa0Xbqa>O`dL|kKyFuNf-S8&kYG4TQQ4k989}8ar zPHt+{gHI{^;z8?2{>kZG$S>J9@%RG~V&bVP@T+n6wA+21_)GBnt@uAe`qm;y`FEH4 z=64o-5Q|V?=wLo>nD)RwqEo2j>G3|J9zD^8BwdI0Mu7~Wi3i|X;Ua1!FV$>n)|7Jb7TiS+zV+&AutMTR#h9O~}y!Fu%ZQZK7V z|8$?*y&FLOvU;@mKT33}CvFQctoRq*%LmU#lmHaZ#~kYKyM_8O=$3l-^b;$k3PJgE zs4tED+tu@zLi!dSHGeYdgZdi7SkhWw3eM-FkpajF<(Jb#kVa@71^%FZYjz>I6ON=f4VN}YzIy|W9YTOLo# z`SueX`k$)C7Q$nAMiaq#JhN&9<+t&PoWNhdnxuFv!f|lZ6Yd%(m-=T=-+)-`CdOwo zOL(>?NI!>0yzn*Smqz4Z=R2m^RT(_y_-KdSOC1Lm!TzVJZ2 zZ&qKLAI?V?W|Pq%;2=xtZv>~D(DZz)H=FA+{#mXov(xLnJb2j$|4;@0jCqXzgb5}H J)9^ps{|jme^EChf literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5c6615a083205f394f8da1374cb30689b69faf56 GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA31jIIKH!*RLMH6Gf`+9{rD$#@s^A%Te9n@w&Pf~DVvq0v88xq zYbE*jDdeGq(1%i%5SBg!QoAv$IRgJIQQJ=nw>H7QTd~SU#TY@%8w<{=OcsugBZV z1lGhZb%a7hsSfU<*07L<-Ij#8|%f!HrZ_OU5G^59(% zk7tWUR+N1I!qI}kHKzSdE)nG7=Wv%j71?K^2`2f}3kNoGTJ|V3SGDa}I-_fs!NcDB zeS+6--Kl1O9k(BRW5RTF)q_SY9ZTueaM~%fU98(g**^T&&6RKLaPFD$TTP!0^$F}P z)`77Qc*Zn3jf{aUiJdl)oNzaS#Mp|qmhbTauZwlzAtTM+>Z3yfeb*BrkZPW7bk@OH z-(@eHd$aUh+tjkTSU$DBX7xBG&>?aL*|6ygm);DT^BdVUER!Bz$?B*)d?~Wqh1h4m z?*~aE>qSIed2Ku2wY;9~IwCdyK6FTG8Zkf3&YIq8^*TTkI9kib@>-&rt##z(s=0DV ze0VeXUuE`L_Kqbn+6FC&(ZKT!Y_;TDtJlg=CwJInXUD;gRB@BpWJ$O<01Zx{b8^*& zCOfOs%B8q#?PjvGR#tzxj(2NlX|l5ht^<;r%*G}=3yHsXWM?f$N}C+TW(-o< z2u?kXDztJb&Bab5JBy10G3q~7ACaBMI%jCNM8Z*MF0!-u%#E?IRR|cl zv{-MoB!$m<^Bp&j(Pj zdkER&lAhU(<)^g#ww`#2vC*JBCd$1gLrNzSZ##&HkB=Oz4+P@lUNZ}jY!|LRKVh6` zAzF(S#eSNxmZN}<3se0XJuy!Nyhjv3KDSqC;f~10@dT@$eMdjd!pU;I`Vp(V` zH*@hM6dP+2`fbk6o3VK2W^N~!*U~rBT0WMD~&21hV$nQST@%kGRPV2()TcbJ`PvZYML zz~oqqrL|lp7T1`KlkM^Kgk~Zmu`p|Dx%_M@t-$~XdK{@-P}4QIFHy}&Y6z+L=ypcS zrnEJXWXJsm+Rwod-7q-}R`%PSc0HBLgC3wZf`ByOGPUw%G9EM)qe~ibAOT~HJvh~3 z?Hn+2ljq*DZk9@Dz#dW90s(fMBTPrsWn)`beGNGZPEp;7l;OH-u?*N@n}mTH?QpDW zJ2y9D`UaW`7zAe>>=e({bJ`J>9|#9I*N#yWB)jaAf9_BO7i*FGg@$(7T`{)~<`lC@ zc4=3-BPz2>lWRb5SGeCR-T5a^3NoJ*AP;clvkAjenBVt;^nEuwExkC!GQyFNN)wsj z+!dwIMCGNZ+!GN|_37Y{!J#6)go*EUH*-l(IvMnpBLQ0KiO}7#N#BXe!#d^nHbhtR zm{IdU;;V@Lp=g4?f1aI{o-_8O2JpP<3HLul=_^rrW>;R+A?{k{ms*CL%4k0=p#2)r ze(?+THAibdEIln9i6H-4T2{WTSMJrjit|$ONfO+JDXO?z$$3KqIA#dav-3=no;q3B z8uh`7l~Udfp$ihEFKyB*h<>B;&wBJ?Ew^mR=vC9OR&p;_GDvzXUgXn&AiY8dKakmv zCFu~Z$*N*Ir#4O&{zghF_nMR!jc6aAGmN>gB9hLFbI|3;y`LyhRd?QISINDHN4j`P zkkX8_t=x7fkDDOU=78xeOr03%gS(aBn33L7Aku$++laIvpBEacw8Y}=L_oY!pYq5t z45o_$jI*f21^o@~aADXK{s@KNbFoX-G@3tnu^CqyU1R12W?ozctM{6fpB(6Fb3QH% z%9ZSsTqTpUApHe>{*8;xSh2VdVlgjA()+QLP(DAQylh6jIC9h6T~S|)*>!$IMR(7z zz7jN~(A--)m8xoIqvOXL;=9RYv)WzzV(mlM4XsPiM+|c?mM(Vz(gLZ&hBj zAR^1FfaSHIa#wSk&e%hpPtLL_t1|{mRMO~xBLgXYWRqTq%H1~QRVykz-pj)A-Z3hh z^1RqUzPo@8Ll@pZ$08*z zw2n~4u?fc|(yB{&<%f1D=Xv^SxMoV7r?$v`j@CY z?@+!xi6H)e7eLv2mmvMl4RQSAJM1-&BX4@O8%UERd3jj6sysL~4AzMvyvAW53q9~+ z>Fb;zedUH`xYx}-5TqwevMWu}(^H-J!sTpHXJ$T%zRzEHoIkaB@blivn5wGN@DD#9 zs_K?HvZ7Al7W}Gefc9YtYgHY`6>OQmu)?E(Q8=c~gH6|NWg= zPhj*qNhGNtPgN&_10K(y7omi4je__mGDyFuDoFB(KQfE_>U+Q+BD%>xdhHgiiIwmT zq5hrV72uB}DM)nqn)VSW6~BXII`N05Ha#BiEb`w#)*}Cci61tJPD+n_2qUzP0)I%q zH8Y>wTAZKA0Piiz*D11(Vp*uCY2P@eAtjl*0zc3n0G5sw5~BXIj%(DcQjmDRQm3G_ zcXlCl%l&Do?*P%E|EX$hJ~E0mnh4Ed&8ks6zl~311pWrpB>7_j%E3cVxNB4{_0N#L z0lwHxjLl@0u(roZK8HrU@HOq{M=77^kxDU%`mO{K?-B7CS@mG+B7Ah-tdElV+Cusk z(gU}a{4+XNkER7S7NR~q-Wy0#%>hcZr+WSe`%s^ypG~1G>W9&>9qop`im{H(&-AOK zL6F4p^Zo*i)8k}61P*Y*y|TsQ!<~SVC`bAI6Q0;i_^7@zOMXbpF!YE6@w3!Ji0A}8 zo*wGG9!06(pF`~#xq)^bM3kw!OL zF)S+@Y5xD5|D5k!|1<8iI_qXuc8_^vAsYI^#Rps0lS$qoQA(Dmr4iJ>f zgrlp8wG|VTywv4HQUj2g^{rmFpL z%VXZDW`7;GAADoLZ0@cHjan+4NLIsXrO9s^~^$R9UXzO=%*$NFzIeHPRwu-8~8 z#zNp3(`eH(2DT)2+(2@~(FhX5E81GF*A2XO){TesG<&U|4hi&KPl!OOakkM`2WQ>q zU2yKz(sON7%Vfj3#QK`q{&oYm%P22J2+EfdaZv1+E)k&~;Y z%0A)Yz2JY9*eBT=ro3n$GUY`BPdBjDlx}SCoe*3i*pTMb-glADaaCfo9lrP^Ju+QMX8Ek}xb9K~iF zQdtj9J#{hJ#(z1ac_16~CZlTs@OLar4Q0H{*rbsmBn~Q8KJac1A zY!w1VE-lnsElJ@vUwntnBY6<6GkKF@X#`UgAs0j*=@PPCnqz|!oR9*7Y?q*s^HmI2 zdv$DxXnT-NcIl~AUw%Z(hxNoujEy?w5m9b6=~B9oc%zw!`1qii^@BisyxYROB-@Uw z+eh>htwd`zqqx_m3wI#xX1l)nmQ=lL*I9dz_`E|GN{XL$G9RhnsO}tNmryzQ;ipr9 z)^G+I%e8DY4#mcrgaM1K>smOPzLwp|=Cstcl$Hy}!nrW(5LoMJi|r`U);BUy&5N_- zMC`OMWk+#HZR72<0;R=hk;Mne2V#CZga z(f8m~tGRQ)#7&-i&AeGGp#ghDehUOxagHz?QI`pCne{bfDL4gnM-sa0E{D@#hh+-p zX|$txRol6?8BT7XsenPS)xl2TTs@~9V0l?M$hlUGnjl$ayZlq9BG_50JRmf5$c~D+ zRi0DKB3Y$f>3va|Q=069f}_IyUg^$1dQ_14qyTw<6Q4yGkpld_7o_ic*m3F4$5>i8 z5K?g>6Pyo4>0?oOAu6{-L{xn`*k^F4h%aW~d)dS6($`%K`pSU-P4z_R-nU3!iOOD` z@~H*U)jVd@Jdn66V&4-D@SmJ!C#9#1J+1*fuX@7$cTxIWRGwOuXLX3XmieWYA*V9h zkMd|gN3@^*jD5k;TK7v&Q%54mzm%4hhxN*>dV67BDm+PoBR@qIb}Kn=co4@7L3(nU zNzykq7BEMBsA8p*c0=fb1nKt{=?92@qw^kOZyY|-dd)38=_FIO^1dL*9Z(|{m- zj}E>gv%eLkL!c(B3hkWQI9d2BDX!dVQl2%UeSFT)=fa9ey3Wo+mm~Lnpg>jKc%5A$ z_wFC*!X-gU)6%wbqgi>>1d%odOj~~H#7LL#R)S+hdQX8!KYCq{v>=}r8mhF!!tF#r zTvEUCpm_vL7X=t+QHLG+8{FZ-h#~w-6#k)|oinG=Lixp}UukrW85fvwapkSP+oJr~ zjIK83pU~%D*x9TZi=!NiaXFIS_oSF|`-t+Q1@+>{O>=ieeXT~< z`2iK(Ji+>l6cydDB{wUn;=TNCtMd2=Dyu%rV=u2V%2x!I@8uty0bmbLl=ku>WctN! zAo`7{+-p;wwIU+ZtAOdXpmJAp7|z&3onN11(`IK36{)1r0Y?T>`oJPR6P26o%J*%k z^l&c=hkM7UY|6KV2J(k9*g$^H{!`RIDp;crJY-N@N-Hnhm9N^-oWhHW>7Auw)Ozx5 z>o4fSPfoF5kqgZuRAFqwaf!4hrJVA-L%G+1T5IS{g?A!B`WGtv>nY|lD~wlG$S1O* z!Zj{{`xfb+aE*2}^4M%p|MQBz<$N3tzZwt?KOTyP=Exg~#?|yAwZe zt&FRxIs^ak^Ny-+siQ0EByPd4s(NW3mataU30%RJ@e3K%h)>H=<$t>B3<_@{18 zXPnM$6rx_nFZg)Ju8>5M3i4ES$~WkA4!ICY0M{spe}Y5wi>iDikNAUg$gjQy{C=Vv z{9~7|)0$Wb-!SUm@m&D^D3W|chp!nofl~3?N2U|Me|ppDbj>0ERb(yjFBSz1_CN0HT^OKS7Op2g6%dx?%_k)B@igI<5q zrCoK$qpEKR@sShq{vQio08XxI)P+w8{NhgQNB+s_U5GE~H}H6UA!6d8%JZvH=d{y# zh4_o`d(HUYMf%nvobv8Q`sQ~QJ>ZLgZ>T&TR}Fn&AJNIx(bRaKo{z4jg*ahW-#Os? zB+)6AVuAUc$Z8T|>zhEl)#EwZk8~gAk3YB>SzAg**Mf2I=O#bTksL`LARSB;ZD7NH!R z^n|;5CjCpW+syRq@vbOdXAg5+~(#0y_D9)6Vai5{sGr>7a`kAMcVjW+PDcB@ajH2$iS|^_e_uc9Gxf7+ltuk8G`^!< zO|D|B!wa(m>X;8Cas0f$0OQOA*$;sOoN%vf@%V5jpd`vse*Xj}H)C$9uXMx%X&HbX zQ5HW_J@|=E(4(p0zAGV=3jW#Gp3$pFr+5WscNRmzOgJ>KMrVVBzk2;$CVCK2L%ruB zp`>djHSSvU#3!lV;HCpG0HR;@ZE?fEZZg%6hYJ06oLfzs@Ca7`KI-+c*yml2dWPN6 zR4`dAA4PeW0rS>EUvMDWH=B&i59gK^X5&kKz(JbSUkElirX_RX-b}Vz|Fc|odZ*WY Ze(<~p{!j&f#yrG-!UPk9Y54!`{{;r3^|$~4 literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a67c066bf0ef661ed2d985dec1c8f0b1a1ff6a2b GIT binary patch literal 8053 zcmc&(U1%Fu7M_vi-#D@3_|9rlCF``##36C?G zbR@c#SYNe3$!lFn=vuF*H@%VT_4NGlvLJW?We1ZiD4V-DO0mucYQGfO$ENzw!}mZv z$`+igsQJN#!xcklO#hplBIw1>!6ADlvd=^lO#GP}1U55T`Y<%twCzYTrE6Cp!tMwC zg4=7|spfwj_a9p~%;%ue)^kU-z{gb1{nvW<>9ko8<~ zgWTJd=lYhG&P1|_jdg3pJqPO>lO5G* zrCNGw?Pjv0R#tDRjrVG3X|kgRt}@L{W@D2b`NvZ0?zin>vZEG};wDGA8HZ9fl2cD@ zOm^^J4rw09CcVjQTL3)Yxk?f&3bI2KO5JHR+LG)rpNKP`Hosg@*v!2@)qUNOr|J8Er>kRC1j^G#|9;kkb;8jl%SFGRSZ{;>evv`9ziuZ zrRR2I`6(^Gt0!J!Y&0lOh;qNl(9(&-JIzGI$H&dA9~9!_UJLV)ZYQojKW0d@60OyW zV*j|Iya#Fb+Kkl;GWDj-VC_cYkL`w1GW?>0`N;%F_3#wChQ=WdznlxUM$*t&Ze^lz zIBcv*7_d1yZ$+Z1TbZ3qR!iPWYS~CEl8vx-fwi8uIZhI7V>2Dqd^k)_#Lf#-PE?1~ zFxEZ6~RBPd7$F4HP+CKG>BD0eeJ#3l!LKjxYmJmyYDD_8KyjoPxPy2_tmZBPsC1HU$GU`q8|m?cCam z=$q&&U=SR2uv3t$=d?pSFG~jn*N#~eB)jaCf9X&JCu@}lgobvxr($iD*A%l!c4=3- zD=KqJlXFn$sc^qny7Nz-6l6XrKpo)dXcI=HAiwVg>4$E1T6%ekrG!Hv6&EtWxhG1W ziOMTcxi2E3>eImigF{7pF%#dLZswGpb~5NIhXS;mCqj4ECVej|kLr{!Y>2MrF{9>z z#8VObBhdu^;5<7ky{0kM;4O2Hc*#4$sVo}Xuu^vuD6)~pXz zJSnB!5V{~i`pPD~hU_;g|E$L-)^f`hjb1ekYen~RC5NOZ;zd3U2-0f|@I#sXR8$VZ zn!GBsb86${;qRola=%G=*@*t}Im4I>D>7pl;7k`T2{M|J zwv{{0%9AF@v^iip@>3^fx_q}195d5<3S{~(?--dDQ^2&kAUl< z0OKs$a6*5BJ6sqsl|M%1_nqvrwTu?ZFE-;!qif8(z|4y)@AY1b@^dqW+MJK`gK{PR zBxlj&EJ%OFn1AbJvsNrUaqt)Vv+-iZY1-)QjFIp((-j8`_uC$gf0YhD2RHtAoY@}fie z>Lh~r|6Kqj?_GlQhaSk|pWbC}cpiDvtKC7GBq_=x(lzDbsS)r_6yY@v6PfRU7mHu# z1nKJ@Xoh>;>;pl1$|SqeBt1LTi7#A^R&{pvqwxFug~#!8n+rdeR>xIUoq>P&`A}7J z>gcLEiCgfisy^C>C9G9-0#~qQ{=y1HePbY|F5vdqDoTvOKXq$5?Q(6S67@QM!N)gt zgEW#>(5I?X{y~>($c<2fxJE_%6B?plH039K#2=bNe)SUY2Z(O+k6ph_YhopQ!)SlU ze--$nNb(aMzGgfGO2uzKxla6n=`EMbJ%{`^k+s0TXyS)WqLYzma5A+_;=T;x!9W)9 zkB#T$RbLe8WG|qv0Ta2p6yDaC!}FUlRVC3_ay%0Ylj#}NH^YOd4jCy}ZGPCh7~b|Qk!S|#sigqe4d`y|rY9a%{X>Y4f{>5@ zSm-Kna#N#jd`jRK4_ZI+PfqVbe#yRx$L9|d6His1UyZt^U9KC%UxeRh#s3k~bBiG5 z+ggjxOY5cx@>Qz^xQ^E=CHI^@(R?g zy`&y}(|s=2t{?eJ>e1r=DAB2&xGli2>RWWJl%J0!0#G;~bEv=XHtNTqTk752Ppp(G z1m(-1z9jN*SI=Jp={Y=VzW9T3*RNo##MFrQrt4?s>RznmU~G(zjh^9S_XvkURu z(!yj4cyCj^PLqF>%lvto@l0SDQjw{v@B{q;VCi^50@QytaE+E#N)qo^>NFhfon0v1 z(s)|Vw~y%1|5P=y5E{cX8Vk(hnN`Cmzl~4i1pfNeIK^WTj)RMyaMw7w)IS6ICd6Vl zHa?qL#g|8Ve7o~clM>>jew0G5ycn`_X=$Z>#7vUp$^L!MYuN=~INcY`d z_Ri|uJi6xFTnu~kXrC`mXAV%JKh^8s-;eez{cIX_Q9lfi?`SvmHOzHnVRk?r^MfXi zpZ6DFoS7j1A#s2c?v)(R4-W!rq8jD*PjGT8=ArYITJ}O&2BAlk<BEy{1iggew3ajq|bC=Ua(-hdt3`NH5lps(dSeIk(Uk z8i@AI>dW)P*`os^HIe LV1h6W|HJ*ir`_^N literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..38a981f2eff4bbab3e975ff1f57d677ebd6b44e7 GIT binary patch literal 8053 zcmc&(T}&HS7M?NYH&6)S&U#WOOIk7^YKVXM2~=#AF$Nn-Tp$Dksy1uv0XvMrYa0wa zMR};As1H@OilV9y5vo@Eh^nfJs#O*3Q=V2)l!rX5R#laUsH#@`un()KTJ1UK&e-EW zaFAB(H1^zk{=V~b&pngQsB<1>XP20Jn%S7pDG1U$d(4Exc|WHOQMds96JB8yLhK18y3&lGnPL&@}*=-puuRdjEJy5WIl0lSvko&D|WOSZ4#ZUyAHwQ+?>cyPzIr z3r<$leE-7HiXk+n|4mL2^y266kUbUIXQBxv{?rWuTNy2V6q>8rP9&MqwaXA;_x%CE z?X~Vy^S_S!53w;}Hh0&9MJ*Xg=+$uADYf0K$3)pNI+$Dg)(&z*#&0!yHnbZCeX>cQOGE>w+2W^pznG@1X@kmMn@gUdM>*` z?#;?`V_QpSBH6^|hBe}7{}4UHY}AZ}Q*Qyw*{w9l!yix4>Zm$=DYD!7+^2sS1WjX` z1w`FhZ718kx|!}iqBQ?La!6?!IX}(Ln$c==H-jYzw3d!!wOBP@>nO@qYvqvq@MiG8 zs_e7u9ZO-f4_gYOf!7=OYN@w2x0Rz#?zG8{&hm~_ag*6(Nw_!!4Nf3AxoSg`9o1>2 zT6$~kX0oGJR&S||cWY>AvZDsBGR;k9W0M{E$5QL=H|=4vqZX0kCP%rMfKoP+Q%`M7 zcJN;gX#vP4y~%7_06gEhN)jvzvO^R~-Dx!1lI$>_h(lJu@+w*|P1HdTk{$S6ssgH` zaI%Yf4OLDRTB(+nLZ^`(g+*D829MQ8WXG}O4C|ImH0PU(>?k~QXDnJR~EoO@>7N`f~g&jMnXn~ytxWnNFM=Gj!0y8nVnm*B}~J>)VvW% zYME3dsxccU+oPK?%|u3OVb;_#+4)3Lg8>fgG$%5CP1oSQL^CI;Ata*XJ1H%l&^ADm z9rqh(KLbN_!}KV4IcRs-^+YBMc7WOl3X*`!w91>wq~A1*A!)>c1e`JUK&sW+Ibh_b z$h~FVEY{F~JtDsa3hX#Xn1QHEM{-ts4H-&K!Q6?25xQ%U6!>AAhJhOWXkOQLZ*E8Q zEp!zy2#z|~Dah4x+7X_YrGtWN$E*pGU3SX9bSi?AwaSA+LxEMvTp(4JRiSKm}b4pLT81$7R0b0%zp}S*~z895;b;=huL|5~eQS(6J zsfhidXoA0go}HDRGxnqg@ch&h?th5V*P`;wuDqy2+_lUvwG26x(SDjo`wgP~@>lE| zj@Euyd0ILWLH@0@s(e?k+^csM=B2`uB=qK|sKRcg;EfF7m?22d&NE4R>R>@@)`u&e zl+tbpU63GsWs_b(_8XOd)?*ZFxn+w+ubPInqILfG$Vj{X~INb?0q%mBM>?rVEz@8BIw$ z%I#+5aT8?P955aEsS`6@zFP^7ndyB6GX0mgjZ6#jd7+_7ODx<@1jH>3D36*)!F5r9 zaTaYjp})Z$E{vMWAEEMlPIk#!MvLVan{lPlHD+F5=EarwdcQ^axfw%k&d2#dxsrd9 zvuJV_q`zX!zjd-XE0*4JEav4%c0ZP4%I7DPmn~=)M{b(CE81%{yUtH&=o286KJgZERVgs$|zqESiYBkashz-`AlgqFG6Nq>=r}^ZOV&QL}Ymt zu)G#j?rObeF!s^rle28b8jRs0lQcTu$UsIP*`ybua<^T1)rLlo_p)%jcZ|xeJTEkm z?=N5j`8E4bQ3I*qjWO_$Lvbmkylz(>x1&3S7ZuApOU119GA^rpf)ks$pW4Zb|b{8oeU$_DvFR&;R93*f*e{YzAycPd|< zL=gYK3!vn^OOXE13wiw0JM1;jBX4@OJ4llxMR`=ZsysL~3f_q#yvAW7^F8ol@#~x* zeccPqaKDFrAV^P`WLKJ`r>DB`h0D>Z&dq%kexJYaIDT$-;pe@z2~|~R;V*tZRMnh1 zwx&+u7W}HJkM>~+YgL`Z6>OQmutHJaIEbl>xIMmx665e!-JVIiTsx>ly@p@#@r_?6 zjieRysp_*^S?2C}-gFydVb?|7C;G=uciasccGbhmcH6OXF?VZ=v4$j5&? zbOkuMsWCS`CGd*}Z21C6My% zt@JPKE_opqLH}@hK5m%yz(1l>sH4e=exn}U%ZqWstbTZe{1nltlw!e!-IaA6a_gT& zyw&qLHh^?L)=wa`y|S^aM>j%oi03w!=lNOkj%>w}3%gP8ggX)**`jc$dn4ub=;fte zQjflwewS;{kNhR|Xz_oH=u}VK7GPNOExA_9&&M(WD4dUZ)Zc##^<&U2_3j=ZR!S9u z@?}t868U$k=P!Zu93C}aeAx^2HIA{QwZ0Nq$S#KmAt#hyP7gvFp>^c>1NyDG#dvOc zaViD8x2Rrc$Un+u{yfckCb0~u$ka9Xf&KunbUYye>OY&fM$0NCiT5jY29Eab9+Ylr zJT2$jM|9|asv21gjpG@O1s3qks$rDh!6$M8e|>74;;{tB!9`EFYn)u_p8yW&T@N91R0-G!}-@R7WEK8ns)4(U0h`);jx z=X7o!UGr@%g*|$--xsGd2Pn~>>hi&LK@-Q% z`wKA6PLltSIKT<_N{;7;2LUxvjq>{^IJF(~(D_QOc%dwV&?Cz7XE_f6q7(FJa-{!y z7_~xt4)tg32GS{C!MWX~a3~!K4{nfbi11fGf7gf}Ley~HvnyfUJ)4|xFL~orbl%{m z126z$K=tRiW8gQr8bF~!zn$P-(`rtwy~go@g?p7wbn=zE!}STkH=F zM*HXVm4%V)^5R^4IRH4wlKLCLA;&a58|h1Dx{ZI9>rU<+&DCj{#bVG;FVliwq(~)ZO5@}QKZR0431yT}!YF$%Ve~NzrIZjx$y3S0C?S*)_E5qIrHnH6Fh&?&UM958pV2G$@5O?C?~S}lSV_Aauw4FM<8GVDj!Q>jF*?^Wg87g>A?QdyPgn%RO4);tq#t* z&O71UtEK1Krk2TubBXmev&T{YK5~ZGMZ*{No)*xY+sLf-xkQVnxWK9bDf^}23z6L@ zgg*0p00fP$7XfwWwC!B?@_MHGfXw`Ec%RJFLw=l{G@RAuYz9r>XDt)XX|Zai){&E| zrpi9y;l1F0mDnfQ8>YNyA2Q`d15Y=w)s${+PBTZH+-Z?*oqHQn#Z6|BCE@HK^f!Ud z$yNKAY^zQym(p8nGm~w#vbxK4yjep>lWjF{?UCGM^flR5cr4fMeAN~v+iE#d+T$oT z7@3VC2$bz15NwF7w5A*gTR4;X0EyDV9bsMGned^V@0uBEhGI2O)@S%<({Pg`t9iMGCxiE3V) zB`0F1g(*9VLuwmurxhqIMvE*xP!GzBJ5~_U0z30?g~J7oVA|7}L@J!gPsCu1NaXU& z$~9RMhGJl9UJIwRY&sm(n1z$A(e;>SAS1CbYHHcsTq32x{04fO6Iq{@)Zo5EHAkr- zB%)*6X)Tk`)W(CI*If>$!4At5 z%+qK`^QxA=wi!-tps9dCu+_m%@mxKp9boyMaFBDY7&Sq%%69puPDQY@R(U{Z=#YCW z=GNYvViw6N?MUy7%AC?<9~62k-0zj{{G&$&nNJFk2RQLrgb^vg?|VV|u7@3${(Ow3 zg##g#CNjbKP?SCvl^3FNOGHG~r-OY4hl==O2ELa)%r1T1#h|Yo2+&kdgzkNd^p&XW z)+wJ_5M9kt0MMY(E$I+X?9Y2%Gl!?!1Jmn+Ce~~9IbV~^fYxOg8WNqS$SBm+^V-1=cVG4B=i=hsN!xV=M4|ym?20{ zPBTgR#>N8Xs1H@Fl=5x}U63IC-Xi?~(Qj1#QIB4%<(4fOy=ofPO77)K21$>^vwRv5 zr0>zecVzarl5_~vWL2@9QyV7>e+r-g3Q64mpfa#(D<1Fg1Lw|!iTo^Hge~H3Bw6k;OG+NkuvFTSDU1P=tW?WnatM9fb zKQ^PQjrq7RC|9yivX@NGg7hc!`4@IJYsS*M7mIN@lHT{Em~#7w@}dRx;>b;NcSU`z zM%VcP72Q0+JSB=sZrHM$l~nOwez#S5d<2zMpXITaR~h9i0xR_L56%FvhbPK=c?mN8 zVmA=|MpW*#DbHFFk?B>y^jc84tMwYr*hQURpJdZ!XAG67q|pIK22%RKB0Uq8o9)W? zZK(8cFAIlz$Ea+|x5WnX$1~VKe$M_=(m*O$qYpe}P+UqYFWZ%`+R>cii;C%;rDD{2 z3T^8z=)zA5WD*Wpy<})jdS60X; zvXa6zE`a+M>7Q_ob}Dy|B8dOr1yJ_hB}lh>A&#HD!T!zT$eUj62GS%+ULKJyDR+*I zfOVn>uW=a2LJvG!`Z^~__j{ok?)I>^1?e%9tV)yg&9N?g;j*==v$OApF7g*1+mG!I z{Jga?uBz$`{KL;Xs=B3)uBel^1;485rF~e!T2&`-1zW~1tnjFJ434P_xIMOlC&u8P zx;dS3IJQxUdKtgq;~l#~5=kn^Q`ITopu;ibL?{7VqagkX4$&{F@{v5^56&UK`WEo} ziEi+ZUA|6hVkLaTs6X$!0Q^xT`G^i*GcE$9;OBa8UQ#tY|FZxrdI&!1ccCNgy?w4IEE<~L%hN=IX<@oX$as%KR1 z40obBq=#UIuVEiP1w7sv!+ur$HR9hwk~g!ou&RzCt3Q|2=7-&jp>5X^9nB&=z2pbI z{-jg8>WW8I-w@&>ClvfY7Q6tQT-B%(pAz`Ro#sLQ$>|-4FX=b%czq#a;-M<=t5L_a z!*PZ9OYnQm_}@kP)*_tp?nL_M^NViqMZh<-Hy&3FePAEa$<@)+c)y;H&ZUJoVOHNc z;QS=fDV1V@`FvzG39dx@qe01|vFUv>obic!~<3s+kd^FiVN_5I6t_v`%co!YZd+TF~02J$E z4(0b>NBQVb|JpCv@n?l-s_aF(_|mTvQSSmt_e&-N-}i?exN@9tRz;5pZd=_ zu2Hi}LE`;NorcoR??CF7`_ok4UZO+)Q`PW7a13iS=AXx!RYQ1w8=uGs{Pn7F^2Z{S zgM*%MSFc>^pZ??q_+lqEKAVnUZBLMV4vl!>YsSrwQa;fmm0}$AUGO2^1L8Bf>cG}T z_~^V*A0_p*h4d|?d#^{_vq`QVP4jLnhFr;Lzc)@b2Pn~=>iO^UpgvPSn?_mG4@2X5 z?P_urV;x?Y9Z<)7Ac^DW{RJ3jCdhsW9N>g|WsAp$I{_t8j`I5_Fu56XQGKN&Zb-`j z^oTw2Gu4Bi=mb5Q8t%UmLaE@NeeD^&igb!sU^c%P3TDEgfi*fCB>dIu?=sPYh#KlW z7YQYuGpTXsqB}lG^#(T`fB_I5)wjhB1G~voKOQRf+i`9+ZNej50r;rb$6~*CIqDvE zMN`3KseBaWT?Wis3;n@?X#Z?7GC!PKT9}P5`2h!MQhy=X-^w(H~gUr{)~Bu|AYx92-EQY+y4uniSp()@@Muz?PS~{J|cfYNiJ&`3RKprBQ+jE1U zd^Q$eNv$qhpcJGorF5;^-<@6C?Do0;a7hsSfU=!Q7L<-oj#8|%f!Hra_K7Jz^58uX zk7tWUR+N1I!r_9!HKzSdE)nG7=U|sT6WQmY2`2f>3kTM7TJ|tBSGBEJI-_fs!NcDB zy@J(?j=38_Vr*Gk&3F5N*Tp*VkdbC5dg+ir-}QtDq?%_Nopo^5 zci9W)-Yz{?H?(XnmQSs%T0M>lbby=zHe~w3r8k4-{Caj3%cR>^vN|jeUy1B?A@dmW$&9Ia(zc`Z@R);jWX)m%9s zKD-(HuQK~Ad)JZ}tpk?CXyEw2C%4;VXZ!w+RB@BpWJ$Q#4-HPBb8^*& zCOfOs%B8q#?PjvGR#tzxj`wP4X|l5hu6>f5%*G}=3yHvYWM?f$N}C+TW)xD{ z2u?kXDztJb&Bab5JBy2bG3q-~ACa9$I%jCNM8Z*MF0!-u%#E?IRR|cl zv{-MoB!$m<^Bpyhp$43s<3j*Nxmt;XsFse03Duy!Hwhc-hfDSpw;f~10@dT@$eMdjd!pU;I`Vp(V` zH*@hM6dP+2`fSdQo3VK2W^Ox|*U~rBT0WMD>=e({bJ`)6?+XVx*N#yWB)jaAe{NR<7i*FGgoZZRT`{-z=M=L^ zc47jsEZI~eqpLjhXqiO}7#N#BXe!#d^nHbhtR zm{IdU;;V@Lp=g4?f1aI{UNH8w2JpP<3HLul>1$DWZdYE`A?{k{ms*CL%4k0;p#28X ze)$Xb4M%G~C_OD5i6H-4T2da@EBESM#d)dtBnj@q6jj`<e%sypwntK{B;BVD{C zNNGmeQf@nxCruD(bHH>KrcR9X{@qG&%t-Gl5b3|XV?HS1XC|?{?UNxg$9Jy)kuBflY>^eW9qPu5U zZ;7Ik8@B9bB~`qa@3tsUkD;>avpn|lDx-WwV1-`(=>-7x=QHKKyabtXv0D)B9amnq zAR^1FfaSHIa#wSk&e%nrPtUSRt1|{lRMO~xBLgXYWRqTs%H3Aw^>I{sw3mgWy<=21 zJ{N*YK7Ym9-13`$BF@YtI)RohAzB+jzvmb zXdR)7V-t=`q*a&l$`5VIUK?typ*I!Zi3I82sPNS}7PKl%R#wO-vXa6zFMvIp^e<6) z(XM=T5<&d`E`YN4EmH;^Vt^74>$Re5k~2&@xDc#XqA7JA^t z($_gb`q~Z6aJP$nC`eD4WLKJ`XQw*wh0EEZPEUUv{eZvlIDcyO;OG72QB_r^;2(ZI zQq@g$cv&6CE%;T{0PVvP)~Y&&E7&rBVTDHnBXCTe!|jn}JTU_Q)Q!ok$Fqe()NA+! zpTNjrE5IK|QjqBIHRU5vDt-sabm9+9Zg@Q28RWl#tVRBL6F+Pcos`7G^gc zbYv0#$Y|lb8i*sE^o8^lU?Ni&qFef6bap+Vs&q7w9?d19qn|!MK%^!7xegQBnkf9;OBXK=KX`~iS+Dt+&}7#MF-c(9qP{D z{(SWFR4>cNz+{idvlB%AvV64IKTLGWC$0-HEC=R2OZ)3%fdCZiV+Q5-+(P;2bW6T_ zdx@1og`fgCl$S>St?KbhA$=1|Es$LBLw=2*FKMkWhGz2%(LRU?#h25=5JqSn1^$qJ zYkDrZxiB}L0p44buajgS#j;ROQ@$}wLrOAr8GfKY04yCVBt-pZ4cDkyr6BQsrA|U= zZ|^|rmiyCE-vOdS|5Mf2Tx0}mG!dG`npLBCehZ(-2>cDGN%F@$l!J$!aM!3@>YpKf z9elBq7@f{6Vr`F+d=8Cx;cLpzk5WF-Bb8zj^<4=f-b3OuyyC&uMfm8vSsx|!wTbji zqz7&-`lofS9!(3Z&qsZFyeE*Pngf(*Pxbr{_M$#ZKbu5Z)DNSh+u99%1!EnXo9^8T|8V~=?nCzP literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..03399d10c5c351359efbcef4d531defcde56fb4b GIT binary patch literal 8053 zcmc&(U1%Fu7M_vi-`KU|_|9rlCF``##G!HYly#sx2PoH~$ z39N&4T>|C~*1#GBp~-0jM!QY0!`}Ps15?47v<&;PjZ`X`>w8l@`yxwBgFZyExZewk za+zpsExEpGfl|=AlGL?6PhVyu-{!SR9e?ALxkP; z2L!j*x>L>nI_^Kj#)R42T@MzuR5Ynq!)d3~cC#K6Wyk2L->rUQ2e~fex0*d0+7sAY ztP^t~@QP`47!?Cs5<6`oInmn)8l$V)dae%>=w#g}WR%%k10*EScRe8jt)^_FqYh*} zm)#)uX63oQrDd|wTykUG8gb9zdWPAk84IW00+w@|8IXq>^pvcQs>7EeyIshA=KDd= zG`3Mh)Sc6|bKNT&neHP>^Y0^vl%|pM)9kDntu}WvSb{)nnP^UnSM#-wqFl9B4#^L1 z2LG$dKFi**6h`~7r7#+Jy@9WmdTVoAIqKw2o9yU3*pVu3GMg+37l)w12_z?1ZD_Ki zI;~twZ>`--cGSx1Ew}M*4J}P})WCH>bCcQFWJlq#+`9WsdzkE~MWnRJQEn!nl#S%n zQyY^V{Fg&o0J2GMGTRmaFLbW51dD>~5QTDg8jZFjJIp8IkQK1Jf)-2@b7PblrqmUNQ zTP;Q5vEF>gts{8|?lXCda%m=06rmPGp6N2OQ<`T(5=clvL3T>e$oVRctA}-Lm}n27 znw-)zyRrO~mfzMBFEKV6l*dH5*JNnvLgMXaBI4ttW;Or{@o}$(`AD}DSD&9SBwC5q zYDKZ%W+?AP+TC_z^_)z-ZZ}wakoZG~p_B|i?__>5!BIUp#jc`ph{Mn4g00aEG?ts$ zSON|kYZ3-+j;@>0So&smC!5n!H&a?J8jt3ptV3X}=WULYMBCWR#55lclaulD!n70B zA+?RS(+ZRpvqhE))PwQjjvZ9Az|I2P;c$Z^nD%rgnTlq1CgU(iBy&5=&Mn!JreR=e zUXP}!8Vw`wg_8g(13OdKA1Iv^(s2GMfWCKy3sCDZph~<;`T$ZyLssG~z%4&KP?j)oSe= zFmhAm-m-3%YG}Y7QP=_ncAO*3K-6WTd8@sKEG4IC?nKfE-L+^M{IE^KK#hJhuW36s zx1#zcx(XNsM;+`G%7a2fhumASwhq=5vq^So zSGprA^GcI*Na(F_zgN2RPo5NHJ}E#Q;OJ-*Mx`LX?*-}m9(G!Kaf+pdBO#R*GQqhk zN}q|!OHsKeBBJWk!6Ab~MSO7+-|HUcl%8}k=qpD8w45hGcgH4uCn^u?lt0)IUCm=g z%>#+2BKC)(3I6_hc2;`M*pnK-^HWc_{~=0WiOMs(@}drL*D}A6xuijalmx@o4&|8?Iio2D9H!_4{h9EsV&m`%og9WWw zAFg;(%DW+SL4x$9O?m~{Z&dzSk5R1UmMs~*Y8uu`?&V4jNsq;gd>Rm>R~X<2GW)Tl z9D+4@RczvCqND1X$lk%bw{o`|nF&9=u(sgkGx*Ub~69rDyowwOl3h&{WE?yF3 zG%al_x0{v6O^|7Gz;qO*PR#Vd-AZuGOz$a>>A$>fWLl8V3k_9TV)1q&AZ}?udDJ`# zu8RVUvuMK!{SEGLVboOq2$kP+vP;%7T0D5M8CM!zW99{BUR(vQ_ga*nnlaSod|Vim zEBPlmOD1PQ`YXo#8z-ByV(C4I#k?HJ?#EJG`TT_PvIXtp$W3#1MSHDg*ZC0*-95tw zN(`0Uuw^$Znc}^CuT^<+0*zIl<*}Dn8RaViEA;YDE&#AUpDFLD0hGOW3DWO-A&-B0hrQ-`>w?)9(_1nCKr>`IgL^i&tVa5-Alxw((R@ADTP$4~7p{Jgh1p{nXE{KL&^+?1?*V^+=qCU8wOh0%R>C)e z_ILbOfIo&LKhfc9)t&#Jyz9z=E6NWm&!!#;cp zczm;_|El_1#Ggl!FB4f@Q^$}skkhq=5${rX+Y=$tEYj1F0N4%aZtaFA5mWuch>wC$ zi2r!#3UG2$V{Uv(;ujCv0P;`G>_UFYzKO@@4-*qlRe@iPxn^9h>%?Dz-)F`D5z_NZ zAm!U#?qAqh@*b33@Cw z(tkaSS|L7%`ZIO|>6EYF+|E)sl!=B1*GV=+_^Y44YeWwrYB=w?<*@FaO-;C$yoo6~ zZ*bEA7yvP#`t#f|@S9u>pir^jPH?Yj6CU9Tz(?bJEcN?VV%`x?EEUpA^`k1^3SiDJ z_J;;z{d4;A!bmQ%IG2b700&u8e+p7D9fI6?u;~& zC0j8pt2ENwd;Y%jbI(1KPN_3KW@nd}caqtd&>;xYEPM$|v3xw)jLw?|D-_`yxwDf<8ntzwZV` z`D`q{np#`2Kq+WlPU#we%&u?s_}qWIBnW;$*})_WN=Fw*Dc0FQ?H3~Z*i;{R@GhuF z*`kvbHQ&E*v|3@?;1iknsH{zGg`n2xS`u&AYDDZLs_JEgXZb(<*LhtK?Op~%;%--rFA%VW@2@z;DWgDG!AnUvA z1-UmX&$Uf0n~UXB>uc7Cqy0nl46xk0)`_Lh!Y2^GgJ8MR()$0ID5NItM%WH{hzSdEctJcaP`Qgpr ze^uFM**liPXdAQ?Mgy-m@YPaptzIifo!nuQogD`|QpHVXlO^He05mv(0a@2pUJ|a7hC1+T-WWrHsF0!-u%$>2YRY(|xw0PcX zDGHzU<~wd3$pdho$(xi*Gnt|YwIK3Lmyuo4EE|wOLJAAAOM*tuR|#A_tYd>jdkEF! zlAhU(<)^g#ww`#2vC*JBCd$1gLrW(TZ##&HkB=Oz4;13#UNZ}jZWpdTKVe9;5Us_E zV!zc;?nc_(He>aiOucS1Si6zgCm%>Og5E{Wp~CCFh`{FJIu~4*;1xqU~;U* z(poMPi)+lr$@chqLNk$(T9`GpTz)o{)?k1GJC0N?sOcKqmuTiBHH1`rbUUMEQ`#D6 zvg3XO?dM>KZkQYfFZ=CIyPnGB!46OxK|vaDnO1o-84sFgP+A(W_WS3p?&mD^3Vl8sN(9kZsE7sP*nqoG|F6~Np zL}gZKat#RX3io@ZJOAWKLFSVJ)B%osHepx_^ZQ<&^p&VQvnwy^5O*!}OD#iAWwf6b(0+|*zxW0F znxnNJR-TrQM38?iEi2#FEBESM#d)dtBnj@q6jj`<6uhAU95V#z*?A^OPn|4m&H7-) zlTzLdp$ihEFKyB*$bO^p&w7kvEw^mR=vC9OR&p;_a!7hCUgXn&Aicr>KakmvCFKyV z$*W>Jr#4O={zghF_nMR!jp!erGmN>gB9hLFbI|1|yq_p=s_wkau2Ogp&vfyUAfp*+ zTemO{m^v}j2X`yMF*CiVK&JolwvlN;J})#>X^F+#iGX;eKIM^P7+els zxk@HyLHaAk{2LdWv0`x_#A05KWcOn!p?rQqdD)D1apb1CyP~}ov+MkbhVGtWeI(NpU;%{@)Bgm#cn~g->ST5K}42U z0n2Ma<*w#7gRzG;pPXe=)?f^ln55AGM+P$b$R@oImAh@qt5!66yqAUJy<=2%<$1Az ze0KpG$S>J{N*YK7Z;XM597;+V<#n6#xDDMYzNlE0hGqyhHi& zB!c+=T>xe8U4ry`H{|h8@37ZAkG$#C?jTK)6y;&*s`B8}FnA}5@EV7SEcC#OrLS{> z^pzW$;a)fUK#-m=$*wd>PfvB?3zxG+otgP4`aXZ*asJfi!OwduW2&l7!(aS-sH$7) z$cj3FTkxx@0osQptW|X!SFmON!U{zLqadcvc{|0gq?Ui%`P2Mn(J+8Khq{6(oJcADKmd^*!JZ5#8h;y>^S%#7g*v(Ed*F z3h>8~6eK!)P5TIxir+zUo%lmjn;wsM7Wr==YmtA!#1ET9CnNFjL}rP^13AQlfh^)5 z9V^JIfjH91UPxaBCUSK#x~(ro=Qa|mN}`GMSS}GI)6;5Tng>xGG*Ym_*RT(t0-nIM z>A$M}7V&Q(DUe;9UsXqtHI&!2xgq~TblbN`qB*2z7DHe+q%*PGW9{5Le3Uxd^)@#(GcX2*RnAH!Dke?trl~N)+x3jdWLvDlP zh_`w^NBWQ+#QF(EHkZ~G_4ryO3Gv+I@;pBa{-KRTdTuA~AM?hdLmLzhb$94sJ^Fd6 zm(^ons@LP$4I+P8JzD%9Av)C)w*?qh0t=qygXd$B02I&1Eb8yQh59k*mU{R05i6w% zK?QQCFOB@$)$^A^`W7CwKyuL!^)-sIq_w^jn#(Un`ynTkUrrB08liO*_(S@wnfc__ z;`~GgcyCd?PLY3<%ffk@_Kjm1Qjw`E@B{q;VCi^5LeziOagCN$N)qo^>J%L9on0v1 z@_1U#cYx^7|5Pg|BHp7o~clM>>j0w09+lc#p`>$f^fh7vUp$^L&(?uPvl+Aw6(w z$v>lW^XOV&V{#393U?1AE^s_0{Mg1^3wxiw9S25SI`I&xoGzgkF ze%@bzaeAEmhr|I+xL3A#es~a26V)icf5H=+2_K!W%#t6jS}Aw+b79#0ST zUXP+yh|i(^jNCvvgVqo(Ibc&&U=0-s(Yu?W8MXSa)Qnq z+;jj2K=i4>E$$fjO|FJesMv4ExYx7^k8lOxqj5eKdIQUG|Bx@9j_9TOQB`0WFmKKG zM*8EuGy2lpP=0ZKCb<{_9AruTmEe>Unx2pKWOH4{Kg)Gxc6xl52QK^JAFAM=F^}<| NFwO*F3jT-te*v4J@ zY)z6a+Ycj@5=tnegp!rA2NOmgLnx($FiM_E9!3eFjIf6iMkr;Jv4=6jC}ZcG?_TLj zmTbi^CK~B}-#Onozw`Gy*PT*l+|0_(G1nxsFri%#q*?e9BH>&#-sA4^xIBG5E_aWs zp9!pub({m{HrBuz1fj`c0YwL|TUZ=vpe3$o0Icockh+Pl7x|GJn4t z1m!Z}=t^RB*#xB^bt#e50Ayw@-{bE7?3^HY0A)LqOek%g9Hm%i0kOXq*}I1L;3sc@ zcsyG)vZCZWXZ9Bit}*Oya)=-oKYP3EiO4<@4KVR1E;z8B)iV2`xuk7|Q|Y939z5*2 z(Engwrch`>fb}oAiHSz!qL?Xnse)!)gHHK=_@XD(tXzZ=>kGxd-kXD1D39d_A36Zl!ngmYS~nyGc< z(O5DSq0{e58V-x^s+OLgnCx zpH2l@!Wn2R*Rs(#6dP+2`Yra3YvE}6T6Qa&(^A(`S}q(5=fbQ_U@fOD_M=2wThBx_ zFV2$VvD3n&1H~crFmI<7D9uKTEIm*U%8NTz5YY@fb8v;j1&&}^)0spnoY@+W!5ERq zZ80m?WJwr`fyuTSPHEY6II1xVCtIUyG0i|mVqw(OvbotrN`v_g^w<(vpO)0%zC<-g zsUakyqnl|hlh9T{k`?zGXg>=xbi?E@SlMs2Ta$@w4)g%E5d@?Fm!Xw6lX0J+7+unc z^9UHD@4=}SbLW7Gn>=^Kyjd!t0eeJY3j|nkjxZcimkH<1`WmtnoT9oz3Eg#IwJXMd@=~ z(v#CnlD@ICfH~@e6)UB@8$uT(NPn1*+=C zYwQxackf6SF9}kbmNu0eHsw(hMA{fI?S-imBfWdK5*#DaI|@Yl(QA671^KklP^Bdn zZzlrclKPYfwqY<`6kwc19S-PkaEA-ShVU;@_y-Pl&YVW`yDv8VN~3GcxWJ5yt6=rr zX5~j4y4skJ3xjec`y@xn$HJKCnp7MCE3y z^8H~{da##;gS}%^Hs#x51Nq|_Y#={p|0!u86|B(*9x^B{rInYh%2%yuPVq&>^v+T- zYF&l4^%r#E$ER4Z#D(S&syH^`xI|i$Qcii^rrc{otu^$f;yaNb{R8 z$A-Z=QH0kx3}m4Po-KWy6Qujy&K%n+>O5|bF5`(& z_@{16Wt`4U6rx_nFZg&zuaHEN3i4ES(l_9A4!RIZ0M{spe}aSbi>iDikNAVL$gjQ$ z{C=Vv{G*qz)0$Wb-w^8G@?8M_D3W|chp%Zjfl~3?N2U|Me`>?&bj>3FRb(ylFBte? zljx)*8kk5&=(snFc+inW{G(%q^Qt$BbkgTft^gC6x)|C_MnZGzF;%6bvD8>L79!Qt zs&|?@Q61Dnu*}!651#@a@3djRs{R`B=aJ;iEY7c}BgpE{CAGOB&q8R^y+}v1NKY^N zL9ajQ(yqGWQPnqy_{a$b|BnVQ04G;9;=-o{esQPuA^*hGHpG|o8+g3F5Hayk75LSt zbIR$wLi{E8y=MIHAU(eTr@Y&d-np#>5BMVB8{8d_tA;+XkLcv;Xlkri&qvqde4H?= zZ|`${g6Nb=vB2C`WF-l)^^GIm>hT=uL%I+1#~<8?tS%;_tHC(2}3KfF#W>H=W`8TV_FM;$tmYO%d=z;tiMPJf78S&5M7DN3I z6N)dV2Ox~lItu*$N5O5e*joXtPnr- zpEX>gW|e}(`;|HcrM1>ejSFgXzL=Pfr zsP|kXlyps}##{@Y_ypA(+;jj2K=i4;JU0yNCR6=*sMv4ExYe`?k8lOxqh22iz22p$ zXUH8*1(T)nQIvNHFz4rcgZ zuy)pY37Fei18We3CZ`P;?KZ&zGVI6K(&<#*^`>(6M3$Tcd5C0wuLlI> zv$6O}YIWHHr66@FrE4yqE4#Mo^7j1kk|6j1We1ZiD4V-DO0mucV!sgCN2d7jgLgnY zo-G<#QS$u@hYJSRnD#e0MUac1gI)GiWS@#AnB-Fr99Yk3*~8FW)wW{kjILb<4}0$S z2_Bzyr<(nB+^DZy&I$0MUGScis9~~0tyPgn%RP$`3qYln` zFMHtJo2BRKhL+96@~O2|tH&{c4v;g*hD~2M^%l^aU(c>$nYg?qtHbi}rO0jn(U~7YoFvMv$4sJ!ehC1&ztrz*-^`p(k4f-8G}?d zf>TdzOm^^J4rva^CcVjQTL8S!xyk}83bI2K%FSss+LG)rpNK|)j^bipjQWq%M`Xv5&KcS*k!UV77uivK=EhjqDg=yN zTCBHPlEQ1f`Hq@L@&Mdt@&?7y45lbTE{HtRWn`x`%LXJkA%z6lDM2IWs|2nd*0DjN zJ%ns>O3&=Z@)KHqTTi^i*l1856Xjl$A*B z3{1_dv9y-U#NryWak4$Wme5ROBo<~(Etj87r8OAfKu>cj7tnMK?n_j2k{UuPKDw3B zvMFs9B-wGlf%bDSL^n(hgO&Yuhh0zQ@}LK(jUXTmxJ<3QnT!Wa#pseo97wIIs;?nO!6~XcmNHy-EtUa0Y?ClhqaDpF z+V;(jn7)pt0tUfR2Rp@c^_+Hy<@>@x&b4FI1j#Nt<)1qg!O2?XexadV?x~nt`*Vuf zB)haD-4T^prO7!U^i;UtE8Y1gPYN=h6d(_9WR?Zu}R;F%ELP4_claV z^O#ZdK;o^4{h?@rzki;cm7X*9qz3T3>IwHhMCmJ0d1hB$)FJL#=9gNAoXTiFDWLrt z(SGp@_BBUqKPWvd9f=_ST3S-Rtyk{VJB#yD@ktVT3R6^Zx03UQ25`&}q-W=uBt3Po zkTvRq6)UB@8$uT(NMG8dR}lS1<)8KF#aeFJlF_TCVXfp|u4It(SiH!m0YQ3&4t^lB zA4}39RFhT3c1~@aEc}g>RPHq?FB;K4K4%znVMQdJ7w4eMk$XQ@psMb?#jcWj4~}&4 zk|3oSX-m1?tUPXlNSgzuqcC-1r1$Suf@4N{SAj_X#sN{w%yIDyU@8!F#%9CTLtokgEy}ZgOUlCZLmw$W#fc^PQc`q+PW?bwRM0>}T z7p;iM@+x3?EvVepdQ4~RqRuC0*_728gC#0ybik2;ls>XaFGS^ToAT;7Dm~iE!qMI_ zDx31W*g(F!fDPoA>^~(9q=GfZz(WQlrHt~rO?ljg<`iF4EblB8qt;z$TYp0r-ap5} zB`&m%P{pwc$0gFLOL^soc4e;}wbsy^itj{%^lw!7@*E3T6(%byDC;zwkJIYIEb~-Q_V=Rj1(} zem+puO?6~hoxm;lRaHOj!xGl2I*u#YGJj!(NByI4Or6K=(Pca_3jfrNsjS<*g+kP8 z_yr&T=yj4vQbC@oP6h_t?m-Vi3E>(A@lSY=eo<9`(7Wvh8fj>xelYjKuEm{*R z;TuBz+kq>6 z#T6Z58yH8t)#EwRhx7pEPcXc(xEj^ttKlU0bAzAf@mcTFjoWPtY;kARD9`E%hspbGB+EYFM1AVB^($A()7WKo(*tT{OH?0(LK}YG0%c8 zIYIRXHywZh5PfQ3lN$zhlc_;GRP487+-ll{N4Ns;(WsAwUjI_uH{^|{!+NQF6y;w6 z%$xJQ;r@8G_x|o9i<$)vBuU5LMM`ANFAtRjWPc+!=fP z2M*F|oyMMf&);``?zv~u8FkLX?CcVAPcs`6It4+Rhc96zl8wguynSA`cc9Ph>2nV< zfpxI1OTgU08d!rMG&ya+XtxP=*n6LSU@AD1l3_o(nM@|KeQ&DgKxFZ0(1%DC4|+jS zHXVtsCpOkBQ1V(=6S@W<)0??IPwyWu34#|;b~4F=vbme16zgoD_6w1HY^o1Eco)>8 zY{AKjn(tpYS}}yi^uNg|f?oU_99ko8=4 zgWQ{y=f<{{&P1|_%?)eB(f%QNhS{hY3#Z-!ma|)FkcU5>qSaA#_)=uI^SMudKM0z} zHVcTlv)WF!dv!D2eMD*gedLhRG;)5Loi(G?=57W{5NItO$!f7`zSdEctJcaP`Qgpr ze^uFM**liPXdku|Mgy-m@YPapZEh<^o!n`Y9i8PJsp2NH$&zq!2pXI~a&py%COfLr zO11RX+RbD~t*qWs8}HW8(qu;sTxFV@%*G}=@{gs~-EZ2%WJfI`#Z8WKGXbS+B&VL* znC#%c9MS@iO?s2rwg7m(bCo1m6l8}el)BSsv?bYLJ`snkfaO)RV4A3d9wa;PyHo{K zN8w}_^%|<2Dzs89Erm`aI|_@k91R|;kI0T=$r;uynP|>87uivG=FV8yDkO|TS~zdD z6oton^BuR25!VTTi^i*l1856Xjl$p`{Cnx0{KGkB^$!04T)A{TAjU-A-J6e!`GwC0eT$ z#X*~)yccPA+l|$8GWELMVC_NT4;_Y5GW@)g`N;%F_23k{ipC)hKc5S>M$*t&Zf2r! zIBcv*7_>RMZbqW1o0;8AR!iPYYS~CEl8vwqfwi8uIZhI7b1NOyd^k)_#?A}VPE?1~ zHr`GvP+H6uSu9Ww#tS=kP|*TA3vh?S4US;iQ|Ux9lHQ$+!5opu?lL>KWJ{QafvI^T zlGHM(NK|7sPPRukW15MK)WWQ(WwP^$qy_^V*lA8={F<)8eTimHQbR~Y$9Ga%I-zZV zCOhso(0&Gn=!WT0@N&@Zu#C0|_`|?15CPwR6D8 zO_6)cx>>BD0eeJ#3l!LKjxYmJmyYDD_8KyjoPxO%2_tmZA}R30HVp$c`q8|u?cUsu z=v(M2U=SR2uv3t$=d>d{FG~jn*N#~eB)jaCf9_NSCu@}lg@z8fw_H8jbT6%GcrGz6P6&EtWxhqPa ziONe+xhEo`>eIm?gF{7pF%#eG9_EyubTQ~FM*_5*Cqj3}CVeL=59^da*brUKV@Ayb ziKim=hoTAo{&{v*dd}FB8o={YPq_afN?(b}GrRJl4sq8qztl41R7U%09_`nN_KRPz zuQ^)#VdZJ*NCf%U(yH=py>hSKS(uj!Pm<7^pP~x8m4Y`igky#vJv+}N>8XPStyv$g zcv4EcA#_24^rcOD1=(*@{#lPvtmT$18og>7){5@sN)Abn#fy9z5TsWa;0H4Mv8Wt^ zHF;HN=hVi@!{118-7IX5eYZ)z;Uu?#eM%S2mfteRq-s}Aq<)>y0wK*T>2jxos zNzS6lS&;sUG5^NN=B!wH%dwc3Bia2}iYcF;P+qp6T^zY-?yhLB)$BSyqM^HI*g%n? zq8qm4W+hX+m+!YKPfnn*>a#rd@+zZzMPT_}{>cRZ_UALDy}SsSaj{zv9keMgS`m@u zRlxFEP`Ru1n!(sdn@`TN8EY_xi%in!fFlDLePokfh|1k|JX0@!l~iyYjry zK)$XKSd3sf;Yy%Lk`8Il=8Y=dEAcf6kb#;?<^Iw){}2rf5Q;oKgU8vA+(NA zg|P|8C9=Zg<)8zt*3g>@??i(1Z#4Mw9P?WZ#w#1-6Is#0H7|ezoAfVHdETjf zc@jbV|1N-%_bx&DeJ|wkPw%kTJdeET)$Slok`(1p>8kSJ)F^l-itrkTiOl!Fi^Z>V zg7j4{G{gNK_JJThVUk^GlAfOG!WS+_t2#ILQTToS!sGa<-G!g`)+SU{orS;n`A}7J z>e!k(gC9G9-5?8Qg{=y1Hed8deF5>q18cK}AUv+yX?Q-p)67?E>!N)g# zoivhG(5I@?{vnrZ*o{zvxJE_%6B?#pH039K#2=bRe)T=z4-nntAHQ~s*2GHqM$rDQ z{|fL&k>n>je9d|Yl#1Vea-H}CGutkgdmj03AZvkt$;1zvL?{me9<%}EpPJc&{E~eWkIx?_CZ4K1zZ!MTxLntXzX-q2ivJ^|=axXq zx3|*2u)E}iSOopU<@vZ_+5`WHPN9w_C;E+gbT2Q)3A6g)5%N<+r&5Xq7j{?Hb;zxM z67g2g=hy(!{a8PN(Dus4vL4+C#UY;CT%PA=$vd(YOD^n2y%X+8cw~#hq3(^8*Q1x0 zdPzO{X8K*OJwNi7)T71!F``pFaa(|4&9~%QEk7U21fXy}=23tDE!2-ex753PfLJM2 z2+Eg1eM#issh+zTwdq#{$-;0O8xz|!%A1gQUP;uQXqJ9{j*wt?6>zM1v;@qG*?gvdA zKkqNVI6F!HL*f7@+$%YrA07nML^aCqpWxJX%tPlZwc>@c3__16%b(>u1c*-1qsfu} z>tWOi@j29=u^ULId{rp`cdI(X&dC#tdb@yy?!oB2;PtkdU zn-0JLhym4~td5r&r MNhSz0@IT!D3p91{%>V!Z literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9227e2a1b48a43d7a032540c98555998ab516769 GIT binary patch literal 8053 zcmc&(TWA|+7M_viYn<9~{LgAqCF``#M4@qX@g;WgmW(W0vg@d}<5;#So0X-prFdj( zCHeZWgglfG`cO&4hW*%jDwWLjys4gjktHTUA0nCGcY&f@ zCK_8!uB}+0;RP(=%`wy`(VK#TwgGDVBP3qNf+9|bNtlLD{KK$?Bt$b?-x!1;THG4L+C$P6z z2j)WH71QW2Dh9SBcFII@%+&}Qqbu54t_Kt7WL+p^l-XN-BqY#xJs|?Erfj354rINT zJRtXG<+--0WwOy+a(&Gjaqq!;2HCI~3#Z-!mU9~!kcS%d7Of7e!xtjEmCt?VhknpB zvR**cmD9F!UCZm4u0u-mZ$k%^rjheg?2H+$ zbreo^QLmxOsX{B&(o*O&vZJsl%TfQ4`iSf}lAK}Pl8NSgbCDf|XYP!JtwO>mq=oZV zOHp{OH{Vg~NFIRuOx~nin#mMJs0EQ{x`gbMX4!xQ5>iNzof0&1zKY}OK^+?;+5@O2 zr}WfrEI*~?clE?ejEx565mD|o8Cp7#c&nL+`1r7y^?^cs+-qTe((T06=f@0*R-(09 zQS2W#l)I32r_ESBBU7*14AyQW{@89PCBx4;Sb$7$RQFG^D`*_z@cL}1HJX9Oaw8i{ zz+q!eLch(?c_SK2-^lJ{b6V;~O3Ov#(Oi_Z3#|2=&2fTg>l>Mv=Eq@jJbq4?bfP+> z9_Q_}0;R=lk;MY_V7#zn2Nf-_GY5A#+~5eNEuBfGqM4oXILr~r+zzvIOSYtG7?_&Z zqA4w#j>a@*<79hmJ+7I^NG;5oS~fSEOldH{ft}`LHlXPm+?QzP1T}V4!fSr=D-e68$m$|aG6$lGZ_zCQiKLXi2S0Cj+)qfHo=Lj1lLq#wH3De3u1mKF|$R9wge=Z+|S zCMqvP<*tZ`s!s<83=S3X#Z7##x|vgY+{vJ?9175Ko(SD-oAkY?Jg8HCZ$or7j~O)& zB;JbHABZOSd*|30=^0~>YXHwrJ>mYlD19v|PwmR{I>cSe{8G!1QyJ~2d9>dk+An{> zzTs%?2bHI#BN60ZOUuf4^~&9PXJK9{JV}BpKSdRGD+O<80LKhLdU}pY(h~;@S+hP^ z@uZY?L+FA8=_{M`60+Z@{G%SDSj#P2GHb$WNV^>GIu5aLi2aDUj(uzinh%kk1JXRa#=JPfXj z0*te0!wLNj?r>q)RQ?c^-*vK!)-swezu1f`jjl2C0y8hJyw`gz%FoRhYI8o$56YGN zlbl79vmpHiWB#p^%~-Lx%CVT2Bia2}iYuQVQ(m;7T^zY-?yhLB)$BSyp`kmcSznQ% zq8qm4W+hX+m+!SIkB_0T>a#rd@+zZzMPT_}{>ga&_NUXOy}SsSaj{zv?H^a3w<03T ztAORTpmJApnZejYn~%@1DQhqWi%in!fFlDLeQ1-Oi^`og<>hfSdbF2?qrGEPcI8>2 zfqZ`+8^|x&e~KDN1#gUjha5^sY2{U$@~92nDZHpy-dQSUtvlbg{)!>Icb0{VLTDYK z3S$$FOJr4-a>|eG%6>aqt)Vv+-iZY1Uuf{fSr)JwOjI_=C$gf0YhD2RHtCI8!L|6Kqj?_GlQg$wfd$9LE(o=4vFYIl$(Ns98YbVa#;au~c5MR<+FMCNMbK~c|l`&OSr{N!dK2+5$ zb!0`Iz%BSyRX^>+64t6Zjw{$Qe_@59{!tK9=W%;<1tmt|pSn4fal5xsiFy^k;Nu^? zMjA;g=u_3nz<}F5=s_qUT%#iX2@ldQnhKCU;t$Uvzxp2V2Z?U-k6yh=YhopQLuh{| za2fbxND2@gzNWndO2zL0xla7SsZF=rGmHG!k+s0TVB&{OqLYzWXd=Bt;{Gh+!9W)9 zkB;T#ReucWWG|?%0u#Bq7}?gBB6AyYRVC4QYAhR%km+gFKh1-v4jL&~;cM83Pacnd z+Vo#le~b9HkmS!S&abK?$QsP)+T4(DA+qgVB+)F=(~Ci{8`M48b#Ef31_luy1tA~* z(eP#9{yi9<)B>pP1T({E~eWk3SG0CZ4K1zZ!E-x!u=@zX-qIivJ^|Z!Lh7 ze|M>OZfC&X6&O zIO46I&yhZ)2e5vE;mxJBMLo6_PCz_2xjfI$f^TReo|@Z<`Nlla$j}CbL){%JuSXv* z^^$t@PxZRpy8+}csYi?dBSfcq;lFD%xy+xZY413eAr+ar0zc3n0G5s?BuM>d9oJ}Cr6lowrB1=o-r0rH zEsdw;eEW$G{ZCb+^WjlEqw(Mzo>?`5^4s`CPT;R!O;9`*;5fMH33rW?OZ_vbZ$K<| z<6|@FB|O{Xq@P10Uig~!aZ#!#dZeS6Kzo-1i1(2EjI6q`brC+2H_u1W`PxGI7SjDU zmwYoiH;=CQHx?pZJ=W__(3u02=uh?f5A>luOFx@JUDOXFV>{Y)eHC*ZouBDfM+2aV z|(Cc50`G&l)R9G+8kE;C3fO%`a zH{2iVoza)(hH{JZGl|6@;2=xtuLOr2*YsSpCzI_m{w&v(-s$mP8o1 literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..aafb639f06e5508ff2c80e215b4f3b2f8053a391 GIT binary patch literal 8052 zcmc&(UuawB6~9-O|HyG-$MJWkn<|;pIyVZ9qd)!=J9s5mmMz(JRNHYZ+mvOpbZse? zY)z6a+b;{Dgc8aqp@dQPV8ZBQ2&I$|M#)pj!zdw?5%y5R2&IfN_Ao{mW$c{u-78(m zlC2oVL?hkrJLfy+cm96ox|8a(hgsP<=AK{{CbSEJGy`A4Vk8%hcYC_MZf|e5+tcmt zX98EyQ{q8Lg zk7tWUR+N1A%>IJGHHQ674iV(yXK$B171_t40Ve*`4F}eyIb-)j0Ss83)= zSUbi-;2G0s*E0sTBzD|Da>&^T5+lpnYOdP@ybjihhx9Z%(o2T~`mQHLAk{eAXs?5_ zp7U-v_j>8Mx}jyVkz8VJ)$DQ9zlWRwcG2*KqpKM-=hic;-5$}>TU=n(fRw$`@TJIZ z6hfbQ(FcNt*NT8TbJ}LEb7?Kpxld+(H@HV;>LEYQP8!ZS?6!d>@UxbQ`hZ%v<{f^qJgIy*lJ3*!)`N2o!oAb?d`i8QpHVXktN}5KlC?& z&dF8#nQX64E0^M|wVBEGT3Nm2I^L?GqsjIfxOPcyGWwcqFFclOcfW26lkK$}DeZ9- zn^8z*JvjB$#bi7G<&b89Y|xvGt_8pgjjJrcq9EHvq1>ECy(`If@4z-{m48JBpQE(rPGks?f@%H1kFSy~QKji;G<`>N`*$k?jXMr)xJw!d7T5vc34s zjWMxR2pGAvSZ}o?g~xpH9W;;Rez?x$4T_}^Oi_ef5P77_$PQ_S^-FL<3JS7Af=13) zFqp(KsGs~XI6dr5iK9p6E86~>XgSsx!t5o=|JKQ8xisGp^f!|KzzK{%zPx< zfvY=*^b;*aYcZp^e^?jpMB1%Zef2G=dey44b|LXan=X_TKWk@xQo&K(J;pAfa`3}X zr-Ch!3^bN&*=QV!jWr2<7JJ9FNHl#dyOqspscR`M7l}o35!NQKmeUsdQKGG_XQG-9 zXUVbHX<@>F;*ffnx6=xgW}`)x9;gT9#T_e%Xoj6xxWeHAM=-7FOd=J@Y>mZWj7a3R zn3Zd?Bn-vCWLu4-v}`&O)tH5otBL2OKNamqMD=B z5E9Xm&9s(DXsaN}iu(<;pM@E^VPXiZ?6caf$wW2>dVtyp0#bm>(8`<1nBP#0E@{Mh z1dP%5;8csbbHKz+o;zaRES1oJJ)*D$0<1Vk7>=mRMDk{R4Ot3KQQe`0?z+p7G}vL8 zfO#72u&roY*ES-_bu<+)2=+SIDW0q6w0$h!6%KN)6{99dR@ou{)UF5))*|-_4Q;Zs zVs7ouDQ1zZ(zbL{RA!VWN59~#aKBf&^N$`CWIicC9^k}h5r(87zwZU~I9lso>1pan1o@ZJlJcltxn1ul&P&B7NpKdXsN!xV=MDDbm?20{ zPcupS#?FG~s1H=El=5x}U63IC!6N+t(Qj1#QIB4%<(4fOy=ofPO77)K21$>_vwRv5 zr0>zecV+gsl5_~xWL2@9QyV7>ex zOXS|YBVD{CNNHNyRBqUm$4wAvW5Bc*rcR9X?%hgoj7aY&5b4Kn=#duW(?Ua)mRP)< z2#8ziRUX=gz;sc7aTawrpufQ#E({sMzeM35I@mdL8qMv#*z_xnt}){RGcK-z)%Ti} zA8qJrV?Hhn%9ZSs93_*pApHq_{)K~0o3S`|V=*pA()+#?Q|=s6UN)m%9Jy)kuBflY z=sG{3qFX0eZ;7Ik8@B9bB~`qa-)m8x971K)XL;=9RYv)WzzV(mgEIi^(TVb2UV==& z*bPL#6_xvkmFF#p$n+{;dM&8j)trVic2MWnC)uRg83QFMX>`Dmfs{V9NY6#(R;%*; zVN`msmxY79V^lWf+hPOx;~8uqKWG0bX&@D>(FYzfC@!UySFOrdt!Pg1MaA^aQZZ^> zg|_t<`JqmHsQELT9Z;vdC{iaZ$qs$^rqrFks$pG75?=U^P3gMD=Xv^ zSxMm<7r+CH^iQ}(+m(Aq5yb!R0w{a$5~Mp$h~sB(vVZe9@}^h2fiy{ymxrWF%H3l_ zV4Wz!Ya9l$&;!qwzRn5K11B`Yoi6r{AU$D{RcVsGIo5$MT=o`qdiuTaMgGEL|FPAD zpSPDsRaKpWfB1PC)k`nUWSfIo^PKhfc9%0r-3{PvUS#2=X4aJk$w$bS`Ci~REje%K^B zDTxNh(~ET6mqk42$Rhrc(ZYGv7ezYh3nW*7iA-GxZzdPRv+FTcrK7RbXf_rm)l;f( ziaSvq&_l4y*RT(t0v_L#VZW;W8u90mKM$vT+lxK3Tk~GSg)po9uD9w*AOomX9X;hlx)4#B~9NW#7DOX?J}r5P)KR z%%J?9>nI0mZSv`IUr022JeDMV@!U(OSz#m9npPq~77v{#(zWafR)4w2~hu8 z!!>GFDM-9usgqFJTicMj<^D9)w~y%1|5P}0)KsKocu8l z<=~_MvUU6aT zB7Ah-sE?BR$|F6Gbl>$w?{t!@N7H=k^I=ai+T)8;%>hcZr+WVTdr_aMpG~4H>WAUc zE$wP@1!EnVo9=iW?m6H2y=UBMb=J%5>=N@#F&h&)1wopFFJU>FizRx!y*`hxzt`jK^$aqB zb+E2Wz}&$aSc4!mIc>mbw+VLGd!Kz^3V1s$!+vZdl}hG%-&D?l$P!Z^50NY!xIj=Y z6OFAU*H3KBnUo0*~uge%I0p4QmnIq*e^u(u_-?M;9U@p zXA4GFlzjif(SpG>ru|J$5#-|MaF;z5*=M2&Ch^n*2R5@><|s5*wVh}xt!tOT!=C&7 zg2!jwsb+s2w;z0C!ffuY2aQ@Pn$)Y|v{PuiS&xabW8`;tR==^sxqHTMHGMYJC$N*O z6JsIpjA?Wj83S7qJ7Xd_;c5hl(N%3d*XspdC+o&TMw*@Mr$YjL*ApU;YMyO$)WKQr zWe=Qtv-Diw(lXg-F1fL8^*AQbA##S;i0KQb-U6C)o0)YilU{Gp>Zm+?DYD!7*k`^U z07;`81w`FBZ70{gvXSXNA~pX$d`M~&VMhbLEiu z@MiG8%Ivf39ZO=g4_OkUf#)08YRR`YkCmfN?zG8{&hm~_ag*6(Nw_!&4NjnQa@B?= zJF3%4rMPPCX0oGJR$r-(cWY>AvZDsBGRaM5W0M{E$5QQ{H|=4vqn0DZO^#wS4ykMe zr=HrF?BKr~(maq&dXw3<0C>K0l>}H6WQQn}n$u{sCD~y<5r?dR<`vXns;Gk=Bs=iC zR0L#4p|Xp54P{OhTB($lLZ^`(g+*D629DK7WXG}28QLw8XwEkm*-?1r##q=Y1dLo- zsJB{@!fUKlg#_6tK_lm@IIbSnu_2;8 zgluw3&+NwXQ(AsoPrSs~Xiy##~1!vrEaFQTr?idMOlZyTF=`Yr--((nTctB9405?=Y=UJ zibHA}Z>JR~EoO@>K2Q(J3p;iY(E>a3aEHSUj$qo;nPe)O*`0{P9FffJGCS8~OPY#- zsd+t`(z5AjOk*}qw#PQ&nu(0W!mO!fb92d*1_K=EX-;MXny$fpiE2(!LrBKPcG6lV zsjY(~JMK5oeinx4hN%&-a=`Ae>&a{m^Z>OH1f&3$sg*aAiGZmXUDAjH2^eGS!Kqej z=YWx$Ja^K%SuCLedqjQ<1lVzoFdb2siEdl0H(_n{f3I=Mlqj^o+ zy}1?DH_=qUAUNt^r*N*G(~hvbEF9!qJ4Q{A?6On-xl<9GtW_Qm8aiZG#oQ{-DQ1)G z(w=ljROXZ>=b+%KaKBf&^G}@;WIicC9^lAl6Go&EzwZU<`yO^idU2Yig(D#qCo;jg zD@vb<%1cqXCnBQi)4?HwLq&XX6W{9|=9HdvG3YBt0<_cO&PPrL-GD7bHku+N4(y{YK@V_2|V~ZrP&ItEOSC=w7a5kn~u*$fp58dW8;tAhREf z(jionRfTp=ZJaFpjg(OCH7PF|(LO$B7;|AoBwZKhq05nbKT)8n?!3jWl6w!2bm5X9 zrDS@-MytwjK@3$yFHKVJ|`8Yo) zSF%rX7ER8A^jGxxH%>Nd#o{W*VqT7<_hTupe11ZC*@Aj;->m{?!L|X zixd^zuq8Jusp7qSzg2m10+m&t<*}Dn8RaVi%lGn6E&#AUzg^nPi;x)?y9LofoARO+ z5m{aZEUyKXyPC^%#y;wNa+Xb7oiS9Tl12v{8A$0PoAg3d?zSth+ED57UKWn`j#1f^ z=YKWBP!=0!O86jBZYuJZR z9*=*<-b73d3?V*p zLf-#l;VZz&RgHS^DT!a)Y5mAQIlTw*CH*EIe;`6kJXCppHRhgnyRQ>}5q`fF|3^sQ zUW8Nrz2(07-9;bxA`}=ZkH-yDAJ|89a&;^<-e=^aXK5ipnAH!DI6p~rN~L&ces_6I zhu8)t5O4K(j`kxxfcX;)Z!NDc>9O^20{pqf&-3^!`i3{-srlWQZ`>1&3~!P<)V<;I zeDv{DFUd#$bf4S37eM}!e6-jd01EXnhw}Swp?q|@CEq>$ z#7dz;Q2s2+OCkSG_4p-`zKx~kPb~Q$zsAs)wAPn{^SPzS0K|mi%jqEqBeae@e^9?Q zyO7vkT9`}&?=8yLX|j)EnXjiA?*yhHC7HSkKhPflmW~w?r2eykYt*b#ka)jRr=hfW z_aJpk{b{LhKhdH8scLi~JcczI56)xFsu4WDgHL1x{`%Df`C}2v!A(!NYg8`v&!D~u zzSxV8&!(5LwkJqFheo{cHRIz)DWB+(N-=@@t^^S85%C#ab7Siwe01KdkD~h8M*23= z{kN8VvpQFgrujD)BVIk$=TA`00ZO!|dj1FcQJ}HxAAAW?(}Z2_wwLnAN-*T{)~By O|AYx92-EOC-2V$7>-GNt literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b695a7486c2960b2d1fb7b60f5ea96119940bc45 GIT binary patch literal 8053 zcmc&(U1%Fu7M_vi-#D@3_|9rlCF``##G!HYuc7Cqx}Q)46zY27EZkdEax^dAP;{$C9A{g@TJIZ7jmEZegHI$ zt``w?<+SZw*YbL%>yXm?`|ts!Y2^GgJ8MSkxVsrFL7=rvG^fR@`C3O&u39Sxf{ca?C99vkt%L7n=ArWqjkY8^%qQZI6|lUF7EBX$(1TL{MLaq_NOFdCOD38N%|&(;pSd#@wh9TOkQUEd zEk)t6-h4-`BY6<+GkKG8X(m$?p%z4*=`ylYnqz|!NJv3Jc1qC5`6`a9hjna-Xb+*9 zoYFJ9vHXOV-_{c^F*X{M$3(f;WN7I`;_YT4;^U)c)(;Btakqu}NVgMLpB*zKT8Y+b zMX`6>P~L;IyKTnmIhlIhX0Uc6@rQOpDH(p=!Te-`qk3?PT}9&%hhNSGTca6hEH|^U z1ROTjBn;Rboj0Sg^v!HOo6}M^Q(7(>kLIGRU0|)}ZH|*fTi?jUG#?I=6Y=xHloQn< z^*C>*6(}uci!2qW2jj&ZJE&-Zoq4#!;RZ)AZRt!h70u));xI=fb9rXxmTXDWFfcW* zMN?Wf9gS(s#>w{BdR#M+ky@BFwQO!KnbKf@13S&htY6bLxG&MnNookm*w}Vj%Otfm z&}7H`2HMZU5Zy2}0$vW-9dIwJHqV$!hJhLk=>JWD=^Ghv5PGz*86wrQ+XutRs z`n_#_EEg(<4ITPb+MgE(dg(zEkSlAbzP(3V1jo$ut^%3<%UedK1^K+tP^BdnZzlrcmim=P%_HEt zD8M+2Hk{Dk;0_l?Oy!SI`8_AQWG$nG{TG{YrO`EJUSQ_MRq%SZMfs^2Lv7B-g+aNJ zf0DCgau%e&V$8pBvRNyZp8Z(N%aQDUB*m4_jwvr&&@PVLGnz9O(fFaP)g0Q>Wq@?Kto%(&Psi1v;vFIo|i zw8GU4vUWm%wHs#fEGS=RCeWg zv4MPd0UOA#*?&qJNCj_w*H18ynl{`NECGZLhN#ud4cJAC|CI)d^g|miY@S6!ndPn7V-5V=E{z27lGf>5R*@jY`yO_yr%| z*mcrKT0x(xPWcC2t|2!<3E~ZWlYi{mEm{*R;TuN# zdH)sQk0HrVboiR_5GWPD{p33F2c|b&F83Vr-$2$P|DuT>Hi=F~V!_Gu5{dh=hzA2% z#6LD(kXL;%q?5gXz6wm_Y9zd^FNNnf;;Krb@zi)W9wyT>s&9q|Q5`Z;u)^1{51#@a z-;C+Ms{S7FZz0K-i7c$DqsSV_>Dv6TcQL%}iI8X(>FG!S>;`nVcEgj1ss16vM?omW ze=KwbIJv1&H$EltiwCVA`6s7$Airea#N+daiHWDGz^}$!(=OL_;xEDPv*P~{>065+ z<=a{6o6j$LAr?XZ(EfbfFztbVM5j>4QsaF_J-Qx*zK&5ZYW?i|DbnPy*t)$>n)|7QMq8@zi`i<{fuO!^0aC4s~aEe?5A6 zsh8EGZ@SOr+VLZQSv^|(A0;}~6SoBzR(y-D<^AU)LI8^AV-EHA-9r5sbW6Rv`-zoO zg`j*{)R#j3?dthUB7F;wnlBOYLVb;4ENQJT1?F>+@Brk5^2_N#NF%h40)IfiHM@}5 ziY!c~f%g{G>ooaCxh$Ng8P5ckAr+ar0zc3n0G5s?BtZRV9oJ}Cr6lowrB1`q&hJ3! zmdDd_zI{Z8{->(Zh0qwD(Rg4U&#W3o`E7h6C-B#&CMX_@a2#CpguBMcrT!VvHy{=} z@$uR85}xe|($Aq0FMQ2-xhT~WJ+udS zfwi;FOTgUD8d!rMG`VcRXtxP=*n6LSU@AD1kzqf+o=&IoJ#VUKUu4Ni(1%Fo_uZf< zpN+*=Q)??0Cw#dVKCbToMF7pzL6h1*M~lqZI3Gp!N%qeQc_aJa`w> zqioU1ikk0VI9f4;#`M3*C4yf393HZ#BKu4 z9heJ&S4^YRs2JFi*l82V33nrCjIC&E`5sK5i*=!pQD$%Tk&r;&^@IqtnzD_~I*|2U z_JZ7-mFL=~md(ZTsr5B$#C?bB8Dzs|EL?gsSk7-`K^|(*SF$>)4quAwb|LrK@B2a1 z$a)b`S6rWqjkY8^%_riJ6|lUF7EBX$(1TL{MYj>LarASaODSOC}tJ<{~?b&)gXcTZM#CNQ>vK zmZI=kZ@%N!kvstRnY>B4G?OWcPzxf@bQ#$t&9VUrB&4t)yCi7je3ih}!#Xxdw1-el zF6o)wSbj>&Z|jMd7#j`BW1`$^GPHCe@wS7A`1r`d`amH*?lrRj>2~4j^Amt5hSU$$u1=e!j<~&KX^^I&?3*az0o;WW|x=#C0|_`|?15B^wR6D8 zO_6)cx>>5B0eeJY3l!LKjxYmJmyK;%?KR{mIYo0PQby>m#WLWBZ4w4*^uw{L?cCgq z=^N-OU=W;juv3()=d>d{KadUzt{t-`NOsvJ|JHBVWT6%GcWrQOkl@>C=xhqPa ziONe+xhEo`>eIm?gF{7p2@~JzZswAnbTa5GM*_5*Cqj3}CVeL=59^fQ+YnvNV@Ayb ziLWB|hoTAo{&{v*dd}FB8o={YPq_afN?(b}GrRJl4sq8qztl41R7U%00qxg__KRPz zuQ^)#VdZJ*NCf%U(z5bxy>hSKRh*ZKPmIm4I>D?(!#@Jtsk2{M|I zww2os<#7{a+8i*Qg{c!WeQ>uD95d5<3S|1vZyT8wQf#$hQW1F zfN>UWxS+qm9WD%;${(TfdoFg#T1N8+FE-;!qif8(z|4!Q;Pqa!@{Xz1=4)>mSv zo207DQxu z6|lS(RPJhSGZ=el^T}B@Wevt)iAfqAaAY8(k8IKlQMucuylO?G$9q{g-aAHRSDqIe z$afd8f&7yFr=)>Y@WvQ;$f2Z^QC_zxkK53l;){yqouy*dx(jXVuNcDn=UAj9gw_$N zI5y$9L{@bvul&%i?6;%U8hTUlok)=WjRs$yV?nFIWMzYVA}cw#<^`~Cll~02(NLN$U+aiSo%6A zNME_38SZtn4+QB6lk7^9^z>9GzHm8P)R~!&qVMw;9_LSO9{jwwGN!8PH2lNQhpM`z zj;yE?xCOte8lZhx!dg|waRpoEFRV~BFbZPoJZ_J!pu{NrQ#Yrw9?v!^QLo__d;+7_ zNh4_meX2Sc9PoGsy$B_YYgEKPkwN-JQ$f;4{E=DYSKkBv5YbKk(QCJ8O{|1(2<`6# zuK<4>NkO8+*R+p7srVfv*NHzgwdwJAXOaH~vKILlO#HA(bTSeTPh^%zJdi^?7|0_2 z(XoQO8i*sE?1l7IU?Nu+qucsYbZ#S|swA37kL40kGCi#Zrg;$6K_dk#d=2~XDc}iA zoBpfnZxR0%k^3kLNn^m*5Xr@qdK$tp$(@ z>@M}r?JW2q7UAIF!F=2>?SX$pr%=b!W4%T_dKc%DgjxOY2>A)3Qz<3Fb303`I^;Gu zj(Dr*bEFUHL9CxpWOHe4QID@hk`T{LF3$Rp`Q`L5q!C(2fj^|*nwd{- zEzVD5fcF;F>lFD%xh$NgY2P@OAr+ar0zc3n0G5s?Bt-pZ9oJ}Cr6lowrB1=o-r0rH zEsv+=d?`D^4s`CPT+4qO;S7-;5c~b33rW?OZ_vXZ$K<| z6Js-(B|O{Xq@P10Uig~!b5W`%dZeS6M0;0)i1jI4UFbrC+2H_u1O`PxGI7SaQ^ zmi#k1H;=9bHWs2jJ>DBg(wPI4=uh?f5B8xwOFx@JUDOYwV>{XneHC*Zo1f`dM}we= zum&&Um=Q@+A8I}6cBHWuw)BiRVyuYUfn5j}#a;k@UUqPllFJ?35TCnxB< z!A%EX07Rb}+~SUb-{fitg^K-ljC)O+@Ca7`J{sp^p*OG`_Ye8v>4;vcA5{gG0rS>; zZ=^roJEJen4doZ-XOfE{z(JPOUkOe*q3QWpPd3+O{8_Fmv(w|dJaE|$f2e{#V;2umM=Da$^ZQc5UG3Hy|%C4@ZWVOdJaLnvk0hkaN=S@xWBXQYuV z*@|IV(MWUeIrp6J{N6L}lse;Oc6Nz*CYg;19fBau!k4fV&BYQu-X5RF*Vp6m_IR!^ zfwi;FOTgUD8d!rMG&ya+XtxP=*!zHeXbL!!mSI1(o=PQiJ#Q;#Ph^QnkcUX-_go+- zmx;z!lWQv$Ct!dc3YbUJ?W!pzL6h1!Z#=M=93XK6M;)B? zUiQGbw@c5pO)Zm+=924cR*z!>9Uy0r4V%7j>Mfu-w~<-HGU@S_tPab=S0cMzh<)aV zevmY>UPRQD)3$S6%j=o0LsIkaLkFa$5%bgRtm&=eo@USlj@B~KoEEQUYaMyHYOWj* zAKnc9SDAg5y=zH~wn0l`H1K=_TP^u^++*dalRIp(qho(Zs<_E)vLsv_fCeYfIk{>> zlO5G*}vGX|+_ z1gD(P_vC*JBA=^Ix>+ir0eeJY3k2A4jxZfjmx*p!^)+NEI7M~Gl7{Q9Mblu1Z4w4*w4-@d z%ir9L>KkY(U=SR2uv0u&&uNEPzAqf)TsuZhknFNk{-r|^oUB#u7aH1SSH;}gpHs{x z*`*!nj;PEkP0j(qRpEZGbmyNuDad?MfIPsF&n65@A%5Qr(huG2wDj^6OACiWDotd9 zb5E2$7nN6{a$iJ5)u)3428W9H;wHW~-OMRH?PSnb4h3kbCqj3}CVej|kLr{^*brUK zV@AybiMJy5N1_S-!FhI8dcoM!8o=|aC*1!KrLRThxm|f#hq!B*Uuqe0Dx>|Zfc6_i z`{l3LHyo|~p!BqKB!c`~X<7NMUb$cIEY3^CCrNM>rl{g>CFczd;Fuvu&(AYSdgfpu zYt#oTR!Vs{gf2*szOqTLA^MHVKkLzpwcN5LqgPGCTFJd!$sp;8c#%&7g7g|4{7_~; zm83(cCaa3=oZ2{9_&X_~+;37|Hllre&M@Y}iby&y&Ow(W_uf^Ys_wkQu9ABXj&$*o zAf;((Te;n=JZXYRn**k!Fm+<2_wQDMV@7&cfk^-59V60$d|qg%(h`fe69Mr^eahqJ zVK7}3V4OuAPUvrNhYQ1|@W&|pzLQnl-Ia>JJ0tfY$f^4(VD=`mDReU`^wUS*W82&~Y{?_L04e?C**%S(_M7rO<~-f`t+ zD5N^}`SdKCvN~h1L?w+5I5LpZ$2RGusN8E)ULQxLM|)W~+B-&N zQ(hDs$oChpf&7~Nr=)>Yu*Mj8$e@IjR^GHJPukF&;){yqouy*bx(jXVZ|K4Y=UBMJ zh1LWT6LM zEPb65q_17j40pTPhl2E!Np_`4dUmQ4U$`8t>def?kq`I_kK^YyH-6q<8B-LrW1c~YSZoZ%p(5{WG(V9nD}9n=%geTnn*9vaeo%^ zpd*X;N5=~1ReucWq%Wwi0u!0K7}?gBB6AyYRi&fx)L1qiA=T5Wf0{c{9W+9)!q>15 zp8_8Lv}wPpev0_FkmS!S&abK?$QsP)+T4(DA+qgVq@!7+rx$~uH>i8G8{R}r4Gbba zazerXqv0#S$yJSb@F|I3+-ZHtKQXlf@g@Bx9)BQ0OgvNtel_Nva=Wh+e+hoS75~Rb z-&%lE{+*@Xx%`3;d=Ux^?vKX}QyDwwhBuei7WLR#I063L+nF2qUzP0)J4y zH8Y>sTAZIq1Me-$*D11(Vp*uCY413uAtjl*0zc3n0G5sw5~TjKj%(DcQjmDRQm3G_ z^E;5b<^HtPx1Z?H|5P00#+%+nf`e#tz z0AK9H$7a$?SliHb?wz8RgXN7MWp3lXm#>-8t7<^UzyQ$7C!eW=gU&!$io^~1nzyuSeB^f=iMfdiayuWa%7a3`Q7%29s*geErQUaGJ3k`K}{1U+J3{4DhlBsxKl zrG|R1M^GyG=RkW#ZXliF6`IK}M8cV9q<@Xhh6#W5`nyK-FrtQf&n-oC&va_cv*1fi zP`$xT2Vek1pBmWWhJoE=Y7h?<`|TLFnl|APt^j;A>SLkTzZ~-od1I-tUMe3&`IiCn z)_iZcKh`^=FU<|*7UyRYi$TCan$+J24mqyrxoA%&+hzP&t}C7I@m?Od?1Mj4!JjdY P@SiZw1YrvPhx>m4xUlty literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8f0ae271cf1815abaaa51757c2e3aaf2533f78cf GIT binary patch literal 8052 zcmc&(U1%Fu7M_vi-`KU|_|9rlCF``##G!HY|~M&Wpg)2Db`s)?C(YPo*_Q)>6;)P z&lZiWDEZErg9U?Y4EvkxBFM$h{w{kWvX4arO#F!x4s2w#%t2@_X1_mw;T3Hy*XIIWJL|?ndYZj9K!*hSt|vqw)i~Q|tAn$y z^G-PTYU#PQsb#X^Tw;CA>~Yk;kDOt4(eQ=6rv)_UHZp5{F3~bjTwv9Jl>O51g~)Cc zLZ5j)2!h7ei-5Xw+E%W6c|Fs8KxTeFvQK8}AwSMe8qR8SHiIVcvz7_xv{*G$>&VGf zQ)Qp<@Lur0O6-&D4O3pU51aC$fu|eTYD%{@r5G2-mIac$+jA}_DF6r`kHJjJeF&BzG@4TZM7UJ?Qs;F z2}or@J0i@#UTk)A2 zV`8fiFmh?J-fBq-m-*s5Y#zx&aGlAU6iXwRq6oPl@<^AF?b19OlHi2o7i7BxjhwGy zxZ15_!$jMKY_dyFt@`pKT0X2NUSe$2DUXP9t4WvAg~S`pM8wAj&1?V!;^W;G<{{a3 zT-`pRpJ*jos~N?;HeGlx(r&ivtM5qF%XXc$2Z_%+bfKj9X(#iN3XbZ|F?I=+gCBlA zne?QBjBB9Yr> zR<6mCFcbq*^IABiWz*rP#w?s{jjqQu0~v{hQB%w2<`XFm<~PvOoXC2$qz3mTsyRvx zArT$lN^6;fwg!@{xZgngS(u?4rbofbL95N0Ok{JQ2dIr8AO*M#t-P5`dJVwasvH15E`Cf~^jAis$M%?EuU7goB)G#i$9ARkq7NcPfINwaSA+LxS6thTHX-9frROXc?`;gFE;eM}l=N~;P$b3?OJiv+1B8*CYe%}ky_dV>m^ygzN zEgT4`G?59;hobbcsJsxBTOuN=J{{~cI8?+JGw{9aVRq@8E(U$&K!B!tB6ROtq_0I~ zw@&%Yg6L`%gV!g2SB%!x3MHP1|Id5bL#|%Mw za+*ofw>IWCM}4?rrIdF==z;|44;JZ1h<>B;k9zcCEw^mR=vC9OR&p;_GDvzPp5@bk zApL+2z9X~0m865eCaa3=oZ2{9_!}v%+-g#uHKKid&d}$=ib%T7E;vVd}(4@7=8g$B6W<0+D|7x*lmkJ}opi(KTjVV8+E&u=;L` z@>4Uq+L(_EgK{PNBzwu^EJ%MspMPm*b7m~Pd$AaoBk6rliYd2`C@)%2FOJ+acURQc zYIL0+QPIs4Y@kF@$qid}vyv*_%kQ=-kB^|T>a#rd@+zZzMPP+q{=pdl_V7e`FE2r+ zU+e~=--*h-Hsx6>A~L-Sm|hDiceP%_8M~J{N*YK7YxIGK42nx>6_bvv3) zPoZu71zq^bDHbSkp?QQVj!if&k=CS?Q=WGy_c~B(4ZW%OP9#YGLWO@l#k^*P@yZJM zL{?I`#szTSBK;Gt(N5*=Q3Ub-y8z1Gy9DWWFU0ZlH`u><9C_2L-9VZo$;+eCCFRbs zQLs)F;WZ8eS?GahOJC;%>3%OX!`&YCwje!bl2vJvzCG53FI=`(b#Ct6;6?tzWBaMy zfuFZlCRA0Og@5>YM^*Fc*or!ZTkxx@9@>W`tW|XqSFmOL!U~Ui#^IQ{h}+{Ucw!v> zshcwyhhqzcsF(2zKA!O_B$1?oJXM|c4mljdPK4seH45UNz%c!yDlf?+{=hu)t8W3n zkLU*f_~q-gCRV~Xg8H|;7l1#CBrnn7Yt}`eRQ&dm>BR4w*>pIZ^T>Y{S&RHj27cHi zIw^_zr_vER?#Ut^bYv0#_(b8n>WLzq^!bvjz(l5of?LT*aA6~+s&q7#n#jh2qWW8I?=a#cClvfY9=HIUT-BHppAz`Roi>2{Q!_gdU(#>j@pyy8#6wl!SEG&@ zhvN$Im*Dr9@xP1o{1Tk<>_qw(wwK)C3%_@GZ#=FV`oKP-ldGetiGDpFouS1zVOHNc z;QSQPDV1XWh3&{{5@PF}M7-7GIW~ZFFXoRguo+nkC8KMBIQVmupXc#ea*u4pQVZKr z_k=SX9N8dus5>Kj^U=*yy(}L+GyM+7ju-jM^3i1f7||)8xGuo3;#qPm@2!sz0VvkT zJj(CCj`GpzrhIn}5G#cWL3y$$FNOSD)#H~ydLB#76A!r|zsAv*v`$8R3%O8m5Mn~{ z<#a!U5n4xq-*wJCM5N{xsFMhv?A%R5iR97{?lo`4+Hd)gYeV!Y48Ue?4lP{ILY( z;Giem)hn0!r!TnyzSxOP%%vk(+mj@pLnB`JnsxJ|luz_Xr5Hzj7rcn~fcT89I8v(uRUW|kxudQ&uuRS1DSAeaE;Ce2!HkZyG--|qK10U zMS@A^Y-+-}dkY*z-}_thlh&&c7j_?oA3x%06yyVvDEKbj=D!& z(NrK=Dj!98mH~5qu|F^v?Vn3V7DjTR#kqLM2RKNR`YXXE$FyWF+?UCA>wlK(PH*?Q Z&JUe;!yl^P&zOh!PncwaFa!U;{l98l^Vk3Y literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9c2f12f2a3ac152c71f92e4e4bb10595098a5ec5 GIT binary patch literal 8053 zcmc&(U1%Fu7M_viA309#IKH!*RLMH6GjV7f{rD$#@s^A%Te9n@w&Pf~D4Ug~v88xq zYbE*jDdZu9(1%i%5K13{Da$^ZQc5UG3Hy|%C4}sfhh-@x522K0ANFAhW!ZDiosmYe zWGjYcl}4I-&);``?zv~uDRsun?Cc`*Ofnl2+66(Hg)d<-mX9ZUd_6voudm1B?eSb@ z0&8O(7lFBrHLwOjXmZ(r(QXs$u=hUuz*KNLBg1}tEuBu~d)`#fp2(7upbwGE?{$Ns zd^Q$eNv$qhpcJ$&rF0EIX4f`*yxqUQC3@?;1iknsH{zGg`n2yeRu&AYDDZLs_JEgXhb(tvJhW~PB`5QaP{nhxbX3vK91ojqd z$6N@!Vj7)B#lV)tPMJuKbvJ^>*s`{o@4*DRSSJb@W%gDd2?_LFPl!OPDck6*16l7S z56Hb)d9H3~*<382T3fY79PJ;VXOIn>v2f|lU^%~@1$p@6Em<8_hc88TtC0Kb_x+%0 zWUYv(Gp}vsJD1k7orjd>--iw;O(W;0*cmfg$2|_P1cBDFvAmY3=4%~AxoWK(kRRR* z{#TWKhP`7cjMhO*VKnf1179umcHCp-sFT}mva@}EN2<8VY_cR=7=Q*Rkepn#p~=qb zv~n%owRSVvSu3lr+{U{#v^3dS1J^#yO=e@0orTA8>z+65VY0Ipk1NSS};x2K@XCh_+737 zs-t+aOL`4eP8C|YmgZuok)6fGz8v)*sgKCcBgq-oEtzl>nv3i#K67U*Y!wnlAuXP_ zT8hGJz4?w>NAdvNXYvN+(oCi(LM@0q(`96rG|L7gkdQ)x?2@37^Hl;@59`<<(H=rI zxuj=yWBCa!zpW=;Vr(=hkBM@x$jQ=OxZBM9q}zq7&yE=qEktXv zqS!lbDDOtv-Bx4uoJ_rLHCVfl_(Pkalng&_X8|(7Q9U@xuAp&*6)4SSi!2qW2jj&ZJE&-eojJI};RZ)At(j~p9m{TyCt!|9<+qugTe780!@%TN zjit3*CKlJ2jg#&1wS;COBegJVYPtMuDy_i)2X-8(TtL$`xG&Mn32F$b_~=$f%citd z&}7H`2HMZT5Zy323|{uzopwEy%Yz-DHiCjQ;4-c9W-=Zy4P!_eaUcO_j6INQv33p^ zxhZmQSvN~HG+>V?Y=Hth&JkuH>awv-tG$LCC8ucaSjq_9)mR4nuuZ~1jea;*wCx)k zF?}6f1q_0-4t9!i^_+Hy=ljw@!L?)71j#PDNurO7oQbXU0FE8Y1gP6#re6rc`p}!tJeo%Q@Iub$twX~#sTd&-!cNOQQ;*%tF7pAD`6 zP)%MH+c~vy^6)oOQn}Zpyl6!K_?%(Pg%y!>T$qC{N8!Drz^S_ZHoHRMJvh_FOM;AM zq%Gx^LwVc;nKlPZXJP8ZOz+>V1jo$ut^%3<)7wU-1^JxNP^BdnZzlrck@}QJj$v?J z6kwc18!qT?aEA-Srt(Lq{GN+lw3gBQ{)^4H(&!p9FEI1sDtNuyto-D_P@D5{VNkB* zpX4f;oCWF681rvjY{rVEdp{QQawNMSO9|z(W6H~Bw2LD*&D|C4wU}M!M>KTzH0vud zRC2?X-K=Da_wwBq<;gKLR(+PoUS4ICuL!Kr%kP{AV1GPa-pfmn85g?+(cW?8MGGRb zyb4%e3o3WDZZjCWX!FS#Hf0UQV2Mc@9dKkIqmOLT3sJe-s=PXmMvwNgaI|-f%C0;w zHjwYmV*~jm`%g&&so;$<@Q_1EDWkk@RUWsZJH;0j%R5WOtaTOI*55FM_s_C$NeHbY zRB>#=afz(zQeOF?P1$Qht2Okd;yaNb{TmIwJj()BgUQMU`9xN7aLo%~&nEp#RGzmh zU!Fh^|Gx{M?7d5ne%B3o{KGr!HP0h&dbK-9lO#oXSh}J-I5`a7i6XqlVIm7X@IvY9 zoFIMG4b5=3i+vzSPncv^nxv;EJMe|e*`m(Od=z<~zwkJJYIWo1z2z}gRj1)Eem+#y zO?6~hoxm;lRaHOj!xGl2I*u#YGJj!(qW)13Q|EDebQvW^;jg+em36zfP>FgKzu@B^ zy+#^IE9g_z$-scyJ?KFwAzY&({s|A#FPaLFKH?A0BER|`@CS)*@{eA5!Kn?m+cS&&*O9fzzhL5rO`?;LcxWQCNaFq+;=w=` z@sExb0~dcuK*Ld8jWn}i;=nYgsPHgB0ZK%M9B2C>YwI8R0oX|Eb}$&!>549 zKW+N2s=r12n@I9!qw_232(kwAx;8iDTZn9Vqa>O`dL|kKyFuNfUH2yAYG4rYQ4k98 z9}QmyPHt+%gHI{^;z8>}{)wp_$S>J9@%RG~V&bVP@T+n6l-qrc_)GBnt@uAe`sM;i z`F9q3=e8Go5Q|V?aDP6ooA$szqEo2j>9Jm;9zD_dBwCvFQcEc+MSOZ(49lmHaZ$1Ljay@~oU=$3l-^bsqi z3PJgEs4tEDTh;TILi#2iHGeYdgZdi9SkhWw49?}Fk$%Vt<(Jb#kVa@71^%FZb7nrd z8J(ZV0Pjtz*D3Oka#=V})826`Ln<mqz4Z=R2m^RVcHb2v^js`#z z$Itr>x;?K_wM{Ab*0b=Je|>>_hdF&h&)1VNgEFJUQ~izRx!yQbzf!z zYiFGofw`SEum(YBa@v5=ZWHXV_dff;RB$>i!+vZ%l}hG%-&D`O$P!ba50NbF_kf~Y zCK_8!uB}+0!6ADpvQI@5Oya2<1U9l-<}fr@wC!jrt!tMc!tVS1 zg4=7|spfwj_a9+U!0VX~tZk>VytxfzF2Hj-0M zZA^CXUk+&=$R@qXY+C?4-?>T>EDEwi6iVG`G}@BvFrSD+R>1NyS};x2K@XB0_+6?3 zs-tkSi+T-JP8C|ImX<=NksXCaS&jyd)JJ5;k>m{PmP|C~n~UryJacC(Y!wnlAuXJ@ zT8hGBz4?w>NAe)tXYwZH(oCi(LM@0q(NA=(&yMo3c4nLm_wnj71SZ-uv z2{>%5Nf@v>I&VZ{=^NRdY)(tvNNKrfJerHLc7e5?vpG%>ZG9sX(|kBgPQ=d%Q%+Qe z)Z@IJR-m+)EwWgk9*h@u?4Y6rcIM#@hZ`Khw52o2R5Y_Q5r;V4%=$H5gZmQAoS=q~jE!xlwMg zk((m-mUXjOLj(4R{1zy%;~ZfIqAnBNvf69NQgRCBjwOxIU5%!}58D(B)aXa^s9(lMDNW8np{K(AUg^$1aYB&!qyTk*qoYk2k%Iib7o_jI*(vG8NtPB4g;ZR~1m~_O zeJUz1MdhA|h^kKq2Mi7s@x@Jiue+I3deX_DuN(@{a-Im?ZJYF+s64DwKDQydn#YWq z2NF+3><>i~{QYz6jP#tbCpCcQr=D>CLzKP}m1lP4MIGXlRVn55$zYh zU|(~z_JhjP(vb-Aucc+>+j`|*y|XYc6`mxaCqG3Mb}I#Mco4@7L3(zMNzzjX3tF>2 zRPm&gc0=fb1nEng^a`@ysQj}Yqgcx=TQqvrG^`cf%at6G9*Y8^TIrIISTI`1y0rNx7ig6@4=ZaToPn7 zEp02enw7^*kZE(kbmXT_%yju~B{*iL_Y}zVpWZeyEy(ADhAJ(wa61tYx74pZY90aC zMFGZHwBdyQ26wnHVk&=x%I`VZMQa%?lwWMdl}6W?d4ZW1SKjNr7UidA47E8Q=Lh9V z{z=ZF$yt#8iZTDj$!4usddjhwmm}HzSc)s39aCPmpj{lfY3{CQuhr~2Kcb<#r&)iI zp`shM7~vFfut_VOyDd_`dSUVi610Q=+V(q3MK%(&Psi1v>wFIo|i z|uxV>BhKfwm=zt>w8GU4vUWm%wHs#fEGS=RCeWg zp@DpN9vjFn*?)=}NCj_TTCJfs72b&i>ECGZLhN#ud4cJAC|CI)d^g|miY@S6!ndPn7V-5V=E{z2LIH}>5R*@jY`z3_yr%| z*fr8fT0x(xPWcC2t|2!<3E~-xV1=(?A3k|J zz8TYhRsAjE-$Ifv6IobQN0Bv<)3y0w?_zk{6Cu$o($kRu*bV4z?YbusQ~g7TkAjeo z|5)fUaB@?lZhT7O7Y|xL@=s3hLVn4!(zg~t z%D21JH@~y!g;)grL*@CnZrTI?h)$u7rN;Y=dUQt?5`3*!AKxlJmEuzQPLJ5fHCYR^=S@aHX#8dM-G4HrL8Xn%DaHzY(<@M<0 zrCw5xzUe-fYuAtbCG}|Wf0XD{Puvz@Sn(~omdnpaga8!I#~kYKyNUWS=$3kS_Y*6n z3PJg@s4s>5+tu@zMEVvUHD4m)h58!9SkhWw3e4vs;Q`1A<(JcgkVa@7dH#TYb9N!I z6ON=f4VN}YzIy|W9Y zTN+Qx`SuYV`k$&s7eZrrM&p5bJhN&T<+t&PoWNh7nxJ?r!f|lX6Yd%(m-=Tw-+)-` z#>Z#VOL(>?NI!>0yzn*S<)Tzi^hifBf%Y!@5$_@S8C`W@>mqz4Z=R2$^Ruk&**ieQ@(<;JB#5^CK?`CBiRt)uYUfn5E zUuYoKH>)qr59cBavx!ImaF8YSSAs*1YkDr)o5^+=f0pY?@AP^u4PNrXAFANbm`C_e Mm|%i14gbUazf0KoVE_OC literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000000.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..c5bc7de5f --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1787122149949,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":56,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"ccbf76d3-a9d2-4def-b738-8ad654a54898","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_acp_system_metadata\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"acp_sourceBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"commitBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trackingId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowVersion\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"primaryIdentity\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"namespace\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"code\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"ingestTime\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"agenticExperience\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"agents\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"agentID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"version\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"conversation\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"conversationID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"turnID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"text\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"raw\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"source\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"identityMap\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"prim\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_DATE\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCHID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_ACP_DATE","_ACP_BATCHID"],"createdTime":1787122149895,"configuration":{"delta.checkpoint.writeStatsAsStruct":"true","delta.checkpoint.writeStatsAsJson":"true"}}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-01","_ACP_BATCHID":"batch-1-2024-01-01"},"size":8026,"modificationTime":1787122149949,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-1-file-1-row-1\",\"ingestTime\":20240101111,\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"isDeleted\":false,\"rowId\":\"row-2024-01-01-batch-1-file-1-row-1\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-01-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-1-file-1-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-1-1-1\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-1-row-2\"},\"turnID\":\"turn-1-1-2\"},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-1-file-1-row-2@example.com\"},\"rowVersion\":1,\"rowId\":\"row-2024-01-01-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-1-file-1-row-2\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"ingestTime\":20240101112},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowVersion\":0,\"isDeleted\":0,\"rowId\":0,\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"trackingId\":0,\"ingestTime\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-01","_ACP_DATE":"2024-01-01"},"size":8027,"modificationTime":1787122149948,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-1-file-2-row-1\",\"ingestTime\":20240101121,\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-01-batch-1-file-2-row-1\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"turnID\":\"turn-1-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-2-row-1\"}}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-2-row-2\"},\"turnID\":\"turn-1-2-2\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-01-batch-1-file-2-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-01-batch-1-file-2-row-2\",\"rowVersion\":1,\"ingestTime\":20240101122,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"commitBatchId\":\"commit-batch-1-2024-01-01\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0,\"rowId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0,\"trackingId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000001.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000001.json new file mode 100644 index 000000000..1253198f4 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149972,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":10,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-387b2ef3-cd3e-403e-8734-c049b9c71d62-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-01","_ACP_DATE":"2024-01-01"},"size":8054,"modificationTime":1787122149972,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-01-batch-2-file-1-row-1\",\"isDeleted\":false,\"ingestTime\":20240101211,\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-1-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-01\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-01-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-01-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-1-row-2\",\"ingestTime\":20240101212,\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"isDeleted\":false},\"conversation\":{\"turnID\":\"turn-2-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-2-file-1-row-2\"},\"conversationID\":\"conversation-2024-01-01-1\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0,\"trackingId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-01","_ACP_BATCHID":"batch-2-2024-01-01"},"size":8054,"modificationTime":1787122149972,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-2-file-2-row-1\"}},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-01-batch-2-file-2-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"ingestTime\":20240101221,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-2-file-2-row-1@example.com\"},\"isDeleted\":false,\"rowVersion\":1}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-2-file-2-row-2\",\"rowId\":\"row-2024-01-01-batch-2-file-2-row-2\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-2-file-2-row-2@example.com\"},\"ingestTime\":20240101222},\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"turnID\":\"turn-2-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-2-file-2-row-2\"}}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"rowVersion\":0,\"isDeleted\":0,\"rowId\":0,\"ingestTime\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000002.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000002.json new file mode 100644 index 000000000..b4f04fd5a --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149981,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-1-2024-01-02"},"size":8052,"modificationTime":1787122149981,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-02-batch-1-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-02-1\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"trackingId\":\"tracking-2024-01-02-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"isDeleted\":false,\"rowVersion\":1,\"rowId\":\"row-2024-01-02-batch-1-file-1-row-1\",\"ingestTime\":20240102111,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-1-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-1-1-2\",\"conversationID\":\"conversation-2024-01-02-1\"},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"ingestTime\":20240102112,\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"rowId\":\"row-2024-01-02-batch-1-file-1-row-2\",\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-02-batch-1-file-1-row-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}},\"_acp_system_metadata\":{\"trackingId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"ingestTime\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"rowId\":0,\"acp_sourceBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-1-2024-01-02"},"size":8053,"modificationTime":1787122149981,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-1-file-2-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowVersion\":1,\"ingestTime\":20240102121,\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"rowId\":\"row-2024-01-02-batch-1-file-2-row-1\"},\"conversation\":{\"turnID\":\"turn-1-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-1-file-2-row-2\",\"rowId\":\"row-2024-01-02-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"ingestTime\":20240102122,\"commitBatchId\":\"commit-batch-1-2024-01-02\"},\"conversation\":{\"turnID\":\"turn-1-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"trackingId\":0,\"commitBatchId\":0,\"rowVersion\":0,\"acp_sourceBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000003.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000003.json new file mode 100644 index 000000000..121a9a1ad --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149986,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-02","_ACP_DATE":"2024-01-02"},"size":8053,"modificationTime":1787122149986,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"ingestTime\":20240102221,\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"rowId\":\"row-2024-01-02-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-02-batch-2-file-2-row-1\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-02-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-02-batch-2-file-2-row-1\"}}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-2-file-2-row-2\",\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-02-batch-2-file-2-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"ingestTime\":20240102222},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-02-batch-2-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-02-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"trackingId\":0,\"rowId\":0,\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-2-2024-01-02"},"size":8053,"modificationTime":1787122149986,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-02-batch-2-file-1-row-1\",\"isDeleted\":false,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-02-batch-2-file-1-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-02-batch-2-file-1-row-1@example.com\"},\"ingestTime\":20240102211,\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"acp_sourceBatchId\":\"batch-2-2024-01-02\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-1\",\"turnID\":\"turn-2-1-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-2-file-1-row-2\",\"rowId\":\"row-2024-01-02-batch-2-file-1-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-02-batch-2-file-1-row-2@example.com\"},\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"isDeleted\":false,\"ingestTime\":20240102212},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-1-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-1\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"trackingId\":0,\"isDeleted\":0,\"rowVersion\":0,\"rowId\":0,\"commitBatchId\":0,\"ingestTime\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000004.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000004.json new file mode 100644 index 000000000..daa289185 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000004.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149991,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-bf68d798-f825-4902-a824-7a704ee3c700-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-03","_ACP_DATE":"2024-01-03"},"size":8052,"modificationTime":1787122149990,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"rowId\":\"row-2024-01-03-batch-1-file-1-row-1\",\"ingestTime\":20240103111,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-03-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-1-file-1-row-1@example.com\"},\"acp_sourceBatchId\":\"batch-1-2024-01-03\"},\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-1-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-03-1\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-1-file-1-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-1-file-1-row-2@example.com\"},\"isDeleted\":false,\"rowId\":\"row-2024-01-03-batch-1-file-1-row-2\",\"ingestTime\":20240103112,\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"trackingId\":\"tracking-2024-01-03-batch-1-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-1-2024-01-03"},"size":8053,"modificationTime":1787122149990,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"ingestTime\":20240103121,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-03-batch-1-file-2-row-1\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-03-batch-1-file-2-row-1\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-1-2-1\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-1-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-1-file-2-row-2\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"trackingId\":\"tracking-2024-01-03-batch-1-file-2-row-2\",\"ingestTime\":20240103122,\"isDeleted\":false,\"rowId\":\"row-2024-01-03-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}},\"_acp_system_metadata\":{\"trackingId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"rowId\":0,\"ingestTime\":0,\"rowVersion\":0,\"commitBatchId\":0,\"isDeleted\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000005.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000005.json new file mode 100644 index 000000000..c13cb2a8b --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000005.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149995,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-2-2024-01-03"},"size":8053,"modificationTime":1787122149995,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-2-file-2-row-1\"},\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-2-2-1\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-03-batch-2-file-2-row-1\",\"rowId\":\"row-2024-01-03-batch-2-file-2-row-1\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"ingestTime\":20240103221}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-03-batch-2-file-2-row-2\",\"ingestTime\":20240103222,\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"rowVersion\":1,\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"rowId\":\"row-2024-01-03-batch-2-file-2-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-2-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-2-2-2\"}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"rowId\":0,\"isDeleted\":0,\"commitBatchId\":0,\"trackingId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-2-2024-01-03"},"size":8053,"modificationTime":1787122149995,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"ingestTime\":20240103211,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-03-batch-2-file-1-row-1\",\"rowId\":\"row-2024-01-03-batch-2-file-1-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-2-file-1-row-1@example.com\"},\"commitBatchId\":\"commit-batch-2-2024-01-03\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-2-file-1-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\"},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-2-file-1-row-2@example.com\"},\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"isDeleted\":false,\"ingestTime\":20240103212,\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"trackingId\":\"tracking-2024-01-03-batch-2-file-1-row-2\",\"rowId\":\"row-2024-01-03-batch-2-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0},\"_acp_system_metadata\":{\"ingestTime\":0,\"acp_sourceBatchId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"isDeleted\":0,\"trackingId\":0,\"rowId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000006.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000006.json new file mode 100644 index 000000000..6e6b02d48 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000006.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150001,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-9abac23b-07d6-48a6-b23b-0710588c47d1-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-1-2024-01-04"},"size":8053,"modificationTime":1787122150000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-2-row-1\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-1-2-1\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-04-batch-1-file-2-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"isDeleted\":false,\"rowVersion\":1,\"ingestTime\":20240104121,\"trackingId\":\"tracking-2024-01-04-batch-1-file-2-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-04\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-1-2-2\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-04-batch-1-file-2-row-2\",\"ingestTime\":20240104122,\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"rowVersion\":1}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"turnID\":0,\"conversationID\":0},\"_acp_system_metadata\":{\"trackingId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0,\"rowVersion\":0,\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-26aa5864-10c1-48e7-a6df-6a294b350546-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-1-2024-01-04"},"size":8052,"modificationTime":1787122150001,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-1-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-1\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"rowId\":\"row-2024-01-04-batch-1-file-1-row-1\",\"rowVersion\":1,\"ingestTime\":20240104111,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-1-file-1-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-04-batch-1-file-1-row-1\",\"isDeleted\":false,\"commitBatchId\":\"commit-batch-1-2024-01-04\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-04-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-1-row-2\"},\"turnID\":\"turn-1-1-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-04-batch-1-file-1-row-2\",\"rowId\":\"row-2024-01-04-batch-1-file-1-row-2\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240104112,\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"rowVersion\":1}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"trackingId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"rowVersion\":0,\"rowId\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.checkpoint.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.checkpoint.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d5373a63fd667338784537bd62dd769d063aa9b9 GIT binary patch literal 38166 zcmeHw4R}*kw&>aCjeG)a>TRFeM3wzQE3TC{rd2QBH3{exEF zT2xdL#|5$_0a zc8kRj4u|XpOUTL@?0&z=U7aR z-e&NtXDu|Ftaf|AQXDjoo~2rDqIPmNp10a<7K7OoFhjdgv4OV*!v-7AIV}DntI2Ay z8TJFvqQO&$Dl(oztAatZHDE2W8EheYk-=gP1Ppd7pLfeb3ug-l{hY&U0V)mw@M1pX z2wNPyA!Ks^a7V~t;O&kggVn*AO*TJY94N9I&Yh)}_i|^co&K=R9xQg)4PiTHh1Mnx zT7$fc`C^kL6ex%9YDJ!U=JDmHfzWb2-%B+R=y|{C@MDW zzlowaOg0B(?<_9nZ2>+QG?;85AjTFpgN^{3jSu)R*Z{~Qb!Cvh8 zwB8Ct@a9mFA>g-~Om=8m6yQf9v==Bpg((l49YJfzZ!_2fJm|<~a~K@O=3+w;AFveJ ztTul!*a7f~1#S+6pmo4uFc;bUU>BAmgWcf>7($k?jSKRjV9->ADOZ>a``=A#Q!GO;_y2H#pa;x;3^gi+|K#^;ed%ZaE@Y!!D8{-4R#CA zW%HXX{^Fv5A24G=DDw+%0pT@fv%_yG0-8^C@ue!n>2xB2ZL7ITsP;57=$ zT-S?$^gBQ~yx(FA8NxOTm~>GP$nb|Oh7cd(iv7GbWU_m2qO_EDp>}Zre57lYTIF1u zV)3Sg9z<@&`;g4T1YZIVF}pTEk?T!)o8N_=1);D8CpcSwg5Gh6v^sz zU?TH$oRd-%W!5>VdAfN{_zhDWS5GOpOl>{6c2Z_&LYckz18DVV>&DQUDZz_j$x4oX zHxrTi%`N-~QB|7oP_5>$NLtw8@0{#j3u`C>Bw@sVb^J zRkf`7G9Vs)){1|DmR#+^mB8{l^X{D$Gz55L#K?<$~bx!7iz_&44&fGY7xmP%?vs2X>7 zoHbE-MP-fQjSm9I+1f13C`Ftt3nBO)fRMkWTk=AQnQy+qY)J!Te30+u@%`@3 zjlNKKPitpKzSCTg-`Z0aiiCPY!F*?!kMx8J^1HfQ+xhN+>R_m&x3zZw)#P^Y?V+A7 zJ`h5wr&o7xs@=aZ#?$%I8 zU{R5xKV-NeCEsi`A-yWl*@0#&_})dg&%QCgH?*lY@0zS`ek0IUI&VIei!M#i4R?0u zX7AE|-`IjM2KBxIcnVNM-8=}eBtoTBmw_TSQ}h0Pu7QIVy?x#4MM!M{5pV``{$`+} zo!_)=ExkvNj-NwlFV*dQiT0{#imUrqTG8FP_fi_Iuhuv|Q>R-uI<`~!qlHYkkfAtR zA@j*v+Wj^{AGv(48$3lcRd8Y(^n39J8ofKWqc0LE?dTOXcAo7oqjqg;X-D2QjKXX_{1b*UKXr!6uiMGAdUyL7=V@lg zFPZEcsjiA+%5BdmQ?KNP@Jk`b%4%il4PzrB-Y% z7(LGppTE{p@fq9v4T!{hj5+WIHz!$r_0f% zMcpZAkhU#l&mo2TVXFCQx_ND??MjL(RCwQ~cip;Z2lo+e+r@H82pPF^njOcP@JV`f zE$v)OA8t<`s45Sl&(vin*V5H%T|e<<(B2&>Cq6^A9OdX|$nn{}->bO+8lClMS>M$R z=VDxI0Z9K^dU##SAw{NL?JQ?^m1myRyl~UdW^`4{7s&V$vtv$5+EvMIHA>H==oj_I z9VtdTt)73h!?{qi`#kGjgmUgoS(K?Tm=T@#`LV8a!#UkP6{_&O#W+Wqye{ru>g7<{ zwaHY0e>?Jg2l;(;p2^i>yq414sBu3_aktX8seJI|J^#X21PGtw((c`Wqt>r_iLuPQ43 zM6+}Hy^1aDETcxDO%5WUkX3 z*qL@*W4w)Z{!EF@{~8d;?k{NLH57Yq>K&gv%uq$`skT#dhSnP+6nY&@ZJ>@S3ZRg{?rM6ghi(2om4Eqcw9>eX`i0B+HdXh_%-zp0zJH(&J;Qj8(f@G?#m#$R zE$-*GeJxt#T$p>F$X^KIFiD6Ye+aDGCa*o0nO!s_Sg{$qo4 zkangkQqvpV=!1_3^SbW76shKAYYSej)j4O4{EN~UW=7v;+TLc^dw|3@-iEl5*_Z}K zVf0e$;TIMdFQrg>w~gKUu4;H^%j>k~TIQ+Wet7hVwTaDKmBN+ILJurGI#=y5W+Xv(T&cn3sF4f80l+4@|s;ECVW%{)px7 zp`0%(Dz0Qp?z_Z;*gg3fzMoJfTdA}vHP^@(hmhywf|i>Z2!Dr%uHe4Q?B!G(kAUZ| zDd!5b$oIm{bePeEit}Fqr;ex6F6eeLPIY$e9Ed>KdUpK8*=9T&*Psem*{j& z>3h72yZzSZTyN$-o^kAD8vUCum+_+cJ7{IbyB$}iIQKEmE146QFt*o}{g*IXH)ZHH zWS=~z(Vf$jfYR6H{4^bDYS)CCxYU$3t;+a0@}JXqs*vaLS;NO(-S*JJBQGqSuQ{Sv z`bxpp@)R)j?O=vGDPQ_L@SL@U;YG@&8`5~z`yLES?XB**InF;PrKW3*eGi~!&o-3T z>HLuOZJ-W)$iDKX($`DXK@@w12A1ai@7wAd-e=W$>px!5@b8~o$+rB6;umvgGFoa` zqZP(8<6-L0Dcnhv_ex1St-o5Elbg1GIgM7H$e<5?pcuSe<)oN>OBiFe!u6@bWac+e z+@4w%-LKEQW3Szx|v^+e*%9fHq?luE?^Hhvz3_FHf8Jiw1>1qzLHz0HXIYdm7kJUB!>Na7~Re3;UU`h1lkgu=X{jj_ZofT9I_o&_Mby_PwV%;MCaXNyoTM+COc1~ zVLJrQ{-=?*oXz=HRoxQS`E>q!XEKZ@C^pQQ6cx*qk1l${qs+W4H)nH-$40}DcJ6y> z!+u?G_Rv$FJE@%W$<9ydd0!vMd;8aP>fp#P6^n|EyHmXX%%p9#c0ZzazDe!4gh~D3 z&;jEE3gaJAcfFi*yXHh8o6&ymPjk{rGq*mgdMbPLoW^-h!>y(7u5z8s+u-Mt)Tfbg z-<`DaIY<({-~G1>TcNRiq45;b!$0O0rFe!F-JfY}S(YRB_J2-+NiRcr zIGE(spx^!5>*m+Ycc9(=z_#8y?fzN$@ zD(yQ99X~i)qyZbO?4%k&#>wv zc^`K*(2LG&@oM}o)Ud+l)s)wCPjkIiGLN}uEsah$to6*J_k1y{xdA!1QbX?! zuBEY_*V1+6Y{na*Mcn%u+d2kqH}Y)T=c&%E$X%fHY^R4G)m`aPCpS8MUu%W~?`>ZQ zrf&QLGju%`jvDB@_Uh`nM{nJAx9!Ktqa5AH>DH+4ysrNYdd4>DRov;#*wkUk?wLFV#S1R+K z^;+q)oGjy~%*nG%``Nr@&s;xxmT{hC(!RT>G}-bwdVuIwueev06EJu<(aNI&e_0b`&s8^s$!4A@E4@2uqm9kQF|&gZ1<`T z?PJiN8;Z*d_GY9wN2nG{rtNf+E+>cI2fkeLI?h{&iOp8gk!uLnIpINXRx|A z)g_>1NLk8Tjv`|a4e!kg3jS=rO}(VtmZsdYgFbc>jfR~ijPD-C z>o@J2t=NC3a*u`qhFhLS#-r45bII@6{3lbn6Rh(`luwhA_iEcM8J@Xl_)+6v)>h)@ zis z3}*Zvp1ea102fSpNEtys(}#y*Wk3cf7q=%Rf@rmj6YC&m1F3_Ug*phXeaI;wnB_NO zg%naWD44L6Ln=IkI#K~4g%qLkA}fuOIz+-!%Z2D1Q(hs!g>p-js~7-~$oN81T%`#v z2@qTiiuH^F8zZ8z1t8oS!VhOQL%}MpJcTvX3MHYq5>#T*k10VFKO5Aka77Y;jvT1s680sqi?l0m`d% zF(pAIQdXC{py5o}iob>CxoK#A0#lx%?P3+g8NO88BoL&F3%CGBOiL>3kvKHDs!VZF zg&3b0*C1ZTTLs)2W14U+85u;?i8U^fk9omiSXN9XSfb@vv4Pw|pn)t%z*Pc;CU8m2 zSt~{df5jLAeG^y>9uBfD6#FMCizk4WKQ5NyVL>DT|BdO;JT{Dp6>n%Q=U?;?vl$B6 zQ}z%MrMzUHKx|k(aS<>vN#D{BVqGEFEj5LQIYg)RV+A80|pR z@!=2;o-r$ztUFe($GQ~i`;(926EP)L^Cyn|@jf@k3FWp!sQ=d4MnXCvjwF%{xjm69 zOx?mU;orgD=KR3&F-TzCaN zM#Ba?6x9mf;>d+N6DSd$Bn-%y3%t5GHur?qavkA#J-ORo{Fo(qGoXyk1jy3qB~^G4 zeJ(7bQ)3qJDJ@+CQw6+$kGDIFS!%W|gq!KhX!u-6k?x48hi0(eKekuCRxg`q;9eOk zgkE?-95xN$tMT2$P+J0JP1cywrfAb+EBC~1 zl(Pj;U@Mb69A`>7+WB!brJ4jl6H;~&T_$5@97)q^d=bQ4M3-ri^xwR3y>?ZNmKOHf zJc5HvAm9%d+k!;~-Vx*smS8AsfDL7^y^IgqtQOvCu{-QwwK{E%%o6h#FPuuhtSPl3 z5m~cX0mRIw7=-8q{x(st!yTp*Or!0AA4v|H#Yuo#+@+a-)g8DCIs_;gi-hnQ>>87W*yD!iSNmsZv6v3N*)jgc$JM8gU75Z|=Vw!DGRF`SJBQKx=NW^>j){dS!V7B%6mSf0q zfa)*FVH7>RI~FTO)+4g{Cj^^+j6=+hOk{LWuVtg`|D}Q{#pxB4BD87vNBg(2Do8aB z{~To>eCnr%S5RAbA|O9|E7f}UzpHBbF755sA5-nDR<$8CQ2kz0VR}E_ilx?S4Iia4 zC#~%80qsu?+7;ePhJBYRJ;5`%(OrSr_o-laR^Er*zfd^-k#(h$orJHk3&9V9BMT3!S5K0Rd4MRwRzfE^3rH;agbSCzs z#IlInW z#WHoqo7sXL=59&l#M9c?c^p5m#gB26*6)PH=~U& z+8Mr0(8R$yN_YM=;zmL@B5ak)a*^peSTtdX(FAz|4vkavANk>f7>ucg^{ko9LpF*` zO=>Y$n8`e%78iBuNyf05%p-1MCQnUbRyC7(;K}by<{_KGCpHwa+AC!OQ*pqV%tM^* z&SV}?U6PJdM`L*Y|IN%JVOefkM=}>{3kqPLn79O!zj)FknX_wf#WEKPka(2_DtgHj zVrW>`N!QMnYICZ&NwFh>S=za=5^9!8dJ_Al%U_(}E@3OBKtw`QLVff>qzy}FO=})o z02Bm6kzU>ySRV>(=<00k=rwNaZtV?S-OKm(xO%Q8HNiY;@Nz1})6>}j6{C1&7fb;_ zR^FtIFl0jR)MSI@hYN}ch*9ACS>$s+f$TA)tzMG}MpBP^3Dj~dS6?icjiK^L?mxU+l_5a&S$MlxY{)=#b z5zARq=#u^QS}T>M>;Z3Y=e22+mCj<|4G3Bt$Y>E5XBXeQei2nO7i<*T&BDjtTk$?t zVH*ng3tdxypDD7`Ftfgn%sI8-Pq;G!cf`JMO=q#>sdPty!S}QdhQRsYYkGTUur=Ho zAiHGoL0s?`xSb4fpD$g{!}(kc6oB;LI3$rogJ<*WdcFl6z;GIbq7Y#!6zJ{j)+P(& zYkJ@U5?)*xkch@?rD2(@?5^&Jw04Bb`lUxS5U1=6hkHW3;HuD0gPE)Da1Zo`dITOd z0lquf+QCQQoHM>dXQ};sPsodRstVV0VAYK8i}b>+QgYTu!W7Un8;VAuj$kNQ0jH^J#qZQQdlwyTuGPT&;&z0f$r8WY_O8{!Gqzjcst)K zxt^*k&OWHDq)yUhTE0TX`7Ad76d_hX9sp(-BOpB*8cI7OeeE3>kFsW&5E~Sc)}CH1 zEXQI1Nn(LnVha0#CeR(ihuVqbL!e28idMLFT_2ou2R5hU-f<*j34<(pI|H2&tx9mr zba-)rH^hVYM7<09E55xNXFAc_I5q%X(69oi>+23pnJL_pWK9>-lj+^mp;ZdPV&TO< z)lq}Mfig*)eF_0X6XAP$8@q6J9K=4F7VLyKC3Hw`D}hl8wELe5bT`k>R%YuJx!{H}(N$s5#Ygkl_>ISH zkZGCjQ=~hE^dp)2w=%F$Cz-^I$NU(ZtzVd%ZBtMyriENc5(xSyS=}hD->TLB>=KOr z!mo-MPsPnigo+B7AV_2=imFhiIeF_pbYNeMn8OQ8tHW?hT^^PFYl-UE|87t97C zB(-b|3vzc8k}m>f+m9LD9ew)k*X!T76l>=~8{y?kV)DZRGl!t!P9lxnz2Fpg&qg== zM>oIYy~eTCX6bD@Q1%xGHS)S0VHLnR_VhsF!%7&O>@7I4^EESvpu{)sB--09oCS~@ z_ap}g#PfCIi|FGIdQE&JCb5!c4nd7IeA(t5+1s9C&J*<~nfJ{cf{LC?oH+y)7f9px zQGujp4ne{CPTW!EnL|+cm|miR0$$06ZztS03&xaipq}7})r99~rg9{2<`5Kq86T|T zCOTV<_qE{n+K~gW6RAo*0KW{)hKW2p@FVvk{ayD?4LmK(ZxfL&mZzg&x8H~KpHBc5=74fhPXbk50g8e?0ib_% zUt)>@v`if06gd&S@_Km`@*DSqsPCT!WKt-K722bKMs|ZxcTX44L>2ld*zFI1P^Sb| zKC@0x+K)m!0T6%nKw?656!Elp<611P2E^)Pt8~r!Hy#AhJva#XZ+#; zp$-H6{zFK=?V-d#v8D+~V)$ZBLIfFKmayrZ`aeE|mSNu?Q;CSuG`4@jWeP$ShO}qT zct|o*MA27?v|O+jworGyF0B7z57NK0Co!v_2ojzIcIA$KVI;C(h0*->VWfZY;Yrc> zL^K3XoUX{SN>n|EA3^%Z9!U&rta>DK6lEf-^vrRF4}2 zF!25Rkp7N+(*Q2T!?$L7I zTEFcHq~HC00QC1hEjd2Ps>169`FJaeY`tJ{lrFLwNhoX9U9Z3O2T1?P{=^IlF@Zoz z3`)L^8HM;KfcV-ErUlU+B~DxejjnwXsC$qOjOG0&k$(S^M1s=UOZg$p@p4E@B!@Ne zsH8R33DVh5o&2Nt&{XdON6HQE~V-|FY%w7rWj@Rg4 zdlq8b(?n)7JMm|B;*;aRF!zOZf+-&Vh)rXsvk`b^C;rS%{3)B5xECU}O?Ya@WTtVX zOun=eMKhHR?edN@X8X;v#k6kx z_5}&qRMbj(&%AwL!nak#1~Pd9F_U>*@WkQF+Xvue>bD7+GjAV|{?WS4o3GuRXoscl zWL~1OP*t=nG;R|-+4wGPc!%1GUiSy2`vmFVI*atD&yd>6w06}lg+&G$ibj+oZk;Ad zf_x{vP|QKPH(;Ub92CL6La@bj8m9%vmBsLaK|U2%9gML*c1!uVqL^Oy2%|em>qowZ zV%*;d^MAjig5zk_-GSnTC9O+Dg*6eu(E^)ZC)5xB1vUrYH%dqzd!TSU+Vps=V)1-& z^c=#t^9Mvv4zGPL3}sRq32X+?k1Z9}t%M@7qf2PY=lR81Q*SvfVt!@*ii2t|I# zBMRxU+hA|}Z4*M0EwD(d3vp*d#4{mLfad_<`x5{p)%+N76F?m6PEo+`Y@_whZ%a&R z9AI&YH;z*2Knj6~S9f>bPEWE-``H_ z4{x6ocyy^%_sJIu-9o)e_sDl?SnAJ8ae2KfS38AcWm&&66yl#vF0aewYJgw-bCt`r z(dAy}s&Kj1xk}fOdv3SORg1rsx?I)xFI-E1;VOV{*~{Rg4}Y(6mE#)#27p4dYTR77 zXI=^T;J&NDL?8n)e1ShmpOwNNXpf<}FcI*B;kyU<)!s-GS25tKYU?j7wAb3}3k&gs zaxdvmh`@+MP>uK-xvuiAZ!RL_ySgxb#~|*HAJvCgh3ylQ&$fdwA&-tvb(lUVWXp-8y){_Ps z@Tb5Zxh72eS35TFtDCz(E+&4pqt90v=_NQ_?hW=Zk&lR}v@s5zOkMuRsB67(f%VxYcEyly1;iTt9bBgxynK3GX);_AcgO@o_6dDT^6B&FB@ zL@DF1RrteA?1?;gS@1RdV`Jv^{Az1m^BT}!Wre8;Fgh^20STU~GO_%Ly$m$3HuneH zeH*}Z0KcPAl7}dNLZD2Zb%4~>#5ww!+an#pH6Gw0((lKu%5ncI%H#Ss;r6orL=?@H zj^3u#kv@20!tG1zYb1Lo@+M3Q@*)Nv#z4vmDMX&wXIxcYi+^1y(cj8w|7!AFkdM$5 zcO{~%+_XMW5g8PP9iU^d5s5r%+cp^mko~TUmop0nZ2(M zRJlFPn5Z(`zOdX5UIth0va9h;R}cOWQMK)53EEyJLyf&=^-ZzQmxInuPoYYw#g20>EmFg=_kPuB$Yi{)SG z>jB$|_&XY01Gv3;P?Se`Er~~^gnojQsM3u)xmMy2zy-GI8}P#4V3ltG`w;j)rr#ve zFZqAG{SehQuig~#ulDr?U9Jtd{hE3Ras<4fe-B0?^EX07rMb%=5qXX@M*h zLi~1%@t@eYEZ&fK-Bbj60$Xpc@OJrEH@b@I*NFNSM$WS6^+qzD$;Mj(iU^&2b8l=~ z<1vBXfbCez{M;t9f3+3-2LBA2L4M^TJyW$`Lcc$XU+_0s{3DP_5Ui?e<12l=O};u) zpgq#pOf1R00psh4_BRoKk@Y8`LTF0ttr7!lZg&j$S66@nOB-?jL8-sHobWG;pM+~c zN?5e+{y-Jseb7}~2K*LUBz{GEAx0y}A5$TGOW402_qTUCg!ng+yhztzdqqzW^aejo zxP4>2U(}xc|Z=kJ6u>XnjA55>iuL)u(!B;bg@tNY}L6{b#NLmq6`PI;O zWf`=0m>kdv11ejEwd^JVL1>SM1$S&Lsl&NbX(JvxIqb_m6jn?4M1PaT2Wakct-&84kM^cDzMf!(r$1<_>-VoVcLghAF*jV_ zHpc%-rR#fbOyK7#>NHLv$q`G_wz9h387{?&dWcT&=kM zra{}}{*GPQJqWy+2yZoI0N+u5g%JNH(kH=3<{Jb(=wE0r&5TS^e{p`7IN!svta7gp zba=Z+z8(>H1v(mW|5f$fW8^Cv|46(crYGo&q%L3|0{}<#6&#x(cPz*J7p{}&5#wdN zea6LeH=(~fivL3KytE0!uc?O*%u@rN?U2bhzQA9}4?$Atvp}BT2eD{_w>{F++-Qz~ z>^DFju%0xkD+7A%)mH$~_}VSpjkjka3No*1tm|xAZSBDG#zIW5wX9gwx8Tq6e3t;k zs&j9^xvC&@oO}4N{(A|M@-j$WNq!_)Zj}VjH9@|F`NR2#m&9CGC2n7666u%XKgKM^ zd=mfH;n=Vq=a=Q{aPX^+#ebY{!Vg4%P55s$7O0Tl2}Vxr8}jr9R}YrAX_ZK?IG&B; zmyGvq;6uQpZvf&(M`IPl|Mtj+<~8taR! zrb1ynnJC`GjUO)i*#7PwvH$ePkIGfR&lQ2V{_*}M)K}uhj}2Jg1FOK)vA(7=ew5lV zK1bAEraOL=5q{UzgXt3cn%ekLf!mt~MS3NFCh;lZ_^}eVw@d9O$`|9tkIFKDe?^m| zpXtP}HiA!*r)z@vg^kAD>*u^(Q0r-fxP#|Bjl7hv)RwFnRQ7wcE#D~Vqu zzaf_pcpxA_UMJ(Cy=oN(6~G@D z=LP4twU9Fv+5u04D4&3ky@CW{MkLsyAP-_+7_>kd@FUe1g)Zdzaw;oV{5PKrfvzqC lyxX^PW3ISqHJ9UzCZh@7Y6Wjs{D1C$YMGa!>Rn#*{{gYs)a3vG literal 0 HcmV?d00001 diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.json new file mode 100644 index 000000000..21750bcdd --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150006,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-92b1f2f5-21dc-4a6a-9e5d-25db1c63d6ff-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-2-2024-01-04"},"size":8053,"modificationTime":1787122150006,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-2-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-1\",\"ingestTime\":20240104221,\"rowId\":\"row-2024-01-04-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-2-2-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-2\",\"rowVersion\":1,\"ingestTime\":20240104222,\"isDeleted\":false,\"rowId\":\"row-2024-01-04-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-2-row-2@example.com\"}},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-04-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"ingestTime\":0,\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-04","_ACP_DATE":"2024-01-04"},"size":8053,"modificationTime":1787122150006,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"ingestTime\":20240104211,\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-1\",\"isDeleted\":false,\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"rowVersion\":1,\"rowId\":\"row-2024-01-04-batch-2-file-1-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-04\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-1\",\"turnID\":\"turn-2-1-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"ingestTime\":20240104212,\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-2\",\"rowVersion\":1,\"rowId\":\"row-2024-01-04-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-1-row-2@example.com\"}},\"conversation\":{\"conversationID\":\"conversation-2024-01-04-1\",\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-2\",\"source\":\"generated\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"rowId\":0,\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000008.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000008.json new file mode 100644 index 000000000..c4c6e68bf --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000008.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150026,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-ecee6b33-3470-4bbf-986e-e08dfe777c7b-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-1-2024-01-05"},"size":8053,"modificationTime":1787122150026,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-1-2-1\",\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-2-row-1\",\"ingestTime\":20240105121,\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-2-row-1@example.com\"},\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-2-row-2\",\"ingestTime\":20240105122,\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-2-row-2@example.com\"},\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-05\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-05-2\",\"turnID\":\"turn-1-2-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"trackingId\":0,\"rowId\":0,\"rowVersion\":0,\"ingestTime\":0,\"isDeleted\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-33f334a1-8c2c-41b2-9dbe-d66dd9ad8ae1-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-1-2024-01-05"},"size":8052,"modificationTime":1787122150026,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-1\",\"rowId\":\"row-2024-01-05-batch-1-file-1-row-1\",\"rowVersion\":1,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-1-row-1@example.com\"},\"ingestTime\":20240105111},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-1\"}}},\"maxValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-05-batch-1-file-1-row-2\",\"isDeleted\":false,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"ingestTime\":20240105112},\"conversation\":{\"turnID\":\"turn-1-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-2\"},\"conversationID\":\"conversation-2024-01-05-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"rowId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"commitBatchId\":0,\"ingestTime\":0,\"rowVersion\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000009.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000009.json new file mode 100644 index 000000000..92dcc908f --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000009.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150033,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-05","_ACP_DATE":"2024-01-05"},"size":8053,"modificationTime":1787122150033,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"ingestTime\":20240105211,\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-1-row-1@example.com\"},\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-2-file-1-row-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-2-1-2\",\"conversationID\":\"conversation-2024-01-05-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-2\"}},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-2\",\"rowVersion\":1,\"isDeleted\":false,\"ingestTime\":20240105212,\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-05-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\"}},\"nullCount\":{\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"trackingId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-3e56e624-1e0b-40c0-b7a0-4ead10a5517e-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-2-2024-01-05"},"size":8053,"modificationTime":1787122150033,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-05-2\"},\"_acp_system_metadata\":{\"ingestTime\":20240105221,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"rowId\":\"row-2024-01-05-batch-2-file-2-row-1\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-2-row-1@example.com\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-05-batch-2-file-2-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240105222,\"rowVersion\":1},\"conversation\":{\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-2-2-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"isDeleted\":0,\"ingestTime\":0,\"rowVersion\":0,\"trackingId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowId\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000010.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000010.json new file mode 100644 index 000000000..7dc16d426 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000010.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150040,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-65f27845-2a8b-4172-a4f7-01e5d41ef9c2-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-1-2024-01-06"},"size":8053,"modificationTime":1787122150040,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-1-2-1\"},\"_acp_system_metadata\":{\"ingestTime\":20240106121,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"isDeleted\":false,\"rowId\":\"row-2024-01-06-batch-1-file-2-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-06-batch-1-file-2-row-2\",\"ingestTime\":20240106122,\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-2-row-2@example.com\"}},\"conversation\":{\"turnID\":\"turn-1-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0}},\"_acp_system_metadata\":{\"commitBatchId\":0,\"ingestTime\":0,\"acp_sourceBatchId\":0,\"isDeleted\":0,\"rowId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-1-2024-01-06"},"size":8052,"modificationTime":1787122150040,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"ingestTime\":20240106111,\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-1-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-1\",\"rowId\":\"row-2024-01-06-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-06\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-1-1-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-2\"}},\"_acp_system_metadata\":{\"rowVersion\":1,\"ingestTime\":20240106112,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-1-row-2@example.com\"},\"rowId\":\"row-2024-01-06-batch-1-file-1-row-2\"}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowVersion\":0,\"isDeleted\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"rowId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000011.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000011.json new file mode 100644 index 000000000..ef44550d8 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000011.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150047,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-06","_ACP_DATE":"2024-01-06"},"size":8053,"modificationTime":1787122150047,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-06-batch-2-file-2-row-1\",\"ingestTime\":20240106221,\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-2-row-1@example.com\"},\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"rowVersion\":1,\"isDeleted\":false}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-2\"},\"turnID\":\"turn-2-2-2\",\"conversationID\":\"conversation-2024-01-06-2\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-2-row-2@example.com\"},\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"isDeleted\":false,\"ingestTime\":20240106222,\"rowId\":\"row-2024-01-06-batch-2-file-2-row-2\",\"rowVersion\":1}},\"nullCount\":{\"_acp_system_metadata\":{\"ingestTime\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"isDeleted\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"rowId\":0,\"commitBatchId\":0,\"rowVersion\":0},\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-77d00763-9ba2-4429-b8b2-ca8b3092ac40-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-2-2024-01-06"},"size":8053,"modificationTime":1787122150047,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"ingestTime\":20240106211,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-06-batch-2-file-1-row-1\",\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-06-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-2\",\"ingestTime\":20240106212,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"rowId\":\"row-2024-01-06-batch-2-file-1-row-2\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"commitBatchId\":\"commit-batch-2-2024-01-06\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-2\"},\"turnID\":\"turn-2-1-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"commitBatchId\":0,\"trackingId\":0,\"ingestTime\":0,\"rowId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000012.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000012.json new file mode 100644 index 000000000..3ef953535 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000012.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150054,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-1-2024-01-07"},"size":8052,"modificationTime":1787122150054,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-07-batch-1-file-1-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-1-file-1-row-1@example.com\"},\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"isDeleted\":false,\"ingestTime\":20240107111,\"acp_sourceBatchId\":\"batch-1-2024-01-07\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-1\",\"source\":\"generated\"}}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-2\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-1-file-1-row-2@example.com\"},\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"rowVersion\":1,\"ingestTime\":20240107112,\"rowId\":\"row-2024-01-07-batch-1-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowVersion\":0,\"commitBatchId\":0,\"rowId\":0,\"isDeleted\":0,\"ingestTime\":0,\"trackingId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-00f3f17d-20f3-4772-9561-1d3b6e0ef2bd-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-1-2024-01-07"},"size":8053,"modificationTime":1787122150054,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"ingestTime\":20240107121,\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-07-batch-1-file-2-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-1\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-1-2-1\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-1-2-2\",\"conversationID\":\"conversation-2024-01-07-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-2\"}},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-07-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-2\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-1-file-2-row-2@example.com\"},\"ingestTime\":20240107122},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000013.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000013.json new file mode 100644 index 000000000..bd2e866f3 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000013.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150061,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-9dca9883-8cd9-4950-87e5-9cd130fbdae7-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-07","_ACP_DATE":"2024-01-07"},"size":8053,"modificationTime":1787122150061,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-07-1\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-07-batch-2-file-1-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-1-row-1@example.com\"},\"ingestTime\":20240107211,\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-2\",\"conversationID\":\"conversation-2024-01-07-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-2\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"ingestTime\":20240107212,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-07-batch-2-file-1-row-2\",\"rowVersion\":1}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0},\"_acp_system_metadata\":{\"rowId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-2-2024-01-07"},"size":8053,"modificationTime":1787122150061,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"rowId\":\"row-2024-01-07-batch-2-file-2-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-1\",\"ingestTime\":20240107221,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"conversation\":{\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-07-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-2-2-2\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-07-batch-2-file-2-row-2\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"ingestTime\":20240107222,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"acp_sourceBatchId\":0,\"ingestTime\":0,\"rowVersion\":0,\"trackingId\":0,\"rowId\":0,\"commitBatchId\":0,\"isDeleted\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/_last_checkpoint b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/_last_checkpoint new file mode 100644 index 000000000..571f53814 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":7,"size":18,"sizeInBytes":38166,"numOfAddFiles":16} \ No newline at end of file diff --git a/crates/core/tests/it_datafusion/integration_datafusion_predicate.rs b/crates/core/tests/it_datafusion/integration_datafusion_predicate.rs new file mode 100644 index 000000000..6085acb4f --- /dev/null +++ b/crates/core/tests/it_datafusion/integration_datafusion_predicate.rs @@ -0,0 +1,762 @@ +use async_trait::async_trait; +use bytes::Bytes; +use datafusion::common::{DataFusionError, Result as DataFusionResult}; +use datafusion::physical_plan::{ + collect, visit_execution_plan, ExecutionPlan, ExecutionPlanVisitor, +}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use delta_kernel::expressions::ColumnName; +use delta_kernel::{ + DeltaResult as KernelResult, DeltaResultIterator, DeltaResultIteratorStatic, Engine, + EngineData, EvaluationHandler, FileDataReadResultIterator, FileMeta, FileSlice, + FilteredEngineData, JsonHandler, ParquetFooter, ParquetHandler, PredicateRef, StorageHandler, +}; +use deltalake_core::delta_datafusion::DeltaScanExec; +use deltalake_core::kernel::DeltaResult; +use deltalake_core::{DeltaTableBuilder, SchemaRef}; +use futures::stream::BoxStream; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetRange, GetResult, ListResult, MultipartUpload, ObjectMeta, + ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; +use std::collections::HashMap; +use std::fmt; +use std::fmt::{Display, Formatter}; +use std::ops::Range; +use std::sync::{Arc, Mutex}; +use tracing::info; +use url::Url; + +/// Records every bounded byte-range read made against the inner object store, keyed by path. +/// +/// Intercepts `get_opts` (for single ranged reads) and `get_ranges` (for multi-range reads, +/// where the `ObjectStore` default impl coalesces ranges into one `get_opts` call — recording +/// at the `get_opts` level alone would hide the caller's per-column intent). `get_range` is +/// not overridden because its default impl routes through `get_opts` without coalescing. +#[derive(Debug)] +pub struct RangeRecordingObjectStore { + inner: Arc, + reads: Arc>>>>, +} + +impl Display for RangeRecordingObjectStore { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "RangeRecording({})", self.inner) + } +} + +#[allow(dead_code)] +impl RangeRecordingObjectStore { + pub fn new(inner: Arc) -> Self { + Self { + inner, + reads: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Returns all byte ranges recorded for any path whose string representation ends with `suffix`. + pub fn reads_for_path_suffix(&self, suffix: &str) -> Vec> { + self.reads + .lock() + .expect("lock not poisoned") + .iter() + .filter(|(k, _)| k.ends_with(suffix)) + .flat_map(|(_, v)| v.clone()) + .collect() + } + + fn clear_reads(&self) { + self.reads.lock().expect("lock not poisoned").clear(); + } + + fn record(&self, path: &Path, range: Range) { + self.reads + .lock() + .expect("lock not poisoned") + .entry(path.to_string()) + .or_default() + .push(range); + } + +} + +#[async_trait] +impl ObjectStore for RangeRecordingObjectStore { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOptions, + ) -> object_store::Result> { + Err(object_store::Error::NotImplemented { + operation: "put_multipart_opts".to_string(), + implementer: "RangeRecordingObjectStore".to_string(), + }) + } + + async fn get_opts( + &self, + location: &Path, + options: GetOptions, + ) -> object_store::Result { + // Only record bounded ranges. Parquet readers issue suffix reads for the footer + // (not column data) and don't issue unranged full reads for column access, so + // skipping those keeps the recorder focused on column-chunk traffic. + if let Some(GetRange::Bounded(r)) = &options.range { + self.record(location, r.clone()); + } + self.inner.get_opts(location, options).await + } + + async fn get_ranges( + &self, + location: &Path, + ranges: &[Range], + ) -> object_store::Result> { + for r in ranges { + self.record(location, r.clone()); + } + self.inner.get_ranges(location, ranges).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// Returns the byte ranges of parquet column chunks whose dotted path matches `predicate`. +fn parquet_column_ranges(parquet_path: &str, predicate: impl Fn(&str) -> bool) -> Vec> { + use parquet::file::reader::{FileReader, SerializedFileReader}; + #[allow(clippy::unwrap_used)] + let reader = SerializedFileReader::new(std::fs::File::open(parquet_path).unwrap()).unwrap(); + reader + .metadata() + .row_groups() + .iter() + .flat_map(|rg| rg.columns()) + .filter(|col| predicate(&col.column_descr().path().parts().join("."))) + .map(|col| { + // `file_offset` is deprecated in the Parquet spec and inconsistent across + // writers (often `0`); `data_page_offset` is the reliable fallback when + // there is no dictionary page. + let start = col + .dictionary_page_offset() + .unwrap_or(col.data_page_offset()) as u64; + start..start + col.compressed_size() as u64 + }) + .collect() +} + +fn ranges_overlap(a: &Range, b: &Range) -> bool { + a.start < b.end && b.start < a.end +} + +#[derive(Clone)] +struct ParquetReadCall { + files: Vec, + physical_schema: SchemaRef, + predicate: Option, +} + +struct TracingParquetHandler { + inner: Arc, + calls: Arc>>, +} + +impl ParquetHandler for TracingParquetHandler { + fn read_parquet_files( + &self, + files: &[FileMeta], + physical_schema: SchemaRef, + predicate: Option, + ) -> KernelResult { + info!( + "TracingEngine::TracingParquetHandler::read_parquet_files\n\tfiles: {:?}\n\tpredicate: {:?}", + &files, &predicate + ); + self.calls + .lock() + .expect("lock not poisoned") + .push(ParquetReadCall { + files: files.to_vec(), + physical_schema: physical_schema.clone(), + predicate: predicate.clone(), + }); + self.inner + .read_parquet_files(files, physical_schema, predicate) + } + + fn write_parquet_file( + &self, + location: Url, + data: DeltaResultIteratorStatic>, + ) -> KernelResult<()> { + self.inner.write_parquet_file(location, data) + } + + fn read_parquet_footer(&self, file: &FileMeta) -> KernelResult { + self.inner.read_parquet_footer(file) + } +} + +struct TracingJsonHandler { + inner: Arc, + predicates: Arc>>>, +} + +impl JsonHandler for TracingJsonHandler { + fn parse_json( + &self, + json_strings: Box, + output_schema: SchemaRef, + ) -> KernelResult> { + self.inner.parse_json(json_strings, output_schema) + } + + fn read_json_files( + &self, + files: &[FileMeta], + physical_schema: SchemaRef, + predicate: Option, + ) -> KernelResult { + info!( + "TracingEngine::TracingJsonHandler::read_json_files\n\tfiles: {:?}\n\tpredicate: {:?}", + &files, predicate + ); + self.predicates + .lock() + .expect("lock not poisoned") + .push(predicate.clone()); + self.inner + .read_json_files(files, physical_schema, predicate) + } + + fn write_json_file( + &self, + path: &Url, + data: DeltaResultIterator<'_, FilteredEngineData>, + overwrite: bool, + ) -> KernelResult<()> { + self.inner.write_json_file(path, data, overwrite) + } +} + +struct TracingStorageHandler { + inner: Arc, +} + +impl StorageHandler for TracingStorageHandler { + fn list_from( + &self, + path: &Url, + ) -> KernelResult>>> { + self.inner.list_from(path) + } + + fn read_files( + &self, + files: Vec, + ) -> KernelResult>>> { + self.inner.read_files(files) + } + + fn copy_atomic(&self, src: &Url, dest: &Url) -> KernelResult<()> { + self.inner.copy_atomic(src, dest) + } + + fn put(&self, path: &Url, data: Bytes, overwrite: bool) -> KernelResult<()> { + self.inner.put(path, data, overwrite) + } + + fn head(&self, path: &Url) -> KernelResult { + self.inner.head(path) + } + + fn delete(&self, path: &Url) -> KernelResult<()> { + self.inner.delete(path) + } +} + +pub struct TracingEngine { + inner: Arc, + parquet_handler: Arc, + storage_handler: Arc, + json_handler: Arc, + parquet_calls: Arc>>, + json_predicates: Arc>>>, +} + +impl TracingEngine { + fn new(inner: Arc) -> Self { + let parquet_calls = Arc::new(Mutex::new(Vec::new())); + let json_predicates = Arc::new(Mutex::new(Vec::new())); + Self { + inner: inner.clone(), + parquet_handler: Arc::new(TracingParquetHandler { + inner: inner.parquet_handler(), + calls: parquet_calls.clone(), + }), + storage_handler: Arc::new(TracingStorageHandler { + inner: inner.storage_handler(), + }), + json_handler: Arc::new(TracingJsonHandler { + inner: inner.json_handler(), + predicates: json_predicates.clone(), + }), + parquet_calls, + json_predicates, + } + } + + pub fn wrap(inner: Arc) -> Arc { + Arc::new(Self::new(inner)) + } + + fn take_parquet_calls(&self) -> Vec { + std::mem::take(&mut *self.parquet_calls.lock().expect("lock not poisoned")) + } + + fn take_json_predicates(&self) -> Vec> { + std::mem::take(&mut *self.json_predicates.lock().expect("lock not poisoned")) + } +} + +impl Engine for TracingEngine { + fn evaluation_handler(&self) -> Arc { + self.inner.evaluation_handler() + } + + fn storage_handler(&self) -> Arc { + self.storage_handler.clone() + } + + fn json_handler(&self) -> Arc { + self.json_handler.clone() + } + + fn parquet_handler(&self) -> Arc { + self.parquet_handler.clone() + } +} + +#[derive(Default)] +struct DeltaScanVisitor { + num_scanned: Option, +} + +impl ExecutionPlanVisitor for DeltaScanVisitor { + type Error = DataFusionError; + + fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> DataFusionResult { + if let Some(delta_scan) = plan.downcast_ref::() { + self.num_scanned = delta_scan + .metrics() + .and_then(|metrics| metrics.sum_by_name("count_files_scanned")) + .map(|value| value.as_usize()); + } + Ok(true) + } +} + +#[derive(Clone, Copy)] +enum QueryColumn { + BatchId, + ConversationId, +} + +struct PredicateQuery { + sql: &'static str, + selected_files: usize, + output_rows: usize, + column: QueryColumn, +} + +struct CheckpointColumnRanges { + stats_parsed: Vec>, + stats: Vec>, + partition_values: Vec>, + partition_values_parsed: Vec>, +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_stats_parsed_without_files() -> DeltaResult<()> { + run_engine_predicate("conversations", true, false).await +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_stats_parsed_with_files() -> DeltaResult<()> { + run_engine_predicate("conversations", true, true).await +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_no_stats_parsed_without_files() -> DeltaResult<()> { + run_engine_predicate("conversations_no_stats_parsed", false, false).await +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_no_stats_parsed_with_files() -> DeltaResult<()> { + run_engine_predicate("conversations_no_stats_parsed", false, true).await +} + +async fn run_engine_predicate( + fixture: &str, + expect_parsed_checkpoint_fields: bool, + require_files: bool, +) -> DeltaResult<()> { + let filter = tracing_subscriber::EnvFilter::from_default_env(); + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_env_filter(filter) + .finish(); + tracing::subscriber::set_global_default(subscriber).ok(); + pretty_env_logger::try_init().ok(); + + let config = + SessionConfig::new().set_bool("datafusion.sql_parser.enable_ident_normalization", false); + + let ctx = SessionContext::new_with_config(config); + + let recording_object_store = Arc::new(RangeRecordingObjectStore::new(Arc::new( + object_store::local::LocalFileSystem::new(), + ))); + + let path = format!( + "file://{}/tests/data/{fixture}/delta/", + env!("CARGO_MANIFEST_DIR") + ); + let url = Url::parse(&path)?; + let mut builder = DeltaTableBuilder::from_url(url.clone()) + .unwrap() + .with_storage_backend(recording_object_store.clone(), url) + .with_checkpoint_stats_json_fallback(true); + let engine = TracingEngine::wrap(builder.build_storage().unwrap().engine(None)); + builder = builder.with_engine(engine.clone()); + if !require_files { + builder = builder.without_files(); + } + + let table = builder.load().await.unwrap(); + table.update_datafusion_session(&ctx.state()).unwrap(); + let provider = table.table_provider().await.unwrap(); + + let construction_parquet_calls = engine.take_parquet_calls(); + let construction_json_predicates = engine.take_json_predicates(); + if require_files { + assert!( + construction_parquet_calls.iter().any(|call| { + call.files.iter().any(|file| { + let path = file.location.path(); + path.contains("/_delta_log/") && path.contains(".checkpoint.") + }) + }), + "eager table construction must read transaction-log parquet" + ); + assert!( + !construction_json_predicates.is_empty(), + "eager table construction must read transaction-log JSON" + ); + } + + ctx.register_table("conversations", provider).unwrap(); + engine.take_parquet_calls(); + engine.take_json_predicates(); + recording_object_store.clear_reads(); + + let checkpoint_name = "00000000000000000007.checkpoint.parquet"; + let checkpoint_path = format!( + "{}/tests/data/{fixture}/delta/_delta_log/{checkpoint_name}", + env!("CARGO_MANIFEST_DIR") + ); + let checkpoint_ranges = CheckpointColumnRanges { + stats_parsed: parquet_column_ranges(&checkpoint_path, |path| { + path.starts_with("add.stats_parsed.") + }), + stats: parquet_column_ranges(&checkpoint_path, |path| path == "add.stats"), + partition_values: parquet_column_ranges(&checkpoint_path, |path| { + path.starts_with("add.partitionValues.") + }), + partition_values_parsed: parquet_column_ranges(&checkpoint_path, |path| { + path.starts_with("add.partitionValues_parsed.") + }), + }; + + let queries = [ + PredicateQuery { + sql: "SELECT conversation['conversationID'] FROM conversations WHERE _ACP_BATCHID = 'batch-1-2024-01-01'", + selected_files: 2, + output_rows: 4, + column: QueryColumn::BatchId, + }, + PredicateQuery { + sql: "SELECT conversation['conversationID'] FROM conversations WHERE _ACP_BATCHID != 'batch-1-2024-01-01'", + selected_files: 26, + output_rows: 52, + column: QueryColumn::BatchId, + }, + PredicateQuery { + sql: "SELECT conversation['conversationID'] FROM conversations WHERE conversation['conversationID'] = 'conversation-2024-01-01-1'", + selected_files: 2, + output_rows: 4, + column: QueryColumn::ConversationId, + }, + ]; + + for query in queries { + run_predicate_query( + &ctx, + &engine, + &recording_object_store, + checkpoint_name, + &checkpoint_ranges, + expect_parsed_checkpoint_fields, + require_files, + &query, + ) + .await; + } + + Ok(()) +} + +async fn run_predicate_query( + ctx: &SessionContext, + engine: &TracingEngine, + recording_object_store: &RangeRecordingObjectStore, + checkpoint_name: &str, + checkpoint_ranges: &CheckpointColumnRanges, + expect_parsed_checkpoint_fields: bool, + require_files: bool, + query: &PredicateQuery, +) { + engine.take_parquet_calls(); + engine.take_json_predicates(); + recording_object_store.clear_reads(); + + let dataframe = ctx.sql(query.sql).await.unwrap(); + let physical_plan = dataframe.create_physical_plan().await.unwrap(); + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(physical_plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(query.selected_files), + "unexpected selected file count for {}", + query.sql + ); + + let result = collect(physical_plan, ctx.task_ctx()).await.unwrap(); + let output_rows = result.iter().map(|batch| batch.num_rows()).sum::(); + assert_eq!( + output_rows, query.output_rows, + "unexpected output row count for {}", + query.sql + ); + + let parquet_calls = engine.take_parquet_calls(); + let json_predicates = engine.take_json_predicates(); + if require_files { + assert_eq!( + parquet_calls.len(), + 0, + "eager query must not make kernel parquet calls: {}", + query.sql + ); + assert_eq!( + json_predicates.len(), + 0, + "eager query must not make kernel JSON calls: {}", + query.sql + ); + } else { + // check we read parquet + json + { + assert_eq!( + json_predicates.len(), + 1, + "expected one kernel JSON call for {}", + query.sql + ); + let predicate = json_predicates[0] + .as_ref() + .expect("query JSON call must have a predicate"); + let query_column = match query.column { + QueryColumn::BatchId => ColumnName::new(["_ACP_BATCHID"]), + QueryColumn::ConversationId => ColumnName::new(["conversation", "conversationID"]), + }; + assert!( + predicate.references().contains(&query_column), + "JSON predicate must reference the query column for {}", + query.sql + ); + } + { + assert_eq!( + parquet_calls.len(), + 2, + "expected sidecar discovery then checkpoint action read for {}", + query.sql + ); + for call in &parquet_calls { + assert_eq!( + call.files.len(), + 1, + "each kernel parquet call must read one checkpoint file" + ); + assert!( + call.files[0].location.path().ends_with(checkpoint_name), + "kernel parquet call must read the version-7 checkpoint" + ); + } + + let sidecar_call = &parquet_calls[0]; + assert!(sidecar_call.physical_schema.field("sidecar").is_some()); + assert!(sidecar_call.physical_schema.field("add").is_none()); + assert!(sidecar_call.predicate.is_none()); + + let action_call = &parquet_calls[1]; + assert!(action_call.physical_schema.field("add").is_some()); + if !expect_parsed_checkpoint_fields { + assert!( + action_call.predicate.is_none(), + "checkpoint without parsed stats must not have an action predicate" + ); + return; + } + + let predicate = action_call + .predicate + .as_ref() + .expect("parsed checkpoint action read must have a predicate"); + let references = predicate.references(); + let partition_column = ColumnName::new(["add", "partitionValues_parsed", "_ACP_BATCHID"]); + match query.column { + QueryColumn::BatchId => assert!( + references.contains(&partition_column), + "checkpoint predicate must reference the parsed partition column" + ), + QueryColumn::ConversationId => { + assert!( + references.contains(&ColumnName::new([ + "add", + "stats_parsed", + "minValues", + "conversation", + "conversationID", + ])), + "checkpoint predicate must reference the nested minimum" + ); + assert!( + references.contains(&ColumnName::new([ + "add", + "stats_parsed", + "maxValues", + "conversation", + "conversationID", + ])), + "checkpoint predicate must reference the nested maximum" + ); + assert!( + !references.contains(&partition_column), + "nested checkpoint predicate must not retain the partition reference" + ); + } + } + } + } + + { + let checkpoint_reads = recording_object_store.reads_for_path_suffix(checkpoint_name); + if require_files { + assert!( + checkpoint_reads.is_empty(), + "eager query must not read checkpoint byte ranges" + ); + return; + } + + let ranges_were_read = |column_ranges: &[Range]| { + checkpoint_reads.iter().any(|read| { + column_ranges + .iter() + .any(|range| ranges_overlap(read, range)) + }) + }; + assert!( + !checkpoint_reads.is_empty(), + "expected checkpoint column reads" + ); + assert!( + !checkpoint_ranges.stats.is_empty(), + "checkpoint must contain add.stats" + ); + assert!( + ranges_were_read(&checkpoint_ranges.partition_values), + "expected add.partitionValues to be read" + ); + + if expect_parsed_checkpoint_fields { + assert!( + !checkpoint_ranges.stats_parsed.is_empty(), + "checkpoint must contain add.stats_parsed" + ); + assert!( + !checkpoint_ranges.partition_values_parsed.is_empty(), + "checkpoint must contain add.partitionValues_parsed" + ); + assert!( + ranges_were_read(&checkpoint_ranges.stats_parsed), + "expected add.stats_parsed to be read" + ); + assert!( + !ranges_were_read(&checkpoint_ranges.stats), + "expected add.stats not to be read" + ); + assert_eq!( + ranges_were_read(&checkpoint_ranges.partition_values_parsed), + matches!(query.column, QueryColumn::BatchId), + "parsed partition-value reads must be query-specific" + ); + } else { + assert!( + checkpoint_ranges.stats_parsed.is_empty(), + "checkpoint must not contain add.stats_parsed" + ); + assert!( + checkpoint_ranges.partition_values_parsed.is_empty(), + "checkpoint must not contain add.partitionValues_parsed" + ); + assert!( + ranges_were_read(&checkpoint_ranges.stats), + "expected add.stats to be read" + ); + } + } +} diff --git a/crates/core/tests/it_datafusion/main.rs b/crates/core/tests/it_datafusion/main.rs index 1c38ec579..e26234101 100644 --- a/crates/core/tests/it_datafusion/main.rs +++ b/crates/core/tests/it_datafusion/main.rs @@ -20,4 +20,5 @@ mod nanosecond_timestamps; mod read_delta_log_test; mod read_delta_partitions_test; mod stats_parsed_checkpoint; -mod integration_datafusion_deep; \ No newline at end of file +mod integration_datafusion_deep; +pub mod integration_datafusion_predicate; \ No newline at end of file