diff --git a/Cargo.toml b/Cargo.toml index dd5011e368..46bc0f82a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,14 +16,14 @@ documentation = "https://docs.rs/deltalake" repository = "https://github.com/delta-io/delta.rs" [workspace.dependencies] -#delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [ -delta_kernel = { package = "buoyant_kernel", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = [ +# delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [ +delta_kernel = { package = "buoyant_kernel", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = [ #delta_kernel = { package = "buoyant_kernel", version = "0.25.0,<0.25.100", features = [ "arrow-58", "internal-api", ] } -#delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false } -delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = ["arrow-58", "rustls"], default-features = false } +# delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false } +delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = ["arrow-58", "rustls"], default-features = false } #delta_kernel_default_engine = { package = "buoyant_kernel_engine", version = "0.25.0,<0.25.100", features = ["arrow-58", "rustls"], default-features = false } @@ -52,11 +52,13 @@ datafusion-proto = { version = "54.0.0" } # serde serde = { version = "1.0.194", features = ["derive"] } serde_json = "1" +strum = { version = "0.28", features = ["derive"] } +prost = { version = "0.14.3" } # "stdlib" bytes = { version = "1" } chrono = { version = "0.4.40", default-features = false, features = ["clock"] } -time = { version = "=0.3.53" } +time = { version = "=0.3.47" } tracing = { version = "0.1", features = ["log"] } regex = { version = "1" } thiserror = { version = "2" } @@ -160,3 +162,76 @@ inherits = "release" opt-level = 3 codegen-units = 1 lto = "thin" + +[patch.crates-io] +datafusion = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-catalog = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-catalog-listing = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-common-runtime = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-avro = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-csv = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-json = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-datasource-parquet = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-execution = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-ffi = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-aggregate = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-aggregate-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-nested = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-table = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-window = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-functions-window-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-expr-adapter = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-pruning = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-physical-plan = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-proto = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-proto-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-session = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-spark = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-sql = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +datafusion-substrait = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' } +#datafusion = { path = "../datafusion/datafusion/core" } +#datafusion-catalog = { path = "../datafusion/datafusion/catalog" } +#datafusion-catalog-listing = { path = "../datafusion/datafusion/catalog-listing" } +#datafusion-common = { path = "../datafusion/datafusion/common" } +#datafusion-common-runtime = { path = "../datafusion/datafusion/common-runtime" } +#datafusion-datasource = { path = "../datafusion/datafusion/datasource" } +#datafusion-datasource-arrow = { path = "../datafusion/datafusion/datasource-arrow" } +#datafusion-datasource-avro = { path = "../datafusion/datafusion/datasource-avro" } +#datafusion-datasource-csv = { path = "../datafusion/datafusion/datasource-csv" } +#datafusion-datasource-json = { path = "../datafusion/datafusion/datasource-json" } +#datafusion-datasource-parquet = { path = "../datafusion/datafusion/datasource-parquet" } +#datafusion-execution = { path = "../datafusion/datafusion/execution" } +#datafusion-expr = { path = "../datafusion/datafusion/expr" } +#datafusion-expr-common = { path = "../datafusion/datafusion/expr-common" } +#datafusion-ffi = { path = "../datafusion/datafusion/ffi" } +#datafusion-functions = { path = "../datafusion/datafusion/functions" } +#datafusion-functions-aggregate = { path = "../datafusion/datafusion/functions-aggregate" } +#datafusion-functions-aggregate-common = { path = "../datafusion/datafusion/functions-aggregate-common" } +#datafusion-functions-nested = { path = "../datafusion/datafusion/functions-nested" } +#datafusion-functions-table = { path = "../datafusion/datafusion/functions-table" } +#datafusion-functions-window = { path = "../datafusion/datafusion/functions-window" } +#datafusion-functions-window-common = { path = "../datafusion/datafusion/functions-window-common" } +#datafusion-optimizer = { path = "../datafusion/datafusion/optimizer" } +#datafusion-physical-expr = { path = "../datafusion/datafusion/physical-expr" } +#datafusion-physical-expr-adapter = { path = "../datafusion/datafusion/physical-expr-adapter" } +#datafusion-physical-expr-common = { path = "../datafusion/datafusion/physical-expr-common" } +#datafusion-physical-optimizer = { path = "../datafusion/datafusion/physical-optimizer" } +#datafusion-physical-plan = { path = "../datafusion/datafusion/physical-plan" } +#datafusion-proto = { path = "../datafusion/datafusion/proto" } +#datafusion-proto-common = { path = "../datafusion/datafusion/proto-common" } +#datafusion-pruning = { path = "../datafusion/datafusion/pruning" } +#datafusion-session = { path = "../datafusion/datafusion/session" } +#datafusion-spark = { path = "../datafusion/datafusion/spark" } +#datafusion-sql = { path = "../datafusion/datafusion/sql" } + +#buoyant_kernel = { git = 'https://github.com/hstack/delta-kernel-rs.git', branch = 'buoyant-v0.25.1-stats' } +#buoyant_kernel = { path = "../delta-kernel-rs/kernel" } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index adb3fc6c3d..66e50c58fe 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -51,6 +51,8 @@ datafusion-proto = { workspace = true, optional = true } # serde serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +strum = { workspace = true, features = ["derive"] } +prost = { workspace = true } # "stdlib" bytes = { workspace = true } @@ -95,6 +97,7 @@ rand = "0.10" sqlparser = { version = "0.61.0" } humantime = { version = "2.1.0", optional = true } validator = { version = "0.19", features = ["derive"] } +ctor = "0.10" [dev-dependencies] arrow = { workspace = true, features = ["prettyprint"]} diff --git a/crates/core/src/delta_datafusion/find_files.rs b/crates/core/src/delta_datafusion/find_files.rs index 0a9988d7fc..f2c37b1722 100644 --- a/crates/core/src/delta_datafusion/find_files.rs +++ b/crates/core/src/delta_datafusion/find_files.rs @@ -215,7 +215,7 @@ impl TreeNodeVisitor<'_> for FindFilesExprProperties { /// rather than materializing every partition column — which avoids coercing values the /// predicate never reads and prevents an unrelated/malformed partition value from /// weakening pruning. -pub(crate) fn extract_partition_only_predicate( +pub fn extract_partition_only_predicate( predicate: Expr, partition_columns: &[String], ) -> DeltaResult)>> { diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs index 0ca6aae559..bf409e0d45 100644 --- a/crates/core/src/delta_datafusion/mod.rs +++ b/crates/core/src/delta_datafusion/mod.rs @@ -73,15 +73,25 @@ pub(crate) use column_mapping::ColumnMappingState; pub(crate) use data_validation::{ DataValidationExec, constraints_to_exprs, generated_columns_to_exprs, validation_predicates, }; +pub use engine::DataFusionEngine; pub(crate) use find_files::*; pub(crate) use table_provider::next::normalize_path_as_file_id; pub use table_provider::{ DeltaScanConfig, DeltaScanConfigBuilder, TableProviderBuilder, next::DeltaScanExec, + next::DeltaNextPhysicalCodec, }; pub(crate) use table_provider::{ next::FILE_ID_COLUMN_DEFAULT, resolve_file_column_name, update_datafusion_session, }; +/// Converts a Delta Kernel expression to a DataFusion expression. +pub fn to_datafusion_expr( + expr: &delta_kernel::expressions::Expression, + output_type: &delta_kernel::schema::DataType, +) -> DataFusionResult { + engine::to_datafusion_expr(expr, output_type) +} + pub(crate) const PATH_COLUMN: &str = "__delta_rs_path"; #[doc(hidden)] @@ -100,8 +110,13 @@ pub mod planner; mod session; pub use session::SessionFallbackPolicy; pub(crate) use session::{SessionResolveContext, resolve_session_state}; +use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields; + mod table_provider; +pub mod udtf; pub(crate) mod utils; +pub mod table_provider_old; +pub mod schema_null; impl From for DataFusionError { fn from(err: DeltaTableError) -> Self { @@ -232,7 +247,7 @@ fn _arrow_schema( partition_columns: &[String], wrap_partitions: bool, ) -> ArrowSchemaRef { - let fields = schema + let mut fields = schema .fields() .into_iter() .filter(|f| !partition_columns.contains(&f.name().to_string())) @@ -261,7 +276,10 @@ fn _arrow_schema( }), ) .collect::>(); - Arc::new(ArrowSchema::new(fields)) + let mut schema = Arc::new(ArrowSchema::new(fields)); + // @Hstack - add the option to have an env var that can nullify fields in the delta schema + schema = rewrite_schema_with_nullable_fields(schema); + schema } pub(crate) fn files_matching_predicate<'a>( diff --git a/crates/core/src/delta_datafusion/schema_null.rs b/crates/core/src/delta_datafusion/schema_null.rs new file mode 100644 index 0000000000..324ffafa32 --- /dev/null +++ b/crates/core/src/delta_datafusion/schema_null.rs @@ -0,0 +1,358 @@ +use std::sync::{Arc, Mutex}; +use arrow_schema::{DataType, Field, FieldRef, Fields, SchemaRef, UnionFields}; +use arrow_schema::DataType::{Dictionary, FixedSizeList, LargeList, LargeListView, List, ListView, Map, RunEndEncoded, Struct, Union}; + +static DELTA_FIELD_PATHS_TO_MAKE_NULLABLE: Mutex> = Mutex::new(Vec::new()); + +#[ctor::ctor] +fn init() { + let paths_flag = if let Ok(v) = std::env::var("DELTA_FIELD_PATHS_TO_MAKE_NULLABLE") { + v + } else { + "_acp_system_metadata.rowId,_acp_system_metadata.rowVersion,_acp_system_metadata.ingestTime,_acp_system_metadata.trackingId".to_string() + }; + + let splits = paths_flag + .split(",") + .map(|s| s.to_string()) + .collect::>(); + *DELTA_FIELD_PATHS_TO_MAKE_NULLABLE.lock().unwrap() = splits; +} + +pub fn rewrite_schema_with_nullable_fields(input: SchemaRef) -> SchemaRef { + let paths = DELTA_FIELD_PATHS_TO_MAKE_NULLABLE.lock().unwrap(); + rewrite_schema_with_nullable_fields_inner(input, &paths) +} + +fn rewrite_schema_with_nullable_fields_inner(input: SchemaRef, paths: &[String]) -> SchemaRef { + if paths.is_empty() { + return input; + } + + let new_fields = rewrite_fields(input.fields(), paths, ""); + Arc::new(arrow_schema::Schema::new_with_metadata(new_fields, input.metadata().clone())) +} + +fn rewrite_fields(fields: &Fields, paths: &[String], parent_path: &str) -> Vec { + fields.iter().map(|field| { + let current_path = if parent_path.is_empty() { + field.name().to_string() + } else { + format!("{}.{}", parent_path, field.name()) + }; + + let make_nullable = !field.is_nullable() && paths.iter().any(|p| *p == current_path); + let new_field = rewrite_field_type(field, paths, ¤t_path); + + if make_nullable { + Arc::new(new_field.with_nullable(true)) + } else { + Arc::new(new_field) + } + }).collect() +} + +fn rewrite_field_type(field: &Field, paths: &[String], current_path: &str) -> Field { + let new_dt = rewrite_data_type(field.data_type(), paths, current_path); + Field::new(field.name(), new_dt, field.is_nullable()) + .with_metadata(field.metadata().clone()) +} + +fn rewrite_data_type(dt: &DataType, paths: &[String], current_path: &str) -> DataType { + match dt { + Struct(fields) => { + Struct(Fields::from(rewrite_fields(fields, paths, current_path))) + }, + List(inner) => { + List(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + LargeList(inner) => { + LargeList(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + FixedSizeList(inner, n) => { + FixedSizeList(Arc::new(rewrite_field_type(inner, paths, current_path)), *n) + }, + ListView(inner) => { + ListView(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + LargeListView(inner) => { + LargeListView(Arc::new(rewrite_field_type(inner, paths, current_path))) + }, + Map(inner, sorted) => { + Map(Arc::new(rewrite_field_type(inner, paths, current_path)), *sorted) + }, + Dictionary(key, value) => { + Dictionary(key.clone(), Box::new(rewrite_data_type(value, paths, current_path))) + }, + RunEndEncoded(run_ends, values) => { + RunEndEncoded( + Arc::clone(run_ends), + Arc::new(rewrite_field_type(values, paths, current_path)), + ) + }, + Union(union_fields, mode) => { + let type_ids: Vec = union_fields.iter().map(|(id, _)| id).collect(); + let fields: Vec = union_fields.iter().map(|(_, field)| { + let field_path = if current_path.is_empty() { + field.name().to_string() + } else { + format!("{}.{}", current_path, field.name()) + }; + let make_nullable = !field.is_nullable() && paths.iter().any(|p| *p == field_path); + let mut new_field = rewrite_field_type(field, paths, &field_path); + if make_nullable { + new_field = new_field.with_nullable(true); + } + Arc::new(new_field) + }).collect(); + Union(UnionFields::new(type_ids, fields), *mode) + } + _ => dt.clone(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::{DataType, Field, Schema}; + + fn make_schema(fields: Vec) -> SchemaRef { + Arc::new(Schema::new(fields)) + } + + fn paths(strs: &[&str]) -> Vec { + strs.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn test_schema_null_empty_paths_returns_input() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema.clone(), &[]); + assert_eq!(schema, result); + } + + #[test] + fn test_schema_null_top_level_field_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Utf8, false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + assert!(result.field_with_name("a").unwrap().is_nullable()); + assert!(!result.field_with_name("b").unwrap().is_nullable()); + } + + #[test] + fn test_schema_null_already_nullable_unchanged() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, true), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + assert!(result.field_with_name("a").unwrap().is_nullable()); + } + + #[test] + fn test_schema_null_path_not_in_schema_unchanged() { + let schema = make_schema(vec![ + Field::new("a", DataType::Int32, false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema.clone(), &paths(&["z"])); + assert_eq!(schema, result); + } + + #[test] + fn test_schema_null_nested_struct_field() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + Field::new("c", DataType::Utf8, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b"])); + let a = result.field_with_name("a").unwrap(); + assert!(!a.is_nullable()); + if let Struct(fields) = a.data_type() { + assert!(fields.find("b").unwrap().1.is_nullable()); + assert!(!fields.find("c").unwrap().1.is_nullable()); + } else { + assert_eq!(true, false, "expected struct"); + } + } + + #[test] + fn test_schema_null_deeply_nested_struct() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", Struct(Fields::from(vec![ + Field::new("c", Struct(Fields::from(vec![ + Field::new("d", DataType::Int32, false), + ])), false), + ])), false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b.c.d"])); + let a = result.field_with_name("a").unwrap(); + assert!(!a.is_nullable()); + let b = match a.data_type() { Struct(f) => f.find("b").unwrap().1, _ => panic!() }; + assert!(!b.is_nullable()); + let c = match b.data_type() { Struct(f) => f.find("c").unwrap().1, _ => panic!() }; + assert!(!c.is_nullable()); + let d = match c.data_type() { Struct(f) => f.find("d").unwrap().1, _ => panic!() }; + assert!(d.is_nullable()); + } + + #[test] + fn test_schema_null_struct_itself_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + let a = result.field_with_name("a").unwrap(); + assert!(a.is_nullable()); + if let Struct(fields) = a.data_type() { + assert!(!fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct"); + } + } + + #[test] + fn test_schema_null_struct_and_child_both_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a", "a.b"])); + let a = result.field_with_name("a").unwrap(); + assert!(a.is_nullable()); + if let Struct(fields) = a.data_type() { + assert!(fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct"); + } + } + + #[test] + fn test_schema_null_list_containing_struct() { + let schema = make_schema(vec![ + Field::new("a", List(Arc::new( + Field::new("item", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + )), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b"])); + let a = result.field_with_name("a").unwrap(); + assert!(!a.is_nullable()); + if let List(inner) = a.data_type() { + if let Struct(fields) = inner.data_type() { + assert!(fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct inside list"); + } + } else { + panic!("expected list"); + } + } + + #[test] + fn test_schema_null_list_itself_made_nullable() { + let schema = make_schema(vec![ + Field::new("a", List(Arc::new( + Field::new("item", DataType::Int32, false), + )), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + assert!(result.field_with_name("a").unwrap().is_nullable()); + } + + #[test] + fn test_schema_null_dictionary_with_nested_value() { + let schema = make_schema(vec![ + Field::new("a", Dictionary( + Box::new(DataType::Int32), + Box::new(Struct(Fields::from(vec![ + Field::new("b", DataType::Utf8, false), + ]))), + ), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.b"])); + let a = result.field_with_name("a").unwrap(); + if let Dictionary(_, value) = a.data_type() { + if let Struct(fields) = value.as_ref() { + assert!(fields.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct in dict value"); + } + } else { + panic!("expected dictionary"); + } + } + + #[test] + fn test_schema_null_map_with_nested_value() { + let schema = make_schema(vec![ + Field::new("a", Map(Arc::new( + Field::new("entries", Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", Struct(Fields::from(vec![ + Field::new("b", DataType::Int32, false), + ])), false), + ])), false), + ), false), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a.value.b"])); + let a = result.field_with_name("a").unwrap(); + if let Map(inner, _) = a.data_type() { + if let Struct(entries) = inner.data_type() { + let value_field = entries.find("value").unwrap().1; + if let Struct(vf) = value_field.data_type() { + assert!(vf.find("b").unwrap().1.is_nullable()); + } else { + panic!("expected struct in value"); + } + } else { + panic!("expected struct in map entries"); + } + } else { + panic!("expected map"); + } + } + + #[test] + fn test_schema_null_multiple_paths_across_tree() { + let schema = make_schema(vec![ + Field::new("x", DataType::Int64, false), + Field::new("y", Struct(Fields::from(vec![ + Field::new("z", DataType::Float64, false), + Field::new("w", DataType::Boolean, false), + ])), false), + ]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["x", "y.z"])); + assert!(result.field_with_name("x").unwrap().is_nullable()); + let y = result.field_with_name("y").unwrap(); + assert!(!y.is_nullable()); + if let Struct(fields) = y.data_type() { + assert!(fields.find("z").unwrap().1.is_nullable()); + assert!(!fields.find("w").unwrap().1.is_nullable()); + } else { + panic!("expected struct"); + } + } + + #[test] + fn test_schema_null_metadata_preserved() { + let mut field = Field::new("a", DataType::Int32, false); + field.set_metadata([("key".to_string(), "val".to_string())].into()); + let schema = make_schema(vec![field]); + let result = rewrite_schema_with_nullable_fields_inner(schema, &paths(&["a"])); + let a = result.field_with_name("a").unwrap(); + assert!(a.is_nullable()); + assert_eq!(a.metadata().get("key").unwrap(), "val"); + } +} diff --git a/crates/core/src/delta_datafusion/table_provider.rs b/crates/core/src/delta_datafusion/table_provider.rs index 7159a99982..a26da01c9f 100644 --- a/crates/core/src/delta_datafusion/table_provider.rs +++ b/crates/core/src/delta_datafusion/table_provider.rs @@ -524,15 +524,15 @@ pub(crate) fn update_datafusion_session( #[derive(Debug)] pub(super) struct DeltaScan { /// The normalized [Url] of the ObjectStore root - table_url: Url, + pub table_url: Url, /// Column that contains an index that maps to the original metadata Add - config: DeltaScanConfig, + pub config: DeltaScanConfig, /// The parquet scan to wrap - parquet_scan: Arc, + pub parquet_scan: Arc, /// The schema of the table to be used when evaluating expressions - logical_schema: Arc, + pub logical_schema: Arc, /// Metrics for scan reported via DataFusion - metrics: ExecutionPlanMetricsSet, + pub metrics: ExecutionPlanMetricsSet, } impl DeltaScan { @@ -556,9 +556,9 @@ impl DeltaScan { #[derive(Debug, Serialize, Deserialize)] pub(super) struct DeltaScanWire { /// This [Url] should have already been passed through [normalize_table_url] - table_url: Url, - config: DeltaScanConfig, - logical_schema: Arc, + pub(crate) table_url: Url, + pub(crate) config: DeltaScanConfig, + pub(crate) logical_schema: Arc, } impl From<&DeltaScan> for DeltaScanWire { diff --git a/crates/core/src/delta_datafusion/table_provider/next/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/mod.rs index 4dc48a5afe..ed2fbd2ba2 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/mod.rs @@ -44,7 +44,7 @@ use serde::{Deserialize, Serialize}; use url::Url; use uuid::Uuid; -pub use self::scan::DeltaScanExec; +pub use self::scan::{DeltaScanExec, DeltaNextPhysicalCodec}; pub(crate) use self::scan::KernelScanPlan; use self::scan::ProjectedScanContract; use super::data_sink::DeltaDataSink; @@ -733,7 +733,11 @@ impl TableProvider for DeltaScan { limit: Option, ) -> Result> { self.ensure_read_ready(session)?; - let engine = DataFusionEngine::new_from_session(session); + // let engine = DataFusionEngine::new_from_session(session); + let engine: Arc = self.snapshot.snapshot().config.engine.as_ref() + .map(|e| e.0.clone()) + .unwrap_or_else(|| DataFusionEngine::new_from_session(session)); + let contract = ProjectedScanContract::try_new( self.scan_schema.clone(), self.full_schema.clone(), @@ -853,6 +857,7 @@ pub(crate) fn test_multi_partitioned_override_schema() -> SchemaRef { #[cfg(test)] mod tests { + use tracing::info; use arrow::{ array::{ BooleanArray, Date32Array, Int32Array, Int64Array, StringArray, @@ -968,6 +973,7 @@ mod tests { struct DeltaScanVisitor { num_scanned: Option, total_bytes_scanned: Option, + num_pruned_by_limit: Option, } impl DeltaScanVisitor { @@ -982,6 +988,9 @@ mod tests { self.num_scanned = metrics .sum_by_name("count_files_scanned") .map(|v| v.as_usize()); + self.num_pruned_by_limit = metrics + .sum_by_name("count_files_pruned_by_limit") + .map(|v| v.as_usize()); Ok(true) } @@ -2957,4 +2966,215 @@ mod tests { Ok(()) } + + + /// LIMIT 1 with no filters: pruning fires, only one file reaches the Parquet reader. + #[tokio::test] + async fn test_limit_pushdown_no_filter_trims_files() -> TestResult { + let id_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + ArrowDataType::Int64, + true, + )])); + + // Three single-row files, one per write. + let table = create_in_memory_id_table_with_rows(vec![1]).await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema.clone(), + vec![Arc::new(Int64Array::from(vec![2]))], + )?]) + .await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema, + vec![Arc::new(Int64Array::from(vec![3]))], + )?]) + .await?; + + let provider = DeltaScan::builder() + .with_log_store(table.log_store()) + .build() + .await?; + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let plan = provider.scan(&state, None, &[], Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + // The headline invariant: file-level pruning kicked in, so only one file + // made it into the scan plan even though three exist in the snapshot. + assert_eq!( + visitor.num_scanned, + Some(1), + "LIMIT 1 should prune to exactly one file" + ); + // The pruned-by-limit metric counts files the stream visited and then + // dropped. Kernel emits scan-metadata in batches, and early-break drops + // the stream before later batches arrive, so the exact value depends on + // kernel batching. We only check that it is non-negative. + assert!(visitor.num_pruned_by_limit.is_some()); + + let batches: Vec<_> = collect_partitioned(plan, session.task_ctx()) + .await? + .into_iter() + .flatten() + .collect(); + let row_count: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(row_count, 1, "LIMIT 1 should return exactly one row"); + + Ok(()) + } + + /// LIMIT 1 + Inexact (data-column) filter: defensive gate disables file pruning even when + /// the caller passes `Some(N)`. DataFusion's own planner would not pass `Some(1)` here in + /// production, but the test exercises the local invariant. + #[tokio::test] + async fn test_limit_pushdown_inexact_filter_does_not_trim_files() -> TestResult { + use datafusion::prelude::{col, lit}; + + let id_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + ArrowDataType::Int64, + true, + )])); + + let table = create_in_memory_id_table_with_rows(vec![1]).await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema.clone(), + vec![Arc::new(Int64Array::from(vec![2]))], + )?]) + .await?; + let table = table + .write(vec![RecordBatch::try_new( + id_schema, + vec![Arc::new(Int64Array::from(vec![3]))], + )?]) + .await?; + + let provider = DeltaScan::builder() + .with_log_store(table.log_store()) + .build() + .await?; + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let filters = vec![col("id").gt(lit(0_i64))]; + let plan = provider.scan(&state, None, &filters, Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(3), + "Inexact filter should leave all three files in the scan plan" + ); + assert_eq!(visitor.num_pruned_by_limit, Some(0)); + + Ok(()) + } + + /// LIMIT 1 with an explicit FileSelection: pinned files win, file-level pruning is bypassed. + #[tokio::test] + async fn test_limit_pushdown_file_selection_bypasses_pruning() -> TestResult { + let log_store = TestTables::Simple.table_builder()?.build_storage()?; + let snapshot = Arc::new(Snapshot::try_new(&log_store, Default::default(), None).await?); + let table_root = snapshot.scan_builder().build()?.table_root().clone(); + + // Pin to the first two files explicitly. + let selected_paths: Vec = snapshot + .file_views(log_store.as_ref(), None) + .take(2) + .map_ok(|v| table_root.join(v.path_raw()).unwrap().to_string()) + .try_collect() + .await?; + assert_eq!(selected_paths.len(), 2); + let selection = FileSelection::from_file_paths(selected_paths); + + let provider = DeltaScan::builder() + .with_snapshot(snapshot) + .with_file_column(FILE_ID_COLUMN_DEFAULT) + .build() + .await? + .with_file_selection(selection); + + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + let plan = provider.scan(&state, None, &[], Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(2), + "FileSelection should override file-level limit pruning" + ); + assert_eq!(visitor.num_pruned_by_limit, Some(0)); + + Ok(()) + } + + /// LIMIT 1 + partition-only Exact filter on a multi-file partition: pruning trims to one + /// matching file. Partition predicates are classified Exact, so the safety gate allows pruning. + #[tokio::test] + async fn test_limit_pushdown_partition_only_filter_trims_files() -> TestResult { + use datafusion::prelude::{col, lit}; + + let row_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("part", ArrowDataType::Utf8, true), + ArrowField::new("value", ArrowDataType::Int32, true), + ])); + let make_batch = |part: &str, value: i32| -> crate::DeltaResult { + Ok(RecordBatch::try_new( + row_schema.clone(), + vec![ + Arc::new(StringArray::from(vec![part])), + Arc::new(Int32Array::from(vec![value])), + ], + )?) + }; + + // Two partitions × one+two files. Partition A has two files, B has one. + let table = crate::DeltaTable::new_in_memory() + .write(vec![make_batch("A", 1)?]) + .with_partition_columns(vec!["part"]) + .await?; + let table = table.write(vec![make_batch("B", 2)?]).await?; + let table = table.write(vec![make_batch("A", 3)?]).await?; + + + info!("TEST START ---------------------------------------"); + let provider = DeltaScan::builder() + .with_log_store(table.log_store()) + .build() + .await?; + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let filters = vec![col("part").eq(lit("A"))]; + let plan = provider.scan(&state, None, &filters, Some(1)).await?; + + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(1), + "Exact partition filter + LIMIT 1 should leave a single matching file" + ); + // Pruned-by-limit count depends on kernel batch shape (see no-filter + // test for the rationale); we assert presence, not value. + assert!(visitor.num_pruned_by_limit.is_some()); + + let batches: Vec<_> = collect_partitioned(plan, session.task_ctx()) + .await? + .into_iter() + .flatten() + .collect(); + let row_count: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(row_count, 1); + + Ok(()) + } } diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs new file mode 100644 index 0000000000..9eb4400f9f --- /dev/null +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/codec.rs @@ -0,0 +1,1186 @@ +//! Codec for serializing and deserializing [`DeltaScanExec`] physical plans. +//! +//! Provides a [`PhysicalExtensionCodec`] implementation for distributed execution. +//! Expressions are serialized via DataFusion protobuf; kernel `Transform` expressions +//! use a custom wire format since they have no DataFusion equivalent. + +use std::collections::VecDeque; +use std::sync::Arc; + +use arrow::datatypes::SchemaRef; +use dashmap::DashMap; +use datafusion::common::HashMap; +use datafusion::error::DataFusionError; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::Expr; +use datafusion_datasource::file_scan_config::{FileScanConfig, FileScanConfigBuilder}; +use datafusion_datasource::source::DataSourceExec; +use datafusion_proto::bytes::Serializeable; +use datafusion_proto::physical_plan::PhysicalExtensionCodec; +use delta_kernel::expressions::{ColumnName, Expression, ExpressionFieldPatch, ExpressionStructPatch}; +use delta_kernel::schema::{DataType as KernelDataType}; +use serde::{Deserialize, Serialize}; +use super::{DeltaScanExec, DeltaScanMetaExec, ProjectedScanContract, PublicFileIdMap}; +use super::plan::KernelScanPlan; +use crate::delta_datafusion::engine::{to_datafusion_expr, to_delta_expression}; +use crate::delta_datafusion::DeltaScanConfig; +use crate::DeltaTableConfig; +use crate::kernel::size_limits::SnapshotLoadMetrics; +use crate::kernel::Snapshot; + +/// Codec for serializing/deserializing [`DeltaScanExec`] physical plans. +/// +/// This codec enables distributed execution by serializing the inputs needed +/// to reconstruct the execution plan rather than the plan itself. This approach +/// avoids the need for serde support in delta-kernel types. +#[derive(Debug, Clone, Default)] +pub struct DeltaNextPhysicalCodec; + +impl PhysicalExtensionCodec for DeltaNextPhysicalCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &TaskContext, + ) -> datafusion::common::Result> { + let wire: ScanExecWire = serde_json::from_slice(buf).map_err(|e| { + DataFusionError::Internal(format!("Failed to decode Delta scan exec: {e}")) + })?; + + match wire { + ScanExecWire::Scan(wire) => wire.into_exec(inputs, ctx), + ScanExecWire::Meta(wire) => wire.into_exec(inputs, ctx), + } + } + + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + ) -> datafusion::common::Result<()> { + let wire = if let Some(delta_scan) = node.downcast_ref::() { + ScanExecWire::Scan(DeltaScanExecWire::try_from(delta_scan)?) + } else if let Some(meta_scan) = node.downcast_ref::() { + ScanExecWire::Meta(DeltaScanMetaExecWire::try_from(meta_scan)?) + } else { + return Err(DataFusionError::Internal( + "Expected DeltaScanExec or DeltaScanMetaExec for encoding".to_string(), + )); + }; + + serde_json::to_writer(buf, &wire).map_err(|e| { + DataFusionError::Internal(format!("Failed to encode Delta scan exec: {e}")) + })?; + Ok(()) + } +} + +/// Tagged wire wrapper dispatching between the two scan exec variants. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) enum ScanExecWire { + Scan(DeltaScanExecWire), + Meta(DeltaScanMetaExecWire), +} + +/// Wire format for a kernel FieldTransform. +#[derive(Debug, Serialize, Deserialize)] +struct ExpressionFieldPatchWire { + keep_input: bool, + insertions: Vec>, + optional: bool, +} + +/// Wire format for a kernel StructPatch expression. +/// +/// StructPatch is a sparse schema modification: specifies which fields to modify, +/// with unmentioned fields passing through unchanged. +#[derive(Debug, Serialize, Deserialize)] +struct StructPatchWire { + input_path: Option>, + field_patches: std::collections::HashMap, + prepended_fields: Vec>, + appended_fields: Vec>, +} + +/// Wire format for serializing [`DeltaScanExec`]. +/// +/// Uses `std::collections::HashMap` instead of `DashMap` for serde compatibility. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct DeltaScanExecWire { + scan: ScanPlanWire, + transforms: std::collections::HashMap, + selection_vectors: std::collections::HashMap>, + input_file_id_column: String, + file_id_column: Option, + public_file_ids: std::collections::HashMap, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct ScanPlanWire { + snapshot: Snapshot, + contract: ProjectedScanContractWire, + filters: Vec>, + skipping_predicate: Option>>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct ProjectedScanContractWire { + table_schema: SchemaRef, + provider_schema: SchemaRef, + projection: Option>, + filters: Vec>, + row_index_column: Option, +} + +/// Build the [`ScanPlanWire`] shared by both scan codecs. +fn scan_plan_wire(scan_plan: &KernelScanPlan) -> Result { + let snapshot = { + let exec_scan_plan_scan_snapshot = scan_plan.scan.snapshot().clone(); + // @HSTACK FIXME AT upgrade + // ATM, DeltaTableConfig is ONLY used with defaults + // The only thing that we set in it is the log_size_limiter - + // which has already been used early in the logical / planning phase + // At upgrade, RECHECK usage sites for DeltaTableConfig, we'll need to re-evaluate if + // stuff begins writing to it + let delta_table_config = DeltaTableConfig::default(); + let load_metrics = SnapshotLoadMetrics::from_snapshot(&exec_scan_plan_scan_snapshot); + Snapshot { + inner: exec_scan_plan_scan_snapshot, + config: delta_table_config, + materialized_files: None, + load_metrics, + } + }; + + let row_index_column = scan_plan + .contract + .row_index_field + .as_ref() + .map(|f| f.name().clone()); + + let projected_scan_contract_wire = ProjectedScanContractWire { + table_schema: scan_plan.contract.table_schema.clone(), + provider_schema: scan_plan.contract.provider_schema.clone(), + projection: scan_plan.contract.projection.clone(), + filters: scan_plan + .contract + .filters + .iter() + .map(|p| p.to_bytes().map(|b| b.to_vec()).unwrap()) + .collect::>(), + row_index_column, + }; + + Ok(ScanPlanWire { + snapshot, + contract: projected_scan_contract_wire, + filters: scan_plan + .filters + .iter() + .map(|p| p.to_bytes().map(|b| b.to_vec()).unwrap()) + .collect::>(), + skipping_predicate: scan_plan.skipping_predicate.clone().map(|sp| { + sp.iter() + .map(|e| e.to_bytes().map(|b| b.to_vec()).unwrap()) + .collect::>() + }), + }) +} + +impl TryFrom<&DeltaScanExec> for DeltaScanExecWire { + type Error = DataFusionError; + + fn try_from(exec: &DeltaScanExec) -> Result { + let scan_plan_wire = scan_plan_wire(&exec.scan_plan)?; + + let transforms: std::collections::HashMap = exec + .transforms + .iter() + .map(|(file_url, kernel_expr)| { + serialize_transform(kernel_expr.as_ref()) + .map(|wire| (file_url.clone(), wire)) + }) + .collect::>()?; + + let selection_vectors: std::collections::HashMap> = exec + .selection_vectors + .iter() + .map(|entry| (entry.key().clone(), entry.value().clone())) + .collect(); + + let public_file_ids = exec.public_file_ids.as_ref().iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + + Ok(Self { + scan: scan_plan_wire, + transforms, + selection_vectors, + input_file_id_column: exec.input_file_id_column.clone(), + file_id_column: exec.file_id_column.clone(), + public_file_ids, + }) + } +} + +/// Converts kernel Expression -> DataFusion Expr -> protobuf bytes. +/// Does not support Transform expressions - use `serialize_transform` instead. +fn serialize_kernel_expression(expr: &Expression) -> Result, DataFusionError> { + let placeholder_type = KernelDataType::STRING; + let df_expr = to_datafusion_expr(expr, &placeholder_type)?; + let bytes = df_expr.to_bytes()?; + Ok(bytes.to_vec()) +} + +/// Converts protobuf bytes -> DataFusion Expr -> kernel Expression. +fn deserialize_kernel_expression(bytes: &[u8]) -> Result { + let df_expr = Expr::from_bytes(bytes)?; + to_delta_expression(&df_expr) + .map_err(|e| DataFusionError::Internal(format!("Failed to convert to kernel expr: {e}"))) +} + +/// Converts a kernel Transform expression to wire format. +fn serialize_transform(expr: &Expression) -> Result { + match expr { + Expression::StructPatch(struct_patch) => { + let input_path = struct_patch + .input_path + .as_ref() + .map(|p| p.iter().map(|s| s.to_string()).collect()); + + let field_patches = struct_patch + .field_patches + .iter() + .map(|(name, ft)| { + let insertions = ft + .insertions + .iter() + .map(|e| serialize_kernel_expression(e)) + .collect::, _>>()?; + Ok(( + name.clone(), + ExpressionFieldPatchWire { + keep_input: ft.keep_input, + insertions, + optional: ft.optional, + }, + )) + }) + .collect::>()?; + + let prepended_fields = struct_patch + .prepended_fields + .iter() + .map(|e| serialize_kernel_expression(e)) + .collect::, _>>()?; + + let appended_fields = struct_patch + .appended_fields + .iter() + .map(|e| serialize_kernel_expression(e)) + .collect::, _>>()?; + + Ok(StructPatchWire { + input_path, + field_patches, + prepended_fields, + appended_fields + }) + } + _ => Err(DataFusionError::Internal(format!( + "Expected Transform expression, got {:?}", + expr + ))), + } +} + +/// Converts wire format to a kernel Transform expression. +fn deserialize_transform(wire: StructPatchWire) -> Result { + let input_path = wire.input_path.map(ColumnName::new); + + let field_patches = wire + .field_patches + .into_iter() + .map(|(name, ft_wire)| { + let exprs = ft_wire + .insertions + .iter() + .map(|bytes| deserialize_kernel_expression(bytes).map(Arc::new)) + .collect::, _>>()?; + Ok(( + name, + ExpressionFieldPatch { + keep_input: ft_wire.keep_input, + insertions: exprs, + optional: ft_wire.optional, + }, + )) + }) + .collect::, DataFusionError>>()?; + + let prepended_fields = wire + .prepended_fields + .iter() + .map(|bytes| deserialize_kernel_expression(bytes).map(Arc::new)) + .collect::, _>>()?; + + let appended_fields = wire + .appended_fields + .iter() + .map(|bytes| deserialize_kernel_expression(bytes).map(Arc::new)) + .collect::, _>>()?; + + Ok(Expression::StructPatch(ExpressionStructPatch { + input_path, + field_patches, + prepended_fields, + appended_fields, + })) +} + +impl DeltaScanExecWire { + /// Reconstruct a [`DeltaScanExec`] from the wire format. + fn into_exec( + self, + inputs: &[Arc], + task: &TaskContext, + ) -> datafusion::common::Result> { + if inputs.len() != 1 { + return Err(DataFusionError::Internal(format!( + "DeltaScanExec expects exactly 1 input, got {}", + inputs.len() + ))); + } + let input = inputs[0].clone(); + + let mut execution_plan = input.clone(); + #[allow(clippy::collapsible_if)] + if let Some(ds_exec) = execution_plan.downcast_ref::() { + if let Some(file_conf) = ds_exec + .data_source() + .downcast_ref::() + { + let new_file_scan_config = FileScanConfigBuilder::from(file_conf.clone()) + .build(); + + // DataSourceExec::from_data_source(new_file_scan_config) + execution_plan = Arc::new( + ds_exec + .clone() + .with_data_source(Arc::new(new_file_scan_config)), + ); + } + } + + let mut delta_scan_config = DeltaScanConfig::new(); + if let Some(fic) = self.file_id_column { + delta_scan_config = delta_scan_config.with_file_column_name(fic); + } + + let filters = self.scan.contract.filters + .iter() + .map(|b| { + Expr::from_bytes_with_ctx(b, task).unwrap() + }) + .collect::>(); + + let row_index_column = self.scan.contract + .row_index_column.clone(); + + let contract = ProjectedScanContract::try_new( + self.scan.contract.table_schema.clone(), + self.scan.contract.provider_schema.clone(), + &delta_scan_config, + row_index_column.as_deref(), + self.scan.contract.projection.as_ref(), + &filters + )?; + + let skipping_predicate = self.scan.skipping_predicate + .map(|osp| { + osp.iter() + .map(|b| Expr::from_bytes_with_ctx(b, task).unwrap()) + .collect::>() + }); + + let scan_plan = KernelScanPlan::try_new_with_contract( + &self.scan.snapshot, + contract, + &filters, + &delta_scan_config, + skipping_predicate + )?; + + let transforms: HashMap> = self + .transforms + .into_iter() + .map(|(file_url, wire)| { + deserialize_transform(wire).map(|expr| (file_url, Arc::new(expr))) + }) + .collect::>()?; + + let selection_vectors: DashMap> = + self.selection_vectors.into_iter().collect(); + + let mut public_file_ids = PublicFileIdMap::default(); + if scan_plan.contract.retain_file_id { + for (k, v) in self.public_file_ids.iter() { + public_file_ids.insert(k.clone(), v.clone()); + } + } + + let exec = DeltaScanExec::new( + Arc::new(scan_plan), + execution_plan, + Arc::new(transforms), + Arc::new(selection_vectors), + Arc::new(public_file_ids), + Default::default(), + Default::default(), + ); + + Ok(Arc::new(exec)) + } +} + +/// Wire format for serializing [`DeltaScanMetaExec`]. +/// +/// Unlike [`DeltaScanExecWire`], there is no child plan: the per-partition +/// `(file_id, row_count)` inputs are serialized directly. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct DeltaScanMetaExecWire { + scan: ScanPlanWire, + input: Vec>, + transforms: std::collections::HashMap, + selection_vectors: std::collections::HashMap>, + file_id_column: Option, + public_file_ids: std::collections::HashMap, +} + +impl TryFrom<&DeltaScanMetaExec> for DeltaScanMetaExecWire { + type Error = DataFusionError; + + fn try_from(exec: &DeltaScanMetaExec) -> Result { + let scan_plan_wire = scan_plan_wire(&exec.scan_plan)?; + + let transforms: std::collections::HashMap = exec + .transforms + .iter() + .map(|(file_id, kernel_expr)| { + serialize_transform(kernel_expr.as_ref()).map(|wire| (file_id.clone(), wire)) + }) + .collect::>()?; + + let selection_vectors: std::collections::HashMap> = exec + .selection_vectors + .iter() + .map(|entry| (entry.key().clone(), entry.value().clone())) + .collect(); + + let public_file_ids = exec + .public_file_ids + .as_ref() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + + Ok(Self { + scan: scan_plan_wire, + input: exec.input.clone(), + transforms, + selection_vectors, + file_id_column: exec.file_id_field.as_ref().map(|f| f.name().clone()), + public_file_ids, + }) + } +} + +impl DeltaScanMetaExecWire { + /// Reconstruct a [`DeltaScanMetaExec`] from the wire format. + fn into_exec( + self, + inputs: &[Arc], + task: &TaskContext, + ) -> datafusion::common::Result> { + if !inputs.is_empty() { + return Err(DataFusionError::Internal(format!( + "DeltaScanMetaExec expects 0 inputs, got {}", + inputs.len() + ))); + } + + let mut delta_scan_config = DeltaScanConfig::new(); + if let Some(fic) = self.file_id_column { + delta_scan_config = delta_scan_config.with_file_column_name(fic); + } + + let filters = self + .scan + .contract + .filters + .iter() + .map(|b| Expr::from_bytes_with_ctx(b, task).unwrap()) + .collect::>(); + + let row_index_column = self.scan.contract.row_index_column.clone(); + + let contract = ProjectedScanContract::try_new( + self.scan.contract.table_schema.clone(), + self.scan.contract.provider_schema.clone(), + &delta_scan_config, + row_index_column.as_deref(), + self.scan.contract.projection.as_ref(), + &filters, + )?; + + let file_id_field = contract + .retain_file_id + .then(|| contract.file_id_field.clone()); + + let skipping_predicate = self.scan.skipping_predicate.map(|osp| { + osp.iter() + .map(|b| Expr::from_bytes_with_ctx(b, task).unwrap()) + .collect::>() + }); + + let scan_plan = KernelScanPlan::try_new_with_contract( + &self.scan.snapshot, + contract, + &filters, + &delta_scan_config, + skipping_predicate, + )?; + + let transforms: HashMap> = self + .transforms + .into_iter() + .map(|(file_id, wire)| { + deserialize_transform(wire).map(|expr| (file_id, Arc::new(expr))) + }) + .collect::>()?; + + let selection_vectors: DashMap> = + self.selection_vectors.into_iter().collect(); + + let mut public_file_ids = PublicFileIdMap::default(); + if scan_plan.contract.retain_file_id { + for (k, v) in self.public_file_ids.iter() { + public_file_ids.insert(k.clone(), v.clone()); + } + } + + let exec = DeltaScanMetaExec::new( + Arc::new(scan_plan), + self.input, + Arc::new(transforms), + Arc::new(selection_vectors), + Arc::new(public_file_ids), + file_id_field, + ExecutionPlanMetricsSet::new(), + ); + + Ok(Arc::new(exec)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use datafusion::physical_plan::joins::CrossJoinExec; + use datafusion::physical_plan::ExecutionPlan; + use datafusion::prelude::{col, lit}; + use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec}; + use datafusion_proto::protobuf; + + use crate::delta_datafusion::session::create_session; + use crate::delta_datafusion::table_provider::next::DeltaScan; + use crate::kernel::Snapshot; + use crate::test_utils::{TestResult, TestTables}; + + use super::*; + + async fn create_delta_scan_exec( + filters: &[Expr], + projection: Option<&Vec>, + ) -> TestResult> { + create_delta_scan_exec_from_table(TestTables::Simple, filters, projection).await + } + + async fn create_delta_scan_exec_from_table( + table: TestTables, + filters: &[Expr], + projection: Option<&Vec>, + ) -> TestResult> { + let log_store = table.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let provider = DeltaScan::builder().with_snapshot(snapshot).await?; + + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let plan = provider.scan(&state, projection, filters, None).await?; + Ok(plan) + } + + fn extract_delta_scan_exec(plan: &Arc) -> Option<&DeltaScanExec> { + plan.downcast_ref::() + } + + /// Build a metadata-only scan by projecting no data columns. + async fn create_delta_scan_meta_exec_from_table( + table: TestTables, + filters: &[Expr], + ) -> TestResult> { + let log_store = table.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let provider = DeltaScan::builder().with_snapshot(snapshot).await?; + + let session = Arc::new(create_session().into_inner()); + let state = session.state_ref().read().clone(); + + let empty_projection = vec![]; + let plan = provider + .scan(&state, Some(&empty_projection), filters, None) + .await?; + Ok(plan) + } + + fn extract_delta_scan_meta_exec(plan: &Arc) -> Option<&DeltaScanMetaExec> { + plan.downcast_ref::() + } + + #[tokio::test] + async fn test_codec_roundtrip_basic() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + assert!(!buf.is_empty(), "Encoded buffer should not be empty"); + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.scan_plan.contract.result_schema, + decoded_delta_scan.scan_plan.contract.result_schema, + "Result schemas should match" + ); + assert_eq!( + delta_scan.file_id_column, + decoded_delta_scan.file_id_column, + "File ID columns should match" + ); + assert_eq!( + delta_scan.input_file_id_column, + decoded_delta_scan.input_file_id_column, + "Retain file IDs should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_projection() -> TestResult { + let projection = vec![0usize]; + let plan = create_delta_scan_exec(&[], Some(&projection)).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.scan_plan.contract.result_schema.fields().len(), + decoded_delta_scan.scan_plan.contract.result_schema.fields().len(), + "Projected schema field count should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_filter() -> TestResult { + let filters = vec![col("id").gt(lit(5i64))]; + let plan = create_delta_scan_exec(&filters, None).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.scan_plan.contract.result_schema, + decoded_delta_scan.scan_plan.contract.result_schema, + "Result schemas should match with filter" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_wire_format_serialization() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let wire = DeltaScanExecWire::try_from(delta_scan)?; + + let json = serde_json::to_string(&wire)?; + assert!(!json.is_empty(), "JSON should not be empty"); + + let deserialized: DeltaScanExecWire = serde_json::from_str(&json)?; + + assert_eq!( + wire.file_id_column, deserialized.file_id_column, + "File ID column should roundtrip" + ); + assert_eq!( + wire.input_file_id_column, deserialized.input_file_id_column, + "Retain file IDs should roundtrip" + ); + assert_eq!( + wire.scan.contract.table_schema, deserialized.scan.contract.table_schema, + "Result schema should roundtrip" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_decode_wrong_input_count() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let result = codec.try_decode(&buf, &[], &task_ctx); + assert!(result.is_err(), "Should fail with 0 inputs"); + + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + let input = delta_scan.children()[0].clone(); + let result = codec.try_decode(&buf, &[input.clone(), input], &task_ctx); + assert!(result.is_err(), "Should fail with 2 inputs"); + + let meta_plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + let mut meta_buf = Vec::new(); + codec.try_encode(meta_plan.clone(), &mut meta_buf)?; + + let result = codec.try_decode(&meta_buf, &[meta_plan], &task_ctx); + assert!(result.is_err(), "DeltaScanMetaExec should fail with 1 input"); + + Ok(()) + } + + #[test] + fn test_kernel_expression_serialization_roundtrip() { + use delta_kernel::expressions::{ColumnName, Expression as KernelExpression, Scalar}; + + let column_expr = KernelExpression::Column(ColumnName::new(["test_column"])); + let serialized = serialize_kernel_expression(&column_expr).unwrap(); + let deserialized = deserialize_kernel_expression(&serialized).unwrap(); + assert_eq!(column_expr, deserialized, "Column expression should roundtrip"); + + let literal_expr = KernelExpression::Literal(Scalar::Integer(42)); + let serialized = serialize_kernel_expression(&literal_expr).unwrap(); + let deserialized = deserialize_kernel_expression(&serialized).unwrap(); + assert_eq!(literal_expr, deserialized, "Literal expression should roundtrip"); + + let string_literal = KernelExpression::Literal(Scalar::String("hello".to_string())); + let serialized = serialize_kernel_expression(&string_literal).unwrap(); + let deserialized = deserialize_kernel_expression(&serialized).unwrap(); + assert_eq!(string_literal, deserialized, "String literal should roundtrip"); + } + + #[tokio::test] + async fn test_wire_format_with_selection_vectors() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let wire = DeltaScanExecWire::try_from(delta_scan)?; + + assert!( + wire.selection_vectors.is_empty() || !wire.selection_vectors.is_empty(), + "Selection vectors should serialize (empty or not)" + ); + + let json = serde_json::to_string(&wire)?; + let deserialized: DeltaScanExecWire = serde_json::from_str(&json)?; + + assert_eq!( + wire.selection_vectors.len(), + deserialized.selection_vectors.len(), + "Selection vectors count should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_preserves_transforms_and_selection_vectors() -> TestResult { + let plan = create_delta_scan_exec(&[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.transforms.len(), + decoded_delta_scan.transforms.len(), + "Transforms count should match" + ); + + assert_eq!( + delta_scan.selection_vectors.len(), + decoded_delta_scan.selection_vectors.len(), + "Selection vectors count should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_deletion_vectors() -> TestResult { + let plan = + create_delta_scan_exec_from_table(TestTables::WithDvSmall, &[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + assert!( + !delta_scan.selection_vectors.is_empty(), + "Table with deletion vectors should have non-empty selection_vectors" + ); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.selection_vectors.len(), + decoded_delta_scan.selection_vectors.len(), + "Selection vectors count should match" + ); + + for entry in delta_scan.selection_vectors.iter() { + let key = entry.key(); + let original_vec = entry.value(); + let decoded_vec = decoded_delta_scan + .selection_vectors + .get(key) + .expect("Decoded should have same keys"); + assert_eq!( + original_vec.as_slice(), + decoded_vec.value().as_slice(), + "Selection vector values should match for key {key}" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_with_column_mapping() -> TestResult { + let plan = + create_delta_scan_exec_from_table(TestTables::WithColumnMapping, &[], None).await?; + let delta_scan = extract_delta_scan_exec(&plan).expect("Expected DeltaScanExec"); + + // Column mapping tables have transforms that inject partition values. + // This test verifies full roundtrip serialization of Transform expressions. + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let input = delta_scan.children()[0].clone(); + let decoded = codec.try_decode(&buf, &[input], &task_ctx)?; + + let decoded_delta_scan = + extract_delta_scan_exec(&decoded).expect("Expected DeltaScanExec after decode"); + + assert_eq!( + delta_scan.transforms.len(), + decoded_delta_scan.transforms.len(), + "Transforms count should match" + ); + + // Verify each transform was correctly serialized and deserialized + for key in delta_scan.transforms.keys() { + assert!( + decoded_delta_scan.transforms.contains_key(key), + "Decoded should have transform for key {key}" + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_basic() -> TestResult { + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + assert!(!buf.is_empty(), "Encoded buffer should not be empty"); + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.scan_plan.contract.result_schema, + decoded_meta.scan_plan.contract.result_schema, + "Result schemas should match" + ); + assert_eq!( + meta_scan.input, decoded_meta.input, + "Per-file (file_id, row_count) inputs should match" + ); + assert_eq!( + meta_scan.partition_statistics(None)?.num_rows, + decoded_meta.partition_statistics(None)?.num_rows, + "Exact row counts should match" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_with_filter() -> TestResult { + // Filter on the partition column so pushdown is Exact and the scan stays + // metadata-only (otherwise data must be read, yielding a DeltaScanExec). + let filters = vec![col("Company Very Short").eq(lit("BMS"))]; + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &filters).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.scan_plan.contract.result_schema, + decoded_meta.scan_plan.contract.result_schema, + "Result schemas should match with filter" + ); + assert_eq!( + meta_scan.partition_statistics(None)?.num_rows, + decoded_meta.partition_statistics(None)?.num_rows, + "Exact row counts should match with filter" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_with_deletion_vectors() -> TestResult { + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithDvSmall, &[]).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + assert!( + !meta_scan.selection_vectors.is_empty(), + "Table with deletion vectors should have non-empty selection_vectors" + ); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.selection_vectors.len(), + decoded_meta.selection_vectors.len(), + "Selection vectors count should match" + ); + for entry in meta_scan.selection_vectors.iter() { + let decoded_vec = decoded_meta + .selection_vectors + .get(entry.key()) + .expect("Decoded should have same keys"); + assert_eq!( + entry.value().as_slice(), + decoded_vec.value().as_slice(), + "Selection vector values should match for key {}", + entry.key() + ); + } + assert_eq!( + meta_scan.partition_statistics(None)?.num_rows, + decoded_meta.partition_statistics(None)?.num_rows, + "Exact row counts should match with deletion vectors" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_meta_codec_roundtrip_with_transforms() -> TestResult { + let plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + let meta_scan = + extract_delta_scan_meta_exec(&plan).expect("Expected DeltaScanMetaExec"); + + let codec = DeltaNextPhysicalCodec; + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + + let decoded = codec.try_decode(&buf, &[], &task_ctx)?; + let decoded_meta = + extract_delta_scan_meta_exec(&decoded).expect("Expected DeltaScanMetaExec after decode"); + + assert_eq!( + meta_scan.transforms.len(), + decoded_meta.transforms.len(), + "Transforms count should match" + ); + for key in meta_scan.transforms.keys() { + assert!( + decoded_meta.transforms.contains_key(key), + "Decoded should have transform for key {key}" + ); + } + + Ok(()) + } + + /// Test with a plan that has both DeltaScanMetaExec and DeltaScanExec + async fn join_roundtrip(meta_first: bool) -> TestResult { + let scan_plan = create_delta_scan_exec_from_table(TestTables::Simple, &[], None).await?; + let meta_plan = + create_delta_scan_meta_exec_from_table(TestTables::WithColumnMapping, &[]).await?; + + let join: Arc = if meta_first { + Arc::new(CrossJoinExec::new(meta_plan.clone(), scan_plan.clone())) + } else { + Arc::new(CrossJoinExec::new(scan_plan.clone(), meta_plan.clone())) + }; + + let codec = DeltaNextPhysicalCodec; + let proto = protobuf::PhysicalPlanNode::try_from_physical_plan(join.clone(), &codec)?; + + let session = create_session().into_inner(); + let task_ctx = session.task_ctx(); + let decoded = proto.try_into_physical_plan(&task_ctx, &codec)?; + + let decoded_children = decoded.children(); + assert_eq!(decoded_children.len(), 2, "Join should have two children"); + + let (meta_idx, scan_idx) = if meta_first { (0, 1) } else { (1, 0) }; + assert!( + decoded_children[meta_idx] + .downcast_ref::() + .is_some(), + "Child {meta_idx} should decode to DeltaScanMetaExec" + ); + assert!( + decoded_children[scan_idx] + .downcast_ref::() + .is_some(), + "Child {scan_idx} should decode to DeltaScanExec" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_codec_roundtrip_join_meta_first() -> TestResult { + join_roundtrip(true).await + } + + #[tokio::test] + async fn test_codec_roundtrip_join_meta_last() -> TestResult { + join_roundtrip(false).await + } +} diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs index 3bcbad79a7..d60b536aa5 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/exec.rs @@ -108,21 +108,21 @@ pub(crate) fn consume_dv_mask( /// 4. Result is cast to the projected scan contract's result schema #[derive(Clone, Debug)] pub struct DeltaScanExec { - scan_plan: Arc, + pub(crate) scan_plan: Arc, /// Execution plan yielding the raw data read from data files. input: Arc, /// Transforms to be applied to data eminating from individual files - transforms: Arc>, + pub(crate) transforms: Arc>, /// Selection vectors to be applied to data read from individual files - selection_vectors: Arc>>, + pub(crate) selection_vectors: Arc>>, /// Public file paths keyed by compact scan file id. - public_file_ids: Arc, + pub(crate) public_file_ids: Arc, /// Execution metrics metrics: ExecutionPlanMetricsSet, /// File id column name carried by the input batches for per file correlation. - input_file_id_column: String, + pub(crate) input_file_id_column: String, /// User-visible file-id column name when projected in the output. - file_id_column: Option, + pub(crate) file_id_column: Option, /// plan properties properties: Arc, /// Aggregated partition column statistics diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs index b67674b89a..5aced90e37 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/exec_meta.rs @@ -62,19 +62,19 @@ use crate::kernel::arrow::engine_ext::ExpressionEvaluatorExt; /// - No deletion vectors affect the results (or they are accounted for) #[derive(Clone, Debug)] pub(crate) struct DeltaScanMetaExec { - scan_plan: Arc, + pub(super) scan_plan: Arc, /// Execution plan yielding the raw data read from data files. - input: Vec>, + pub(super) input: Vec>, /// Transforms to be applied to data eminating from individual files - transforms: Arc>, + pub(super) transforms: Arc>, /// Deletion vectors for the table - selection_vectors: Arc>>, + pub(super) selection_vectors: Arc>>, /// Public file paths keyed by compact scan file id. - public_file_ids: Arc, + pub(super) public_file_ids: Arc, /// Execution metrics metrics: ExecutionPlanMetricsSet, /// Column name for the file id - file_id_field: Option, + pub(super) file_id_field: Option, /// plan properties properties: Arc, } diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/limit.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/limit.rs new file mode 100644 index 0000000000..9703b5fd1c --- /dev/null +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/limit.rs @@ -0,0 +1,232 @@ +//! File-level row-count accumulator for `LIMIT` pushdown. +//! +//! This module owns the correctness logic that decides — given a stream of +//! [`ScanFileContext`]s and a row cap — which files to keep and which to drop. +//! It is intentionally pure (no async, no I/O) so it can be unit-tested in +//! isolation from `replay_files` and the kernel scan-metadata stream. +//! +//! Rules: +//! - Files with a deletion vector (`has_dv == true`) go into a set-aside pool, +//! because a DV may delete an arbitrary number of rows and `num_records` +//! would overstate the file's contribution to the limit. +//! - Files missing `num_records` stats go into the same set-aside pool for the +//! same reason. +//! - Files with exact stats and no DV accumulate; once `rows_collected >= +//! limit` the outer loop should break. +//! - At finalize time, the set-aside pool is drained into the kept list iff +//! the accumulator is still short of the limit; otherwise its entries are +//! counted as pruned. + +use datafusion::common::stats::Precision; +use tracing::info; +use super::replay::ScanFileContext; + +/// Per-scan accumulator for file-level limit pruning. See module docs. +#[derive(Debug)] +pub(crate) struct LimitPruneState { + limit: usize, + rows_collected: usize, + files_with_exact_num_records: Vec, + files_with_unknown_num_records: Vec, +} + +impl LimitPruneState { + pub(crate) fn new(limit: usize) -> Self { + Self { + limit, + rows_collected: 0, + files_with_exact_num_records: Vec::new(), + files_with_unknown_num_records: Vec::new(), + } + } + + /// Try to accept one file. Returns `true` when the row cap has been reached + /// and the outer scan-metadata loop should break. + pub(crate) fn accept(&mut self, file: ScanFileContext) -> bool { + if file.has_deletion_vector { + self.files_with_unknown_num_records.push(file); + return false; + } + info!("ACCEPT {} = {:?}", file.file_url.as_str(), file.stats); + match file.stats.num_rows { + Precision::Exact(n) => { + self.rows_collected = self.rows_collected.saturating_add(n); + self.files_with_exact_num_records.push(file); + self.rows_collected >= self.limit + } + _ => { + self.files_with_unknown_num_records.push(file); + false + } + } + } + + /// Consume the state, returning `(kept_files, count_files_pruned_by_limit)`. + /// + /// The set-aside pool is drained into `kept_files` only when the accumulator + /// is still short of the row cap; otherwise its entries are counted as pruned. + pub(crate) fn finalize(mut self) -> (Vec, usize) { + if self.rows_collected < self.limit { + self.files_with_exact_num_records.extend(self.files_with_unknown_num_records.drain(..)); + (self.files_with_exact_num_records, 0) + } else { + let pruned = self.files_with_unknown_num_records.len(); + (self.files_with_exact_num_records, pruned) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use datafusion::common::Statistics; + use url::Url; + + use crate::delta_datafusion::table_provider::next::scan::replay::ScanFileContext; + + fn stub_file(url: &str, num_rows: Precision, has_deletion_vector: bool) -> ScanFileContext { + let stats = Statistics { + num_rows, + total_byte_size: Precision::Absent, + column_statistics: Vec::new(), + }; + ScanFileContext { + file_url: Url::parse(url).unwrap(), + size: 0, + transform: None, + stats, + partitions: None, + has_deletion_vector, + } + } + + #[test] + fn accept_exact_stats_accumulates_until_cap_reached() { + let mut state = LimitPruneState::new(10); + + let stop = state.accept(stub_file("file:///a", Precision::Exact(4), false)); + assert!( + !stop, + "accumulator at 4 rows should not signal stop with cap=10" + ); + + let stop = state.accept(stub_file("file:///b", Precision::Exact(7), false)); + assert!( + stop, + "accumulator at 11 rows should signal stop with cap=10" + ); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 2); + assert_eq!(pruned, 0); + } + + #[test] + fn dv_files_go_to_unknown_pool() { + let mut state = LimitPruneState::new(5); + + let stop = state.accept(stub_file("file:///dv", Precision::Exact(1000), true)); + assert!( + !stop, + "DV files must not contribute to row count regardless of stats" + ); + + let (files, pruned) = state.finalize(); + assert_eq!( + files.len(), + 1, + "DV file should be drained from unknown pool when limit unmet" + ); + assert_eq!(pruned, 0); + } + + #[test] + fn absent_stats_go_to_unknown_pool() { + let mut state = LimitPruneState::new(5); + + let stop = state.accept(stub_file("file:///stats-less", Precision::Absent, false)); + assert!(!stop); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 1); + assert_eq!(pruned, 0); + } + + #[test] + fn unknown_pool_is_dropped_when_cap_already_satisfied() { + let mut state = LimitPruneState::new(3); + + // DV file goes into unknown pool (does not contribute to rows_collected). + let _ = state.accept(stub_file("file:///dv", Precision::Exact(10), true)); + // Exact-stats file saturates the cap. + let stop = state.accept(stub_file("file:///exact", Precision::Exact(5), false)); + assert!(stop, "exact 5 rows should saturate cap=3"); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 1, "only the exact-stats file should be kept"); + assert_eq!(pruned, 1, "the unknown-pool DV file should count as pruned"); + } + + #[test] + fn unknown_pool_is_drained_when_cap_unmet() { + let mut state = LimitPruneState::new(100); + + let _ = state.accept(stub_file("file:///exact", Precision::Exact(5), false)); + let _ = state.accept(stub_file("file:///dv", Precision::Exact(10), true)); + let _ = state.accept(stub_file("file:///no-stats", Precision::Absent, false)); + + let (files, pruned) = state.finalize(); + assert_eq!( + files.len(), + 3, + "all files should be returned when cap is not reached" + ); + assert_eq!(pruned, 0); + } + + #[test] + fn inexact_stats_go_to_unknown_pool() { + let mut state = LimitPruneState::new(5); + + let stop = state.accept(stub_file("file:///inexact", Precision::Inexact(3), false)); + assert!(!stop); + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 1); + assert_eq!(pruned, 0); + } + + #[test] + fn saturating_add_handles_pathological_num_records() { + let mut state = LimitPruneState::new(10); + let stop = state.accept(stub_file( + "file:///big", + Precision::Exact(usize::MAX), + false, + )); + assert!( + stop, + "even saturated accumulator must signal stop once cap is reached" + ); + + let (files, _) = state.finalize(); + assert_eq!(files.len(), 1); + } + + #[test] + fn limit_greater_than_total_keeps_everything() { + let mut state = LimitPruneState::new(1_000_000); + + for i in 0..3 { + let url = format!("file:///f{i}"); + let stop = state.accept(stub_file(&url, Precision::Exact(10), false)); + assert!(!stop, "small files should never trip a generous cap"); + } + + let (files, pruned) = state.finalize(); + assert_eq!(files.len(), 3); + assert_eq!(pruned, 0); + } +} + diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs index 2ff3ad39b8..42f415a1bd 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/mod.rs @@ -46,6 +46,7 @@ use datafusion::{ }, prelude::Expr, }; +use datafusion::logical_expr::TableProviderFilterPushDown; use datafusion_datasource::{ PartitionedFile, TableSchema, compute_all_files_statistics, file_groups::FileGroup, file_scan_config::FileScanConfigBuilder, source::DataSourceExec, @@ -63,8 +64,9 @@ use object_store::{ObjectMeta, path::Path}; use tracing::debug; use url::Url; +pub use self::codec::DeltaNextPhysicalCodec; pub use self::exec::DeltaScanExec; -use self::exec_meta::DeltaScanMetaExec; +pub(crate) use self::exec_meta::DeltaScanMetaExec; use self::expr_adapter::{DeltaPhysicalExprAdapterFactory, relax_schema_nested_nullability}; pub(crate) use self::plan::{KernelScanPlan, ProjectedScanContract, supports_filters_pushdown}; use self::replay::{ScanFileContext, ScanFileStream}; @@ -79,12 +81,16 @@ use crate::{ }, kernel::LogicalFileView, }; +use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields; +use crate::delta_datafusion::table_provider::next::scan::limit::LimitPruneState; mod exec; mod exec_meta; mod expr_adapter; mod plan; mod replay; +mod codec; +pub mod limit; type ScanMetadataStream = Pin> + Send>>; type PublicFileIdMap = HashMap; @@ -114,7 +120,24 @@ pub(super) async fn execution_plan( ))); } - let replayed = replay_files(engine, &scan_plan, config.clone(), stream, file_selection).await?; + // File-level limit pruning is safe iff: + // - the caller asked for a row cap, + // - no explicit FileSelection is in play (explicit pins win), and + // - every filter is marked Exact (DataFusion's own contract already forbids + // passing `Some(N)` here when an Inexact filter would survive above the + // scan; this check is a defensive duplicate so the invariant is local). + let prune_by_limit: Option = limit.filter(|_| { + file_selection.is_none() && { + let filter_refs: Vec<&Expr> = scan_plan.filters.iter().collect(); + let pushdown = + supports_filters_pushdown(&filter_refs, scan_plan.table_configuration(), config); + pushdown + .iter() + .all(|p| matches!(p, TableProviderFilterPushDown::Exact)) + } + }); + + let replayed = replay_files(engine, &scan_plan, config.clone(), stream, file_selection, prune_by_limit).await?; let file_id_field = scan_plan.contract.file_id_field.clone(); if scan_plan.is_metadata_only() && !scan_plan.contract.retain_row_index { @@ -306,6 +329,7 @@ async fn replay_files( scan_config: DeltaScanConfig, stream: ScanMetadataStream, file_selection: Option<&ResolvedFileSelection>, + prune_by_limit: Option, ) -> Result { let mut stream = ScanFileStream::new( engine, @@ -314,10 +338,34 @@ async fn replay_files( file_selection.map(|selection| &selection.active_file_ids), stream, ); - let mut files = Vec::new(); - while let Some(file) = stream.try_next().await? { - files.extend(file); - } + // let mut files = Vec::new(); + // while let Some(file) = stream.try_next().await? { + // files.extend(file); + // } + + let mut files = if let Some(cap) = prune_by_limit { + let mut state = LimitPruneState::new(cap); + 'outer: while let Some(batch_files) = stream.try_next().await? { + for file in batch_files { + if state.accept(file) { + break 'outer; + } + } + } + // The second tuple element (unknown-pool drops) is a subset of + // `count_files_pruned_by_limit`, which we compute below from the + // stream's total visit count minus the final file set. That formula + // also captures files in the partially-consumed final batch that we + // dropped on early-break without going through the unknown pool. + let (kept, _) = state.finalize(); + kept + } else { + let mut files = Vec::new(); + while let Some(batch_files) = stream.try_next().await? { + files.extend(batch_files); + } + files + }; let mut public_file_ids = PublicFileIdMap::default(); if scan_plan.contract.retain_file_id { @@ -350,6 +398,10 @@ async fn replay_files( MetricBuilder::new(&metrics) .global_counter("count_files_scanned") .add(stream.metrics.num_scanned); + let pruned_by_limit = stream.metrics.num_scanned.saturating_sub(files.len()); + MetricBuilder::new(&metrics) + .global_counter("count_files_pruned_by_limit") + .add(pruned_by_limit); Ok(ReplayedScanFiles { files, @@ -781,9 +833,12 @@ fn finalize_transformed_batch( // The batch adapter targets a nullability-relaxed schema (see // `expr_adapter`). Restamp to the strict logical schema; this // validates actual data nulls rather than declared nullability. + let output_schema = rewrite_schema_with_nullable_fields( + scan_plan.contract.result_schema.clone() + ); crate::kernel::cast_record_batch( &adapted, - scan_plan.contract.result_schema.clone(), + output_schema, false, false, ) @@ -1045,6 +1100,7 @@ mod tests { transform: None, stats: Statistics::new_unknown(&Schema::empty()), partitions: None, + has_deletion_vector: false, } } diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs index 7de7e81426..c7333e2b73 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/plan.rs @@ -37,6 +37,7 @@ use crate::delta_datafusion::DeltaScanConfig; use crate::delta_datafusion::engine::{ to_datafusion_expr, to_delta_expression, to_delta_predicate, }; +use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields; use crate::delta_datafusion::table_provider::next::FILE_ID_COLUMN_DEFAULT; use crate::kernel::{Scan, Snapshot}; @@ -46,6 +47,10 @@ use crate::kernel::{Scan, Snapshot}; /// `TableProvider::scan` request so planning and execution do not rederive them independently. #[derive(Clone, Debug)] pub(crate) struct ProjectedScanContract { + pub(crate) table_schema: SchemaRef, + pub(crate) provider_schema: SchemaRef, + pub(crate) projection: Option>, + pub(crate) filters: Vec, /// Logical data schema after removing temporary predicate only columns and internal metadata. pub(crate) result_schema: SchemaRef, /// Logical schema produced by the kernel scan before dropping filter-only columns. @@ -201,6 +206,13 @@ impl ProjectedScanContract { }; Ok(Self { + table_schema, + provider_schema, + projection: projection.cloned(), + filters: filters + .iter() + .map(|x| x.clone()) + .collect::>(), result_schema, scan_schema, output_schema, @@ -221,6 +233,8 @@ impl ProjectedScanContract { /// be pushed to kernel file skipping vs. Parquet readers. #[derive(Clone, Debug)] pub(crate) struct KernelScanPlan { + pub(crate) filters: Vec, + pub(crate) skipping_predicate: Option>, /// Wrapped kernel scan to produce logical file stream pub(crate) scan: Arc, /// Query scoped contract shared across planning and execution. @@ -277,7 +291,7 @@ impl KernelScanPlan { // if some dedicated file skipping predicate is supplied, // we do not push the scan filters into the kernel scan. - let scan_predicate = if let Some(sp) = skipping_predicate { + let scan_predicate = if let Some(sp) = skipping_predicate.clone() { let (scan_predicate, _) = process_filters(&sp, table_config, config)?; if scan_predicate.is_none() { debug!( @@ -308,13 +322,21 @@ impl KernelScanPlan { } else { Arc::new(scan_builder.build()?) }; - let parquet_read_schema = config.physical_arrow_schema( + // let parquet_read_schema = config.physical_arrow_schema( + // scan.snapshot().table_configuration(), + // &scan.physical_schema().as_ref().try_into_arrow()?, + // )?; + let mut parquet_read_schema = config.physical_arrow_schema( scan.snapshot().table_configuration(), &scan.physical_schema().as_ref().try_into_arrow()?, - )?; + )?; + parquet_read_schema = rewrite_schema_with_nullable_fields(parquet_read_schema); + let parquet_predicate_schema = build_parquet_predicate_schema(&parquet_read_schema, &contract.file_id_field); Ok(Self { + filters: filters.iter().map(|x| x.clone()).collect::>(), + skipping_predicate: skipping_predicate.clone(), scan, contract, parquet_read_schema, @@ -1326,6 +1348,10 @@ mod tests { fn test_retained_row_index_field_ignores_incomplete_internal_contract() { let schema = Arc::new(Schema::empty()); let contract = ProjectedScanContract { + table_schema: Arc::new(Schema::empty()), + provider_schema: Arc::new(Schema::empty()), + projection: None, + filters: vec![], result_schema: Arc::clone(&schema), scan_schema: Arc::clone(&schema), output_schema: schema, diff --git a/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs b/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs index 4ee4c17171..b1809c5658 100644 --- a/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs +++ b/crates/core/src/delta_datafusion/table_provider/next/scan/replay.rs @@ -372,6 +372,9 @@ pub(crate) struct ScanFileContext { pub stats: Statistics, /// Partition values for the file. pub partitions: Option, + /// Whether this file carries a deletion vector. Cached at construction time so consumers + /// can branch on DV presence without waiting on the asynchronous DV-load task. + pub has_deletion_vector: bool, } impl ScanFileContext { @@ -383,6 +386,7 @@ impl ScanFileContext { transform: inner.transform, stats, partitions, + has_deletion_vector: inner.dv_info.has_vector() } } } @@ -500,10 +504,16 @@ fn visit_scan_file(ctx: &mut ScanContext, scan_file: ScanFile) { mod tests { use std::collections::HashMap; + use arrow_schema::Schema; + use datafusion::common::Statistics; + use delta_kernel::actions::deletion_vector::{ + DeletionVectorDescriptor, DeletionVectorStorageType, + }; + use delta_kernel::scan::state::{DvInfo, ScanFile}; use url::Url; - use super::{ScanContext, visit_scan_file}; + use super::{ScanContext, visit_scan_file, ScanFileContext, ScanFileContextInner}; fn scan_file(path: impl Into) -> ScanFile { ScanFile { @@ -536,4 +546,48 @@ mod tests { "file:///tmp/delta/part-000.parquet" ); } + + #[test] + fn test_scan_file_context_propagates_has_dv_from_inner() { + let url = Url::parse("file:///tmp/f.parquet").unwrap(); + let descriptor = DeletionVectorDescriptor { + storage_type: DeletionVectorStorageType::Inline, + path_or_inline_dv: String::new(), + offset: None, + size_in_bytes: 0, + cardinality: 0, + }; + let inner_with_dv = ScanFileContextInner { + file_url: url.clone(), + size: 0, + transform: None, + num_records: None, + dv_info: DvInfo::from(descriptor), + }; + let inner_without_dv = ScanFileContextInner { + file_url: url, + size: 0, + transform: None, + num_records: None, + dv_info: DvInfo::default(), + }; + + let empty_schema = Schema::empty(); + let ctx_with = + ScanFileContext::new(inner_with_dv, Statistics::new_unknown(&empty_schema), None); + let ctx_without = ScanFileContext::new( + inner_without_dv, + Statistics::new_unknown(&empty_schema), + None, + ); + + assert!( + ctx_with.has_deletion_vector, + "ScanFileContext built from inner with DV should expose has_dv=true" + ); + assert!( + !ctx_without.has_deletion_vector, + "ScanFileContext built from inner without DV should expose has_dv=false" + ); + } } diff --git a/crates/core/src/delta_datafusion/table_provider_old.rs b/crates/core/src/delta_datafusion/table_provider_old.rs new file mode 100644 index 0000000000..339dbebc51 --- /dev/null +++ b/crates/core/src/delta_datafusion/table_provider_old.rs @@ -0,0 +1,649 @@ +use std::borrow::Cow; +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; +use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef}; +use chrono::{DateTime, TimeZone, Utc}; +use datafusion::catalog::{ScanArgs, ScanResult, Session, TableProvider}; +use datafusion::common::{Column, Result, ScalarValue, Statistics, ToDFSchema}; +use datafusion::common::pruning::PruningStatistics; +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion::config::TableParquetOptions; +use datafusion::datasource::physical_plan::ParquetSource; +use datafusion::datasource::TableType; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::logical_expr::{BinaryExpr, Expr, LogicalPlan, Operator, TableProviderFilterPushDown}; +use datafusion::logical_expr::utils::{conjunction, split_conjunction}; +use datafusion::physical_expr_common::metrics::{ExecutionPlanMetricsSet, MetricBuilder}; +use datafusion::physical_optimizer::pruning::PruningPredicate; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_datasource::{FileExtensions, PartitionedFile, TableSchema}; +use datafusion_datasource::file_groups::FileGroup; +use datafusion_datasource::file_scan_config::FileScanConfigBuilder; +use datafusion_datasource::source::DataSourceExec; +use object_store::ObjectMeta; +use url::Url; +use crate::delta_datafusion::{get_null_of_arrow_type, to_correct_scalar_value, DataFusionMixins, DeltaScanConfig, DeltaScanConfigBuilder}; +use crate::delta_datafusion::table_provider::{simplify_expr, DeltaScan}; +use crate::{DeltaResult, DeltaTable, DeltaTableConfig, DeltaTableError}; +use crate::delta_datafusion::file_id::{file_id_data_type, wrap_file_id_value}; +use crate::kernel::{Add, EagerSnapshot}; +use crate::kernel::transaction::PROTOCOL; +use crate::logstore::LogStoreRef; +use crate::table::state::DeltaTableState; + + +pub(crate) fn get_pushdown_filters( + filter: &[&Expr], + partition_cols: &[String], +) -> Vec { + filter + .iter() + .cloned() + .map(|expr| { + let applicable = expr_is_exact_predicate_for_cols(partition_cols, expr); + if !expr.column_refs().is_empty() && applicable { + TableProviderFilterPushDown::Exact + } else { + TableProviderFilterPushDown::Inexact + } + }) + .collect() +} + +// inspired from datafusion::listing::helpers, but adapted to only stats based pruning +fn expr_is_exact_predicate_for_cols(partition_cols: &[String], expr: &Expr) -> bool { + let mut is_applicable = true; + expr.apply(|expr| match expr { + Expr::Column(Column { name, .. }) => { + is_applicable &= partition_cols.contains(name); + + // TODO: decide if we should constrain this to Utf8 columns (including views, dicts etc) + + if is_applicable { + Ok(TreeNodeRecursion::Jump) + } else { + Ok(TreeNodeRecursion::Stop) + } + } + Expr::BinaryExpr(BinaryExpr { op, .. }) => { + is_applicable &= matches!( + op, + Operator::And + | Operator::Or + | Operator::NotEq + | Operator::Eq + | Operator::Gt + | Operator::GtEq + | Operator::Lt + | Operator::LtEq + ); + if is_applicable { + Ok(TreeNodeRecursion::Continue) + } else { + Ok(TreeNodeRecursion::Stop) + } + } + Expr::Literal(_, _) + | Expr::Not(_) + | Expr::IsNotNull(_) + | Expr::IsNull(_) + | Expr::Between(_) + | Expr::InList(_) => Ok(TreeNodeRecursion::Continue), + _ => { + is_applicable = false; + Ok(TreeNodeRecursion::Stop) + } + }) + .unwrap(); + is_applicable +} + +fn partitioned_file_from_action( + action: &Add, + partition_columns: &[String], + schema: &Schema, +) -> PartitionedFile { + let partition_values = partition_columns + .iter() + .map(|part| { + action + .partition_values + .get(part) + .map(|val| { + schema + .field_with_name(part) + .map(|field| match val { + Some(value) => to_correct_scalar_value( + &serde_json::Value::String(value.to_string()), + field.data_type(), + ) + .unwrap_or(Some(ScalarValue::Null)) + .unwrap_or(ScalarValue::Null), + None => get_null_of_arrow_type(field.data_type()) + .unwrap_or(ScalarValue::Null), + }) + .unwrap_or(ScalarValue::Null) + }) + .unwrap_or(ScalarValue::Null) + }) + .collect::>(); + + let ts_secs = action.modification_time / 1000; + let ts_ns = (action.modification_time % 1000) * 1_000_000; + let last_modified = Utc.from_utc_datetime( + &DateTime::from_timestamp(ts_secs, ts_ns as u32) + .unwrap() + .naive_utc(), + ); + PartitionedFile { + object_meta: ObjectMeta { + last_modified, + ..action.try_into().unwrap() + }, + partition_values, + range: None, + statistics: None, + ordering: None, + extensions: FileExtensions::new(), + metadata_size_hint: None, + table_reference: None, + } +} + +/// The logical schema for a Deltatable is different from the protocol level schema since partition +/// columns must appear at the end of the schema. This is to align with how partition are handled +/// at the physical level +pub(crate) fn df_logical_schema( + snapshot: &EagerSnapshot, + file_column_name: &Option, + schema: Option, +) -> DeltaResult { + let input_schema = match schema { + Some(schema) => schema, + None => snapshot.input_schema(), + }; + let table_partition_cols = snapshot.metadata().partition_columns(); + + let mut fields: Vec> = input_schema + .fields() + .iter() + .filter(|f| !table_partition_cols.contains(f.name())) + .cloned() + .collect(); + + for partition_col in table_partition_cols.iter() { + fields.push(Arc::new( + input_schema + .field_with_name(partition_col)? + .to_owned(), + )); + } + + if let Some(file_column_name) = file_column_name { + fields.push(Arc::new(Field::new(file_column_name, DataType::Utf8, true))); + } + + Ok(Arc::new(Schema::new(fields))) +} + +pub(crate) struct DeltaScanBuilder<'a> { + snapshot: &'a EagerSnapshot, + log_store: LogStoreRef, + filter: Option, + session: &'a dyn Session, + projection: Option<&'a Vec>, + limit: Option, + files: Option<&'a [Add]>, + config: Option, +} + +impl<'a> DeltaScanBuilder<'a> { + pub fn new( + snapshot: &'a EagerSnapshot, + log_store: LogStoreRef, + session: &'a dyn Session, + ) -> Self { + DeltaScanBuilder { + snapshot, + log_store, + filter: None, + session, + projection: None, + limit: None, + files: None, + config: None, + } + } + + pub fn with_filter(mut self, filter: Option) -> Self { + self.filter = filter; + self + } + + pub fn with_files(mut self, files: &'a [Add]) -> Self { + self.files = Some(files); + self + } + + pub fn with_projection(mut self, projection: Option<&'a Vec>) -> Self { + self.projection = projection; + self + } + + pub fn with_limit(mut self, limit: Option) -> Self { + self.limit = limit; + self + } + + pub fn with_scan_config(mut self, config: DeltaScanConfig) -> Self { + self.config = Some(config); + self + } + + pub async fn build(self) -> DeltaResult { + PROTOCOL.can_read_from(self.snapshot)?; + let config = match self.config { + Some(config) => config, + None => DeltaScanConfigBuilder::new().build(self.snapshot)?, + }; + + let schema = match config.schema.clone() { + Some(value) => value, + None => self.snapshot.read_schema(), + }; + + let logical_schema = df_logical_schema( + self.snapshot, + &config.file_column_name, + Some(schema.clone()), + )?; + + let logical_schema = if let Some(used_columns) = self.projection { + let mut fields = Vec::with_capacity(used_columns.len()); + for idx in used_columns { + fields.push(logical_schema.field(*idx).to_owned()); + } + // partition filters with Exact pushdown were removed from projection by DF optimizer, + // we need to add them back for the predicate pruning to work + if let Some(expr) = &self.filter { + // Avoid non-determinism of HashSet (affects the expressions in pushed-down filters) + let column_refs = expr.column_refs().into_iter().collect::>(); + for c in column_refs { + let idx = logical_schema.index_of(c.name.as_str())?; + if !used_columns.contains(&idx) { + fields.push(logical_schema.field(idx).to_owned()); + } + } + } + Arc::new(Schema::new(fields)) + } else { + logical_schema + }; + + let df_schema = Arc::new(logical_schema.clone().to_dfschema()?); + + let logical_filter = self + .filter + .clone() + .map(|expr| simplify_expr(self.session, df_schema.clone(), expr)) + .transpose()?; + // only inexact filters should be pushed down to the data source, doing otherwise + // will make stats inexact and disable datafusion optimizations like AggregateStatistics + let pushdown_filter = self + .filter + .and_then(|expr| { + let predicates = split_conjunction(&expr); + let pushdown_filters = + get_pushdown_filters(&predicates, self.snapshot.metadata().partition_columns()); + + let filtered_predicates = predicates + .into_iter() + .zip(pushdown_filters.into_iter()) + .filter_map(|(filter, pushdown)| { + if pushdown == TableProviderFilterPushDown::Inexact { + Some(filter.clone()) + } else { + None + } + }); + conjunction(filtered_predicates) + }) + .map(|expr| simplify_expr(self.session, df_schema.clone(), expr)) + .transpose()?; + + // Perform Pruning of files to scan + let (files, files_scanned, files_pruned, _) = match self.files { + Some(files) => { + let files = files.to_owned(); + let files_scanned = files.len(); + (files, files_scanned, 0, None) + } + None => { + // early return in case we have no push down filters or limit + if logical_filter.is_none() && self.limit.is_none() { + // let files = self + // .snapshot + // .file_views(&self.log_store, None) + // .map_ok(|f| f.to_add()) + // .try_collect::>() + // .await?; + let files = self + .snapshot + .log_data() + .iter() + .map(|f| f.to_add()) + .collect::>(); + + let files_scanned = files.len(); + (files, files_scanned, 0, None) + } else { + let num_containers = self.snapshot.num_containers(); + + let files_to_prune = if let Some(predicate) = &logical_filter { + let pruning_predicate = + PruningPredicate::try_new(predicate.clone(), logical_schema.clone())?; + pruning_predicate.prune(self.snapshot)? + } else { + vec![true; num_containers] + }; + + // needed to enforce limit and deal with missing statistics + // rust port of https://github.com/delta-io/delta/pull/1495 + let mut pruned_without_stats = Vec::new(); + let mut rows_collected = 0; + let mut files = Vec::with_capacity(num_containers); + + // let file_actions: Vec<_> = self + // .snapshot + // .file_views(&self.log_store, None) + // .map_ok(|f| f.to_add()) + // .try_collect::>() + // .await?; + + use rand::seq::SliceRandom; + + let mut indices = (0..num_containers).collect::>(); + if self.limit.is_some() && std::env::var("DELTA_RS_SHUFFLE_FILES").is_ok() { + let mut rng = rand::rng(); + indices.shuffle(&mut rng); + } + + let log_data_handler = self.snapshot.log_data(); + for i in indices { + let file_view = log_data_handler.get(i).unwrap(); + let keep = files_to_prune[i]; + + // prune file based on predicate pushdown + let action = file_view.add_action_no_stats(); + let num_records = file_view.num_records(); + if keep { + // prune file based on limit pushdown + if let Some(limit) = self.limit { + if let Some(num_records) = num_records { + if rows_collected < limit as i64 { + rows_collected += num_records as i64; + files.push(action.to_owned()); + } else { + break; + } + } else { + // some files are missing stats; skipping but storing them + // in a list in case we can't reach the target limit + pruned_without_stats.push(action.to_owned()); + } + } else { + files.push(action.to_owned()); + } + } + } + + if let Some(limit) = self.limit + && rows_collected < limit as i64 + { + files.extend(pruned_without_stats); + } + + let files_scanned = files.len(); + let files_pruned = num_containers - files_scanned; + (files, files_scanned, files_pruned, Some(files_to_prune)) + } + } + }; + + // TODO we group files together by their partition values. If the table is partitioned + // and partitions are somewhat evenly distributed, probably not the worst choice ... + // However we may want to do some additional balancing in case we are far off from the above. + let mut file_groups: HashMap, Vec> = HashMap::new(); + + let table_partition_cols = &self.snapshot.metadata().partition_columns(); + + for action in files.iter() { + let mut part = partitioned_file_from_action(action, table_partition_cols, &schema); + + if config.file_column_name.is_some() { + let partition_value = if config.wrap_partition_values { + wrap_file_id_value(action.path.clone()) + } else { + ScalarValue::Utf8(Some(action.path.clone())) + }; + part.partition_values.push(partition_value); + } + + file_groups + .entry(part.partition_values.clone()) + .or_default() + .push(part); + } + + let file_schema = Arc::new(Schema::new( + schema + .fields() + .iter() + .filter(|f| !table_partition_cols.contains(f.name())) + .cloned() + .collect::>(), + )); + + let mut table_partition_cols = table_partition_cols + .iter() + .map(|name| schema.field_with_name(name).map(|f| f.to_owned())) + .collect::, ArrowError>>()?; + + if let Some(file_column_name) = &config.file_column_name { + let field_name_datatype = if config.wrap_partition_values { + file_id_data_type() + } else { + DataType::Utf8 + }; + table_partition_cols.push(Field::new( + file_column_name.clone(), + field_name_datatype, + false, + )); + } + + let parquet_options = TableParquetOptions { + global: self.session.config().options().execution.parquet.clone(), + ..Default::default() + }; + + let partition_fields: Vec> = + table_partition_cols.into_iter().map(Arc::new).collect(); + let table_schema = TableSchema::new(file_schema, partition_fields); + + let mut file_source = + ParquetSource::new(table_schema).with_table_parquet_options(parquet_options); + + // Sometimes (i.e Merge) we want to prune files that don't make the + // filter and read the entire contents for files that do match the + // filter + if let Some(predicate) = pushdown_filter + && config.enable_parquet_pushdown + { + file_source = file_source.with_predicate(predicate); + }; + + let file_scan_config = + FileScanConfigBuilder::new(self.log_store.object_store_url(), Arc::new(file_source)) + .with_file_groups( + // If all files were filtered out, we still need to emit at least one partition to + // pass datafusion sanity checks. + // + // See https://github.com/apache/datafusion/issues/11322 + if file_groups.is_empty() { + vec![FileGroup::from(vec![])] + } else { + file_groups.into_values().map(FileGroup::from).collect() + }, + ) + .with_projection_indices(self.projection.cloned())? + .with_limit(self.limit) + .build(); + + let metrics = ExecutionPlanMetricsSet::new(); + MetricBuilder::new(&metrics) + .global_counter("files_scanned") + .add(files_scanned); + MetricBuilder::new(&metrics) + .global_counter("files_pruned") + .add(files_pruned); + + Ok(DeltaScan { + table_url: self.log_store.root_url().clone(), + config, + parquet_scan: DataSourceExec::from_data_source(file_scan_config), + logical_schema, + metrics, + }) + } +} + + + +impl DeltaTable { + pub fn table_provider_old(&self) -> DeltaTableOldProvider { + self.clone().into() + } +} + +// each delta table must register a specific object store, since paths are internally +// handled relative to the table root. +pub(crate) fn register_store(store: LogStoreRef, env: &RuntimeEnv) { + let object_store_url = store.object_store_url(); + let url: &Url = object_store_url.as_ref(); + env.register_object_store(url, store.object_store(None)); +} + +#[derive(Debug, Clone)] +pub struct DeltaTableOldProvider { + /// The state of the table as of the most recent loaded Delta log entry. + pub state: Option, + /// the load options used during load + pub config: DeltaTableConfig, + /// log store + pub(crate) log_store: LogStoreRef, + /// Optional schema override for scanning + pub(crate) schema: Option, +} + +impl DeltaTableOldProvider { + pub fn snapshot(&self) -> DeltaResult<&DeltaTableState> { + self.state.as_ref().ok_or(DeltaTableError::NotInitialized) + } + pub fn log_store(&self) -> LogStoreRef { + self.log_store.clone() + } + pub fn with_schema(mut self, schema: SchemaRef) -> Self { + self.schema = Some(schema); + self + } +} + +impl From for DeltaTableOldProvider { + fn from(value: DeltaTable) -> Self { + Self { + state: value.state.clone(), + config: value.config.clone(), + log_store: value.log_store.clone(), + schema: None, + } + } +} + +#[async_trait::async_trait] +impl TableProvider for DeltaTableOldProvider { + fn schema(&self) -> Arc { + match &self.schema { + Some(s) => df_logical_schema( + self.snapshot().unwrap().snapshot(), + &None, + Some(s.clone()), + ) + .unwrap_or_else(|_| s.clone()), + None => self.snapshot().unwrap().snapshot().read_schema(), + } + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + fn get_table_definition(&self) -> Option<&str> { + None + } + + fn get_logical_plan(&self) -> Option> { + None + } + + async fn scan( + &self, + _session: &dyn Session, + _projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result> { + unimplemented!("scan is not available for this table provider; use scan_with_args") + } + + async fn scan_with_args<'a>(&self, state: &dyn Session, args: ScanArgs<'a>) -> Result { + register_store(self.log_store(), state.runtime_env().as_ref()); + let filters = args.filters().unwrap_or(&[]); + let filter_expr = conjunction(filters.iter().cloned()); + + let config = DeltaScanConfigBuilder { + include_file_column: false, + file_column_name: None, + wrap_partition_values: None, + enable_parquet_pushdown: true, + schema: self.schema.clone(), + }; + + let config = config + .build(self.snapshot()?.snapshot())?; + + let projection = args.projection().map(|p| p.to_vec()); + let scan = DeltaScanBuilder::new(self.snapshot()?.snapshot(), self.log_store(), state) + .with_projection(projection.as_ref()) + .with_limit(args.limit()) + .with_filter(filter_expr) + .with_scan_config(config) + .build() + .await?; + + Ok(ScanResult::new(Arc::new(scan))) + } + + fn supports_filters_pushdown( + &self, + filter: &[&Expr], + ) -> Result> { + let partition_cols = self.snapshot()?.metadata().partition_columns(); + Ok(get_pushdown_filters(filter, partition_cols)) + } + + fn statistics(&self) -> Option { + // HSTACK: removed in upstream as well + None + } +} diff --git a/crates/core/src/delta_datafusion/udtf.rs b/crates/core/src/delta_datafusion/udtf.rs new file mode 100644 index 0000000000..2baa3fc65f --- /dev/null +++ b/crates/core/src/delta_datafusion/udtf.rs @@ -0,0 +1,144 @@ +use async_trait::async_trait; +use delta_kernel::Engine; +use datafusion::catalog::{TableFunctionImpl, TableProvider}; +use datafusion::common::{internal_datafusion_err, DataFusionError, Result, ScalarValue}; +use datafusion::logical_expr::Expr; +use datafusion::prelude::SessionContext; +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; +use tokio::runtime::Runtime; +use url::Url; +use crate::{builder_from_valid_url, open_table_with_storage_options}; + +pub fn register_delta_table_udtf(ctx: &SessionContext, name: Option<&str>, settings: Option<&HashMap>) { + register_delta_table_udtf_with_engine(ctx, name, settings, None); +} + +pub fn register_delta_table_udtf_with_engine( + ctx: &SessionContext, + name: Option<&str>, + settings: Option<&HashMap>, + engine: Option>, +) { + let prefix = name + .or_else(|| Some("delta_table")).unwrap(); + + ctx.register_udtf( + prefix, + Arc::new(DeltaTableUdtf { + flavor: DeltaTableUdtfFlavor::Old, + settings: settings.cloned(), + engine: None, + }), + ); + ctx.register_udtf( + format!("{prefix}_next").as_str(), + Arc::new(DeltaTableUdtf { + flavor: DeltaTableUdtfFlavor::Next, + settings: settings.cloned(), + engine, + }), + ); +} + +#[derive(Debug, Clone)] +pub enum DeltaTableUdtfFlavor { + Old, + Next +} + +pub struct DeltaTableUdtf { + flavor: DeltaTableUdtfFlavor, + settings: Option>, + engine: Option>, +} + +impl std::fmt::Debug for DeltaTableUdtf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeltaTableUdtf") + .field("flavor", &self.flavor) + .field("settings", &self.settings) + .field("has_engine", &self.engine.is_some()) + .finish() + } +} + +#[async_trait] +impl TableFunctionImpl for DeltaTableUdtf { + fn call(&self, args: &[Expr]) -> Result> { + if args.len() != 1 { + return Err(DataFusionError::Execution( + "Delta table function expects table_uri as argument".to_string(), + )); + } + + let mut args_string = args + .iter() + .map(|arg| match arg.clone() { + Expr::Literal(ScalarValue::Utf8(Some(path)), _) + | Expr::Literal(ScalarValue::LargeUtf8(Some(path)), _) + | Expr::Literal(ScalarValue::Utf8View(Some(path)), _) => Ok(path), + _ => Err(DataFusionError::Execution(format!( + "Unexpected argument type: {:?}", + arg + ))), + }) + .collect::>>()?; + + let path = args_string + .pop_front() + .expect("DeltaTableUdtf missing path"); + assert_eq!(args_string.len() % 2, 0, "DeltaTableUdtf: Can't build hashmap out of odd-sized args"); + + let settings = if let Some(global_settings) = &self.settings { + global_settings.clone() + } else { + HashMap::new() + }; + + let flavor = self.flavor.clone(); + let engine = self.engine.clone(); + let table = std::thread::spawn(move || { + let rt = Runtime::new().unwrap(); + let table_uri = Url::parse(&path) + .expect(&format!("Invalid table uri: {}", path)); + rt.block_on(async { + match flavor { + DeltaTableUdtfFlavor::Old => { + let delta_table = open_table_with_storage_options(table_uri, settings) + .await + .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) + .unwrap(); + Arc::new(delta_table.table_provider_old()) as Arc + } + DeltaTableUdtfFlavor::Next => { + let mut builder = builder_from_valid_url(table_uri) + .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) + .unwrap() + .with_storage_options(settings) + .without_files() + .with_checkpoint_stats_json_fallback(true); + if let Some(engine) = engine { + builder = builder.with_engine(engine); + } + let delta_table = builder + .load() + .await + .map_err(|e| { internal_datafusion_err!("DeltaTableUdtf could not open table at {}: {}",&path,e.to_string()) }) + .unwrap(); + let provider = delta_table + .table_provider() + .build() + .await + .unwrap(); + Arc::new(provider) as Arc + } + } + }) + }) + .join() + .map_err(|_e| internal_datafusion_err!("DeltaTableFunc error opening table"))?; + + Ok(table.clone()) + } +} diff --git a/crates/core/src/hacks.rs b/crates/core/src/hacks.rs new file mode 100644 index 0000000000..2b6ff32a5c --- /dev/null +++ b/crates/core/src/hacks.rs @@ -0,0 +1,18 @@ +use std::sync::Arc; +use crate::kernel::{EagerSnapshot, Snapshot}; +use delta_kernel::snapshot::Snapshot as KernelSnapshot; + +pub fn new_delta_eager_snapshot(snapshot: Snapshot) -> EagerSnapshot { + EagerSnapshot { + snapshot: Arc::new(snapshot), + } +} + +pub fn new_delta_snapshot(input: &Snapshot, kernel_snapshot: KernelSnapshot) -> Snapshot { + Snapshot { + inner: Arc::new(kernel_snapshot), + config: input.config.clone(), + materialized_files: input.materialized_files.clone(), + load_metrics: input.load_metrics.clone(), + } +} \ No newline at end of file diff --git a/crates/core/src/kernel/schema/cast/mod.rs b/crates/core/src/kernel/schema/cast/mod.rs index 36a10cc267..325f188c4f 100644 --- a/crates/core/src/kernel/schema/cast/mod.rs +++ b/crates/core/src/kernel/schema/cast/mod.rs @@ -35,7 +35,7 @@ fn should_cast_nanos_timestamps_to_micros() -> bool { CAST_NANOS_TS_TO_MICROS.load(Ordering::Acquire) } -fn cast_struct( +pub fn cast_struct( struct_array: &StructArray, fields: &Fields, cast_options: &CastOptions, @@ -68,7 +68,7 @@ fn cast_struct( ) } -fn cast_list( +pub fn cast_list( array: &GenericListArray, field: &FieldRef, cast_options: &CastOptions, @@ -83,7 +83,7 @@ fn cast_list( ) } -fn cast_map( +pub fn cast_map( array: &MapArray, entries_field: &FieldRef, sorted: bool, @@ -107,7 +107,7 @@ fn cast_map( } } -fn cast_field( +pub fn cast_field( col: &ArrayRef, field: &FieldRef, cast_options: &CastOptions, @@ -220,15 +220,25 @@ pub fn cast_record_batch( ..Default::default() }; - // Can be simplified with StructArray::try_new_with_length in arrow 55.1 - let col_arrays = batch.columns().to_owned(); - let s = if col_arrays.is_empty() { - StructArray::new_empty_fields(batch.num_rows(), None) - } else { - StructArray::new(batch.schema().as_ref().to_owned().fields, col_arrays, None) - }; + // // Can be simplified with StructArray::try_new_with_length in arrow 55.1 + // let col_arrays = batch.columns().to_owned(); + // let s = if col_arrays.is_empty() { + // StructArray::new_empty_fields(batch.num_rows(), None) + // } else { + // StructArray::new(batch.schema().as_ref().to_owned().fields, col_arrays, None) + // }; + // + // let struct_array = cast_struct(&s, target_schema.fields(), &cast_options, add_missing)?; + + // @HStack Fix schema mapping for record batches with an empty schema + let mut struct_array = StructArray::try_new_with_length( + batch.schema().as_ref().to_owned().fields, + batch.columns().to_owned(), + None, + batch.num_rows(), + )?; + struct_array = cast_struct(&struct_array, target_schema.fields(), &cast_options, add_missing)?; - let struct_array = cast_struct(&s, target_schema.fields(), &cast_options, add_missing)?; Ok(RecordBatch::try_new_with_options( target_schema, diff --git a/crates/core/src/kernel/snapshot/iterators.rs b/crates/core/src/kernel/snapshot/iterators.rs index 91023c0f47..a81a84a7b6 100644 --- a/crates/core/src/kernel/snapshot/iterators.rs +++ b/crates/core/src/kernel/snapshot/iterators.rs @@ -27,10 +27,12 @@ use crate::kernel::scalars::ScalarExt; use crate::kernel::{Add, DeletionVectorDescriptor, Remove}; use crate::{DeltaResult, DeltaTableError}; -pub(crate) use self::scan_row::{ScanRowOutStream, scan_row_in_eval}; +pub(crate) use self::scan_row::ScanRowOutStream; +pub(crate) use self::scan_row_cached::CachedScanRowEvaluator; mod scan_row; mod tombstones; +mod scan_row_cached; const FIELD_NAME_PATH: &str = "path"; const FIELD_NAME_SIZE: &str = "size"; @@ -340,6 +342,24 @@ impl LogicalFileView { pub fn add_action(&self) -> Add { self.to_add() } + + /// Converts this file view into an Add action for log operations. + /// FIXME: Double json > stats conversion TOO EXPENSIVE + pub(crate) fn add_action_no_stats(&self) -> Add { + Add { + path: self.path().to_string(), + partition_values: self.partition_values_map(), + size: self.size(), + modification_time: self.modification_time(), + data_change: true, + stats: None, + tags: None, + deletion_vector: None, + base_row_id: None, + default_row_commit_version: None, + clustering_provider: None, + } + } /// Converts this file view into a Remove action for log operations. pub fn remove_action(&self, data_change: bool) -> Remove { diff --git a/crates/core/src/kernel/snapshot/iterators/scan_row.rs b/crates/core/src/kernel/snapshot/iterators/scan_row.rs index 6f01f64857..7b561b8e4f 100644 --- a/crates/core/src/kernel/snapshot/iterators/scan_row.rs +++ b/crates/core/src/kernel/snapshot/iterators/scan_row.rs @@ -9,6 +9,7 @@ use arrow_schema::{Field, Schema}; use delta_kernel::engine::arrow_conversion::TryIntoArrow as _; use delta_kernel::engine::arrow_conversion::TryIntoKernel; use delta_kernel::engine::arrow_data::ArrowEngineData; +use delta_kernel::engine::arrow_expression::evaluate_expression::to_json; use delta_kernel::engine::parse_json; use delta_kernel::expressions::Scalar; use delta_kernel::scan::scan_row_schema; @@ -142,7 +143,7 @@ pub(crate) fn parse_stats_column_with_schema( ) } -fn parse_stats_column_impl( +pub(crate) fn parse_stats_column_impl( batch: &RecordBatch, stats_schema: KernelSchemaRef, stats_arrow_fields: Option<&Fields>, @@ -157,20 +158,55 @@ fn parse_stats_column_impl( stats_arrow_schema = stats_schema.as_ref().try_into_arrow()?; stats_arrow_schema.fields() }; + let generated_stats = if stats_materialization.preserves_raw_stats() { + batch + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .filter(|parsed| { + project_struct_array(parsed, stats_arrow_fields, StructProjectionMode::Strict) + .is_ok() + }) + .map(|parsed| to_json(parsed)) + .transpose()? + } else { + None + }; let mut columns = Vec::with_capacity(batch.num_columns() + 2); let mut fields = Vec::with_capacity(batch.num_columns() + 2); + let mut emitted_stats = false; for (field, column) in batch.schema().fields().iter().zip(batch.columns()) { if field.name() == FIELD_STATS_PARSED { continue; } - if field.name() == FIELD_STATS && !stats_materialization.preserves_raw_stats() { + if field.name() == FIELD_STATS { + if !stats_materialization.preserves_raw_stats() { + continue; + } + emitted_stats = true; + fields.push(field.clone()); + columns.push(match generated_stats.as_ref() { + Some(generated_stats) => { + generated_stats_json_with_fallback(generated_stats, Some(column))? + } + None => column.clone(), + }); continue; } fields.push(field.clone()); columns.push(column.clone()); } + if !emitted_stats && let Some(generated_stats) = generated_stats { + let generated_stats = generated_stats_json_with_fallback(&generated_stats, None)?; + fields.push(Arc::new(Field::new( + FIELD_STATS, + generated_stats.data_type().clone(), + true, + ))); + columns.push(generated_stats); + } + if let Some(stats_array) = materialize_stats_array( batch, stats_schema, @@ -206,6 +242,40 @@ fn parse_stats_column_impl( )?) } +fn generated_stats_json_with_fallback( + generated_stats: &ArrayRef, + fallback: Option<&ArrayRef>, +) -> DeltaResult { + let generated_stats = generated_stats + .as_any() + .downcast_ref::() + .ok_or_else(|| DeltaTableError::SchemaMismatch { + msg: "generated stats column is not a string".to_string(), + })?; + let fallback = fallback + .map(|fallback| { + fallback + .as_string_opt::() + .ok_or_else(|| DeltaTableError::SchemaMismatch { + msg: "stats column is not a string".to_string(), + }) + }) + .transpose()?; + let mut output = StringBuilder::new(); + + for row in 0..generated_stats.len() { + if generated_stats.is_valid(row) && generated_stats.value(row) != "{}" { + output.append_value(generated_stats.value(row)); + } else if let Some(fallback) = fallback.filter(|fallback| fallback.is_valid(row)) { + output.append_value(fallback.value(row)); + } else { + output.append_null(); + } + } + + Ok(Arc::new(output.finish())) +} + fn materialize_stats_array( batch: &RecordBatch, stats_schema: KernelSchemaRef, @@ -224,7 +294,11 @@ fn materialize_stats_array( .ok_or_else(|| DeltaTableError::SchemaMismatch { msg: "stats_parsed column is not a struct".to_string(), })?; - match project_struct_array(parsed, stats_arrow_fields) { + match project_struct_array( + parsed, + stats_arrow_fields, + StructProjectionMode::AddMissingNullable, + ) { Ok(projected) => return Ok(Some(Arc::new(projected))), Err(err) => { debug!( @@ -244,6 +318,25 @@ fn materialize_stats_array( } } +// fn parse_raw_stats_array( +// batch: &RecordBatch, +// stats_schema: KernelSchemaRef, +// ) -> DeltaResult> { +// let Some((stats_idx, _)) = batch.schema_ref().column_with_name(FIELD_STATS) else { +// return Err(DeltaTableError::SchemaMismatch { +// msg: "stats column not found".to_string(), +// }); +// }; +// +// let stats_batch = batch.project(&[stats_idx])?; +// let stats_data = Box::new(ArrowEngineData::new(stats_batch)); +// +// let parsed = parse_json(stats_data, stats_schema)?; +// let parsed: RecordBatch = ArrowEngineData::try_from_engine_data(parsed)?.into(); +// +// Ok(Arc::new(parsed.into())) +// } + fn parse_raw_stats_array( batch: &RecordBatch, stats_schema: KernelSchemaRef, @@ -253,8 +346,28 @@ fn parse_raw_stats_array( msg: "stats column not found".to_string(), }); }; + parse_raw_stats_array_at(batch, stats_schema, stats_idx) +} - let stats_batch = batch.project(&[stats_idx])?; +pub(crate) fn parse_raw_stats_array_at( + batch: &RecordBatch, + stats_schema: KernelSchemaRef, + stats_idx: usize, +) -> DeltaResult> { + let stats_column = batch.columns().get(stats_idx).ok_or_else(|| { + DeltaTableError::SchemaMismatch { + msg: "stats column not found".to_string(), + } + })?; + let stats_field = Arc::new(Field::new( + FIELD_STATS, + stats_column.data_type().clone(), + true, + )); + let stats_batch = RecordBatch::try_new( + Arc::new(Schema::new([stats_field])), + vec![stats_column.clone()], + )?; let stats_data = Box::new(ArrowEngineData::new(stats_batch)); let parsed = parse_json(stats_data, stats_schema)?; @@ -263,9 +376,16 @@ fn parse_raw_stats_array( Ok(Arc::new(parsed.into())) } +#[derive(Clone, Copy)] +enum StructProjectionMode { + Strict, + AddMissingNullable, +} + fn project_struct_array( array: &StructArray, requested_fields: &Fields, + mode: StructProjectionMode, ) -> DeltaResult { let ArrowDataType::Struct(existing_fields) = array.data_type() else { return Err(DeltaTableError::SchemaMismatch { @@ -275,10 +395,17 @@ fn project_struct_array( let mut columns = Vec::with_capacity(requested_fields.len()); for requested_field in requested_fields { - let existing_idx = existing_fields + let Some(existing_idx) = existing_fields .iter() .position(|field| field.name() == requested_field.name()) - .ok_or_else(|| DeltaTableError::SchemaMismatch { + else { + if matches!(mode, StructProjectionMode::AddMissingNullable) + && requested_field.is_nullable() + { + columns.push(new_null_array(requested_field.data_type(), array.len())); + continue; + } + return Err(DeltaTableError::SchemaMismatch { msg: format!( "stats_parsed field {} not found; existing fields: [{}]", requested_field.name(), @@ -288,7 +415,8 @@ fn project_struct_array( .collect::>() .join(", ") ), - })?; + }); + }; let existing_column = array.column(existing_idx); let column: ArrayRef = match (requested_field.data_type(), existing_column.data_type()) { (ArrowDataType::Struct(requested_child_fields), ArrowDataType::Struct(_)) => { @@ -301,7 +429,7 @@ fn project_struct_array( requested_field.name() ), })?; - Arc::new(project_struct_array(child, requested_child_fields)?) + Arc::new(project_struct_array(child, requested_child_fields, mode)?) } (requested, existing) if requested == existing => existing_column.clone(), (requested, existing) => { @@ -676,7 +804,7 @@ fn collect_map(val: &StructArray) -> Option RecordBatch { + pub(crate) fn scan_row_batch_with_stats(raw_stats: &str) -> RecordBatch { let schema: ArrowSchema = scan_row_schema().as_ref().try_into_arrow().unwrap(); let mut columns: Vec = schema .fields() @@ -712,20 +841,20 @@ mod tests { RecordBatch::try_new(Arc::new(schema), columns).unwrap() } - fn raw_stats_string(batch: RecordBatch, row: usize) -> Option { + pub(crate) fn raw_stats_string(batch: RecordBatch, row: usize) -> Option { batch .column_by_name("stats") .and_then(|col| col.as_string_opt::()) .and_then(|col| col.is_valid(row).then(|| col.value(row).to_string())) } - fn num_records_stats_schema() -> SchemaRef { + pub(crate) fn num_records_stats_schema() -> SchemaRef { Arc::new( StructType::try_new([StructField::nullable("numRecords", DataType::LONG)]).unwrap(), ) } - fn value_stats_schema() -> SchemaRef { + pub(crate) fn value_stats_schema() -> SchemaRef { Arc::new( StructType::try_new([ StructField::nullable("numRecords", DataType::LONG), @@ -748,7 +877,7 @@ mod tests { ) } - fn append_stats_parsed(batch: &RecordBatch, stats_parsed: StructArray) -> RecordBatch { + pub(crate) fn append_stats_parsed(batch: &RecordBatch, stats_parsed: StructArray) -> RecordBatch { let mut fields = batch.schema().fields().to_vec(); let mut columns = batch.columns().to_vec(); fields.push(Arc::new(Field::new( @@ -778,7 +907,7 @@ mod tests { RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() } - fn num_records_stats_parsed(num_records: i64) -> StructArray { + pub(crate) fn num_records_stats_parsed(num_records: i64) -> StructArray { StructArray::from(vec![( Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), Arc::new(Int64Array::from(vec![Some(num_records)])) as ArrayRef, @@ -792,7 +921,7 @@ mod tests { )]) } - fn value_stats_parsed() -> StructArray { + pub(crate) fn value_stats_parsed() -> StructArray { let min_values = StructArray::from(vec![( Arc::new(Field::new("value", ArrowDataType::Int32, true)), Arc::new(Int32Array::from(vec![Some(1)])) as ArrayRef, @@ -838,7 +967,7 @@ mod tests { ]) } - fn stats_parsed_field_names(batch: &RecordBatch) -> Vec { + pub(crate) fn stats_parsed_field_names(batch: &RecordBatch) -> Vec { let schema = batch.schema(); let field = schema.field_with_name("stats_parsed").unwrap(); let ArrowDataType::Struct(fields) = field.data_type() else { @@ -1099,9 +1228,9 @@ mod tests { } #[test] - fn parse_stats_column_impl_falls_back_to_raw_stats_when_existing_parsed_is_partial() + fn parse_stats_column_impl_adds_missing_nullable_fields_without_parsing_raw_stats() -> DeltaResult<()> { - let raw_stats = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#; + let raw_stats = "not valid JSON"; let batch = scan_row_batch_with_stats(raw_stats); let batch = append_stats_parsed(&batch, num_records_stats_parsed(11)); @@ -1118,11 +1247,110 @@ mod tests { stats_parsed_field_names(&projected), vec!["numRecords", "minValues", "maxValues", "nullCount"] ); - assert_eq!(raw_stats_string(projected, 0), Some(raw_stats.to_string())); + assert_eq!( + raw_stats_string(projected.clone(), 0), + Some(raw_stats.to_string()) + ); + + let stats = projected + .column_by_name(FIELD_STATS_PARSED) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + stats + .column_by_name("numRecords") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 11 + ); + for (field_name, value_type) in [ + ("minValues", ArrowDataType::Int32), + ("maxValues", ArrowDataType::Int32), + ("nullCount", ArrowDataType::Int64), + ] { + let field = stats.column_by_name(field_name).unwrap(); + assert!(field.is_null(0)); + let field = field.as_any().downcast_ref::().unwrap(); + let value = field.column_by_name("value").unwrap(); + assert_eq!(value.data_type(), &value_type); + assert!(value.is_null(0)); + } Ok(()) } + #[test] + fn project_struct_array_adds_nested_nullable_fields_and_preserves_null_buffers() + -> DeltaResult<()> { + let keep: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)])); + let nested = StructArray::new( + Fields::from(vec![Field::new("keep", ArrowDataType::Int32, true)]), + vec![keep.clone()], + Some(NullBuffer::from_iter([true, false, true])), + ); + let array = StructArray::new( + Fields::from(vec![Field::new("nested", nested.data_type().clone(), true)]), + vec![Arc::new(nested)], + Some(NullBuffer::from_iter([false, true, true])), + ); + let requested_nested = Fields::from(vec![ + Field::new("added", ArrowDataType::Int64, true), + Field::new("keep", ArrowDataType::Int32, true), + ]); + let requested = Fields::from(vec![Field::new( + "nested", + ArrowDataType::Struct(requested_nested), + true, + )]); + + let projected = + project_struct_array(&array, &requested, StructProjectionMode::AddMissingNullable)?; + + assert_eq!( + projected.nulls().unwrap(), + &NullBuffer::from_iter([false, true, true]) + ); + let nested = projected + .column_by_name("nested") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + nested.nulls().unwrap(), + &NullBuffer::from_iter([true, false, true]) + ); + assert_eq!( + nested.column_by_name("added").unwrap().data_type(), + &ArrowDataType::Int64 + ); + assert_eq!(nested.column_by_name("added").unwrap().null_count(), 3); + assert!(Arc::ptr_eq(nested.column_by_name("keep").unwrap(), &keep)); + + Ok(()) + } + + #[test] + fn project_struct_array_rejects_missing_non_nullable_field() { + let array = num_records_stats_parsed(11); + let requested = Fields::from(vec![Field::new("required", ArrowDataType::Int64, false)]); + + let err = + project_struct_array(&array, &requested, StructProjectionMode::AddMissingNullable) + .expect_err("missing non-nullable field should fail projection"); + + assert!( + err.to_string() + .contains("stats_parsed field required not found"), + "unexpected error: {err}" + ); + } + #[test] fn parse_stats_column_impl_accepts_raw_stats_without_min_max_values() -> DeltaResult<()> { let raw_stats = r#"{"numRecords":11}"#; @@ -1146,6 +1374,8 @@ mod tests { Ok(()) } + // HSTACK - changed behavior + #[ignore = "Changed behavior, superseded by parse_stats_column_impl_regenerates_raw_stats_when_raw_policy_preserves"] #[test] fn parse_stats_column_impl_keeps_raw_stats_when_raw_policy_preserves() -> DeltaResult<()> { let raw_stats = r#"{"maxValues":{"value":9},"numRecords":11,"nullCount":{"value":0},"minValues":{"value":1}}"#; diff --git a/crates/core/src/kernel/snapshot/iterators/scan_row_cached.rs b/crates/core/src/kernel/snapshot/iterators/scan_row_cached.rs new file mode 100644 index 0000000000..2760b27ecd --- /dev/null +++ b/crates/core/src/kernel/snapshot/iterators/scan_row_cached.rs @@ -0,0 +1,1219 @@ +use std::sync::Arc; + +use arrow_array::{Array, ArrayRef, RecordBatch, StructArray}; +use arrow_cast::CastOptions; +use arrow_schema::{DataType as ArrowDataType, Field, Fields, Schema}; +use delta_kernel::engine::arrow_conversion::TryIntoArrow; +use delta_kernel::expressions::Scalar; +use delta_kernel::scan::scan_row_schema; +use delta_kernel::schema::{DataType, SchemaRef as KernelSchemaRef, StructField, StructType}; +use delta_kernel::{EvaluationHandler, Expression, ExpressionEvaluator}; + +use crate::kernel::arrow::engine_ext::ExpressionEvaluatorExt; +use crate::kernel::schema::cast::cast_struct; +use crate::kernel::snapshot::stats_projection::{FIELD_PARTITION_VALUES_PARSED, FIELD_STATS}; +use crate::kernel::{ARROW_HANDLER, FIELD_STATS_PARSED}; +use crate::{DeltaResult, DeltaTableError}; + +struct CachedStructProjection { + path: String, + input_fields: Fields, + output_fields: Fields, +} + +impl CachedStructProjection { + fn try_new(input_fields: Fields, output_fields: Fields, path: &str) -> DeltaResult { + Self::validate_fields(&input_fields, &output_fields, path)?; + Ok(Self { + path: path.to_string(), + input_fields, + output_fields, + }) + } + + fn validate_fields( + input_fields: &Fields, + output_fields: &Fields, + path: &str, + ) -> DeltaResult<()> { + for output_field in output_fields { + let field_path = format!("{path}.{}", output_field.name()); + let Some(input_field) = input_fields + .iter() + .find(|field| field.name() == output_field.name()) + else { + if output_field.is_nullable() { + continue; + } + return Err(DeltaTableError::SchemaMismatch { + msg: format!("cached {field_path} is not available"), + }); + }; + + match (input_field.data_type(), output_field.data_type()) { + (ArrowDataType::Struct(input_child), ArrowDataType::Struct(output_child)) => { + Self::validate_fields(input_child, output_child, &field_path)?; + } + (input_type, output_type) + if input_type == output_type + || Self::is_legal_primitive_widening(input_type, output_type) => {} + (input_type, output_type) => { + return Err(DeltaTableError::SchemaMismatch { + msg: format!( + "cached {field_path} has type {input_type:?} but requested {output_type:?}" + ), + }); + } + } + } + Ok(()) + } + + fn is_legal_primitive_widening( + input_type: &ArrowDataType, + output_type: &ArrowDataType, + ) -> bool { + matches!( + (input_type, output_type), + ( + ArrowDataType::Int8, + ArrowDataType::Int16 | ArrowDataType::Int32 | ArrowDataType::Int64 + ) | ( + ArrowDataType::Int16, + ArrowDataType::Int32 | ArrowDataType::Int64 + ) | (ArrowDataType::Int32, ArrowDataType::Int64) + | (ArrowDataType::Float32, ArrowDataType::Float64) + ) + } + + /// Reshape a seed `stats_parsed` array to the requested schema with a targeted arrow cast: + /// reorder / drop / add-nullable / widen / nested, preserving null buffers. + fn cast_to_requested(&self, array: &StructArray) -> DeltaResult { + let expected_type = ArrowDataType::Struct(self.input_fields.clone()); + if array.data_type() != &expected_type { + return Err(DeltaTableError::SchemaMismatch { + msg: format!( + "cached {} type {:?} does not match declared type {:?}", + self.path, + array.data_type(), + expected_type + ), + }); + } + + Ok(cast_struct( + array, + &self.output_fields, + &CastOptions { + safe: true, + ..Default::default() + }, + true, + )?) + } +} + +/// How the requested `stats_parsed` column is produced from a materialized-file seed batch. +enum StatsParsedSource { + /// The seed's typed `stats_parsed` column is compatible with the request: reshape it with a + /// targeted arrow cast ([`CachedStructProjection`]). A kernel expression cannot do this + /// because its evaluator never widens primitives, so this one step stays imperative. + Reshape(CachedStructProjection), + /// The seed's typed stats are missing or incompatible: parse the raw `stats` JSON instead. + /// This path is expressed entirely as a kernel `parse_json` in the assembly expression. + ParseJson, +} + +impl StatsParsedSource { + /// The assembly sub-expression that yields `stats_parsed` for this source. The reshape path + /// references the column that [`CachedScanRowEvaluator::reshape_cached_stats`] casts into + /// place; the JSON path parses raw `stats` directly, fully inside the kernel expression. + fn stats_expression(&self, requested_schema: &KernelSchemaRef) -> Expression { + match self { + StatsParsedSource::Reshape(_) => Expression::column([FIELD_STATS_PARSED]), + StatsParsedSource::ParseJson => { + Expression::parse_json(Expression::column([FIELD_STATS]), requested_schema.clone()) + } + } + } +} + +/// The chosen `stats_parsed` production strategy plus the exact schema it emits. +struct StatsPlan { + source: StatsParsedSource, + schema: KernelSchemaRef, +} + +/// One fixed transformation for every batch in a materialized-file seed. +/// +/// Assembly is a single kernel [`ExpressionEvaluator`]: base scan-row columns pass through by +/// name, a missing raw `stats` column becomes a typed NULL, `stats_parsed` is produced per the +/// [`StatsPlan`] (a by-name column reference on the reshape path, or `parse_json` on the raw-JSON +/// path), and `partitionValues_parsed` passes through by name. Columns resolve by name, so the +/// cached batch may carry extra columns or a different field order. +/// +/// The only step the expression cannot perform is the reshape cast (kernel expressions never +/// widen primitives); [`Self::reshape_cached_stats`] applies it as a per-batch pre-pass, and only +/// on the reshape path. `partition_fields`, when set, guards that the seed carries a +/// `partitionValues_parsed` column with exactly the expected fields, surfacing a typed +/// [`DeltaTableError::SchemaMismatch`] rather than a kernel column-not-found error. +pub(crate) struct CachedScanRowEvaluator { + evaluator: Arc, + stats_plan: Option, + partition_fields: Option, +} + +impl CachedScanRowEvaluator { + pub(crate) fn try_new( + available_stats_schema: Option<&KernelSchemaRef>, + raw_stats_available: bool, + effective_replay_stats_schema: Option, + partition_schema: Option<&KernelSchemaRef>, + ) -> DeltaResult { + let base_schema = scan_row_schema(); + + let available_stats_fields = available_stats_schema + .map(|available| { + let available_arrow: Schema = available.as_ref().try_into_arrow()?; + Ok::<_, DeltaTableError>(available_arrow.fields().clone()) + }) + .transpose()?; + let partition_fields = partition_schema + .map(|schema| { + let partition_arrow: Schema = schema.as_ref().try_into_arrow()?; + Ok::<_, DeltaTableError>(partition_arrow.fields().clone()) + }) + .transpose()?; + + // Decide once how `stats_parsed` is produced (reshape cached typed stats, or parse raw + // JSON). The choice is fixed for every batch of this seed. + let stats_plan = Self::plan_stats( + available_stats_fields.as_ref(), + raw_stats_available, + effective_replay_stats_schema.as_ref(), + )?; + + // Assemble the scan-row output expression: base columns (or a typed NULL for a missing + // raw `stats`), then `stats_parsed`, then `partitionValues_parsed`. Column references are + // resolved by name, so a positional reordering of the seed cannot silently mis-map. + let mut expr_fields: Vec = Vec::with_capacity(base_schema.num_fields() + 2); + let mut output_fields: Vec = + Vec::with_capacity(base_schema.num_fields() + 2); + for field in base_schema.fields() { + if field.name().as_str() == FIELD_STATS && !raw_stats_available { + expr_fields.push(Expression::literal(Scalar::Null(field.data_type().clone()))); + } else { + expr_fields.push(Expression::column([field.name().clone()])); + } + output_fields.push(field.clone()); + } + if let Some(plan) = &stats_plan { + expr_fields.push(plan.source.stats_expression(&plan.schema)); + output_fields.push(StructField::nullable( + FIELD_STATS_PARSED, + DataType::Struct(Box::new(plan.schema.as_ref().clone())), + )); + } + if let Some(partition_schema) = partition_schema { + expr_fields.push(Expression::column([FIELD_PARTITION_VALUES_PARSED])); + output_fields.push(StructField::not_null( + FIELD_PARTITION_VALUES_PARSED, + DataType::Struct(Box::new(partition_schema.as_ref().clone())), + )); + } + + let output_type = DataType::Struct(Box::new(StructType::try_new(output_fields)?)); + let expression = Expression::struct_from(expr_fields); + // The evaluator ignores its declared input schema at runtime (it resolves columns by name + // from each batch), so the base scan-row schema is a sufficient placeholder here. + let evaluator = ARROW_HANDLER.new_expression_evaluator( + base_schema.clone(), + Arc::new(expression), + output_type, + )?; + + Ok(Self { + evaluator, + stats_plan, + partition_fields, + }) + } + + /// Chooses the `stats_parsed` strategy: reshape the seed's typed stats when they are present + /// and compatible (cheapest — no JSON parsing), otherwise parse the raw `stats` JSON, and + /// error when neither is available. The decision is made here rather than at evaluation time + /// because a `cast`/parse can silently produce nulls instead of failing, so a runtime failure + /// cannot be used to trigger the fallback. + fn plan_stats( + available_stats_fields: Option<&Fields>, + raw_stats_available: bool, + effective_replay_stats_schema: Option<&KernelSchemaRef>, + ) -> DeltaResult> { + let Some(requested_schema) = effective_replay_stats_schema else { + return Ok(None); + }; + + // Can the seed's typed `stats_parsed` be reshaped to the request? `Ok(None)` means there + // were no typed stats at all; `Err` means they exist but are incompatible. + let reshape = available_stats_fields + .map(|available_fields| { + let requested_arrow: Schema = requested_schema.as_ref().try_into_arrow()?; + CachedStructProjection::try_new( + available_fields.clone(), + requested_arrow.fields().clone(), + FIELD_STATS_PARSED, + ) + }) + .transpose(); + + let source = match reshape { + Ok(Some(projection)) => StatsParsedSource::Reshape(projection), + Ok(None) | Err(_) if raw_stats_available => StatsParsedSource::ParseJson, + Err(err) => return Err(err), + Ok(None) => { + return Err(DeltaTableError::SchemaMismatch { + msg: "cached seed has neither parsed nor raw statistics required by the scan" + .to_string(), + }); + } + }; + + Ok(Some(StatsPlan { + source, + schema: requested_schema.clone(), + })) + } + + /// Exact `stats_parsed` schema emitted by this evaluator. + pub(crate) fn stats_parsed_schema(&self) -> Option { + self.stats_plan.as_ref().map(|plan| plan.schema.clone()) + } + + pub(crate) fn evaluate(&self, batch: RecordBatch) -> DeltaResult { + self.validate_partition_layout(&batch)?; + let batch = self.reshape_cached_stats(batch)?; + Ok(self.evaluator.evaluate_arrow(batch)?) + } + + /// When a partition schema was declared, require the seed to carry a `partitionValues_parsed` + /// column with exactly the expected fields. This turns a missing / mismatched partition column + /// into a typed [`DeltaTableError::SchemaMismatch`] instead of the kernel's generic + /// column-not-found error. Base and stats columns are intentionally not checked here — they are + /// resolved by name by the expression, so extra or reordered columns are harmless. + fn validate_partition_layout(&self, batch: &RecordBatch) -> DeltaResult<()> { + let Some(expected_fields) = &self.partition_fields else { + return Ok(()); + }; + let column = batch.column_by_name(FIELD_PARTITION_VALUES_PARSED).ok_or_else(|| { + DeltaTableError::SchemaMismatch { + msg: format!("cached batch is missing required {FIELD_PARTITION_VALUES_PARSED} column"), + } + })?; + let ArrowDataType::Struct(actual_fields) = column.data_type() else { + return Err(DeltaTableError::SchemaMismatch { + msg: format!("cached {FIELD_PARTITION_VALUES_PARSED} column is not a struct"), + }); + }; + if actual_fields != expected_fields { + return Err(DeltaTableError::SchemaMismatch { + msg: format!( + "cached {FIELD_PARTITION_VALUES_PARSED} has fields {actual_fields:?} but expected {expected_fields:?}" + ), + }); + } + Ok(()) + } + + /// On the reshape path, cast the seed's `stats_parsed` column to the requested schema + /// (reorder / drop / add-nullable / widen / nested, preserving null buffers) so the assembly + /// expression can reference it by name. Every other path (raw-JSON parse, no stats) is handled + /// entirely inside the assembly expression and passes through untouched here. + fn reshape_cached_stats(&self, batch: RecordBatch) -> DeltaResult { + let Some(StatsPlan { + source: StatsParsedSource::Reshape(projection), + .. + }) = &self.stats_plan + else { + return Ok(batch); + }; + let index = batch.schema().index_of(FIELD_STATS_PARSED)?; + let parsed = batch + .column(index) + .as_any() + .downcast_ref::() + .ok_or_else(|| DeltaTableError::SchemaMismatch { + msg: "stats_parsed column is not a struct".to_string(), + })?; + let reshaped: ArrayRef = Arc::new(projection.cast_to_requested(parsed)?); + + let mut fields = batch.schema().fields().to_vec(); + fields[index] = Arc::new(Field::new( + FIELD_STATS_PARSED, + reshaped.data_type().clone(), + true, + )); + let mut columns = batch.columns().to_vec(); + columns[index] = reshaped; + Ok(RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?) + } + +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ArrayRef, new_null_array}; + use arrow::datatypes::DataType as ArrowDataType; + use arrow::datatypes::Schema as ArrowSchema; + use arrow_array::{Int32Array, Int64Array, StringArray}; + use arrow_buffer::NullBuffer; + use arrow_schema::Field; + use delta_kernel::expressions::Scalar; + use delta_kernel::scan::scan_row_schema; + use delta_kernel::schema::{SchemaRef, StructField, StructType}; + use delta_kernel::table_features::ColumnMappingMode; + use pretty_assertions::assert_eq; + + use crate::kernel::arrow::engine_ext::SnapshotExt; + use crate::kernel::snapshot::iterators::scan_row::parse_stats_column_with_schema; + use crate::kernel::scalars::ScalarExt; + use crate::kernel::snapshot::Snapshot; + use crate::kernel::snapshot::iterators::scan_row::parse_stats_column_impl; + use crate::kernel::snapshot::iterators::scan_row::tests::{append_stats_parsed, num_records_stats_parsed, num_records_stats_schema, raw_stats_string, scan_row_batch_with_stats, stats_parsed_field_names, value_stats_parsed, value_stats_schema}; + use crate::kernel::snapshot::stats_projection::{FileStatsMaterialization, StatsProjection}; + use crate::test_utils::TestTables; + + fn empty_num_records_stats_parsed() -> StructArray { + StructArray::from(vec![( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Option::::None])) as ArrayRef, + )]) + } + + fn scan_row_batch_with_optional_stats(raw_stats: Option<&str>) -> RecordBatch { + let schema: ArrowSchema = scan_row_schema().as_ref().try_into_arrow().unwrap(); + let mut columns: Vec = schema + .fields() + .iter() + .map(|field| new_null_array(field.data_type(), 1)) + .collect(); + columns[schema.index_of("path").unwrap()] = + Arc::new(StringArray::from(vec![Some("part-000.parquet")])); + columns[schema.index_of("size").unwrap()] = Arc::new(Int64Array::from(vec![1])); + columns[schema.index_of("modificationTime").unwrap()] = Arc::new(Int64Array::from(vec![1])); + columns[schema.index_of("stats").unwrap()] = Arc::new(StringArray::from(vec![raw_stats])); + + RecordBatch::try_new(Arc::new(schema), columns).unwrap() + } + + fn wide_min_values_stats_parsed( + num_records: i64, + min_value: i32, + min_values_valid: bool, + ) -> StructArray { + let min_values = StructArray::new( + Fields::from(vec![ + Field::new("value", ArrowDataType::Int32, true), + Field::new("other", ArrowDataType::Int32, true), + ]), + vec![ + Arc::new(Int32Array::from(vec![Some(min_value)])) as ArrayRef, + Arc::new(Int32Array::from(vec![Some(99)])) as ArrayRef, + ], + Some(NullBuffer::from_iter([min_values_valid])), + ); + + StructArray::from(vec![ + ( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(num_records)])) as ArrayRef, + ), + ( + Arc::new(Field::new( + "minValues", + min_values.data_type().clone(), + true, + )), + Arc::new(min_values) as ArrayRef, + ), + ]) + } + + fn wide_min_values_stats_schema() -> SchemaRef { + Arc::new( + StructType::try_new([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable( + "minValues", + StructType::try_new([ + StructField::nullable("value", DataType::INTEGER), + StructField::nullable("other", DataType::INTEGER), + ]) + .unwrap(), + ), + ]) + .unwrap(), + ) + } + + fn selected_min_values_stats_schema() -> SchemaRef { + Arc::new( + StructType::try_new([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable( + "minValues", + StructType::try_new([StructField::nullable("value", DataType::INTEGER)]) + .unwrap(), + ), + ]) + .unwrap(), + ) + } + + pub(crate) fn append_partition_values_parsed( + batch: &RecordBatch, + partition_values: StructArray, + ) -> RecordBatch { + let mut fields = batch.schema().fields().to_vec(); + let mut columns = batch.columns().to_vec(); + fields.push(Arc::new(Field::new( + FIELD_PARTITION_VALUES_PARSED, + partition_values.data_type().clone(), + false, + ))); + columns.push(Arc::new(partition_values)); + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() + } + + #[test] + fn parse_stats_column_impl_regenerates_raw_stats_when_raw_policy_preserves() -> DeltaResult<()> + { + let batch = scan_row_batch_with_stats("stale raw stats"); + let batch = append_stats_parsed(&batch, value_stats_parsed()); + let projected = parse_stats_column_impl( + &batch, + value_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::full()), + )?; + + let raw_stats = raw_stats_string(projected, 0) + .expect("full typed stats should generate compatibility JSON"); + let raw_stats: serde_json::Value = serde_json::from_str(&raw_stats)?; + assert_eq!(raw_stats["numRecords"], serde_json::json!(11)); + assert_eq!(raw_stats["minValues"]["value"], serde_json::json!(1)); + assert_eq!(raw_stats["maxValues"]["value"], serde_json::json!(9)); + assert_eq!(raw_stats["nullCount"]["value"], serde_json::json!(0)); + + Ok(()) + } + + #[test] + fn cached_uniform_evaluator_maps_reordered_base_fields_by_name() -> DeltaResult<()> { + // Give `size` and `modificationTime` distinct values so a positional (rather than by-name) + // read would route the wrong value to the wrong output field. + let batch = scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)); + let schema = batch.schema(); + let size_index = schema.index_of("size").unwrap(); + let modification_time_index = schema.index_of("modificationTime").unwrap(); + let mut columns = batch.columns().to_vec(); + columns[size_index] = Arc::new(Int64Array::from(vec![10])); + columns[modification_time_index] = Arc::new(Int64Array::from(vec![20])); + + // Physically swap the two same-typed columns (field + data together). By-name column + // resolution must still route each value back to its correctly named output field. + let mut fields = schema.fields().to_vec(); + fields.swap(size_index, modification_time_index); + columns.swap(size_index, modification_time_index); + let reordered = RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap(); + + let evaluator = CachedScanRowEvaluator::try_new(None, true, None, None)?; + let projected = evaluator.evaluate(reordered)?; + + let size = projected + .column_by_name("size") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("size should be an Int64 column"); + let modification_time = projected + .column_by_name("modificationTime") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("modificationTime should be an Int64 column"); + assert_eq!(size.value(0), 10); + assert_eq!(modification_time.value(0), 20); + Ok(()) + } + + #[test] + fn cached_uniform_evaluator_requires_exact_declared_partition_layout() -> DeltaResult<()> { + let partition_schema = Arc::new(StructType::try_new([StructField::nullable( + "part", + DataType::STRING, + )])?); + let evaluator = CachedScanRowEvaluator::try_new(None, true, None, Some(&partition_schema))?; + let wrong_partition = StructArray::from(vec![( + Arc::new(Field::new("wrong", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("value")])) as ArrayRef, + )]); + + for batch in [ + scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)), + append_partition_values_parsed( + &scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)), + wrong_partition, + ), + ] { + let error = evaluator + .evaluate(batch) + .expect_err("missing or wrong partition layout must be rejected"); + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + } + + let partition_values = StructArray::from(vec![( + Arc::new(Field::new("part", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("value")])) as ArrayRef, + )]); + let projected = evaluator.evaluate(append_partition_values_parsed( + &scan_row_batch_with_optional_stats(Some(r#"{"numRecords":11}"#)), + partition_values, + ))?; + assert!( + projected + .schema() + .field_with_name(FIELD_PARTITION_VALUES_PARSED) + .is_ok() + ); + + Ok(()) + } + + #[test] + fn seeded_input_wider_parsed_stats_project_to_selected_nested_columns() -> DeltaResult<()> { + let available_schema = wide_min_values_stats_schema(); + let requested_schema = selected_min_values_stats_schema(); + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_schema), + true, + Some(requested_schema.clone()), + None, + )?; + + assert_eq!(evaluator.stats_parsed_schema(), Some(requested_schema)); + for (num_records, min_value, min_values_valid) in [(11, 1, true), (12, 2, false)] { + let raw_stats = format!( + r#"{{"numRecords":{num_records},"minValues":{{"value":{min_value},"other":99}}}}"# + ); + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(&raw_stats)), + wide_min_values_stats_parsed(num_records, min_value, min_values_valid), + ); + let projected = evaluator.evaluate(batch)?; + + assert_eq!( + stats_parsed_field_names(&projected), + vec!["numRecords", "minValues"] + ); + assert_eq!( + raw_stats_string(projected.clone(), 0), + Some(raw_stats.clone()) + ); + let stats = projected + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("stats_parsed should be a struct"); + let min_values = stats + .column_by_name("minValues") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("minValues should be a struct"); + let ArrowDataType::Struct(fields) = min_values.data_type() else { + panic!("minValues should have a struct type"); + }; + assert_eq!( + fields + .iter() + .map(|field| field.name().as_str()) + .collect::>(), + vec!["value"] + ); + assert_eq!(min_values.is_valid(0), min_values_valid); + if min_values_valid { + assert_eq!( + Scalar::from_array(min_values.column_by_name("value").unwrap().as_ref(), 0), + Some(Scalar::Integer(min_value)) + ); + } + } + + Ok(()) + } + + #[test] + fn seeded_input_stats_schema_change_between_batches_returns_typed_error() { + let available_schema = wide_min_values_stats_schema(); + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_schema), + true, + Some(available_schema.clone()), + None, + ) + .unwrap(); + + let matching_batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some( + r#"{"numRecords":11,"minValues":{"value":1,"other":99}}"#, + )), + wide_min_values_stats_parsed(11, 1, true), + ); + evaluator + .evaluate(matching_batch) + .expect("matching declared stats_parsed schema should be accepted"); + + let changed_batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(r#"{"numRecords":12}"#)), + num_records_stats_parsed(12), + ); + let error = evaluator + .evaluate(changed_batch) + .expect_err("changed stats_parsed schema should return an error"); + + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + } + + #[test] + fn cached_uniform_evaluator_falls_back_to_exact_raw_json() -> DeltaResult<()> { + let raw_stats = r#"{ "minValues": {"value": 1}, "numRecords": 11 }"#; + let available_schema = Arc::new(StructType::try_new([StructField::nullable( + "minValues", + DataType::STRING, + )])?); + let requested_schema = selected_min_values_stats_schema(); + let incompatible_parsed = StructArray::from(vec![( + Arc::new(Field::new("minValues", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("incompatible")])) as ArrayRef, + )]); + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(raw_stats)), + incompatible_parsed, + ); + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_schema), + true, + Some(requested_schema.clone()), + None, + )?; + let projected = evaluator.evaluate(batch)?; + + assert_eq!(evaluator.stats_parsed_schema(), Some(requested_schema)); + assert_eq!( + raw_stats_string(projected.clone(), 0), + Some(raw_stats.to_string()) + ); + assert_eq!( + stats_parsed_field_names(&projected), + vec!["numRecords", "minValues"] + ); + let min_values = projected + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .and_then(|stats| stats.column_by_name("minValues")) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("raw fallback should produce minValues"); + assert_eq!( + Scalar::from_array(min_values.column_by_name("value").unwrap().as_ref(), 0), + Some(Scalar::Integer(1)) + ); + + Ok(()) + } + + #[test] + fn cached_uniform_evaluator_rejects_incompatible_seed_without_raw() { + let available_schema = Arc::new( + StructType::try_new([StructField::nullable("minValues", DataType::STRING)]).unwrap(), + ); + let error = match CachedScanRowEvaluator::try_new( + Some(&available_schema), + false, + Some(selected_min_values_stats_schema()), + None, + ) { + Ok(_) => panic!("incompatible parsed seed should fail without raw statistics"), + Err(error) => error, + }; + + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + assert!( + error.to_string().contains("stats_parsed.minValues"), + "unexpected error: {error}" + ); + } + + #[test] + fn parse_stats_column_impl_generates_full_json_before_narrowing_parsed() -> DeltaResult<()> { + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(None), + value_stats_parsed(), + ); + let projected = parse_stats_column_impl( + &batch, + num_records_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::NumRecordsOnly), + )?; + + let raw_stats = raw_stats_string(projected.clone(), 0) + .expect("full typed stats should generate compatibility JSON"); + let raw_stats: serde_json::Value = serde_json::from_str(&raw_stats)?; + assert_eq!( + raw_stats, + serde_json::json!({ + "numRecords": 11, + "minValues": {"value": 1}, + "maxValues": {"value": 9}, + "nullCount": {"value": 0} + }) + ); + assert_eq!(stats_parsed_field_names(&projected), vec!["numRecords"]); + + Ok(()) + } + + #[test] + fn parse_stats_column_impl_keeps_raw_null_when_typed_stats_are_empty() -> DeltaResult<()> { + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(None), + empty_num_records_stats_parsed(), + ); + let projected = parse_stats_column_impl( + &batch, + num_records_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::NumRecordsOnly), + )?; + + assert_eq!(raw_stats_string(projected, 0), None); + + let raw_stats = r#"{"numRecords":11}"#; + let batch = append_stats_parsed( + &scan_row_batch_with_optional_stats(Some(raw_stats)), + empty_num_records_stats_parsed(), + ); + let projected = parse_stats_column_impl( + &batch, + num_records_stats_schema(), + None, + None, + ColumnMappingMode::None, + &FileStatsMaterialization::compatibility(StatsProjection::NumRecordsOnly), + )?; + assert_eq!(raw_stats_string(projected, 0), Some(raw_stats.to_string())); + + Ok(()) + } + + fn stats_schema(fields: impl IntoIterator) -> SchemaRef { + Arc::new(StructType::try_new(fields).unwrap()) + } + + fn cached_batch( + rows: usize, + raw_stats: Option<&[Option<&str>]>, + stats_parsed: StructArray, + ) -> RecordBatch { + assert_eq!(stats_parsed.len(), rows); + let base_schema: ArrowSchema = scan_row_schema().as_ref().try_into_arrow().unwrap(); + let mut fields = Vec::new(); + let mut columns = Vec::new(); + + for field in base_schema.fields() { + let column: ArrayRef = match field.name().as_str() { + FIELD_STATS if raw_stats.is_none() => continue, + "path" => Arc::new(StringArray::from(vec!["file.parquet"; rows])), + "size" | "modificationTime" => Arc::new(Int64Array::from(vec![1; rows])), + FIELD_STATS => Arc::new(StringArray::from(raw_stats.unwrap().to_vec())), + _ => new_null_array(field.data_type(), rows), + }; + fields.push(field.clone()); + columns.push(column); + } + + fields.push(Arc::new(Field::new( + FIELD_STATS_PARSED, + stats_parsed.data_type().clone(), + true, + ))); + columns.push(Arc::new(stats_parsed)); + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() + } + + fn projected_stats(batch: &RecordBatch) -> &StructArray { + batch + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("stats_parsed should be a struct") + } + + #[tokio::test] + async fn cached_batches_preserve_raw_and_parsed_stats_for_kernel_roundtrip() -> DeltaResult<()> + { + let raw_stats = + r#"{"maxValues":{"id":9},"numRecords":11,"nullCount":{"id":0},"minValues":{"id":1}}"#; + let batch = scan_row_batch_with_optional_stats(Some(raw_stats)); + let log_store = TestTables::Simple.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let projected = parse_stats_column_with_schema( + snapshot.inner.as_ref(), + &batch, + snapshot.inner.stats_schema()?, + )?; + // Keep this guard so cached batches still pass through the uniform evaluator + // without rebuilding raw `stats` from `ToJson`. + let available_stats_schema = snapshot.inner.stats_schema()?; + let partition_schema = snapshot.inner.partitions_schema()?; + let evaluator = CachedScanRowEvaluator::try_new( + Some(&available_stats_schema), + true, + Some(available_stats_schema.clone()), + partition_schema.as_ref(), + )?; + let reparsed = evaluator.evaluate(projected)?; + + assert_eq!( + raw_stats_string(reparsed.clone(), 0), + Some(raw_stats.to_string()) + ); + let stats = reparsed + .column_by_name(FIELD_STATS_PARSED) + .and_then(|column| column.as_any().downcast_ref::()) + .expect("stats_parsed should be preserved"); + let num_records = stats + .column_by_name("numRecords") + .expect("numRecords should be preserved"); + let min_values = stats + .column_by_name("minValues") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("minValues should be a struct"); + let max_values = stats + .column_by_name("maxValues") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("maxValues should be a struct"); + let null_count = stats + .column_by_name("nullCount") + .and_then(|column| column.as_any().downcast_ref::()) + .expect("nullCount should be a struct"); + let min_value = min_values + .column_by_name("id") + .expect("min value should be preserved"); + let max_value = max_values + .column_by_name("id") + .expect("max value should be preserved"); + let null_value = null_count + .column_by_name("id") + .expect("null count should be preserved"); + + assert_eq!( + Scalar::from_array(num_records.as_ref(), 0), + Some(Scalar::Long(11)) + ); + assert_eq!( + Scalar::from_array(min_value.as_ref(), 0), + Some(Scalar::Long(1)) + ); + assert_eq!( + Scalar::from_array(max_value.as_ref(), 0), + Some(Scalar::Long(9)) + ); + assert_eq!( + Scalar::from_array(null_value.as_ref(), 0), + Some(Scalar::Long(0)) + ); + + Ok(()) + } + + #[tokio::test] + async fn cached_batches_without_stats_parsed_use_uniform_evaluator() -> DeltaResult<()> { + let raw_stats = r#"{"numRecords":11}"#; + let batch = scan_row_batch_with_optional_stats(Some(raw_stats)); + let evaluator = CachedScanRowEvaluator::try_new(None, true, None, None)?; + let projected = evaluator.evaluate(batch)?; + + assert!(projected.column_by_name(FIELD_STATS_PARSED).is_none()); + assert_eq!(raw_stats_string(projected, 0), Some(raw_stats.to_string())); + + Ok(()) + } + + #[test] + fn cached_stats_add_missing_nullable_field_without_raw() -> DeltaResult<()> { + let available = stats_schema([StructField::nullable("numRecords", DataType::LONG)]); + let requested = stats_schema([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable("added", DataType::INTEGER), + ]); + let parsed = StructArray::from(vec![( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef, + )]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, None, parsed))?; + let stats = projected_stats(&projected); + + assert_eq!( + stats + .column_by_name("numRecords") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 11 + ); + let added = stats.column_by_name("added").unwrap(); + assert_eq!(added.data_type(), &ArrowDataType::Int32); + assert!(added.is_null(0)); + Ok(()) + } + + #[test] + fn cached_stats_drop_extra_field_without_raw() -> DeltaResult<()> { + let available = stats_schema([ + StructField::nullable("keep", DataType::INTEGER), + StructField::nullable("dropped", DataType::STRING), + ]); + let requested = stats_schema([StructField::nullable("keep", DataType::INTEGER)]); + let parsed = StructArray::from(vec![ + ( + Arc::new(Field::new("keep", ArrowDataType::Int32, true)), + Arc::new(Int32Array::from(vec![Some(3)])) as ArrayRef, + ), + ( + Arc::new(Field::new("dropped", ArrowDataType::Utf8, true)), + Arc::new(StringArray::from(vec![Some("unused")])) as ArrayRef, + ), + ]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, None, parsed))?; + let stats = projected_stats(&projected); + let ArrowDataType::Struct(fields) = stats.data_type() else { + unreachable!() + }; + assert_eq!( + fields.iter().map(|field| field.name()).collect::>(), + ["keep"] + ); + Ok(()) + } + + #[test] + fn cached_stats_reorder_fields_by_name_without_raw() -> DeltaResult<()> { + let available = stats_schema([ + StructField::nullable("first", DataType::INTEGER), + StructField::nullable("second", DataType::LONG), + ]); + let requested = stats_schema([ + StructField::nullable("second", DataType::LONG), + StructField::nullable("first", DataType::INTEGER), + ]); + let parsed = StructArray::from(vec![ + ( + Arc::new(Field::new("first", ArrowDataType::Int32, true)), + Arc::new(Int32Array::from(vec![Some(1)])) as ArrayRef, + ), + ( + Arc::new(Field::new("second", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(2)])) as ArrayRef, + ), + ]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, None, parsed))?; + let stats = projected_stats(&projected); + let ArrowDataType::Struct(fields) = stats.data_type() else { + unreachable!() + }; + assert_eq!( + fields.iter().map(|field| field.name()).collect::>(), + ["second", "first"] + ); + assert_eq!( + stats + .column_by_name("second") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 2 + ); + assert_eq!( + stats + .column_by_name("first") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 1 + ); + Ok(()) + } + + #[test] + fn cached_stats_evolve_nested_fields_without_raw() -> DeltaResult<()> { + let available_nested = StructType::try_new([ + StructField::nullable("dropped", DataType::INTEGER), + StructField::nullable("keep", DataType::INTEGER), + ])?; + let requested_nested = StructType::try_new([ + StructField::nullable("added", DataType::INTEGER), + StructField::nullable("keep", DataType::INTEGER), + ])?; + let available = stats_schema([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable("minValues", available_nested), + ]); + let requested = stats_schema([ + StructField::nullable("minValues", requested_nested), + StructField::nullable("numRecords", DataType::LONG), + ]); + + let min_values = StructArray::new( + Fields::from(vec![ + Field::new("dropped", ArrowDataType::Int32, true), + Field::new("keep", ArrowDataType::Int32, true), + ]), + vec![ + Arc::new(Int32Array::from(vec![Some(9), Some(8), Some(7)])), + Arc::new(Int32Array::from(vec![Some(1), Some(2), Some(3)])), + ], + Some(NullBuffer::from_iter([true, true, false])), + ); + let parsed = StructArray::new( + Fields::from(vec![ + Field::new("numRecords", ArrowDataType::Int64, true), + Field::new("minValues", min_values.data_type().clone(), true), + ]), + vec![ + Arc::new(Int64Array::from(vec![Some(10), Some(20), Some(30)])), + Arc::new(min_values), + ], + Some(NullBuffer::from_iter([true, false, true])), + ); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(3, None, parsed))?; + let stats = projected_stats(&projected); + assert_eq!( + stats.nulls().unwrap(), + &NullBuffer::from_iter([true, false, true]) + ); + + let min_values = stats + .column_by_name("minValues") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + min_values.nulls().unwrap(), + &NullBuffer::from_iter([true, true, false]) + ); + let ArrowDataType::Struct(fields) = min_values.data_type() else { + unreachable!() + }; + assert_eq!( + fields.iter().map(|field| field.name()).collect::>(), + ["added", "keep"] + ); + assert_eq!(min_values.column_by_name("added").unwrap().null_count(), 3); + assert_eq!( + min_values + .column_by_name("keep") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values(), + &[1, 2, 3] + ); + Ok(()) + } + + #[test] + fn cached_stats_widen_primitive_without_raw() -> DeltaResult<()> { + let available = stats_schema([StructField::nullable("value", DataType::INTEGER)]); + let requested = stats_schema([StructField::nullable("value", DataType::LONG)]); + let parsed = StructArray::from(vec![( + Arc::new(Field::new("value", ArrowDataType::Int32, true)), + Arc::new(Int32Array::from(vec![Some(7), None])) as ArrayRef, + )]); + + let evaluator = + CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(2, None, parsed))?; + let values = projected_stats(&projected) + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(0), 7); + assert!(values.is_null(1)); + Ok(()) + } + + #[test] + fn cached_stats_normal_evolution_does_not_parse_raw() -> DeltaResult<()> { + let available = stats_schema([StructField::nullable("numRecords", DataType::LONG)]); + let requested = stats_schema([ + StructField::nullable("numRecords", DataType::LONG), + StructField::nullable("added", DataType::INTEGER), + ]); + let parsed = StructArray::from(vec![( + Arc::new(Field::new("numRecords", ArrowDataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(11)])) as ArrayRef, + )]); + let invalid_raw = [Some("not valid JSON")]; + + let evaluator = CachedScanRowEvaluator::try_new(Some(&available), true, Some(requested), None)?; + let projected = evaluator.evaluate(cached_batch(1, Some(&invalid_raw), parsed))?; + assert!( + projected_stats(&projected) + .column_by_name("added") + .unwrap() + .is_null(0) + ); + assert_eq!( + raw_stats_string(projected, 0), + Some("not valid JSON".to_string()) + ); + Ok(()) + } + + #[test] + fn cached_stats_reject_truly_incompatible_present_type() { + let available = stats_schema([StructField::nullable("value", DataType::STRING)]); + let requested = stats_schema([StructField::nullable("value", DataType::INTEGER)]); + let error = + match CachedScanRowEvaluator::try_new(Some(&available), false, Some(requested), None) { + Ok(_) => panic!("string statistics should not cast to integer"), + Err(error) => error, + }; + + assert!(matches!(error, DeltaTableError::SchemaMismatch { .. })); + assert!(error.to_string().contains("Utf8")); + assert!(error.to_string().contains("Int32")); + } +} diff --git a/crates/core/src/kernel/snapshot/log_data.rs b/crates/core/src/kernel/snapshot/log_data.rs index 0ffff1f674..876871e3c1 100644 --- a/crates/core/src/kernel/snapshot/log_data.rs +++ b/crates/core/src/kernel/snapshot/log_data.rs @@ -88,6 +88,16 @@ impl<'a> LogDataHandler<'a> { self.data.iter().map(|batch| batch.num_rows()).sum() } + pub fn get(&self, mut index: usize) -> Option { + for batch in self.data { + if index < batch.num_rows() { + return Some(LogicalFileView::new(batch.clone(), index)); + } + index -= batch.num_rows(); + } + None + } + /// Iterate over each file as a [`LogicalFileView`] without materializing all of them at once. pub fn iter(&self) -> impl Iterator + '_ { self.data.iter().flat_map(|batch| { diff --git a/crates/core/src/kernel/snapshot/mod.rs b/crates/core/src/kernel/snapshot/mod.rs index d9b60144f1..0fe2ef96cd 100644 --- a/crates/core/src/kernel/snapshot/mod.rs +++ b/crates/core/src/kernel/snapshot/mod.rs @@ -20,6 +20,8 @@ use std::sync::{Arc, LazyLock}; use arrow::array::RecordBatch; use arrow::compute::{filter_record_batch, is_not_null}; use arrow::datatypes::SchemaRef; +use arrow_arith::aggregate::sum_array_checked; +use arrow_array::{Int64Array, StructArray}; use delta_kernel::actions::{Remove, Sidecar}; use delta_kernel::engine::arrow_conversion::TryIntoArrow as _; use delta_kernel::engine::arrow_data::ArrowEngineData; @@ -41,9 +43,11 @@ use object_store::{ObjectStore, ObjectStoreExt as _}; use serde_json::Deserializer; use url::Url; +use crate::kernel::arrow::extract::{self as ex, ProvidesColumnByName}; + use super::{Action, CommitInfo, Metadata, Protocol}; use crate::checkpoints::parse_last_checkpoint_hint; -use crate::kernel::arrow::engine_ext::{ExpressionEvaluatorExt, rb_from_scan_meta}; +use crate::kernel::arrow::engine_ext::{ExpressionEvaluatorExt, SnapshotExt, rb_from_scan_meta}; use crate::kernel::{ARROW_HANDLER, StructType, spawn_blocking_with_span}; use crate::logstore::{LogStore, LogStoreExt}; use crate::{DeltaResult, DeltaTableConfig, DeltaTableError, PartitionFilter, to_kernel_predicate}; @@ -55,6 +59,7 @@ pub(crate) use self::stats_projection::{ pub use iterators::*; pub use scan::*; pub use stream::*; +use crate::kernel::size_limits::SnapshotLoadMetrics; mod iterators; mod log_data; @@ -62,6 +67,7 @@ mod scan; mod serde; mod stats_projection; mod stream; +pub mod size_limits; pub(crate) static SCAN_ROW_ARROW_SCHEMA: LazyLock = LazyLock::new(|| Arc::new(scan_row_schema().as_ref().try_into_arrow().unwrap())); @@ -197,11 +203,13 @@ impl<'a> ConflictReadSet<'a> { #[derive(Debug, Clone, PartialEq)] pub struct Snapshot { /// Log segment containing all log files in the snapshot - pub(crate) inner: Arc, + pub inner: Arc, /// Configuration for the current session - config: DeltaTableConfig, + pub(crate) config: DeltaTableConfig, /// Optional materialized replay state owned by this snapshot. - materialized_files: Option>, + pub(crate) materialized_files: Option>, + /// Metrics captured during snapshot loading + pub(crate) load_metrics: SnapshotLoadMetrics, } impl Snapshot { @@ -215,8 +223,10 @@ impl Snapshot { config: DeltaTableConfig, version: Option, ) -> DeltaResult { + let engine2 = engine.clone(); + let table_root2 = table_root.clone(); let snapshot = match spawn_blocking_with_span(move || { - let mut builder = KernelSnapshot::builder_for(table_root); + let mut builder = KernelSnapshot::builder_for(table_root.clone()); if let Some(version) = version { builder = builder.at_version(version); } @@ -236,13 +246,23 @@ impl Snapshot { } }; + let (snapshot, load_metrics) = + size_limits::apply_optional_log_limiter(snapshot, config.log_size_limiter.as_ref(), engine2.as_ref(), &table_root2).await?; + Ok(Self { inner: snapshot, config, materialized_files: None, + load_metrics, }) } + /// Returns the engine from the config, or falls back to the log store's default engine. + fn resolve_engine(config: &DeltaTableConfig, log_store: &dyn LogStore) -> Arc { + config.engine.as_ref().map(|e| e.0.clone()) + .unwrap_or_else(|| log_store.engine(None)) + } + /// Create a new [`Snapshot`] instance pub async fn try_new( log_store: &dyn LogStore, @@ -250,7 +270,8 @@ impl Snapshot { version: Option, ) -> DeltaResult { // TODO: bundle operation_id with logstore ... - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = Self::resolve_engine(&config, log_store); // NB: kernel engine uses Url::join to construct paths, // if the path does not end with a slash, the would override the entire path. @@ -263,14 +284,21 @@ impl Snapshot { Self::try_new_with_engine(engine, table_root, config, version).await } + /// Returns the configured engine, falling back to the log store's default engine. + pub(crate) fn engine(&self, log_store: &dyn LogStore) -> Arc { + Self::resolve_engine(&self.config, log_store) + } + /// Create a [`ScanBuilder`] borrowing this snapshot to configure a read of the table. pub fn scan_builder(&self) -> ScanBuilder { ScanBuilder::new(self.inner.clone()) + .with_checkpoint_stats_json_fallback(self.config.checkpoint_stats_json_fallback) } /// Consume this snapshot and create a [`ScanBuilder`] that owns the underlying state. pub fn into_scan_builder(self) -> ScanBuilder { ScanBuilder::new(self.inner) + .with_checkpoint_stats_json_fallback(self.config.checkpoint_stats_json_fallback) } /// Update the snapshot to the given version @@ -309,10 +337,13 @@ impl Snapshot { .await .map_err(|e| DeltaTableError::Generic(e.to_string()))??; + let load_metrics = SnapshotLoadMetrics::from_snapshot(snapshot.as_ref()); + let snapshot = Arc::new(Self { inner: snapshot, config: self.config.clone(), materialized_files: None, + load_metrics, }); if snapshot.version() as u64 == current_version { // `target_version` was `None`, but there were no new commits. @@ -322,8 +353,12 @@ impl Snapshot { } else { match self.materialized_files() { Some(materialized_files) => { + // Cached partitionValues_parsed uses the source snapshot's layout. + let materialized_seed = (self.inner.partitions_schema()? + == snapshot.inner.partitions_schema()?) + .then_some(materialized_files); snapshot - .materialize_files_with_engine(engine, Some(materialized_files)) + .materialize_files_with_engine(engine, materialized_seed) .await } None => Ok(snapshot), @@ -442,6 +477,13 @@ impl Snapshot { } } + fn materialized_stats_parsed_schema(&self) -> DeltaResult> { + match self.materialized_files_policy() { + MaterializedFilesPolicy::FullTablePreserveRaw => Ok(Some(self.inner.stats_schema()?)), + MaterializedFilesPolicy::FullTableWithoutStats => Ok(None), + } + } + pub(crate) fn materialized_files(&self) -> Option<&Arc> { self.materialized_files .as_ref() @@ -461,7 +503,8 @@ impl Snapshot { return Ok(self); } - self.materialize_files_with_engine(log_store.engine(None), None) + let engine = self.engine(log_store); + self.materialize_files_with_engine(engine, None) .await } @@ -543,6 +586,11 @@ impl Snapshot { } } + /// Get the metrics captured during snapshot loading + pub fn load_metrics(&self) -> &SnapshotLoadMetrics { + &self.load_metrics + } + /// Get the table root of the snapshot pub(crate) fn table_root_path(&self) -> DeltaResult { Ok(Path::from_url_path(self.inner.table_root().path())?) @@ -596,8 +644,11 @@ impl Snapshot { { return cached; } + + let engine = self.engine(log_store); + if predicate.is_some() && self.config.skip_stats { - return self.files_with_engine(log_store.engine(None), predicate); + return self.files_with_engine(engine, predicate); } match self @@ -605,17 +656,24 @@ impl Snapshot { .and_then(|materialized_files| materialized_files.full_table_seed()) { Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); self.files_from( - log_store.engine(None), + engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, Box::new(existing_data), existing_predicate, ) } - None => self.files_with_engine(log_store.engine(None), predicate), + None => self.files_with_engine(engine, predicate), } } @@ -764,12 +822,19 @@ impl Snapshot { .and_then(|materialized_files| materialized_files.full_table_seed()) { Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); self.files_from_materialized_with_options( log_store.engine(None), predicate, existing_version, + available_stats_schema, + raw_stats_available, Box::new(existing_data), existing_predicate, FileMaterializationOptions { @@ -792,6 +857,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, ) -> SendableRBStream { @@ -799,6 +866,8 @@ impl Snapshot { engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, existing_data, existing_predicate, FileStatsMode::PreserveRaw, @@ -810,6 +879,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, ) -> SendableRBStream { @@ -817,6 +888,8 @@ impl Snapshot { engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, existing_data, existing_predicate, FileStatsMode::FullPreserveRaw, @@ -828,6 +901,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, stats_mode: FileStatsMode, @@ -836,6 +911,8 @@ impl Snapshot { engine, predicate, existing_version, + available_stats_schema, + raw_stats_available, existing_data, existing_predicate, FileMaterializationOptions { @@ -850,6 +927,8 @@ impl Snapshot { engine: Arc, predicate: Option, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, options: FileMaterializationOptions, @@ -862,7 +941,14 @@ impl Snapshot { let stats_materialization = scan.stats_materialization().clone(); let stream = scan - .scan_metadata_from(engine, existing_version, existing_data, existing_predicate) + .scan_metadata_from( + engine, + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) .map(|d| Ok(rb_from_scan_meta(d?)?)); match ScanRowOutStream::try_new_with_materialization( @@ -908,6 +994,7 @@ impl Snapshot { inner: self.inner.clone(), config: self.config.clone(), materialized_files, + load_metrics: self.load_metrics.clone(), } } @@ -920,14 +1007,23 @@ impl Snapshot { return Ok(self); } + // TODO: Compatible parsed checkpoints avoid physical `stats` JSON I/O, but FullTablePreserveRaw still retains both + // synthesized `stats` JSON and full `stats_parsed`; removing duplication requires a narrower or lazy cache representation. let batches = match materialized_seed.and_then(|seed| seed.full_table_seed()) { Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); self.files_from_preserving_raw( engine, None, existing_version, + available_stats_schema, + raw_stats_available, Box::new(existing_data), existing_predicate, ) @@ -940,7 +1036,7 @@ impl Snapshot { .await? } }; - let materialized_files = Arc::new(MaterializedFiles::full(self.as_ref(), batches)); + let materialized_files = Arc::new(MaterializedFiles::full(self.as_ref(), batches)?); Ok(Arc::new( self.with_materialized_files(Some(materialized_files)), )) @@ -1047,7 +1143,8 @@ impl Snapshot { let tx = builder.tx(); // TODO: bundle operation id with log store ... - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = self.engine(log_store); let remove_data = match self .inner @@ -1094,7 +1191,8 @@ impl Snapshot { app_id: String, ) -> DeltaResult> { // TODO: bundle operation id with log store ... - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = self.engine(log_store); let inner = self.inner.clone(); let version = spawn_blocking_with_span(move || inner.get_app_id_version(&app_id, engine.as_ref())) @@ -1116,7 +1214,8 @@ impl Snapshot { log_store: &dyn LogStore, domain: impl ToString, ) -> DeltaResult> { - let engine = log_store.engine(None); + // let engine = log_store.engine(None); + let engine = self.engine(log_store); let inner = self.inner.clone(); let domain = domain.to_string(); let metadata = @@ -1152,6 +1251,7 @@ pub(crate) enum MaterializedFilesScope { pub(crate) struct MaterializedFiles { identity: SnapshotIdentity, policy: MaterializedFilesPolicy, + stats_parsed_schema: Option, pub(crate) scope: MaterializedFilesScope, pub(crate) existing_predicate: Option, pub(crate) batches: Arc<[RecordBatch]>, @@ -1160,14 +1260,26 @@ pub(crate) struct MaterializedFiles { #[derive(Debug, Clone)] struct MaterializedFilesSeed { version: Version, + stats_parsed_schema: Option, + raw_stats_available: bool, existing_predicate: Option, batches: Arc<[RecordBatch]>, } impl MaterializedFilesSeed { - fn into_parts(self) -> (Version, SharedRecordBatchIter, Option) { + fn into_parts( + self, + ) -> ( + Version, + Option, + bool, + SharedRecordBatchIter, + Option, + ) { ( self.version, + self.stats_parsed_schema, + self.raw_stats_available, SharedRecordBatchIter::new(self.batches), self.existing_predicate, ) @@ -1209,15 +1321,16 @@ impl Iterator for SharedRecordBatchIter { } impl MaterializedFiles { - fn full(snapshot: &Snapshot, batches: Vec) -> Self { + fn full(snapshot: &Snapshot, batches: Vec) -> DeltaResult { let identity = snapshot.identity(); - Self { + Ok(Self { identity, policy: snapshot.materialized_files_policy(), + stats_parsed_schema: snapshot.materialized_stats_parsed_schema()?, scope: MaterializedFilesScope::FullTable, existing_predicate: None, batches: batches.into(), - } + }) } fn is_cache_for(&self, snapshot: &Snapshot) -> bool { @@ -1228,6 +1341,8 @@ impl MaterializedFiles { match self.scope { MaterializedFilesScope::FullTable => Some(MaterializedFilesSeed { version: self.identity.version, + stats_parsed_schema: self.stats_parsed_schema.clone(), + raw_stats_available: matches!(self.policy, MaterializedFilesPolicy::FullTablePreserveRaw), existing_predicate: self.existing_predicate.clone(), batches: self.batches.clone(), }), @@ -1238,7 +1353,7 @@ impl MaterializedFiles { /// A snapshot of a Delta table that has been eagerly loaded into memory. #[derive(Debug, Clone, PartialEq)] pub struct EagerSnapshot { - snapshot: Arc, + pub snapshot: Arc, } /// Read `_last_checkpoint` and return the hinted version when present. @@ -1295,6 +1410,14 @@ pub(crate) async fn resolve_snapshot( } } +fn read_adds_size(array: &dyn ProvidesColumnByName) -> usize { + if let Some(size) = ex::extract_and_cast_opt::(array, "size") { + sum_array_checked::(size).unwrap().unwrap_or_default() as usize + } else { + 0 + } +} + impl EagerSnapshot { /// Create a new [`EagerSnapshot`] instance pub async fn try_new( @@ -1345,9 +1468,10 @@ impl EagerSnapshot { target_version: Option, ) -> DeltaResult<()> { let previous_snapshot = self.snapshot.clone(); + let engine = self.snapshot.engine(log_store); let updated_snapshot = previous_snapshot .clone() - .update(log_store.engine(None), target_version) + .update(engine, target_version) .await?; if Arc::ptr_eq(&updated_snapshot, &previous_snapshot) { return Ok(()); @@ -1408,6 +1532,11 @@ impl EagerSnapshot { self.snapshot.load_config() } + /// Get the metrics captured during snapshot loading + pub fn load_metrics(&self) -> &SnapshotLoadMetrics { + &self.snapshot.load_metrics() + } + /// Well known table configuration pub fn table_properties(&self) -> &TableProperties { self.snapshot.table_properties() @@ -1442,6 +1571,22 @@ impl EagerSnapshot { } } + /// Get the metadata size in the snapshot + pub fn files_metadata_size(&self) -> usize { + self.snapshot + .materialized_files() + .map(|m| m.batches.iter().map(|frb| frb.get_array_memory_size()).sum()) + .unwrap_or(0) + } + + /// Get the total size of files in the snapshot + pub fn files_total_size(&self) -> usize { + self.snapshot + .materialized_files() + .map(|m| m.batches.iter().map(|frb| read_adds_size(frb)).sum()) + .unwrap_or(0) + } + /// Stream the active files in the snapshot /// /// This function returns a stream of [`LogicalFileView`] objects, @@ -1584,11 +1729,13 @@ mod tests { let engine = log_store.engine(None); let snapshot = KernelSnapshot::builder_for(table_url.clone()).build(engine.as_ref())?; + let load_metrics = SnapshotLoadMetrics::from_snapshot(snapshot.as_ref()); Ok(( Self { inner: snapshot, config: Default::default(), materialized_files: None, + load_metrics, }, log_store, )) @@ -2084,7 +2231,7 @@ mod tests { let target = Snapshot::try_new(log_store.as_ref(), Default::default(), Some(12)).await?; let expected = active_add_paths(&target, log_store.as_ref()).await?; - let mut mismatched_cache = MaterializedFiles::full(&target, vec![]); + let mut mismatched_cache = MaterializedFiles::full(&target, vec![])?; mismatched_cache.identity.table_root = Url::parse("memory:///other/")?; let cached = Arc::new(target.with_materialized_files(Some(Arc::new(mismatched_cache)))); @@ -2300,6 +2447,7 @@ mod tests { let value = serde_json::to_value(snapshot.as_ref())?; assert_eq!(value[10]["policy"], json!("FullTablePreserveRaw")); + assert!(value[10].get("stats_parsed_schema").is_none()); let bytes = serde_json::to_vec(snapshot.as_ref())?; let actual: Snapshot = serde_json::from_slice(&bytes)?; @@ -2317,6 +2465,10 @@ mod tests { materialized_files.policy, MaterializedFilesPolicy::FullTablePreserveRaw ); + assert_eq!( + materialized_files.stats_parsed_schema, + Some(actual.inner.stats_schema()?) + ); Ok(()) } @@ -2366,6 +2518,10 @@ mod tests { MaterializedFilesPolicy::FullTablePreserveRaw ); assert_eq!(materialized_files.scope, MaterializedFilesScope::FullTable); + assert_eq!( + materialized_files.stats_parsed_schema, + Some(actual.snapshot().inner.stats_schema()?) + ); assert_eq!(actual.try_log_data()?.num_files(), 1); assert_eq!( actual @@ -2430,6 +2586,7 @@ mod tests { materialized_files.policy, MaterializedFilesPolicy::FullTableWithoutStats ); + assert!(materialized_files.stats_parsed_schema.is_none()); assert!( actual .try_log_data()? @@ -3079,12 +3236,43 @@ mod tests { Ok(()) } + #[tokio::test] + async fn cached_seed_carries_exact_schema_for_empty_cache() -> TestResult { + let log_store = TestTables::Simple.table_builder()?.build_storage()?; + let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; + let expected_schema = snapshot.inner.stats_schema()?; + let materialized_files = MaterializedFiles::full(&snapshot, vec![])?; + assert_eq!( + materialized_files.stats_parsed_schema, + Some(expected_schema.clone()) + ); + let seed = materialized_files + .full_table_seed() + .expect("full-table cache must produce a seed"); + assert_eq!(seed.stats_parsed_schema, Some(expected_schema)); + assert!(seed.raw_stats_available); + + let config = DeltaTableConfig { + skip_stats: true, + ..Default::default() + }; + let snapshot = Snapshot::try_new(&log_store, config, None).await?; + let materialized_files = MaterializedFiles::full(&snapshot, vec![])?; + assert!(materialized_files.stats_parsed_schema.is_none()); + let seed = materialized_files + .full_table_seed() + .expect("full-table cache must produce a seed"); + assert!(seed.stats_parsed_schema.is_none()); + assert!(!seed.raw_stats_available); + Ok(()) + } + #[tokio::test] async fn test_materialized_files_full_table_seed_shares_batches() -> TestResult { let log_store = TestTables::Simple.table_builder()?.build_storage()?; let snapshot = Snapshot::try_new(&log_store, Default::default(), None).await?; let batch = RecordBatch::new_empty(Arc::new(arrow_schema::Schema::empty())); - let materialized_files = MaterializedFiles::full(&snapshot, vec![batch]); + let materialized_files = MaterializedFiles::full(&snapshot, vec![batch])?; let seed = materialized_files .full_table_seed() diff --git a/crates/core/src/kernel/snapshot/scan.rs b/crates/core/src/kernel/snapshot/scan.rs index b1b259fe66..639e674512 100644 --- a/crates/core/src/kernel/snapshot/scan.rs +++ b/crates/core/src/kernel/snapshot/scan.rs @@ -3,7 +3,10 @@ use std::sync::Arc; use arrow_array::RecordBatch; use delta_kernel::engine::arrow_data::ArrowEngineData; -use delta_kernel::scan::{Scan as KernelScan, ScanBuilder as KernelScanBuilder, ScanMetadata}; +use delta_kernel::scan::{ + Scan as KernelScan, ScanBuilder as KernelScanBuilder, ScanMetadata, + StatsOptions, +}; use delta_kernel::schema::SchemaRef; use delta_kernel::snapshot::Snapshot as KernelSnapshot; use delta_kernel::{Engine, EngineData, PredicateRef, SnapshotRef, Version}; @@ -14,9 +17,12 @@ use url::Url; #[cfg(feature = "datafusion")] use super::MaterializedFiles; -use super::stats_projection::{FileStatsMaterialization, StatsProjection}; +use super::stats_projection::{ + FileStatsMaterialization, RawStatsPolicy, StatsProjection, StatsSourcePolicy, +}; use crate::DeltaResult; -use crate::kernel::{ReceiverStreamBuilder, scan_row_in_eval}; +use crate::kernel::arrow::engine_ext::SnapshotExt; +use crate::kernel::{CachedScanRowEvaluator, ReceiverStreamBuilder}; /// A boxed, `Send`able stream of [`ScanMetadata`] results produced while scanning a snapshot. pub type SendableScanMetadataStream = Pin> + Send>>; @@ -28,6 +34,7 @@ pub struct ScanBuilder { schema: Option, predicate: Option, stats_materialization: Option, + checkpoint_stats_json_fallback: bool, } impl ScanBuilder { @@ -38,9 +45,15 @@ impl ScanBuilder { schema: None, predicate: None, stats_materialization: None, + checkpoint_stats_json_fallback: false, } } + pub(super) fn with_checkpoint_stats_json_fallback(mut self, enabled: bool) -> Self { + self.checkpoint_stats_json_fallback = enabled; + self + } + /// Provide [`Schema`] for columns to select from the [`Snapshot`]. /// /// A table with columns `[a, b, c]` could have a scan which reads only the first @@ -108,6 +121,7 @@ impl ScanBuilder { schema, predicate, stats_materialization, + checkpoint_stats_json_fallback, } = self; let stats_materialization = match stats_materialization { @@ -122,7 +136,13 @@ impl ScanBuilder { // Modernization: Use kernel's StatsOptions when available // TODO: Add condition to use kernel StatsOptions API once fully integrated - let inner = build_kernel_scan(snapshot, schema, predicate, Some(&stats_materialization))?; + let inner = build_kernel_scan( + snapshot, + schema, + predicate, + Some(&stats_materialization), + checkpoint_stats_json_fallback, + )?; Ok(Scan::new(Arc::new(inner), stats_materialization)) } @@ -142,41 +162,73 @@ fn build_kernel_scan( schema: Option, predicate: Option, stats_materialization: Option<&FileStatsMaterialization>, + checkpoint_stats_json_fallback: bool, ) -> DeltaResult { let mut builder = KernelScanBuilder::new(snapshot) .with_schema_opt(schema) .with_predicate(predicate); if let Some(stats_materialization) = stats_materialization { - builder = with_kernel_stats_output(builder, stats_materialization); + builder = with_kernel_stats_output( + builder, + stats_materialization, + checkpoint_stats_json_fallback, + ); } Ok(builder.build()?) } +fn kernel_stats_options( + materialization: &FileStatsMaterialization, + checkpoint_stats_json_fallback: bool, +) -> StatsOptions { + match ( + materialization.stats_source_policy(), + materialization.raw_stats_policy(), + materialization.stats_projection(), + ) { + (StatsSourcePolicy::None, _, _) | (_, _, StatsProjection::None) => StatsOptions::none(), + (StatsSourcePolicy::ParsedWithJsonFallback, RawStatsPolicy::Preserve, _) => { + StatsOptions::all_struct() + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback) + } + ( + StatsSourcePolicy::ParsedWithJsonFallback, + RawStatsPolicy::Omit, + StatsProjection::Full, + ) => StatsOptions::all_struct() + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback), + ( + StatsSourcePolicy::ParsedWithJsonFallback, + RawStatsPolicy::Omit, + StatsProjection::PredicateColumns(columns), + ) => StatsOptions::struct_columns(columns.iter().cloned().collect()) + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback), + ( + StatsSourcePolicy::ParsedWithJsonFallback, + RawStatsPolicy::Omit, + StatsProjection::NumRecordsOnly, + ) => StatsOptions::struct_columns(vec![]) + .with_checkpoint_stats_json_fallback(checkpoint_stats_json_fallback), + } +} + fn with_kernel_stats_output( builder: KernelScanBuilder, materialization: &FileStatsMaterialization, + checkpoint_stats_json_fallback: bool, ) -> KernelScanBuilder { - // Modernize to use kernel's StatsOptions API - // Map our legacy StatsProjection to kernel's modern StatsOptions - - let stats_opts = match materialization.stats_projection() { - StatsProjection::None => delta_kernel::scan::StatsOptions::json_only(), - StatsProjection::Full => delta_kernel::scan::StatsOptions::all_struct(), - StatsProjection::PredicateColumns(_columns) => { - // Use kernel's efficient column-based stats - // TODO: Use specific column selection: StructStats::Columns(_columns.iter().cloned().collect()) - delta_kernel::scan::StatsOptions::all_struct() - } - StatsProjection::NumRecordsOnly => delta_kernel::scan::StatsOptions::json_only(), - }; - - // Apply the kernel's native stats options - // This reuses the kernel's efficient StructStats implementation - builder.with_stats(stats_opts) + builder.with_stats(kernel_stats_options( + materialization, + checkpoint_stats_json_fallback, + )) } +#[cfg(test)] +#[path = "scan/stats_options_tests.rs"] +mod stats_options_tests; + #[cfg(test)] mod tests { use delta_kernel::expressions::{ColumnName, Scalar}; @@ -207,6 +259,13 @@ mod tests { super::super::Snapshot::try_new(table.log_store().as_ref(), Default::default(), None).await } + fn assert_stats_options_eq(actual: StatsOptions, expected: StatsOptions) { + assert_eq!( + serde_json::to_value(actual).expect("actual StatsOptions should serialize"), + serde_json::to_value(expected).expect("expected StatsOptions should serialize") + ); + } + #[tokio::test] async fn scan_builder_infers_num_records_only_for_default_query_scan() -> DeltaResult<()> { let snapshot = synthetic_snapshot().await?; @@ -276,6 +335,16 @@ mod tests { Ok(()) } + #[test] + fn scan_builder_compatibility_materialization_maps_to_all_struct_stats() { + let materialization = FileStatsMaterialization::compatibility(StatsProjection::full()); + + assert_stats_options_eq( + kernel_stats_options(&materialization, false), + StatsOptions::all_struct().with_checkpoint_stats_json_fallback(false), + ); + } + #[tokio::test] async fn scan_builder_preserves_explicit_without_stats_materialization() -> DeltaResult<()> { let snapshot = synthetic_snapshot().await?; @@ -434,20 +503,26 @@ impl Scan { engine: Arc, materialized_files: Option<&Arc>, ) -> SendableScanMetadataStream { - match materialized_files.and_then(|materialized_files| materialized_files.full_table_seed()) - { - Some(materialized_seed) => { - let (existing_version, existing_data, existing_predicate) = - materialized_seed.into_parts(); - self.scan_metadata_from( - engine, - existing_version, - Box::new(existing_data), - existing_predicate, - ) - } - None => self.scan_metadata(engine), - } + let Some(materialized_seed) = + materialized_files.and_then(|materialized_files| materialized_files.full_table_seed()) + else { + return self.scan_metadata(engine); + }; + let ( + existing_version, + available_stats_schema, + raw_stats_available, + existing_data, + existing_predicate, + ) = materialized_seed.into_parts(); + self.scan_metadata_from( + engine, + existing_version, + available_stats_schema, + raw_stats_available, + Box::new(existing_data), + existing_predicate, + ) } /// Stream [`ScanMetadata`] incrementally starting from a previously observed state. @@ -459,24 +534,33 @@ impl Scan { &self, engine: Arc, existing_version: Version, + available_stats_schema: Option, + raw_stats_available: bool, existing_data: Box, existing_predicate: Option, ) -> SendableScanMetadataStream { let inner = self.inner.clone(); - let snapshot = self.inner.snapshot().clone(); - - // process our stored / cached data to conform to the expected input for log replay - let evaluator = match scan_row_in_eval(&snapshot) { - Ok(scan_row_in_eval) => scan_row_in_eval, + let effective_replay_stats_schema = inner.effective_replay_stats_schema().cloned(); + let partition_schema = match inner.snapshot().partitions_schema() { + Ok(partition_schema) => partition_schema, + Err(err) => return Box::pin(once(ready(Err(err)))), + }; + let evaluator = match CachedScanRowEvaluator::try_new( + available_stats_schema.as_ref(), + raw_stats_available, + effective_replay_stats_schema, + partition_schema.as_ref(), + ) { + Ok(evaluator) => evaluator, Err(err) => return Box::pin(once(ready(Err(err)))), }; - let scan_row_iter = existing_data - .map(|batch| Box::new(ArrowEngineData::new(batch)) as Box) - .map(move |b| { - evaluator - .evaluate(b.as_ref()) - .expect("malformed cached log data") - }); + let stats_parsed_schema = evaluator.stats_parsed_schema(); + let scan_row_iter = existing_data.map(move |batch| { + evaluator + .evaluate(batch) + .map(|batch| Box::new(ArrowEngineData::new(batch)) as Box) + .map_err(|err| delta_kernel::Error::Generic(err.to_string())) + }); // TODO: which capacity to choose? let mut builder = ReceiverStreamBuilder::::new(100); @@ -485,6 +569,7 @@ impl Scan { for res in inner.scan_metadata_from( engine.as_ref(), existing_version, + stats_parsed_schema, Box::new(scan_row_iter), existing_predicate, )? { diff --git a/crates/core/src/kernel/snapshot/scan/stats_options_tests.rs b/crates/core/src/kernel/snapshot/scan/stats_options_tests.rs new file mode 100644 index 0000000000..67a072d7c2 --- /dev/null +++ b/crates/core/src/kernel/snapshot/scan/stats_options_tests.rs @@ -0,0 +1,77 @@ +use delta_kernel::expressions::ColumnName; +use serde_json::{Value, json}; + +use super::*; + +fn serialized(options: StatsOptions) -> Value { + serde_json::to_value(options).expect("StatsOptions should serialize") +} + +fn assert_typed_options(materialization: FileStatsMaterialization, expected: StatsOptions) { + let actual = serialized(kernel_stats_options(&materialization, false)); + assert_eq!(actual, serialized(expected)); + assert_eq!( + actual.get("checkpoint_stats_json_fallback"), + Some(&json!(false)) + ); +} + +#[test] +fn compatibility_full_disables_checkpoint_json_fallback() { + assert_typed_options( + FileStatsMaterialization::compatibility(StatsProjection::Full), + StatsOptions::all_struct().with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn query_full_disables_checkpoint_json_fallback() { + assert_typed_options( + FileStatsMaterialization::query(StatsProjection::Full), + StatsOptions::all_struct().with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn predicate_columns_disable_checkpoint_json_fallback() { + let columns = [ColumnName::new(["value"])].into(); + assert_typed_options( + FileStatsMaterialization::query(StatsProjection::PredicateColumns(columns)), + StatsOptions::struct_columns(vec![ColumnName::new(["value"])]) + .with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn row_count_only_disables_checkpoint_json_fallback() { + assert_typed_options( + FileStatsMaterialization::query(StatsProjection::NumRecordsOnly), + StatsOptions::struct_columns(vec![]).with_checkpoint_stats_json_fallback(false), + ); +} + +#[test] +fn disabled_stats_map_to_none() { + let none = serialized(StatsOptions::none()); + assert_eq!( + serialized(kernel_stats_options( + &FileStatsMaterialization::query(StatsProjection::None), + false, + )), + none + ); + assert_eq!( + serialized(kernel_stats_options( + &FileStatsMaterialization::compatibility(StatsProjection::None), + false, + )), + none + ); + assert_eq!( + serialized(kernel_stats_options( + &FileStatsMaterialization::without_stats(), + false, + )), + none + ); +} diff --git a/crates/core/src/kernel/snapshot/serde.rs b/crates/core/src/kernel/snapshot/serde.rs index fc0b75ba24..3eddf3de75 100644 --- a/crates/core/src/kernel/snapshot/serde.rs +++ b/crates/core/src/kernel/snapshot/serde.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize, ser::SerializeSeq}; use url::Url; use crate::DeltaTableConfig; - +use crate::kernel::size_limits::SnapshotLoadMetrics; use super::{ EagerSnapshot, MaterializedFiles, MaterializedFilesPolicy, MaterializedFilesScope, Snapshot, SnapshotIdentity, SnapshotMaterializationMode, @@ -162,6 +162,9 @@ impl MaterializedFilesWire { let materialized_files = MaterializedFiles { identity, policy, + stats_parsed_schema: owning_snapshot + .materialized_stats_parsed_schema() + .map_err(|err| err.to_string())?, scope, existing_predicate: None, batches: deserialize_batches(self.batches)?.into(), @@ -189,10 +192,10 @@ fn materialized_files_from_legacy_eager_payload( return Ok(None); } - Ok(Some(Arc::new(MaterializedFiles::full( - snapshot, - deserialize_batches(legacy_payload)?, - )))) + Ok(Some(Arc::new( + MaterializedFiles::full(snapshot, deserialize_batches(legacy_payload)?) + .map_err(|err| err.to_string())?, + ))) } fn serialize_batches(batches: &[arrow_array::RecordBatch]) -> Result, String> { @@ -432,11 +435,13 @@ impl<'de> Visitor<'de> for SnapshotVisitor { .map_err(de::Error::custom)?; let snapshot = KernelSnapshot::new(log_segment, table_configuration); + let load_metrics = SnapshotLoadMetrics::from_snapshot(&snapshot); let snapshot = Snapshot { inner: Arc::new(snapshot), config, materialized_files: None, + load_metrics, }; let materialized_files = materialized_files .map(|value| { diff --git a/crates/core/src/kernel/snapshot/size_limits.rs b/crates/core/src/kernel/snapshot/size_limits.rs new file mode 100644 index 0000000000..79db6083fc --- /dev/null +++ b/crates/core/src/kernel/snapshot/size_limits.rs @@ -0,0 +1,992 @@ +use crate::logstore::{LogStore, LogStoreExt}; +use crate::{DeltaResult, DeltaTableError}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::convert::identity; +use std::num::{NonZeroU64, NonZeroUsize}; +use std::ops::RangeInclusive; +use std::sync::Arc; +use delta_kernel::log_segment::{LogSegment, LogSegmentFiles}; +use delta_kernel::path::{LogPathFileType, ParsedLogPath}; +use delta_kernel::{Engine, Snapshot}; +use itertools::Itertools; +use strum::Display; +use tracing::{debug, info, trace, warn}; +use url::Url; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SnapshotLoadMetrics { + /// Current version of the snapshot + pub current_version: i64, + + /// Oldest commit version included in the loaded snapshot + /// None if only checkpoint was loaded (no commits) + pub oldest_version_loaded: Option, + + /// Final log segment size in bytes (after truncation if applied) + pub loaded_log_segment_size: u64, + + /// Total number of commit and checkpoint files in the loaded log segment + pub num_log_files: usize, + + /// Number of commit files in the loaded log segment + pub num_commit_files: usize, + + /// Number of checkpoint files in the loaded log segment + pub num_checkpoint_files: usize, + + /// Whether log size limiting was applied during load + pub log_size_limiter_applied: bool, + + /// Original log segment size in bytes (before any truncation) + /// None if no limiter was configured + pub original_log_segment_size: Option, + + /// Number of commits discarded due to truncation + /// None if no truncation occurred + pub num_commits_discarded: Option, +} + +impl Default for SnapshotLoadMetrics { + fn default() -> Self { + Self { + current_version: 0, + oldest_version_loaded: None, + loaded_log_segment_size: 0, + num_log_files: 0, + num_commit_files: 0, + num_checkpoint_files: 0, + log_size_limiter_applied: false, + original_log_segment_size: None, + num_commits_discarded: None, + } + } +} + +pub(crate) fn log_segment_total_byte_size(segment: &LogSegment) -> u64 { + segment + .listed + .checkpoint_parts + .iter() + .chain(segment.listed.ascending_commit_files.iter()) + .map(|p| p.location.size) + .sum() +} + +pub(crate) async fn apply_optional_log_limiter( + snapshot: Arc, + limiter: Option<&LogSizeLimiter>, + engine: &dyn Engine, + root_url: &Url, +) -> DeltaResult<(Arc, SnapshotLoadMetrics)> { + let current_version = snapshot.version() as i64; + + if let Some(limiter) = limiter { + let original_segment = snapshot.log_segment().clone(); + let original_size = log_segment_total_byte_size(&original_segment); + + let (truncated_segment, truncation_info) = + limiter.truncate(original_segment, engine, root_url).await?; + let table_configuration = snapshot.table_configuration().clone(); + + let oldest_version = truncated_segment + .listed + .ascending_commit_files + .first() + .map(|p| p.version as i64); + + let metrics = match truncation_info { + Some(info) => SnapshotLoadMetrics::with_truncation( + current_version, + oldest_version, + info.truncated_size, + info.original_size, + info.commits_discarded, + ), + None => SnapshotLoadMetrics::no_truncation( + current_version, + oldest_version, + original_size, + ), + } + .with_log_segment_file_counts(&truncated_segment); + + Ok(( + Arc::new(Snapshot::new(truncated_segment, table_configuration)), + metrics, + )) + } else { + let metrics = SnapshotLoadMetrics::from_snapshot(snapshot.as_ref()); + Ok((snapshot, metrics)) + } +} + +impl SnapshotLoadMetrics { + pub fn from_snapshot(snapshot: &Snapshot) -> Self { + let log_segment = snapshot.log_segment(); + let log_size = log_segment_total_byte_size(log_segment); + let oldest_version = log_segment + .listed + .ascending_commit_files + .first() + .map(|p| p.version as i64); + + SnapshotLoadMetrics::no_truncation( + snapshot.version() as i64, + oldest_version, + log_size, + ) + .with_log_segment_file_counts(log_segment) + } + + fn with_log_segment_file_counts(mut self, log_segment: &LogSegment) -> Self { + self.num_commit_files = log_segment.listed.ascending_commit_files.len(); + self.num_checkpoint_files = log_segment.listed.checkpoint_parts.len(); + self.num_log_files = self.num_commit_files + self.num_checkpoint_files; + self + } + + pub fn no_truncation( + version: i64, + oldest_version: Option, + log_segment_size: u64, + ) -> Self { + Self { + current_version: version, + oldest_version_loaded: oldest_version, + loaded_log_segment_size: log_segment_size, + num_log_files: 0, + num_commit_files: 0, + num_checkpoint_files: 0, + log_size_limiter_applied: false, + original_log_segment_size: None, + num_commits_discarded: None, + } + } + + pub fn with_truncation( + version: i64, + oldest_version: Option, + truncated_size: u64, + original_size: u64, + commits_discarded: usize, + ) -> Self { + Self { + current_version: version, + oldest_version_loaded: oldest_version, + loaded_log_segment_size: truncated_size, + num_log_files: 0, + num_commit_files: 0, + num_checkpoint_files: 0, + log_size_limiter_applied: true, + original_log_segment_size: Some(original_size), + num_commits_discarded: Some(commits_discarded), + } + } + + pub fn was_truncated(&self) -> bool { + self.log_size_limiter_applied && self.num_commits_discarded.is_some() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TruncationInfo { + /// Original log segment size in bytes (before truncation) + pub original_size: u64, + /// Final log segment size in bytes (after truncation) + pub truncated_size: u64, + /// Number of commits discarded + pub commits_discarded: usize, +} + +#[derive(Debug, Clone, PartialEq, Display, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OversizePolicy { + Reject, + /// Skip checkpoints and only load JSON commits. + UseTruncatedCommitLog(NonZeroUsize), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LogSizeLimiter { + /// Maximum allowed size in bytes for the total log segment (checkpoint + commit files). + size_limit: NonZeroU64, + oversize_policy: OversizePolicy, +} + + +impl LogSizeLimiter { + + pub fn new(size_limit: NonZeroU64, oversize_policy: OversizePolicy) -> Self { + Self { + size_limit, + oversize_policy, + } + } + + pub fn try_new(size_limit: u64, truncated_commit_log_size: Option) -> DeltaResult { + let size_limit = NonZeroU64::new(size_limit) + .ok_or_else(|| DeltaTableError::Generic("max_log_bytes must be nonzero".into()))?; + let oversize_policy = if let Some(num_commits) = truncated_commit_log_size { + let num_commits = NonZeroUsize::new(num_commits) + .ok_or_else(|| DeltaTableError::Generic("pseudo_cdf_lookback_count must be nonzero".into()))?; + OversizePolicy::UseTruncatedCommitLog(num_commits) + } else { + OversizePolicy::Reject + }; + Ok(Self { + size_limit, + oversize_policy, + }) + } + + pub fn from_storage_options(opts: &mut HashMap) -> DeltaResult> { + let prefix = "log_size_limiter"; + let size_limit_key = &format!("{prefix}.size_limit"); + let use_commit_log_key = &format!("{prefix}.use_truncated_commit_log"); + let num_commits_key = &format!("{prefix}.truncated_commit_log_size"); + + let size_limit: Option = opts.remove(size_limit_key) + .map(|opt| opt.parse() + .expect(&format!("{size_limit_key} must be a positive int; got {opt}"))); + let use_commit_log: bool = opts.remove(use_commit_log_key) + .map(|opt| opt.parse() + .expect(&format!("{use_commit_log_key} must be a boolean; got {opt}"))) + .unwrap_or(false); + let num_commits: usize = opts.remove(num_commits_key) + .map(|opt| opt.parse() + .expect(&format!("{num_commits_key} must be a positive int; got {opt}"))) + .unwrap_or(24); // default number of commits to use when commit log is enabled with no size specified + size_limit + .map(|limit| LogSizeLimiter::try_new(limit, use_commit_log.then_some(num_commits))) + .transpose() + } + + pub(super) async fn truncate(&self, log_segment: LogSegment, engine: &dyn Engine, root_url: &Url) -> DeltaResult<(LogSegment, Option)> { + let total_size: u64 = log_segment_total_byte_size(&log_segment); + let total_size = total_size; + let size_limit = self.size_limit.get(); + let original_commit_count = log_segment.listed.ascending_commit_files.len(); + + if total_size > size_limit { + warn!( + "Log segment size in bytes: {} > {}. Applying policy: {:?}", + total_size, size_limit, self.oversize_policy + ); + trace!("Oversized log segment: {:?}", log_segment); + match &self.oversize_policy { + OversizePolicy::Reject => + Err(DeltaTableError::Generic(format!(r#" + Table log segment size ({} bytes) exceeds maximum allowed size ({} bytes). + Consider increasing the size limit or using an oversize policy other than {}. + "#, total_size, self.size_limit, self.oversize_policy))), + OversizePolicy::UseTruncatedCommitLog(num_commits) => { + let (truncated_segment, truncated_size) = + truncated_commit_log(log_segment, engine, root_url, num_commits, size_limit).await?; + let final_commit_count = truncated_segment.listed.ascending_commit_files.len(); + let commits_discarded = original_commit_count.saturating_sub(final_commit_count); + + let truncation_info = TruncationInfo { + original_size: total_size, + truncated_size, + commits_discarded, + }; + Ok((truncated_segment, Some(truncation_info))) + } + } + } else { + debug!("Log segment size ({} bytes) is within the limit of {} bytes", total_size, size_limit); + Ok((log_segment, None)) + } + } +} + +async fn truncated_commit_log(log_segment: LogSegment, engine: &dyn Engine, root_url: &Url, num_commits: &NonZeroUsize, size_limit: u64) -> DeltaResult<(LogSegment, u64)> { + let num_commits = num_commits.get(); + let truncated_log: Vec = if log_segment.listed.ascending_commit_files.len() < num_commits { + let segment_version = log_segment.end_version as usize; + let first_missing_version = segment_version.saturating_sub(num_commits - 1); // start from zero if num_commits > segment_version + let last_missing_version = segment_version - log_segment.listed.ascending_commit_files.len(); // cannot overflow + info!("Extending the segment commit log with versions {}-{}", first_missing_version, last_missing_version); + let missing_versions = first_missing_version..=last_missing_version; + let previous_commits = list_commit_files(engine, root_url, missing_versions).await?; + previous_commits.into_iter() + .chain(log_segment.listed.ascending_commit_files) + .collect() + } else { + info!("Discarding the last {} entries from the segment commit log", log_segment.listed.ascending_commit_files.len() - num_commits); + log_segment.listed.ascending_commit_files[log_segment.listed.ascending_commit_files.len() - num_commits..] + .to_vec() + }; + let mut truncated_log_size = 0_u64; // keep track of the total size to cut it shorter if needed + let latest_commit_file = truncated_log.last().cloned(); + let final_commits: Vec = truncated_log.into_iter() + .rev() + .take_while(|file_meta| { + truncated_log_size += file_meta.location.size; + truncated_log_size <= size_limit + }) + .collect::>() + .into_iter() + .rev() + .collect(); + info!("final_commits: {:?}", &final_commits); + + // Calculate the actual final size + let final_size: u64 = final_commits.iter().map(|f| f.location.size).sum(); + + let segment_files = LogSegmentFiles { + ascending_commit_files: final_commits, + ascending_compaction_files: vec![], + checkpoint_parts: vec![], + latest_crc_file: None, + latest_commit_file, + max_published_version: Some(log_segment.end_version), + }; + + let segment = LogSegment { + end_version: log_segment.end_version, + listed: segment_files, + // ascending_commit_files: final_commits, + // checkpoint_parts: vec![], + // ascending_compaction_files: vec![], + log_root: root_url.clone(), + checkpoint_version: None, + last_checkpoint_metadata: log_segment.last_checkpoint_metadata.clone() + // latest_crc_file: None, + // latest_commit_file, + }; + + Ok((segment, final_size)) +} + +async fn list_commit_files( + engine: &dyn Engine, + root_url: &Url, + version_range: RangeInclusive, +) -> DeltaResult> { + // @HSTack see LogStore::log_root_url + let delta_log_root_url = root_url.join("_delta_log/").unwrap(); + let log_path = delta_log_root_url.clone(); + let lower_bound = log_path.join(&format!("{:020}", version_range.start())) + .map_err(|_| DeltaTableError::InvalidTableLocation(log_path.to_string()))?; + let upper_bound = log_path.join(&format!("{:020}", version_range.end() + 1)) + .map_err(|_| DeltaTableError::InvalidTableLocation(log_path.to_string()))?; + let commit_files = engine.storage_handler() + .list_from(&lower_bound)? + .map(|meta| { + ParsedLogPath::try_from(meta?) + }) + // TODO this filters out .crc files etc which start with "." - how do we want to use these kind of files? + .filter_map_ok(identity) + .take_while(move |path_res| match path_res { + Ok(path) => version_range.end() >= &(path.version as usize), + Err(_) => true, + }) + .filter_ok(|log_file| matches!(log_file.file_type, LogPathFileType::Commit)) + .try_collect()?; + Ok(commit_files) +} + + +#[cfg(test)] +mod tests { + use delta_kernel::Version; + use delta_kernel::metrics::SnapshotLoadMetricContext; + use super::*; + use test_doubles::*; + use crate::DeltaTableBuilder; + + + async fn create_log_segment(log_store: &TestLogStore, version: Option) -> DeltaResult { + let storage = log_store.engine(None).storage_handler(); + let log_root = log_store.log_root_url(); + Ok(LogSegment::for_snapshot(storage.as_ref(), log_root, Vec::new(), version, SnapshotLoadMetricContext::default())?) + } + + #[test] + fn test_serde() -> DeltaResult<()> { + let json = r#"{ + "size_limit": 10055, + "oversize_policy": "reject" + }"#; + assert_eq!( + serde_json::from_str::(json)?, + LogSizeLimiter::new( + NonZeroU64::new(10055).unwrap(), + OversizePolicy::Reject, + ) + ); + + let json = r#"{ + "size_limit": 10055, + "oversize_policy": { + "use_truncated_commit_log": 100 + } + }"#; + assert_eq!( + serde_json::from_str::(json)?, + LogSizeLimiter::new( + NonZeroU64::new(10055).unwrap(), + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(100).unwrap()), + ) + ); + + Ok(()) + } + + #[test] + fn test_from_storage_opts() -> DeltaResult<()> { + assert_eq!( + LogSizeLimiter::from_storage_options(&mut HashMap::new())?, + None + ); + let mut opts = HashMap::from([ + ("log_size_limiter.size_limit".into(), "10".into()), + ("log_size_limiter.use_truncated_commit_log".into(), "false".into()), + ("log_size_limiter.truncated_commit_log_size".into(), "5".into()), // should be ignored + ("test".into(), "1".into()), + ]); + assert_eq!( + LogSizeLimiter::from_storage_options(&mut opts)?, + Some(LogSizeLimiter::new( + NonZeroU64::new(10).unwrap(), + OversizePolicy::Reject + )) + ); + assert_eq!(opts.len(), 1); + assert!(opts.contains_key("test")); + Ok(()) + } + + #[test] + fn test_storage_opts_propagation() -> DeltaResult<()> { + let url = "memory:///".parse().map_err(|e: url::ParseError| DeltaTableError::Generic(e.to_string()))?; + let table = DeltaTableBuilder::from_url(url)? + .with_storage_options(HashMap::from([ + ("log_size_limiter.size_limit".into(), "10".into()), + ("log_size_limiter.use_truncated_commit_log".into(), "true".into()), + ("log_size_limiter.truncated_commit_log_size".into(), "5".into()), + ])).build()?; + assert_eq!( + table.config.log_size_limiter.expect("LogSizeLimiter should be set"), + LogSizeLimiter::new( + NonZeroU64::new(10).unwrap(), + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(5).unwrap()) + ) + ); + + Ok(()) + } + + #[tokio::test] + async fn test_noop_within_limits() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(10), CommitFsize(100), CheckpointFsize(3000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(5000).unwrap(), + OversizePolicy::Reject, + ); + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 90, 10); + // total size < size limit + // let tmp = log_store.engine(None); + let (truncated_segment, truncation_info) = limiter + .truncate( + segment.clone(), + log_store.engine(None).as_ref(), + &log_store.table_root_url() + ).await?; + assert_eq!(truncated_segment, segment); + assert_eq!(truncation_info, None); + + Ok(()) + } + + #[tokio::test] + async fn test_reject_policy() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(10), CommitFsize(100), CheckpointFsize(3000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(2500).unwrap(), + OversizePolicy::Reject, + ); + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 90, 10); + let result = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await; + + assert!(result.is_err()); + let error_msg = result.unwrap_err().to_string(); + assert!(error_msg.contains("exceeds maximum allowed size")); + assert!(error_msg.contains("4000 bytes"), "`{}` does not contain '4000 bytes'", error_msg); + assert!(error_msg.contains("2500 bytes"), "`{}` does not contain '2500 bytes'", error_msg); + + Ok(()) + } + + #[tokio::test] + async fn test_commit_log_truncation_with_regular_delta_log() -> DeltaResult<()> { + let _ = pretty_env_logger::try_init(); + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(5), CommitFsize(10), CheckpointFsize(1000) + ); + info!("LOG STORE ROOT URL: {}", log_store.root_url().to_string()); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(10).unwrap()), + ); + + let segment = create_log_segment(&log_store, Some(25)).await?; + info!("segment 1: {:#?}", &segment); + assert_segment_with_checkpoint(&segment, 25, 0); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 16..=25); + + let segment = create_log_segment(&log_store, Some(7)).await?; + assert_segment_with_checkpoint(&segment, 5, 2); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 0..=7); + + let segment = create_log_segment(&log_store, Some(19)).await?; + assert_segment_with_checkpoint(&segment, 15, 4); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 10..=19); + + Ok(()) + } + + #[tokio::test] + async fn test_commit_log_truncation_with_no_checkpoints_in_log() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(200), CommitFsize(10), CheckpointFsize(1000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(10).unwrap()), + ); + + let segment = create_log_segment(&log_store, Some(30)).await?; + assert_segment_with_commits_only(&segment, 0..=30); + // size limit not exceeded: 31 commits * 10 bytes < 500 bytes, segment not truncated + let (truncated_segment, _truncation_info) = limiter + .truncate(segment.clone(), log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_eq!(truncated_segment, segment); + + let segment = create_log_segment(&log_store, Some(75)).await?; + assert_segment_with_commits_only(&segment, 0..=75); + // size limit exceeded: 75 commits * 10 bytes > 500 bytes; keeps the last 10 commits + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 66..=75); + + Ok(()) + } + + #[tokio::test] + async fn test_commit_log_truncation_with_vacuumed_log() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(30..=150), CheckpointCadence(25), CommitFsize(10), CheckpointFsize(1000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(50).unwrap()), + ); + + let segment = create_log_segment(&log_store, Some(70)).await?; + assert_segment_with_checkpoint(&segment, 50, 20); + // less than 50 commits available in the vacuumed store + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 30..=70); + + Ok(()) + } + + #[tokio::test] + async fn test_truncated_log_gets_cut_off_to_enforce_size_limit() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(30..=150), CheckpointCadence(25), CommitFsize(10), CheckpointFsize(1000) + ); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(100).unwrap()), // go back 100 commits + ); + + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 125, 25); + // only loads 50 commits instead of the configured 100 to stay within the size limit + let (truncated_segment, _truncation_info) = limiter.truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()).await?; + assert_segment_with_commits_only(&truncated_segment, 101..=150); + + Ok(()) + } + + #[tokio::test] + async fn test_compacted_json_files_are_ignored() -> DeltaResult<()> { + let extra_files = vec![ + format!("{:020}.{:020}.compacted.json", 15, 19), + ]; + let log_store = TestLogStore::new( + CommitRange(0..=100), CheckpointCadence(10), CommitFsize(10), CheckpointFsize(1000) + ).with_additional_files(extra_files, 200); + let limiter = LogSizeLimiter::new( + NonZeroU64::new(500).unwrap(), // smaller than the checkpoint size, can fit 50 commits + OversizePolicy::UseTruncatedCommitLog(NonZeroUsize::new(20).unwrap()), // go back 100 commits + ); + + let segment = create_log_segment(&log_store, Some(23)).await?; + assert_segment_with_checkpoint(&segment, 20, 3); + let (truncated_segment, _truncation_info) = limiter + .truncate(segment, log_store.engine(None).as_ref(), &log_store.table_root_url()) + .await?; + assert_segment_with_commits_only(&truncated_segment, 4..=23 ); + Ok(()) + } + + fn commit_file_name(version: Version) -> String { + format!("{:020}.json", version) + } + + fn checkpoint_file_name(version: Version) -> String { + format!("{:020}.checkpoint.parquet", version) + } + + fn extract_file_names<'a>(stored_objects: impl IntoIterator) -> Vec { + stored_objects.into_iter() + .filter_map(|parsed_path| { + let path_str = parsed_path.location.location.path(); + path_str.split('/').last().map(ToString::to_string) + }) + .collect() + } + + fn assert_segment_with_checkpoint(segment: &LogSegment, checkpoint_version: Version, num_subsequent_commits: u64) { + assert_eq!(segment.end_version, checkpoint_version + num_subsequent_commits); + assert_eq!( + extract_file_names(&segment.listed.checkpoint_parts), + vec![ checkpoint_file_name(checkpoint_version) ], + ); + assert_eq!( + extract_file_names(&segment.listed.ascending_commit_files), + (checkpoint_version + 1 ..= checkpoint_version + num_subsequent_commits) + .map(|v| commit_file_name(v as u64)) + .collect::>(), + ); + } + + fn assert_segment_with_commits_only(log_segment: &LogSegment, versions: RangeInclusive) { + assert_eq!(log_segment.end_version, *versions.end() as u64); + assert_eq!(log_segment.listed.checkpoint_parts, vec![]); + assert_eq!( + extract_file_names(&log_segment.listed.ascending_commit_files), + versions.map(|v| commit_file_name(v as u64)).collect::>(), + ); + } + + + mod test_doubles { + use object_store::PutMultipartOptions; +use super::*; + use crate::DeltaResult; + use crate::kernel::transaction::TransactionError; + use crate::logstore::{object_store_path, CommitOrBytes, LogStore, LogStoreConfig, LogStoreExt}; + use async_trait::async_trait; + use bytes::Bytes; + use futures::stream; + use futures::stream::BoxStream; + use object_store::path::Path; + use object_store::{CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutOptions, PutPayload, PutResult, Result as ObjectStoreResult}; + use std::ops::RangeInclusive; + use std::sync::Arc; + use url::Url; + use uuid::Uuid; + + // substitute for named arguments to make the test code self documenting + pub(super) struct CommitRange(pub(super) RangeInclusive); + pub(super) struct CheckpointCadence(pub(super) usize); + pub(super) struct CommitFsize(pub(super) u64); + pub(super) struct CheckpointFsize(pub(super) u64); + + #[derive(Debug, Clone)] + pub(super) struct TestLogStore { + config: LogStoreConfig, + files: Vec, + } + + impl TestLogStore { + /// Commit files are generated to span the entire `CommitRange`, and checkpoints are + /// created according to the configured `CheckpointCadence`, starting from 0 (exclusive) + /// up to the last version in the store (also exclusive) and only for versions + /// that are also in the `CommitRange`. + /// E.g. commits: 15 up to 100, cadence: 10 => checkpoints at versions 20, 30, ..., 90 + pub(super) fn new( + commit_range: CommitRange, + checkpoint_cadence: CheckpointCadence, + commit_fsize: CommitFsize, + checkpoint_fsize: CheckpointFsize, + ) -> Self { + // get rid of the self-documenting superfluous types + let commit_range = commit_range.0; + let checkpoint_cadence = checkpoint_cadence.0; + let commit_fsize = commit_fsize.0; + let checkpoint_fsize = checkpoint_fsize.0; + + let mut store = TestLogStore { + config: LogStoreConfig::new( + &Url::parse("memory://test/delta_table").unwrap(), + Default::default(), + ), + files: vec![] + }; + let path = object_store_path(&store.log_root_url()).unwrap(); + let commit_files = commit_range.clone() + .map(|v| commit_file_name(v as u64)) + .map(|f| obj_meta(path.clone().join(f), commit_fsize)); + let checkpoint_files = (0..*commit_range.end()) + .skip(checkpoint_cadence) + .step_by(checkpoint_cadence) + .filter(|version| commit_range.contains(version)) + .map(|v| checkpoint_file_name(v as u64)) + .map(|f| obj_meta(path.clone().join(f), checkpoint_fsize)); + let mut files = commit_files + .chain(checkpoint_files) + .collect::>(); + files.sort_unstable_by(|a, b| a.location.cmp(&b.location)); + // files.shuffle(&mut thread_rng()); // no order guarantees for store listing + store.files = files; + store + } + + pub(super) fn with_additional_files(mut self, fnames: Vec, fsize: u64) -> Self { + let log_path = object_store_path(&self.log_root_url()).unwrap(); + let mut files: Vec = fnames + .into_iter() + .map(|fname| obj_meta(log_path.clone().join(fname), fsize)) + .collect(); + self.files.append(&mut files); + self.files.sort_unstable_by(|a, b| a.location.cmp(&b.location)); + self + } + + } + + #[async_trait] + impl LogStore for TestLogStore { + fn name(&self) -> String { + "TestLogStore".to_string() + } + + async fn read_commit_entry(&self, _version: Version) -> DeltaResult> { + unimplemented!("TestLogStore::read_commit_entry not implemented for tests") + } + + async fn write_commit_entry(&self, _version: Version, _commit_or_bytes: CommitOrBytes, _operation_id: Uuid) -> Result<(), TransactionError> { + unimplemented!("TestLogStore::write_commit_entry not implemented for tests") + } + + async fn abort_commit_entry(&self, _version: Version, _commit_or_bytes: CommitOrBytes, _operation_id: Uuid) -> Result<(), TransactionError> { + unimplemented!("TestLogStore::abort_commit_entry not implemented for tests") + } + + async fn get_latest_version(&self, _start_version: Version) -> DeltaResult { + unimplemented!("TestLogStore::get_latest_version not implemented for tests") + } + + fn object_store(&self, operation_id: Option) -> Arc { + self.root_object_store(operation_id) + } + + fn root_object_store(&self, _operation_id: Option) -> Arc { + Arc::new(self.clone()) + } + + fn config(&self) -> &LogStoreConfig { + &self.config + } + } + + impl std::fmt::Display for TestLogStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } + } + + #[async_trait] + impl ObjectStore for TestLogStore { + + async fn put_opts(&self, _location: &Path, _bytes: PutPayload, _options: PutOptions) -> ObjectStoreResult { + unimplemented!("TestLogStore::put_opts not implemented for tests") + } + + async fn put_multipart_opts(&self, _location: &Path, _opts: PutMultipartOptions) -> ObjectStoreResult> { + unimplemented!("TestLogStore::put_multipart_opts not implemented for tests") + } + + async fn get_opts(&self, location: &Path, _options: GetOptions) -> ObjectStoreResult { + self.files.iter().find(|obj_meta| obj_meta.location == *location) + .map(|obj_meta| GetResult { + payload: GetResultPayload::Stream(Box::pin(futures::stream::once(async { Ok(Bytes::new()) }))), + meta: obj_meta.clone(), + range: 0..obj_meta.size, + attributes: Default::default(), + }) + .ok_or_else(|| object_store::Error::NotFound { + path: location.to_string(), + source: Box::new(std::io::Error::new(std::io::ErrorKind::NotFound, "Not found")), + }) + } + + fn delete_stream(&self, _locations: BoxStream<'static, ObjectStoreResult>) -> BoxStream<'static, ObjectStoreResult> { + todo!() + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { + let log_path = object_store_path(&self.log_root_url()); + // Be more permissive - return files if prefix is None or matches the log path + if prefix.is_none() || prefix == log_path.ok().as_ref() { + Box::pin(stream::iter(self.files.clone().into_iter().map(Ok))) + } else { + Box::pin(stream::empty()) + } + } + + async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> ObjectStoreResult { + unimplemented!("TestLogStore::list_with_delimiter not implemented for tests") + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _options: CopyOptions) -> ObjectStoreResult<()> { + todo!() + } + } + + fn obj_meta(path: impl Into, size: u64) -> ObjectMeta { + ObjectMeta { + location: path.into(), + size, + last_modified: "2025-07-18T15:30:00Z".parse().unwrap(), + e_tag: None, + version: None, + } + } + + fn parsed_log_path(path: impl Into, size: u64) -> ParsedLogPath { + let path = path.into(); + let dummy_url = Url::parse("memory://test/").unwrap(); + let file_url = dummy_url.join(path.as_ref()).unwrap(); + let parsed_url_path = ParsedLogPath::try_from(file_url).unwrap().unwrap(); + // Convert to FileMeta-based ParsedLogPath + let file_meta = delta_kernel::FileMeta { + location: parsed_url_path.location, + last_modified: 1752852600000, + size, + }; + ParsedLogPath::try_from(file_meta).unwrap().unwrap() + } + + #[tokio::test] + async fn test_fake_log_store() -> DeltaResult<()> { + let log_store = TestLogStore::new( + CommitRange(2..=97), CheckpointCadence(10), CommitFsize(128), CheckpointFsize(1024) + ); + + // before the first checkpoint + let segment = create_log_segment(&log_store, Some(5)).await?; + assert_segment_with_commits_only(&segment, 2..=5); + assert_eq!( + segment, + LogSegment { + end_version: 5, + listed: LogSegmentFiles { + ascending_commit_files: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000002.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000003.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000004.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000005.json", 128), + ], + checkpoint_parts: vec![], + ascending_compaction_files: vec![], + latest_crc_file: None, + latest_commit_file: Some(parsed_log_path("delta_table/_delta_log/00000000000000000005.json", 128)), + max_published_version: Some(5), + }, + log_root: log_store.log_root_url(), + checkpoint_version: None, + last_checkpoint_metadata: None, + }, + ); + + // with checkpoint + let segment = create_log_segment(&log_store, Some(32)).await?; + assert_segment_with_checkpoint(&segment, 30, 2); + assert_eq!( + segment, + LogSegment { + end_version: 32, + listed: LogSegmentFiles { + ascending_commit_files: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000031.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000032.json", 128), + ], + checkpoint_parts: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000030.checkpoint.parquet", 1024), + ], + ascending_compaction_files: vec![], + latest_crc_file: None, + latest_commit_file: Some(parsed_log_path("delta_table/_delta_log/00000000000000000032.json", 128)), + max_published_version: Some(32), + }, + log_root: log_store.log_root_url(), + checkpoint_version: Some(30), + last_checkpoint_metadata: None, + } + ); + + // latest version + let segment = create_log_segment(&log_store, None).await?; + assert_segment_with_checkpoint(&segment, 90, 7); + assert_eq!( + segment, + LogSegment { + end_version: 97, + listed: LogSegmentFiles { + ascending_commit_files: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000091.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000092.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000093.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000094.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000095.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000096.json", 128), + parsed_log_path("delta_table/_delta_log/00000000000000000097.json", 128), + ], + checkpoint_parts: vec![ + parsed_log_path("delta_table/_delta_log/00000000000000000090.checkpoint.parquet", 1024), + ], + ascending_compaction_files: vec![], + latest_crc_file: None, + latest_commit_file: Some(parsed_log_path("delta_table/_delta_log/00000000000000000097.json", 128),), + max_published_version: Some(97), + }, + log_root: log_store.log_root_url(), + checkpoint_version: Some(90), + last_checkpoint_metadata: None, + } + ); + + Ok(()) + } + } +} diff --git a/crates/core/src/kernel/snapshot/stats_projection.rs b/crates/core/src/kernel/snapshot/stats_projection.rs index 0093357537..02392828a5 100644 --- a/crates/core/src/kernel/snapshot/stats_projection.rs +++ b/crates/core/src/kernel/snapshot/stats_projection.rs @@ -117,6 +117,10 @@ impl FileStatsMaterialization { self.stats_source_policy } + pub(crate) fn raw_stats_policy(&self) -> RawStatsPolicy { + self.raw_stats_policy + } + pub(crate) fn preserves_raw_stats(&self) -> bool { self.raw_stats_policy == RawStatsPolicy::Preserve } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 5795e78bdd..77fd551bbc 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -87,6 +87,7 @@ pub(crate) mod parquet_utils; pub mod protocol; pub use kernel::schema; pub mod table; +pub mod hacks; /// Test fixtures and helpers (reference tables, env scoping, batch assertions) exposed for /// integration tests of this crate and downstream crates. diff --git a/crates/core/src/logstore/mod.rs b/crates/core/src/logstore/mod.rs index 327036a834..b1a2f351f6 100644 --- a/crates/core/src/logstore/mod.rs +++ b/crates/core/src/logstore/mod.rs @@ -605,7 +605,7 @@ pub(crate) fn get_engine(store: Arc) -> Arc { } #[cfg(feature = "datafusion")] -fn object_store_url(location: &Url) -> ObjectStoreUrl { +pub fn object_store_url(location: &Url) -> ObjectStoreUrl { use object_store::path::DELIMITER; // azure storage urls encode the container as user in the url diff --git a/crates/core/src/table/builder.rs b/crates/core/src/table/builder.rs index 1ecd405386..08581b969f 100644 --- a/crates/core/src/table/builder.rs +++ b/crates/core/src/table/builder.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use std::sync::Arc; use chrono::{DateTime, FixedOffset, Utc}; +use delta_kernel::Engine; use deltalake_derive::DeltaConfig; use object_store::DynObjectStore; use serde::{Deserialize, Serialize}; @@ -16,6 +17,7 @@ use crate::kernel::Version; use crate::logstore::storage::IORuntime; use crate::logstore::{LogStoreRef, StorageConfig, object_store_factories}; use crate::{DeltaResult, DeltaTable, DeltaTableError}; +use crate::kernel::size_limits::LogSizeLimiter; /// possible version specifications for loading a delta table #[derive(Debug, Copy, Clone, PartialEq, Eq, Default)] @@ -29,6 +31,16 @@ pub enum DeltaVersion { Timestamp(DateTime), } +/// Wrapper around `Arc` that implements `Debug` and `Clone`. +#[derive(Clone)] +pub struct EngineRef(pub Arc); + +impl std::fmt::Debug for EngineRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "EngineRef()") + } +} + /// Configuration options for delta table #[derive(Debug, Serialize, Deserialize, Clone, DeltaConfig)] #[serde(rename_all = "camelCase")] @@ -62,10 +74,25 @@ pub struct DeltaTableConfig { #[serde(default)] pub skip_stats: bool, + /// Allow parsed file statistics to fall back to JSON stats stored in checkpoints. + /// This defaults to `false`. + #[serde(default)] + #[delta(skip)] + pub checkpoint_stats_json_fallback: bool, + #[serde(skip_serializing, skip_deserializing)] #[delta(skip)] /// When a runtime handler is provided, all IO tasks are spawn in that handle pub io_runtime: Option, + + #[delta(skip)] + pub log_size_limiter: Option, + + /// Custom kernel engine to use for snapshot loading and scan operations. + /// When set, this engine is used instead of the default engine derived from the log store. + #[serde(skip_serializing, skip_deserializing)] + #[delta(skip)] + pub engine: Option, } impl Default for DeltaTableConfig { @@ -75,7 +102,10 @@ impl Default for DeltaTableConfig { log_buffer_size: num_cpus::get() * 4, log_batch_size: 1024, skip_stats: false, + checkpoint_stats_json_fallback: false, io_runtime: None, + log_size_limiter: None, + engine: None, } } } @@ -86,6 +116,8 @@ impl PartialEq for DeltaTableConfig { && self.log_buffer_size == other.log_buffer_size && self.log_batch_size == other.log_batch_size && self.skip_stats == other.skip_stats + && self.checkpoint_stats_json_fallback == other.checkpoint_stats_json_fallback + && self.log_size_limiter == other.log_size_limiter } } @@ -123,9 +155,10 @@ impl DeltaTableBuilder { })?; debug!("creating table builder with {table_url}"); + let actual_table_url = parse_table_uri(&table_url)?; Ok(Self { - table_url, + table_url: actual_table_url, storage_backend: None, version: DeltaVersion::default(), storage_options: None, @@ -147,6 +180,24 @@ impl DeltaTableBuilder { self } + /// Sets whether checkpoint stats parsing may fall back to JSON stats. + pub fn with_checkpoint_stats_json_fallback(mut self, enabled: bool) -> Self { + self.table_config.checkpoint_stats_json_fallback = enabled; + self + } + + /// Sets `log_size_limiter` to the builder + pub fn with_log_size_limiter(mut self, limiter: LogSizeLimiter) -> Self { + self.table_config.log_size_limiter = Some(limiter); + self + } + + /// Sets a custom kernel `Engine` to use for snapshot loading and scan operations. + pub fn with_engine(mut self, engine: Arc) -> Self { + self.table_config.engine = Some(EngineRef(engine)); + self + } + /// Sets `version` to the builder pub fn with_version(mut self, version: Version) -> Self { self.version = DeltaVersion::Version(version); @@ -213,6 +264,7 @@ impl DeltaTableBuilder { storage_options .clone() .into_iter() + .map(|(k, v)| (k.strip_prefix("deltalake.").map(ToString::to_string).unwrap_or(k), v)) .map(|(k, v)| { let needs_trim = v.starts_with("http://") || v.starts_with("https://") @@ -225,6 +277,12 @@ impl DeltaTableBuilder { }) .collect(), ); + let mut opts = self.storage_options.unwrap().clone(); + self.table_config.log_size_limiter = LogSizeLimiter::from_storage_options(&mut opts) + .expect("Invalid log_size_limiter options"); + + self.storage_options = Some(opts); + self } @@ -257,6 +315,7 @@ impl DeltaTableBuilder { /// Build a delta storage backend for the given config pub fn build_storage(&self) -> DeltaResult { debug!("build_storage() with {}", self.table_url); + let location = self.table_url.clone(); let mut storage_config = StorageConfig::parse_options(self.storage_options())?; if let Some(io_runtime) = self.table_config.io_runtime.clone() { @@ -265,14 +324,14 @@ impl DeltaTableBuilder { if let Some((store, _url)) = self.storage_backend.as_ref() { debug!("Loading a logstore with a custom store: {store:?}"); - crate::logstore::logstore_with(store.clone(), &self.table_url, storage_config) + crate::logstore::logstore_with(store.clone(), &location, storage_config) } else { // If there has been no backend defined just default to the normal logstore look up debug!( "Loading a logstore based off the location: {:?}", - self.table_url + location ); - crate::logstore::logstore_for(&self.table_url, storage_config) + crate::logstore::logstore_for(&location, storage_config) } } @@ -389,13 +448,24 @@ pub fn parse_table_uri(table_uri: impl AsRef) -> DeltaResult { let mut url = match uri_type { UriType::LocalPath(path) => { - let path = std::fs::canonicalize(&path).map_err(|err| { - let msg = format!( - "Local path \"{}\" does not exist or you don't have access!\nError: {err:?}", - path.display(), - ); - DeltaTableError::InvalidTableLocation(msg) - })?; + // let path = std::fs::canonicalize(&path).map_err(|err| { + // let msg = format!( + // "Local path \"{}\" does not exist or you don't have access!\nError: {err:?}", + // path.display(), + // ); + // DeltaTableError::InvalidTableLocation(msg) + // })?; + // Tolerate non-existent paths: callers like `DeltaTableBuilder::from_url` + // must work for tables that don't exist on disk yet. Existence is + // enforced separately by `open_table` (see `builder_from_valid_url`). + let path = match std::fs::canonicalize(&path) { + Ok(canonical) => canonical, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => path, + Err(err) => { + let msg = format!("Invalid table location: {table_uri}\nError: {err:?}"); + return Err(DeltaTableError::InvalidTableLocation(msg)); + } + }; Url::from_directory_path(path).map_err(|_| { let msg = format!( "Could not construct a URL from the canonical path: {table_uri}.\n\ @@ -571,6 +641,13 @@ mod tests { assert_eq!(expected.as_str(), url.as_str()); } + #[test] + fn test_invalid_uri() { + // Urls should round trips as-is + DeltaTableBuilder::from_url(Url::parse("this://is.nonsense").unwrap()) + .expect_err("this should be an error"); + } + #[test] fn test_writer_storage_opts_url_trim() { let cases = [ @@ -810,9 +887,12 @@ mod tests { ); let builder = builder_result.unwrap(); - assert_eq!( - builder.table_url.as_str(), - Url::from_directory_path(&new_path).unwrap().as_str() - ); + // `from_url` normalizes the path by trimming the trailing `/` so the + // URL is consistent across object-store registrations (see HSTACK + // commit "FIX table constructor"). Compare against the trimmed form. + let mut expected = Url::from_directory_path(&new_path).unwrap(); + let trimmed = expected.path().trim_end_matches('/').to_owned(); + expected.set_path(&trimmed); + assert_eq!(builder.table_url.as_str(), expected.as_str()); } } diff --git a/crates/core/src/table/mod.rs b/crates/core/src/table/mod.rs index c286661a59..508e891e45 100644 --- a/crates/core/src/table/mod.rs +++ b/crates/core/src/table/mod.rs @@ -17,6 +17,7 @@ use url::Url; use self::builder::DeltaTableConfig; use self::state::DeltaTableState; +use crate::kernel::size_limits::SnapshotLoadMetrics; use crate::kernel::{CommitInfo, DataCheck, LogicalFileView, Version}; use crate::logstore::{ LogStoreConfig, LogStoreExt, LogStoreRef, ObjectStoreRef, commit_uri_from_version, @@ -153,6 +154,19 @@ impl DeltaTable { } } + /// Update the current [`DeltaTable`] with an updated [`LogStore`] + /// + /// NOTE: This is for advanced users and allows swapping settings like AZURE_PROXY_URL + /// before passing the table to Datafusion, allowing for example to conditionally change or + /// remove proxy usage for metadata and data + pub fn with_new_store(&self, log_store: LogStoreRef) -> Self { + Self { + state: self.state.clone(), + log_store, + config: self.config.clone(), + } + } + /// get a shared reference to the delta object store pub fn object_store(&self) -> ObjectStoreRef { self.log_store.object_store(None) @@ -188,6 +202,18 @@ impl DeltaTable { self.state.as_ref().map(|s| s.version()) } + /// Returns the metrics captured during snapshot loading. + /// + /// This method provides access to information about how the snapshot was loaded, + /// including whether log size limiting was applied and if truncation occurred. + /// + /// ## Returns + /// + /// A reference to the snapshot load metrics if the table has been loaded, `None` otherwise. + pub fn snapshot_load_metrics(&self) -> Option<&SnapshotLoadMetrics> { + self.state.as_ref().map(|s| s.snapshot().load_metrics()) + } + /// Load DeltaTable with data from latest checkpoint pub async fn load(&mut self) -> Result<(), DeltaTableError> { self.update_incremental(None).await diff --git a/crates/core/src/test_utils/mod.rs b/crates/core/src/test_utils/mod.rs index 3ade2520da..8bb0f58924 100644 --- a/crates/core/src/test_utils/mod.rs +++ b/crates/core/src/test_utils/mod.rs @@ -129,6 +129,7 @@ pub enum TestTables { LatestNotCheckpointed, /// A small table that uses deletion vectors. WithDvSmall, + WithColumnMapping, /// A custom, caller-named table (path resolution is not provided). Custom(String), } @@ -148,6 +149,7 @@ impl TestTables { Self::Checkpoints => data_path.join("checkpoints"), Self::LatestNotCheckpointed => data_path.join("latest_not_checkpointed"), Self::WithDvSmall => data_path.join("table-with-dv-small"), + Self::WithColumnMapping => data_path.join("table_with_column_mapping"), // the data path for upload does not apply to custom tables. Self::Custom(_) => todo!(), } @@ -166,6 +168,7 @@ impl TestTables { Self::Checkpoints => "checkpoints".into(), Self::LatestNotCheckpointed => "latest_not_checkpointed".into(), Self::WithDvSmall => "table-with-dv-small".into(), + Self::WithColumnMapping => "table_with_column_mapping".into(), Self::Custom(name) => name.to_owned(), } } diff --git a/crates/core/tests/data/conversations/create_delta.py b/crates/core/tests/data/conversations/create_delta.py new file mode 100644 index 0000000000..d4b2be3369 --- /dev/null +++ b/crates/core/tests/data/conversations/create_delta.py @@ -0,0 +1,281 @@ +from datetime import date, timedelta +from pathlib import Path +import json +import shutil + +import pyarrow as pa +import pyarrow.parquet as pq +from deltalake import DeltaTable, write_deltalake + +TABLE_PATH = Path("delta") +START_DATE = date.fromisoformat("2024-01-01") +CHECKPOINT_DATE = date.fromisoformat("2024-01-04") # Use None for the final version. +DAYS = 7 +BATCHES_PER_DAY = 2 +FILES_PER_PARTITION = 2 +ROWS_PER_FILE = 2 + +SCHEMA = pa.schema( + [ + pa.field( + "_acp_system_metadata", + pa.struct( + [ + pa.field("acp_sourceBatchId", pa.string()), + pa.field("commitBatchId", pa.string()), + pa.field("trackingId", pa.string()), + pa.field("rowId", pa.string()), + pa.field("rowVersion", pa.int64()), + pa.field( + "primaryIdentity", + pa.struct( + [ + pa.field("id", pa.string()), + pa.field( + "namespace", + pa.struct([pa.field("code", pa.string())]), + ), + ] + ), + ), + pa.field("ingestTime", pa.int64()), + pa.field("isDeleted", pa.bool_()), + ] + ), + ), + pa.field( + "agenticExperience", + pa.struct( + [ + pa.field( + "agents", + pa.list_(pa.struct([pa.field("agentID", pa.string())])), + ), + pa.field("version", pa.string()), + ] + ), + ), + pa.field( + "conversation", + pa.struct( + [ + pa.field("conversationID", pa.string()), + pa.field("turnID", pa.string()), + pa.field( + "text", + pa.struct( + [ + pa.field("raw", pa.string()), + pa.field("source", pa.string()), + ] + ), + ), + ] + ), + ), + pa.field( + "identityMap", + pa.map_( + pa.string(), + pa.list_( + pa.struct( + [ + pa.field("id", pa.string()), + pa.field("prim", pa.bool_()), + ] + ) + ), + ), + ), + pa.field("_ACP_DATE", pa.date32()), + pa.field("_ACP_BATCHID", pa.string()), + ] +) + +shutil.rmtree(TABLE_PATH, ignore_errors=True) + +commit = 0 +for day_index in range(DAYS): + current_date = START_DATE + timedelta(days=day_index) + for batch_number in range(1, BATCHES_PER_DAY + 1): + batch_id = f"batch-{batch_number}-{current_date.isoformat()}" + file_batches = [] + + for file_number in range(1, FILES_PER_PARTITION + 1): + file_batch_id = f"{batch_id}--file-{file_number}" + rows = [] + for row_number in range(1, ROWS_PER_FILE + 1): + row_key = ( + f"{current_date.isoformat()}-batch-{batch_number}" + f"-file-{file_number}-row-{row_number}" + ) + identity = f"user-{row_key}@example.com" + rows.append( + { + "_acp_system_metadata": { + "acp_sourceBatchId": batch_id, + "commitBatchId": f"commit-{batch_id}", + "trackingId": f"tracking-{row_key}", + "rowId": f"row-{row_key}", + "rowVersion": 1, + "primaryIdentity": { + "id": identity, + "namespace": {"code": "email"}, + }, + "ingestTime": int( + f"{current_date:%Y%m%d}" + f"{batch_number}{file_number}{row_number}" + ), + "isDeleted": False, + }, + "agenticExperience": { + "agents": [{"agentID": f"agent-{batch_number}"}], + "version": "1.0", + }, + "conversation": { + "conversationID": f"conversation-{current_date}-{file_number}", + "turnID": f"turn-{batch_number}-{file_number}-{row_number}", + "text": { + "raw": f"Conversation text for {row_key}", + "source": "generated", + }, + }, + "identityMap": [ + ("email", [{"id": identity, "prim": True}]) + ], + "_ACP_DATE": current_date, + # A temporary partition forces delta-rs to create this file. + "_ACP_BATCHID": file_batch_id, + } + ) + + file_batches.append(pa.RecordBatch.from_pylist(rows, schema=SCHEMA)) + + write_deltalake( + TABLE_PATH, + pa.RecordBatchReader.from_batches(SCHEMA, file_batches), + mode="overwrite" if commit == 0 else "append", + partition_by=["_ACP_DATE", "_ACP_BATCHID"], + configuration=( + { + "delta.checkpoint.writeStatsAsJson": "true", + "delta.checkpoint.writeStatsAsStruct": "true", + } + if commit == 0 + else None + ), + ) + + # Move both files into the real partition and normalize their add actions. + transaction_path = TABLE_PATH / "_delta_log" / f"{commit:020}.json" + actions = [json.loads(line) for line in transaction_path.read_text().splitlines()] + add_actions = [action["add"] for action in actions if "add" in action] + if len(add_actions) != FILES_PER_PARTITION: + raise RuntimeError( + f"Expected {FILES_PER_PARTITION} files, got {len(add_actions)}" + ) + + destination_directory = ( + TABLE_PATH + / f"_ACP_DATE={current_date.isoformat()}" + / f"_ACP_BATCHID={batch_id}" + ) + destination_directory.mkdir(parents=True, exist_ok=True) + for add in add_actions: + source = TABLE_PATH / add["path"] + destination = destination_directory / source.name + source.rename(destination) + source.parent.rmdir() + add["path"] = destination.relative_to(TABLE_PATH).as_posix() + add["partitionValues"]["_ACP_BATCHID"] = batch_id + + transaction_path.write_text( + "\n".join(json.dumps(action, separators=(",", ":")) for action in actions) + + "\n" + ) + commit += 1 + + if CHECKPOINT_DATE == current_date: + DeltaTable(TABLE_PATH).create_checkpoint() + +if CHECKPOINT_DATE is None: + DeltaTable(TABLE_PATH).create_checkpoint() + +# delta-rs does not currently materialize these configured checkpoint fields. +checkpoint_path = next((TABLE_PATH / "_delta_log").glob("*.checkpoint.parquet")) +checkpoint = pq.read_table(checkpoint_path) +add_index = checkpoint.schema.get_field_index("add") +add_field = checkpoint.schema.field(add_index) +adds = checkpoint.column(add_index).to_pylist() + +stats = [ + json.loads(add["stats"]) if add and add.get("stats") else None + for add in adds +] +for parsed_stats in stats: + if parsed_stats is not None: + parsed_stats["tightBounds"] = True + +inferred_stats_type = pa.array(stats).type +stats_field_order = [ + "numRecords", + "minValues", + "maxValues", + "nullCount", + "tightBounds", +] +stats_type = pa.struct( + [ + inferred_stats_type.field(name) + for name in stats_field_order + if inferred_stats_type.get_field_index(name) != -1 + ] +) +partition_values_type = pa.struct( + [ + pa.field("_ACP_DATE", pa.date32()), + pa.field("_ACP_BATCHID", pa.string()), + ] +) + +for add, parsed_stats in zip(adds, stats): + if add is None: + continue + partition_values = dict(add["partitionValues"]) + add["stats_parsed"] = parsed_stats + add["partitionValues_parsed"] = { + "_ACP_DATE": date.fromisoformat(partition_values["_ACP_DATE"]), + "_ACP_BATCHID": partition_values["_ACP_BATCHID"], + } + +original_add_type = add_field.type +patched_add_type = pa.struct( + [ + field + for field in original_add_type + if field.name not in {"stats_parsed", "partitionValues_parsed"} + ] + + [ + pa.field("stats_parsed", stats_type), + pa.field("partitionValues_parsed", partition_values_type), + ] +) +patched_add_field = pa.field( + "add", + patched_add_type, + nullable=add_field.nullable, + metadata=add_field.metadata, +) +checkpoint = checkpoint.set_column( + add_index, + patched_add_field, + pa.array(adds, type=patched_add_type), +) +pq.write_table(checkpoint, checkpoint_path, compression="snappy") + +last_checkpoint_path = TABLE_PATH / "_delta_log" / "_last_checkpoint" +last_checkpoint = json.loads(last_checkpoint_path.read_text()) +last_checkpoint["sizeInBytes"] = checkpoint_path.stat().st_size +last_checkpoint_path.write_text(json.dumps(last_checkpoint, separators=(",", ":"))) + +print(f"Created {TABLE_PATH} with {commit} versions and a patched checkpoint") diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-5af5ddd7-48ea-413c-94b6-5625d1dfdfba-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-5af5ddd7-48ea-413c-94b6-5625d1dfdfba-c000.snappy.parquet new file mode 100644 index 0000000000..f449af9924 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-5af5ddd7-48ea-413c-94b6-5625d1dfdfba-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-68246fa2-b36c-4f20-8b24-9fa3894dfcb6-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-68246fa2-b36c-4f20-8b24-9fa3894dfcb6-c000.snappy.parquet new file mode 100644 index 0000000000..5c2b6adcea Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-68246fa2-b36c-4f20-8b24-9fa3894dfcb6-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-66777adb-10ca-4f3d-a838-3254776dad75-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-66777adb-10ca-4f3d-a838-3254776dad75-c000.snappy.parquet new file mode 100644 index 0000000000..1d840d0c8e Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-66777adb-10ca-4f3d-a838-3254776dad75-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-e5cb40ab-fba4-401d-b9d7-9f063567174a-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-e5cb40ab-fba4-401d-b9d7-9f063567174a-c000.snappy.parquet new file mode 100644 index 0000000000..36483e4750 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-e5cb40ab-fba4-401d-b9d7-9f063567174a-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-d4320b85-10f7-454a-acdf-f9a8496b4f70-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-d4320b85-10f7-454a-acdf-f9a8496b4f70-c000.snappy.parquet new file mode 100644 index 0000000000..e68796950c Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-d4320b85-10f7-454a-acdf-f9a8496b4f70-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f442863b-8e12-49ae-8b4d-3df43530c506-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f442863b-8e12-49ae-8b4d-3df43530c506-c000.snappy.parquet new file mode 100644 index 0000000000..f63e35f632 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f442863b-8e12-49ae-8b4d-3df43530c506-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-82e0525d-9555-4568-8215-0e3f5881f549-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-82e0525d-9555-4568-8215-0e3f5881f549-c000.snappy.parquet new file mode 100644 index 0000000000..9f0aa51380 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-82e0525d-9555-4568-8215-0e3f5881f549-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-8ce2f2b0-a7c6-4e8a-ab1c-d7c71928a942-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-8ce2f2b0-a7c6-4e8a-ab1c-d7c71928a942-c000.snappy.parquet new file mode 100644 index 0000000000..6055c50dfd Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-8ce2f2b0-a7c6-4e8a-ab1c-d7c71928a942-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1b24d95a-1de7-4ac9-8af1-cb19b66c0edb-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1b24d95a-1de7-4ac9-8af1-cb19b66c0edb-c000.snappy.parquet new file mode 100644 index 0000000000..abc87527a2 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1b24d95a-1de7-4ac9-8af1-cb19b66c0edb-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-81dd5d9f-3e71-4aea-a46c-86905984e372-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-81dd5d9f-3e71-4aea-a46c-86905984e372-c000.snappy.parquet new file mode 100644 index 0000000000..5c6615a083 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-81dd5d9f-3e71-4aea-a46c-86905984e372-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-95c2849a-4d95-4a71-b1bb-8a894fa5d7dd-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-95c2849a-4d95-4a71-b1bb-8a894fa5d7dd-c000.snappy.parquet new file mode 100644 index 0000000000..a67c066bf0 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-95c2849a-4d95-4a71-b1bb-8a894fa5d7dd-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-c4450a76-c10c-46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-c4450a76-c10c-46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet new file mode 100644 index 0000000000..38a981f2ef Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-c4450a76-c10c-46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-801b3b87-b9d3-4796-a4cb-323deb17569b-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-801b3b87-b9d3-4796-a4cb-323deb17569b-c000.snappy.parquet new file mode 100644 index 0000000000..e585a0621f Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-801b3b87-b9d3-4796-a4cb-323deb17569b-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-bfc1b106-122a-47b8-8e47-3a70c6a54b19-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-bfc1b106-122a-47b8-8e47-3a70c6a54b19-c000.snappy.parquet new file mode 100644 index 0000000000..23ec1a561f Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-bfc1b106-122a-47b8-8e47-3a70c6a54b19-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet new file mode 100644 index 0000000000..894ff640e2 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-59765f2d-40bb-4972-a250-b641daa5a6a7-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-59765f2d-40bb-4972-a250-b641daa5a6a7-c000.snappy.parquet new file mode 100644 index 0000000000..03399d10c5 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-59765f2d-40bb-4972-a250-b641daa5a6a7-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-06c535fb-4c81-4033-bedc-026d5b184b78-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-06c535fb-4c81-4033-bedc-026d5b184b78-c000.snappy.parquet new file mode 100644 index 0000000000..9586fca7a7 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-06c535fb-4c81-4033-bedc-026d5b184b78-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-2ffd1456-206d-4884-a7e2-87e358e8e103-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-2ffd1456-206d-4884-a7e2-87e358e8e103-c000.snappy.parquet new file mode 100644 index 0000000000..5fde4fd54a Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-2ffd1456-206d-4884-a7e2-87e358e8e103-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-b95c7946-cd89-4a7d-aad6-8664717be548-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-b95c7946-cd89-4a7d-aad6-8664717be548-c000.snappy.parquet new file mode 100644 index 0000000000..89386ccd02 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-b95c7946-cd89-4a7d-aad6-8664717be548-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet new file mode 100644 index 0000000000..9227e2a1b4 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet new file mode 100644 index 0000000000..d80a4dfdea Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet new file mode 100644 index 0000000000..aafb639f06 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-6c8cf883-cdf4-48ca-907c-fa042d2e83fa-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-6c8cf883-cdf4-48ca-907c-fa042d2e83fa-c000.snappy.parquet new file mode 100644 index 0000000000..7b3db24b09 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-6c8cf883-cdf4-48ca-907c-fa042d2e83fa-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-e6b6c2fb-201a-45b7-a88a-3662756e970b-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-e6b6c2fb-201a-45b7-a88a-3662756e970b-c000.snappy.parquet new file mode 100644 index 0000000000..b695a7486c Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-e6b6c2fb-201a-45b7-a88a-3662756e970b-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-01aa1a67-ddba-489c-bcaf-6406017afe40-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-01aa1a67-ddba-489c-bcaf-6406017afe40-c000.snappy.parquet new file mode 100644 index 0000000000..8f0ae271cf Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-01aa1a67-ddba-489c-bcaf-6406017afe40-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet new file mode 100644 index 0000000000..18e63d0bef Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet new file mode 100644 index 0000000000..d9cb622560 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet new file mode 100644 index 0000000000..9c2f12f2a3 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000000.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..2d87f37774 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1786523126154,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Overwrite"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":54,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"81326f7e-b8b8-4f94-806e-7deb802a0ebe","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_acp_system_metadata\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"acp_sourceBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"commitBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trackingId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowVersion\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"primaryIdentity\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"namespace\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"code\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"ingestTime\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"agenticExperience\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"agents\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"agentID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"version\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"conversation\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"conversationID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"turnID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"text\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"raw\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"source\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"identityMap\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"prim\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_DATE\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCHID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_ACP_DATE","_ACP_BATCHID"],"createdTime":1786523126103,"configuration":{"delta.checkpoint.writeStatsAsStruct":"true","delta.checkpoint.writeStatsAsJson":"true"}}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-5af5ddd7-48ea-413c-94b6-5625d1dfdfba-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-01","_ACP_DATE":"2024-01-01"},"size":8027,"modificationTime":1786523126154,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-2-row-1\"},\"turnID\":\"turn-1-2-1\"},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowId\":\"row-2024-01-01-batch-1-file-2-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-01-batch-1-file-2-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"ingestTime\":20240101121}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-1-file-2-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-1-2-2\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-1-file-2-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-1-file-2-row-2@example.com\"},\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-1-file-2-row-2\",\"ingestTime\":20240101122},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"turnID\":0,\"conversationID\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"rowId\":0,\"isDeleted\":0,\"rowVersion\":0,\"ingestTime\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"commitBatchId\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-68246fa2-b36c-4f20-8b24-9fa3894dfcb6-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-01","_ACP_BATCHID":"batch-1-2024-01-01"},"size":8026,"modificationTime":1786523126154,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-01-batch-1-file-1-row-1\",\"ingestTime\":20240101111,\"rowId\":\"row-2024-01-01-batch-1-file-1-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-01-1\",\"turnID\":\"turn-1-1-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-1-row-2\"},\"conversationID\":\"conversation-2024-01-01-1\",\"turnID\":\"turn-1-1-2\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-1-file-1-row-2\",\"rowVersion\":1,\"ingestTime\":20240101112,\"rowId\":\"row-2024-01-01-batch-1-file-1-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0}},\"_acp_system_metadata\":{\"trackingId\":0,\"commitBatchId\":0,\"rowId\":0,\"acp_sourceBatchId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"isDeleted\":0,\"ingestTime\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000001.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000001.json new file mode 100644 index 0000000000..bbd2aca092 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126173,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":8,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-66777adb-10ca-4f3d-a838-3254776dad75-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-01","_ACP_DATE":"2024-01-01"},"size":8054,"modificationTime":1786523126173,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"isDeleted\":false,\"ingestTime\":20240101211,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-2-file-1-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-01-batch-2-file-1-row-1\",\"rowVersion\":1,\"rowId\":\"row-2024-01-01-batch-2-file-1-row-1\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-01-1\",\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-2-file-1-row-1\",\"source\":\"generated\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-1-row-2\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-2-file-1-row-2@example.com\"},\"ingestTime\":20240101212,\"trackingId\":\"tracking-2024-01-01-batch-2-file-1-row-2\"},\"conversation\":{\"turnID\":\"turn-2-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-2-file-1-row-2\"},\"conversationID\":\"conversation-2024-01-01-1\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}},\"_acp_system_metadata\":{\"trackingId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"isDeleted\":0,\"rowVersion\":0,\"rowId\":0,\"ingestTime\":0,\"acp_sourceBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-e5cb40ab-fba4-401d-b9d7-9f063567174a-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-01","_ACP_BATCHID":"batch-2-2024-01-01"},"size":8054,"modificationTime":1786523126173,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-2-2-1\",\"conversationID\":\"conversation-2024-01-01-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-2-file-2-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-01-batch-2-file-2-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"rowVersion\":1,\"ingestTime\":20240101221,\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-01-batch-2-file-2-row-2\",\"ingestTime\":20240101222,\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"rowVersion\":1,\"rowId\":\"row-2024-01-01-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-01\"},\"conversation\":{\"turnID\":\"turn-2-2-2\",\"conversationID\":\"conversation-2024-01-01-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-2-file-2-row-2\",\"source\":\"generated\"}}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"isDeleted\":0,\"commitBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000002.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000002.json new file mode 100644 index 0000000000..1ce865fcbf --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126181,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-d4320b85-10f7-454a-acdf-f9a8496b4f70-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-1-2024-01-02"},"size":8053,"modificationTime":1786523126181,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"rowId\":\"row-2024-01-02-batch-1-file-2-row-1\",\"ingestTime\":20240102121,\"trackingId\":\"tracking-2024-01-02-batch-1-file-2-row-1\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-2\",\"turnID\":\"turn-1-2-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-02-batch-1-file-2-row-2\",\"ingestTime\":20240102122,\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"trackingId\":\"tracking-2024-01-02-batch-1-file-2-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-1-2-2\",\"conversationID\":\"conversation-2024-01-02-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-2-row-2\",\"source\":\"generated\"}}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"rowVersion\":0,\"isDeleted\":0,\"ingestTime\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f442863b-8e12-49ae-8b4d-3df43530c506-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-1-2024-01-02"},"size":8052,"modificationTime":1786523126181,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-02-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-1-row-1\",\"source\":\"generated\"}},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-02-batch-1-file-1-row-1\",\"rowId\":\"row-2024-01-02-batch-1-file-1-row-1\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240102111,\"commitBatchId\":\"commit-batch-1-2024-01-02\"}},\"maxValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"isDeleted\":false,\"ingestTime\":20240102112,\"trackingId\":\"tracking-2024-01-02-batch-1-file-1-row-2\",\"rowId\":\"row-2024-01-02-batch-1-file-1-row-2\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}}},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-02-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-1-row-2\",\"source\":\"generated\"}}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0},\"_acp_system_metadata\":{\"ingestTime\":0,\"rowVersion\":0,\"trackingId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowId\":0,\"commitBatchId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000003.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000003.json new file mode 100644 index 0000000000..abb1f46496 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126185,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-8ce2f2b0-a7c6-4e8a-ab1c-d7c71928a942-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-02","_ACP_DATE":"2024-01-02"},"size":8053,"modificationTime":1786523126185,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"isDeleted\":false,\"rowVersion\":1,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-02-batch-2-file-1-row-1@example.com\"},\"ingestTime\":20240102211,\"trackingId\":\"tracking-2024-01-02-batch-2-file-1-row-1\",\"rowId\":\"row-2024-01-02-batch-2-file-1-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"commitBatchId\":\"commit-batch-2-2024-01-02\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"conversationID\":\"conversation-2024-01-02-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-1-row-1\",\"source\":\"generated\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-1-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-1\",\"turnID\":\"turn-2-1-2\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-02-batch-2-file-1-row-2\",\"ingestTime\":20240102212,\"isDeleted\":false,\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"trackingId\":\"tracking-2024-01-02-batch-2-file-1-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}}}},\"nullCount\":{\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0},\"_acp_system_metadata\":{\"isDeleted\":0,\"rowId\":0,\"commitBatchId\":0,\"ingestTime\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"rowVersion\":0,\"acp_sourceBatchId\":0,\"trackingId\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-82e0525d-9555-4568-8215-0e3f5881f549-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-02","_ACP_DATE":"2024-01-02"},"size":8053,"modificationTime":1786523126185,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-2\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-02-batch-2-file-2-row-1\",\"trackingId\":\"tracking-2024-01-02-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"rowVersion\":1,\"ingestTime\":20240102221,\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-02-batch-2-file-2-row-1@example.com\"}}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-02-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-2-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-2-2-2\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"ingestTime\":20240102222,\"trackingId\":\"tracking-2024-01-02-batch-2-file-2-row-2\",\"rowId\":\"row-2024-01-02-batch-2-file-2-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"rowVersion\":1,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-02-batch-2-file-2-row-2@example.com\"},\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"rowVersion\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}}},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000004.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000004.json new file mode 100644 index 0000000000..97dc5a715f --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000004.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126190,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1b24d95a-1de7-4ac9-8af1-cb19b66c0edb-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-03","_ACP_DATE":"2024-01-03"},"size":8052,"modificationTime":1786523126190,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"ingestTime\":20240103111,\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"trackingId\":\"tracking-2024-01-03-batch-1-file-1-row-1\",\"isDeleted\":false,\"rowId\":\"row-2024-01-03-batch-1-file-1-row-1\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-1-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\",\"turnID\":\"turn-1-1-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"ingestTime\":20240103112,\"trackingId\":\"tracking-2024-01-03-batch-1-file-1-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"rowId\":\"row-2024-01-03-batch-1-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-03-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-1-file-1-row-2\"}}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"commitBatchId\":0,\"trackingId\":0,\"rowId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-81dd5d9f-3e71-4aea-a46c-86905984e372-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-03","_ACP_DATE":"2024-01-03"},"size":8053,"modificationTime":1786523126190,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-03-batch-1-file-2-row-1\",\"ingestTime\":20240103121,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-1-file-2-row-1@example.com\"},\"rowId\":\"row-2024-01-03-batch-1-file-2-row-1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-03\"},\"conversation\":{\"turnID\":\"turn-1-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-1-file-2-row-1\"},\"conversationID\":\"conversation-2024-01-03-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"ingestTime\":20240103122,\"trackingId\":\"tracking-2024-01-03-batch-1-file-2-row-2\",\"rowId\":\"row-2024-01-03-batch-1-file-2-row-2\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-03\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-1-file-2-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-1-2-2\",\"conversationID\":\"conversation-2024-01-03-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}},\"_acp_system_metadata\":{\"rowId\":0,\"ingestTime\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"isDeleted\":0,\"acp_sourceBatchId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"trackingId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000005.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000005.json new file mode 100644 index 0000000000..375aecd74c --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000005.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126195,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-c4450a76-c10c-46ff-ad4d-1bcd4216d7d5-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-2-2024-01-03"},"size":8053,"modificationTime":1786523126195,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"ingestTime\":20240103221,\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"rowId\":\"row-2024-01-03-batch-2-file-2-row-1\",\"isDeleted\":false,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-03-batch-2-file-2-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-2-2-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-03-batch-2-file-2-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"rowId\":\"row-2024-01-03-batch-2-file-2-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"rowVersion\":1,\"isDeleted\":false,\"ingestTime\":20240103222},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-2-file-2-row-2\"},\"turnID\":\"turn-2-2-2\",\"conversationID\":\"conversation-2024-01-03-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0},\"_acp_system_metadata\":{\"commitBatchId\":0,\"rowId\":0,\"rowVersion\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0,\"trackingId\":0,\"isDeleted\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-95c2849a-4d95-4a71-b1bb-8a894fa5d7dd-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-2-2024-01-03"},"size":8053,"modificationTime":1786523126195,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"ingestTime\":20240103211,\"trackingId\":\"tracking-2024-01-03-batch-2-file-1-row-1\",\"rowVersion\":1,\"rowId\":\"row-2024-01-03-batch-2-file-1-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"isDeleted\":false}},\"maxValues\":{\"_acp_system_metadata\":{\"isDeleted\":false,\"ingestTime\":20240103212,\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-03-batch-2-file-1-row-2\",\"trackingId\":\"tracking-2024-01-03-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"rowVersion\":1},\"conversation\":{\"conversationID\":\"conversation-2024-01-03-1\",\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-2-file-1-row-2\",\"source\":\"generated\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"ingestTime\":0,\"rowVersion\":0,\"acp_sourceBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowId\":0,\"commitBatchId\":0,\"trackingId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0},\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000006.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000006.json new file mode 100644 index 0000000000..f1b957b9f1 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000006.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126200,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-bfc1b106-122a-47b8-8e47-3a70c6a54b19-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-04","_ACP_DATE":"2024-01-04"},"size":8053,"modificationTime":1786523126200,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-1-2-1\"},\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-04-batch-1-file-2-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-1-file-2-row-1\",\"rowVersion\":1,\"ingestTime\":20240104121,\"isDeleted\":false}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-2-row-2\"},\"turnID\":\"turn-1-2-2\",\"conversationID\":\"conversation-2024-01-04-2\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowId\":\"row-2024-01-04-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"ingestTime\":20240104122,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-1-file-2-row-2@example.com\"},\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-04-batch-1-file-2-row-2\"}},\"nullCount\":{\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0},\"_acp_system_metadata\":{\"rowVersion\":0,\"ingestTime\":0,\"commitBatchId\":0,\"trackingId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-801b3b87-b9d3-4796-a4cb-323deb17569b-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-04","_ACP_DATE":"2024-01-04"},"size":8052,"modificationTime":1786523126200,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-04-batch-1-file-1-row-1\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"rowId\":\"row-2024-01-04-batch-1-file-1-row-1\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"ingestTime\":20240104111},\"conversation\":{\"conversationID\":\"conversation-2024-01-04-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-1-row-1\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-1-file-1-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-1-1-2\",\"conversationID\":\"conversation-2024-01-04-1\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-04-batch-1-file-1-row-2\",\"ingestTime\":20240104112,\"rowId\":\"row-2024-01-04-batch-1-file-1-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"isDeleted\":false,\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}}}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"ingestTime\":0,\"isDeleted\":0,\"rowId\":0,\"commitBatchId\":0,\"acp_sourceBatchId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"trackingId\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.checkpoint.parquet b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.checkpoint.parquet new file mode 100644 index 0000000000..004f188b65 Binary files /dev/null and b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.checkpoint.parquet differ diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.json new file mode 100644 index 0000000000..6261840020 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000007.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126205,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-32361681-d3e5-43d5-b09e-3317e6b53514-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-2-2024-01-04"},"size":8053,"modificationTime":1786523126205,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-04-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-1\"},\"turnID\":\"turn-2-2-1\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-04-batch-2-file-2-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-1\",\"isDeleted\":false,\"ingestTime\":20240104221,\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-2-2-2\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-2\",\"ingestTime\":20240104222,\"rowId\":\"row-2024-01-04-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"isDeleted\":false,\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"isDeleted\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"trackingId\":0,\"commitBatchId\":0,\"rowVersion\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-59765f2d-40bb-4972-a250-b641daa5a6a7-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-04","_ACP_DATE":"2024-01-04"},"size":8053,"modificationTime":1786523126205,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240104211,\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"rowId\":\"row-2024-01-04-batch-2-file-1-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-04\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-04-1\",\"turnID\":\"turn-2-1-1\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-04-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-2\"},\"turnID\":\"turn-2-1-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"rowVersion\":1,\"rowId\":\"row-2024-01-04-batch-2-file-1-row-2\",\"isDeleted\":false,\"ingestTime\":20240104212,\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-1-row-2@example.com\"}}},\"nullCount\":{\"_acp_system_metadata\":{\"commitBatchId\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowVersion\":0,\"ingestTime\":0,\"rowId\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000008.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000008.json new file mode 100644 index 0000000000..272d2a2698 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000008.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126223,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-2ffd1456-206d-4884-a7e2-87e358e8e103-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-1-2024-01-05"},"size":8052,"modificationTime":1786523126223,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-1-row-1@example.com\"},\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-1\",\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-1-row-1\",\"ingestTime\":20240105111},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"rowVersion\":1,\"ingestTime\":20240105112,\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-1-1-2\",\"conversationID\":\"conversation-2024-01-05-1\"}},\"nullCount\":{\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"ingestTime\":0,\"rowId\":0,\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-06c535fb-4c81-4033-bedc-026d5b184b78-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-05","_ACP_DATE":"2024-01-05"},"size":8053,"modificationTime":1786523126223,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"rowId\":\"row-2024-01-05-batch-1-file-2-row-1\",\"rowVersion\":1,\"ingestTime\":20240105121,\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-1\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-1-2-1\",\"conversationID\":\"conversation-2024-01-05-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-1-2-2\",\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-2\",\"ingestTime\":20240105122,\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-05-batch-1-file-2-row-2\",\"isDeleted\":false,\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-05\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0,\"trackingId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000009.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000009.json new file mode 100644 index 0000000000..82174c9b78 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000009.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126230,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-d3cd7fe1-9960-472a-8d3f-2890d998f099-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-05","_ACP_DATE":"2024-01-05"},"size":8053,"modificationTime":1786523126230,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-2-1-1\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-05-batch-2-file-1-row-1\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"rowVersion\":1,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"ingestTime\":20240105211,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-1-row-1@example.com\"}}},\"maxValues\":{\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-2\"},\"turnID\":\"turn-2-1-2\",\"conversationID\":\"conversation-2024-01-05-1\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"ingestTime\":20240105212,\"rowId\":\"row-2024-01-05-batch-2-file-1-row-2\",\"isDeleted\":false,\"rowVersion\":1,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-1-row-2@example.com\"},\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowId\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"isDeleted\":0,\"ingestTime\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-b95c7946-cd89-4a7d-aad6-8664717be548-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-2-2024-01-05"},"size":8053,"modificationTime":1786523126230,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-1\"},\"turnID\":\"turn-2-2-1\"},\"_acp_system_metadata\":{\"ingestTime\":20240105221,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-2-row-1@example.com\"},\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-1\",\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-2-file-2-row-1\",\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-2\",\"turnID\":\"turn-2-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"isDeleted\":false,\"rowVersion\":1,\"ingestTime\":20240105222,\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-2\",\"rowId\":\"row-2024-01-05-batch-2-file-2-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-2-row-2@example.com\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000010.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000010.json new file mode 100644 index 0000000000..7de9d78f90 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000010.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126237,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-93d5962a-0908-4ee7-8cfe-2925fa05fc9c-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-06","_ACP_DATE":"2024-01-06"},"size":8053,"modificationTime":1786523126237,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"ingestTime\":20240106121,\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-2-row-1@example.com\"},\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-1-file-2-row-1\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-1-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-1\"}}},\"maxValues\":{\"_acp_system_metadata\":{\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-1-file-2-row-2\",\"ingestTime\":20240106122,\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-2\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-2-row-2@example.com\"}},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-1-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-2\",\"source\":\"generated\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"commitBatchId\":0,\"isDeleted\":0,\"ingestTime\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-f5fa5ab0-7bbe-4340-807a-719ec6048133-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-1-2024-01-06"},"size":8052,"modificationTime":1786523126237,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-1\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"ingestTime\":20240106111,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-1-file-1-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-06-batch-1-file-1-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240106112,\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-2\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-2\",\"source\":\"generated\"}}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowVersion\":0,\"isDeleted\":0,\"rowId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"commitBatchId\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"ingestTime\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000011.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000011.json new file mode 100644 index 0000000000..0041d5186e --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000011.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126244,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-e6b6c2fb-201a-45b7-a88a-3662756e970b-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-2-2024-01-06"},"size":8053,"modificationTime":1786523126244,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-06-batch-2-file-2-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-1\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-2-row-1@example.com\"},\"rowVersion\":1,\"ingestTime\":20240106221,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"commitBatchId\":\"commit-batch-2-2024-01-06\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-1\"},\"turnID\":\"turn-2-2-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-2\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"ingestTime\":20240106222,\"rowVersion\":1,\"rowId\":\"row-2024-01-06-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-06\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-2-2-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"isDeleted\":0,\"rowId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"ingestTime\":0,\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-6c8cf883-cdf4-48ca-907c-fa042d2e83fa-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-06","_ACP_DATE":"2024-01-06"},"size":8053,"modificationTime":1786523126244,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-1-row-1@example.com\"},\"ingestTime\":20240106211,\"isDeleted\":false,\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"rowId\":\"row-2024-01-06-batch-2-file-1-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-06\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-2-1-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240106212,\"rowId\":\"row-2024-01-06-batch-2-file-1-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-2\",\"isDeleted\":false}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"rowVersion\":0,\"ingestTime\":0,\"trackingId\":0,\"rowId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000012.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000012.json new file mode 100644 index 0000000000..5d01a55367 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000012.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126251,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-01aa1a67-ddba-489c-bcaf-6406017afe40-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-07","_ACP_DATE":"2024-01-07"},"size":8052,"modificationTime":1786523126251,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-1\",\"rowVersion\":1,\"ingestTime\":20240107111,\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"rowId\":\"row-2024-01-07-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"ingestTime\":20240107112,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-2\",\"rowId\":\"row-2024-01-07-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0,\"rowId\":0,\"isDeleted\":0,\"trackingId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"rowVersion\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-51a52697-5a0e-4be9-b318-4b06a02001fd-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-1-2024-01-07"},"size":8053,"modificationTime":1786523126251,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"rowId\":\"row-2024-01-07-batch-1-file-2-row-1\",\"rowVersion\":1,\"isDeleted\":false,\"ingestTime\":20240107121,\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-07\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-1-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-1\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"rowId\":\"row-2024-01-07-batch-1-file-2-row-2\",\"rowVersion\":1,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"ingestTime\":20240107122,\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}}},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-1-2-2\"}},\"nullCount\":{\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"isDeleted\":0,\"rowId\":0,\"trackingId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"ingestTime\":0,\"rowVersion\":0,\"acp_sourceBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000013.json b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000013.json new file mode 100644 index 0000000000..b350d85218 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/00000000000000000013.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1786523126258,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-dfbabb74-56c8-4b72-a482-566f29c82b5c-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-07","_ACP_DATE":"2024-01-07"},"size":8053,"modificationTime":1786523126258,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowId\":\"row-2024-01-07-batch-2-file-2-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-2-row-1@example.com\"},\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"rowVersion\":1,\"ingestTime\":20240107221},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-1\",\"source\":\"generated\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-2-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-2\",\"ingestTime\":20240107222,\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-2-row-2@example.com\"},\"rowVersion\":1,\"rowId\":\"row-2024-01-07-batch-2-file-2-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-07\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"acp_sourceBatchId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"trackingId\":0,\"rowId\":0,\"rowVersion\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-3db9d197-f732-45a7-ae4d-7d5ff915786e-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-2-2024-01-07"},"size":8053,"modificationTime":1786523126258,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"ingestTime\":20240107211,\"isDeleted\":false,\"rowVersion\":1,\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-1-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-1\",\"rowId\":\"row-2024-01-07-batch-2-file-1-row-1\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-07-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-2-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-2\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-1-row-2@example.com\"},\"rowId\":\"row-2024-01-07-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"rowVersion\":1,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-2\",\"ingestTime\":20240107212},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowVersion\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"ingestTime\":0,\"commitBatchId\":0,\"rowId\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0},\"turnID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations/delta/_delta_log/_last_checkpoint b/crates/core/tests/data/conversations/delta/_delta_log/_last_checkpoint new file mode 100644 index 0000000000..68e3317f34 --- /dev/null +++ b/crates/core/tests/data/conversations/delta/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":7,"size":18,"sizeInBytes":58489,"numOfAddFiles":16} \ No newline at end of file diff --git a/crates/core/tests/data/conversations/schema.sql b/crates/core/tests/data/conversations/schema.sql new file mode 100644 index 0000000000..85fd1d1df3 --- /dev/null +++ b/crates/core/tests/data/conversations/schema.sql @@ -0,0 +1,41 @@ +CREATE OR REPLACE TABLE conversations( + _acp_system_metadata STRUCT( -- 1 + acp_sourceBatchId VARCHAR, -- 2 + commitBatchId VARCHAR, --3 + trackingId VARCHAR, + rowId VARCHAR, + rowVersion INT8, + primaryIdentity STRUCT( + id VARCHAR, + namespace STRUCT( + code VARCHAR + ) + ), + ingestTime INT8, -- 4 + isDeleted BOOL -- 5 + ), + agenticExperience STRUCT( + agents STRUCT( + agentID VARCHAR + )[], + version VARCHAR + ), + conversation STRUCT( + conversationID VARCHAR, + turnID VARCHAR, + text STRUCT( + raw VARCHAR, + source VARCHAR + ) + ), + identityMap MAP( + VARCHAR, + STRUCT( + id VARCHAR, + prim BOOLEAN + )[] + ), + _ACP_DATE DATE, + _ACP_BATCHID VARCHAR, +); + diff --git a/crates/core/tests/data/conversations_no_stats_parsed/create_delta.py b/crates/core/tests/data/conversations_no_stats_parsed/create_delta.py new file mode 100644 index 0000000000..52018358f1 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/create_delta.py @@ -0,0 +1,245 @@ +from datetime import date, timedelta +from pathlib import Path +import json +import shutil + +import pyarrow as pa +import pyarrow.parquet as pq +from deltalake import DeltaTable, write_deltalake + +TABLE_PATH = Path("delta") +START_DATE = date.fromisoformat("2024-01-01") +CHECKPOINT_DATE = date.fromisoformat("2024-01-04") # Use None for the final version. +DAYS = 7 +BATCHES_PER_DAY = 2 +FILES_PER_PARTITION = 2 +ROWS_PER_FILE = 2 + +SCHEMA = pa.schema( + [ + pa.field( + "_acp_system_metadata", + pa.struct( + [ + pa.field("acp_sourceBatchId", pa.string()), + pa.field("commitBatchId", pa.string()), + pa.field("trackingId", pa.string()), + pa.field("rowId", pa.string()), + pa.field("rowVersion", pa.int64()), + pa.field( + "primaryIdentity", + pa.struct( + [ + pa.field("id", pa.string()), + pa.field( + "namespace", + pa.struct([pa.field("code", pa.string())]), + ), + ] + ), + ), + pa.field("ingestTime", pa.int64()), + pa.field("isDeleted", pa.bool_()), + ] + ), + ), + pa.field( + "agenticExperience", + pa.struct( + [ + pa.field( + "agents", + pa.list_(pa.struct([pa.field("agentID", pa.string())])), + ), + pa.field("version", pa.string()), + ] + ), + ), + pa.field( + "conversation", + pa.struct( + [ + pa.field("conversationID", pa.string()), + pa.field("turnID", pa.string()), + pa.field( + "text", + pa.struct( + [ + pa.field("raw", pa.string()), + pa.field("source", pa.string()), + ] + ), + ), + ] + ), + ), + pa.field( + "identityMap", + pa.map_( + pa.string(), + pa.list_( + pa.struct( + [ + pa.field("id", pa.string()), + pa.field("prim", pa.bool_()), + ] + ) + ), + ), + ), + pa.field("_ACP_DATE", pa.date32()), + pa.field("_ACP_BATCHID", pa.string()), + ] +) + +shutil.rmtree(TABLE_PATH, ignore_errors=True) + +commit = 0 +for day_index in range(DAYS): + current_date = START_DATE + timedelta(days=day_index) + for batch_number in range(1, BATCHES_PER_DAY + 1): + batch_id = f"batch-{batch_number}-{current_date.isoformat()}" + file_batches = [] + + for file_number in range(1, FILES_PER_PARTITION + 1): + file_batch_id = f"{batch_id}--file-{file_number}" + rows = [] + for row_number in range(1, ROWS_PER_FILE + 1): + row_key = ( + f"{current_date.isoformat()}-batch-{batch_number}" + f"-file-{file_number}-row-{row_number}" + ) + identity = f"user-{row_key}@example.com" + rows.append( + { + "_acp_system_metadata": { + "acp_sourceBatchId": batch_id, + "commitBatchId": f"commit-{batch_id}", + "trackingId": f"tracking-{row_key}", + "rowId": f"row-{row_key}", + "rowVersion": 1, + "primaryIdentity": { + "id": identity, + "namespace": {"code": "email"}, + }, + "ingestTime": int( + f"{current_date:%Y%m%d}" + f"{batch_number}{file_number}{row_number}" + ), + "isDeleted": False, + }, + "agenticExperience": { + "agents": [{"agentID": f"agent-{batch_number}"}], + "version": "1.0", + }, + "conversation": { + "conversationID": f"conversation-{current_date}-{file_number}", + "turnID": f"turn-{batch_number}-{file_number}-{row_number}", + "text": { + "raw": f"Conversation text for {row_key}", + "source": "generated", + }, + }, + "identityMap": [ + ("email", [{"id": identity, "prim": True}]) + ], + "_ACP_DATE": current_date, + # A temporary partition forces delta-rs to create this file. + "_ACP_BATCHID": file_batch_id, + } + ) + + file_batches.append(pa.RecordBatch.from_pylist(rows, schema=SCHEMA)) + + write_deltalake( + TABLE_PATH, + pa.RecordBatchReader.from_batches(SCHEMA, file_batches), + mode="overwrite" if commit == 0 else "append", + partition_by=["_ACP_DATE", "_ACP_BATCHID"], + configuration=( + { + "delta.checkpoint.writeStatsAsJson": "true", + "delta.checkpoint.writeStatsAsStruct": "true", + } + if commit == 0 + else None + ), + ) + + # Move both files into the real partition and normalize their add actions. + transaction_path = TABLE_PATH / "_delta_log" / f"{commit:020}.json" + actions = [json.loads(line) for line in transaction_path.read_text().splitlines()] + add_actions = [action["add"] for action in actions if "add" in action] + if len(add_actions) != FILES_PER_PARTITION: + raise RuntimeError( + f"Expected {FILES_PER_PARTITION} files, got {len(add_actions)}" + ) + + destination_directory = ( + TABLE_PATH + / f"_ACP_DATE={current_date.isoformat()}" + / f"_ACP_BATCHID={batch_id}" + ) + destination_directory.mkdir(parents=True, exist_ok=True) + for add in add_actions: + source = TABLE_PATH / add["path"] + destination = destination_directory / source.name + source.rename(destination) + source.parent.rmdir() + add["path"] = destination.relative_to(TABLE_PATH).as_posix() + add["partitionValues"]["_ACP_BATCHID"] = batch_id + + transaction_path.write_text( + "\n".join(json.dumps(action, separators=(",", ":")) for action in actions) + + "\n" + ) + commit += 1 + + if CHECKPOINT_DATE == current_date: + DeltaTable(TABLE_PATH).create_checkpoint() + +if CHECKPOINT_DATE is None: + DeltaTable(TABLE_PATH).create_checkpoint() + +# Ensure this checkpoint has only raw stats and raw partition values. +checkpoint_path = next((TABLE_PATH / "_delta_log").glob("*.checkpoint.parquet")) +checkpoint = pq.read_table(checkpoint_path) +add_index = checkpoint.schema.get_field_index("add") +add_field = checkpoint.schema.field(add_index) +adds = checkpoint.column(add_index).to_pylist() + +for add in adds: + if add is None: + continue + add.pop("stats_parsed", None) + add.pop("partitionValues_parsed", None) + +original_add_type = add_field.type +patched_add_type = pa.struct( + [ + field + for field in original_add_type + if field.name not in {"stats_parsed", "partitionValues_parsed"} + ] +) +patched_add_field = pa.field( + "add", + patched_add_type, + nullable=add_field.nullable, + metadata=add_field.metadata, +) +checkpoint = checkpoint.set_column( + add_index, + patched_add_field, + pa.array(adds, type=patched_add_type), +) +pq.write_table(checkpoint, checkpoint_path, compression="snappy") + +last_checkpoint_path = TABLE_PATH / "_delta_log" / "_last_checkpoint" +last_checkpoint = json.loads(last_checkpoint_path.read_text()) +last_checkpoint["sizeInBytes"] = checkpoint_path.stat().st_size +last_checkpoint_path.write_text(json.dumps(last_checkpoint, separators=(",", ":"))) + +print( + f"Created {TABLE_PATH} with {commit} versions and a raw-only checkpoint" +) diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet new file mode 100644 index 0000000000..5c2b6adcea Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet new file mode 100644 index 0000000000..f449af9924 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-387b2ef3-cd3e-403e-8734-c049b9c71d62-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-387b2ef3-cd3e-403e-8734-c049b9c71d62-c000.snappy.parquet new file mode 100644 index 0000000000..1d840d0c8e Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-387b2ef3-cd3e-403e-8734-c049b9c71d62-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet new file mode 100644 index 0000000000..36483e4750 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet new file mode 100644 index 0000000000..e68796950c Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet new file mode 100644 index 0000000000..f63e35f632 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet new file mode 100644 index 0000000000..6055c50dfd Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet new file mode 100644 index 0000000000..9f0aa51380 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet new file mode 100644 index 0000000000..5c6615a083 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-bf68d798-f825-4902-a824-7a704ee3c700-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-bf68d798-f825-4902-a824-7a704ee3c700-c000.snappy.parquet new file mode 100644 index 0000000000..abc87527a2 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-bf68d798-f825-4902-a824-7a704ee3c700-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet new file mode 100644 index 0000000000..a67c066bf0 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet new file mode 100644 index 0000000000..38a981f2ef Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-26aa5864-10c1-48e7-a6df-6a294b350546-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-26aa5864-10c1-48e7-a6df-6a294b350546-c000.snappy.parquet new file mode 100644 index 0000000000..e585a0621f Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-26aa5864-10c1-48e7-a6df-6a294b350546-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-9abac23b-07d6-48a6-b23b-0710588c47d1-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-9abac23b-07d6-48a6-b23b-0710588c47d1-c000.snappy.parquet new file mode 100644 index 0000000000..23ec1a561f Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-9abac23b-07d6-48a6-b23b-0710588c47d1-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet new file mode 100644 index 0000000000..03399d10c5 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-92b1f2f5-21dc-4a6a-9e5d-25db1c63d6ff-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-92b1f2f5-21dc-4a6a-9e5d-25db1c63d6ff-c000.snappy.parquet new file mode 100644 index 0000000000..894ff640e2 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-92b1f2f5-21dc-4a6a-9e5d-25db1c63d6ff-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-33f334a1-8c2c-41b2-9dbe-d66dd9ad8ae1-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-33f334a1-8c2c-41b2-9dbe-d66dd9ad8ae1-c000.snappy.parquet new file mode 100644 index 0000000000..5fde4fd54a Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-33f334a1-8c2c-41b2-9dbe-d66dd9ad8ae1-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-ecee6b33-3470-4bbf-986e-e08dfe777c7b-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-ecee6b33-3470-4bbf-986e-e08dfe777c7b-c000.snappy.parquet new file mode 100644 index 0000000000..9586fca7a7 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-ecee6b33-3470-4bbf-986e-e08dfe777c7b-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-3e56e624-1e0b-40c0-b7a0-4ead10a5517e-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-3e56e624-1e0b-40c0-b7a0-4ead10a5517e-c000.snappy.parquet new file mode 100644 index 0000000000..89386ccd02 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-3e56e624-1e0b-40c0-b7a0-4ead10a5517e-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet new file mode 100644 index 0000000000..9227e2a1b4 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet new file mode 100644 index 0000000000..aafb639f06 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-65f27845-2a8b-4172-a4f7-01e5d41ef9c2-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-65f27845-2a8b-4172-a4f7-01e5d41ef9c2-c000.snappy.parquet new file mode 100644 index 0000000000..d80a4dfdea Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-65f27845-2a8b-4172-a4f7-01e5d41ef9c2-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet new file mode 100644 index 0000000000..b695a7486c Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-77d00763-9ba2-4429-b8b2-ca8b3092ac40-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-77d00763-9ba2-4429-b8b2-ca8b3092ac40-c000.snappy.parquet new file mode 100644 index 0000000000..7b3db24b09 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-77d00763-9ba2-4429-b8b2-ca8b3092ac40-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-00f3f17d-20f3-4772-9561-1d3b6e0ef2bd-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-00f3f17d-20f3-4772-9561-1d3b6e0ef2bd-c000.snappy.parquet new file mode 100644 index 0000000000..18e63d0bef Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-00f3f17d-20f3-4772-9561-1d3b6e0ef2bd-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet new file mode 100644 index 0000000000..8f0ae271cf Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet new file mode 100644 index 0000000000..9c2f12f2a3 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-9dca9883-8cd9-4950-87e5-9cd130fbdae7-c000.snappy.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-9dca9883-8cd9-4950-87e5-9cd130fbdae7-c000.snappy.parquet new file mode 100644 index 0000000000..d9cb622560 Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-9dca9883-8cd9-4950-87e5-9cd130fbdae7-c000.snappy.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000000.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..c5bc7de5f3 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1787122149949,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":56,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"ccbf76d3-a9d2-4def-b738-8ad654a54898","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_acp_system_metadata\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"acp_sourceBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"commitBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trackingId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowVersion\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"primaryIdentity\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"namespace\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"code\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"ingestTime\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"agenticExperience\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"agents\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"agentID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"version\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"conversation\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"conversationID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"turnID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"text\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"raw\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"source\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"identityMap\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"prim\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_DATE\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCHID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_ACP_DATE","_ACP_BATCHID"],"createdTime":1787122149895,"configuration":{"delta.checkpoint.writeStatsAsStruct":"true","delta.checkpoint.writeStatsAsJson":"true"}}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-119b43a0-0ff6-401b-91cc-cb6b8aeb1388-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-01","_ACP_BATCHID":"batch-1-2024-01-01"},"size":8026,"modificationTime":1787122149949,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-1-file-1-row-1\",\"ingestTime\":20240101111,\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"isDeleted\":false,\"rowId\":\"row-2024-01-01-batch-1-file-1-row-1\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-01-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-1-file-1-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-1-1-1\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-1-row-2\"},\"turnID\":\"turn-1-1-2\"},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-1-file-1-row-2@example.com\"},\"rowVersion\":1,\"rowId\":\"row-2024-01-01-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-1-file-1-row-2\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"ingestTime\":20240101112},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowVersion\":0,\"isDeleted\":0,\"rowId\":0,\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"trackingId\":0,\"ingestTime\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-1-2024-01-01/part-00000-d79ab46e-f64c-43de-8be4-eae27ba5e088-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-01","_ACP_DATE":"2024-01-01"},"size":8027,"modificationTime":1787122149948,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-1-file-2-row-1\",\"ingestTime\":20240101121,\"commitBatchId\":\"commit-batch-1-2024-01-01\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-01-batch-1-file-2-row-1\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"turnID\":\"turn-1-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-2-row-1\"}}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-1-file-2-row-2\"},\"turnID\":\"turn-1-2-2\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-01-batch-1-file-2-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-01-batch-1-file-2-row-2\",\"rowVersion\":1,\"ingestTime\":20240101122,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-01\",\"commitBatchId\":\"commit-batch-1-2024-01-01\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0,\"rowId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0,\"trackingId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000001.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000001.json new file mode 100644 index 0000000000..1253198f46 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000001.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149972,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":10,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-387b2ef3-cd3e-403e-8734-c049b9c71d62-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-01","_ACP_DATE":"2024-01-01"},"size":8054,"modificationTime":1787122149972,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-01-batch-2-file-1-row-1\",\"isDeleted\":false,\"ingestTime\":20240101211,\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-1-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-01\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-01-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-01-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-01-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-1-row-2\",\"ingestTime\":20240101212,\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-01-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"isDeleted\":false},\"conversation\":{\"turnID\":\"turn-2-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-2-file-1-row-2\"},\"conversationID\":\"conversation-2024-01-01-1\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0,\"trackingId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-01/_ACP_BATCHID=batch-2-2024-01-01/part-00000-82bbfc0a-2979-44b8-84b6-6b04b73cb9dc-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-01","_ACP_BATCHID":"batch-2-2024-01-01"},"size":8054,"modificationTime":1787122149972,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-2-file-2-row-1\"}},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-01-batch-2-file-2-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"rowId\":\"row-2024-01-01-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"ingestTime\":20240101221,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-2-file-2-row-1@example.com\"},\"isDeleted\":false,\"rowVersion\":1}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"commitBatchId\":\"commit-batch-2-2024-01-01\",\"trackingId\":\"tracking-2024-01-01-batch-2-file-2-row-2\",\"rowId\":\"row-2024-01-01-batch-2-file-2-row-2\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-01\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-01-batch-2-file-2-row-2@example.com\"},\"ingestTime\":20240101222},\"conversation\":{\"conversationID\":\"conversation-2024-01-01-2\",\"turnID\":\"turn-2-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-01-batch-2-file-2-row-2\"}}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"rowVersion\":0,\"isDeleted\":0,\"rowId\":0,\"ingestTime\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000002.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000002.json new file mode 100644 index 0000000000..b4f04fd5a5 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000002.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149981,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-f19d5eb6-8ca0-4669-9717-3ac43656b7f5-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-1-2024-01-02"},"size":8052,"modificationTime":1787122149981,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-02-batch-1-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-02-1\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"trackingId\":\"tracking-2024-01-02-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"isDeleted\":false,\"rowVersion\":1,\"rowId\":\"row-2024-01-02-batch-1-file-1-row-1\",\"ingestTime\":20240102111,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-1-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-1-1-2\",\"conversationID\":\"conversation-2024-01-02-1\"},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"ingestTime\":20240102112,\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"rowId\":\"row-2024-01-02-batch-1-file-1-row-2\",\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-02-batch-1-file-1-row-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}},\"_acp_system_metadata\":{\"trackingId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"ingestTime\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"rowId\":0,\"acp_sourceBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-1-2024-01-02/part-00000-1ce19dc9-136b-4a43-899c-e4f62daedd03-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-1-2024-01-02"},"size":8053,"modificationTime":1787122149981,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-1-file-2-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowVersion\":1,\"ingestTime\":20240102121,\"commitBatchId\":\"commit-batch-1-2024-01-02\",\"rowId\":\"row-2024-01-02-batch-1-file-2-row-1\"},\"conversation\":{\"turnID\":\"turn-1-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-1-file-2-row-2\",\"rowId\":\"row-2024-01-02-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-02\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"ingestTime\":20240102122,\"commitBatchId\":\"commit-batch-1-2024-01-02\"},\"conversation\":{\"turnID\":\"turn-1-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-1-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowId\":0,\"trackingId\":0,\"commitBatchId\":0,\"rowVersion\":0,\"acp_sourceBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000003.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000003.json new file mode 100644 index 0000000000..121a9a1ade --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000003.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149986,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-77a6cadd-06e1-46f1-9298-6aeb8c34eab9-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-02","_ACP_DATE":"2024-01-02"},"size":8053,"modificationTime":1787122149986,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"ingestTime\":20240102221,\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"rowId\":\"row-2024-01-02-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-02-batch-2-file-2-row-1\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-02-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-02-batch-2-file-2-row-1\"}}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-2-file-2-row-2\",\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-02-batch-2-file-2-row-2\",\"primaryIdentity\":{\"id\":\"user-2024-01-02-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"ingestTime\":20240102222},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-02-batch-2-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-02-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"trackingId\":0,\"rowId\":0,\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-02/_ACP_BATCHID=batch-2-2024-01-02/part-00000-508a12a5-87c1-4524-a1e3-cb500834e3ca-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-02","_ACP_BATCHID":"batch-2-2024-01-02"},"size":8053,"modificationTime":1787122149986,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-02-batch-2-file-1-row-1\",\"isDeleted\":false,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-02-batch-2-file-1-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-02-batch-2-file-1-row-1@example.com\"},\"ingestTime\":20240102211,\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"acp_sourceBatchId\":\"batch-2-2024-01-02\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-1\",\"turnID\":\"turn-2-1-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-02-batch-2-file-1-row-2\",\"rowId\":\"row-2024-01-02-batch-2-file-1-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-02-batch-2-file-1-row-2@example.com\"},\"acp_sourceBatchId\":\"batch-2-2024-01-02\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-2-2024-01-02\",\"isDeleted\":false,\"ingestTime\":20240102212},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-02-batch-2-file-1-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-02-1\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"trackingId\":0,\"isDeleted\":0,\"rowVersion\":0,\"rowId\":0,\"commitBatchId\":0,\"ingestTime\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"turnID\":0,\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000004.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000004.json new file mode 100644 index 0000000000..daa289185f --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000004.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149991,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-bf68d798-f825-4902-a824-7a704ee3c700-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-1-2024-01-03","_ACP_DATE":"2024-01-03"},"size":8052,"modificationTime":1787122149990,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"rowId\":\"row-2024-01-03-batch-1-file-1-row-1\",\"ingestTime\":20240103111,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-03-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-1-file-1-row-1@example.com\"},\"acp_sourceBatchId\":\"batch-1-2024-01-03\"},\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-1-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-03-1\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-1-file-1-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-1-file-1-row-2@example.com\"},\"isDeleted\":false,\"rowId\":\"row-2024-01-03-batch-1-file-1-row-2\",\"ingestTime\":20240103112,\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"trackingId\":\"tracking-2024-01-03-batch-1-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"commitBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-1-2024-01-03/part-00000-1bc809cd-f964-4c8e-b65e-ce87d5a3ec37-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-1-2024-01-03"},"size":8053,"modificationTime":1787122149990,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"ingestTime\":20240103121,\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-03-batch-1-file-2-row-1\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-03-batch-1-file-2-row-1\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-1-2-1\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-1-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-1-file-2-row-2\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-03\",\"trackingId\":\"tracking-2024-01-03-batch-1-file-2-row-2\",\"ingestTime\":20240103122,\"isDeleted\":false,\"rowId\":\"row-2024-01-03-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-03\",\"rowVersion\":1,\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"turnID\":0,\"text\":{\"raw\":0,\"source\":0}},\"_acp_system_metadata\":{\"trackingId\":0,\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"rowId\":0,\"ingestTime\":0,\"rowVersion\":0,\"commitBatchId\":0,\"isDeleted\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000005.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000005.json new file mode 100644 index 0000000000..c13cb2a8b1 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000005.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122149995,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-dd15c536-6e83-41cc-85a4-ff426fdb2954-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-2-2024-01-03"},"size":8053,"modificationTime":1787122149995,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-2-file-2-row-1\"},\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-2-2-1\"},\"_acp_system_metadata\":{\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-03-batch-2-file-2-row-1\",\"rowId\":\"row-2024-01-03-batch-2-file-2-row-1\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"ingestTime\":20240103221}},\"maxValues\":{\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-03-batch-2-file-2-row-2\",\"ingestTime\":20240103222,\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"primaryIdentity\":{\"id\":\"user-2024-01-03-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"rowVersion\":1,\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"rowId\":\"row-2024-01-03-batch-2-file-2-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-03-batch-2-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-03-2\",\"turnID\":\"turn-2-2-2\"}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"raw\":0,\"source\":0}},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"rowId\":0,\"isDeleted\":0,\"commitBatchId\":0,\"trackingId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-03/_ACP_BATCHID=batch-2-2024-01-03/part-00000-7ae9f49a-e691-49e9-a893-592106ba7c38-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-03","_ACP_BATCHID":"batch-2-2024-01-03"},"size":8053,"modificationTime":1787122149995,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"ingestTime\":20240103211,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-03-batch-2-file-1-row-1\",\"rowId\":\"row-2024-01-03-batch-2-file-1-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-2-file-1-row-1@example.com\"},\"commitBatchId\":\"commit-batch-2-2024-01-03\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-03-batch-2-file-1-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-03-1\"},\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-03-batch-2-file-1-row-2@example.com\"},\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-03\",\"isDeleted\":false,\"ingestTime\":20240103212,\"commitBatchId\":\"commit-batch-2-2024-01-03\",\"trackingId\":\"tracking-2024-01-03-batch-2-file-1-row-2\",\"rowId\":\"row-2024-01-03-batch-2-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0},\"_acp_system_metadata\":{\"ingestTime\":0,\"acp_sourceBatchId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"isDeleted\":0,\"trackingId\":0,\"rowId\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000006.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000006.json new file mode 100644 index 0000000000..6e6b02d484 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000006.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150001,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-9abac23b-07d6-48a6-b23b-0710588c47d1-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-1-2024-01-04"},"size":8053,"modificationTime":1787122150000,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-2-row-1\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-1-2-1\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-04-batch-1-file-2-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"isDeleted\":false,\"rowVersion\":1,\"ingestTime\":20240104121,\"trackingId\":\"tracking-2024-01-04-batch-1-file-2-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-04\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-1-2-2\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-04-batch-1-file-2-row-2\",\"ingestTime\":20240104122,\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"rowVersion\":1}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"turnID\":0,\"conversationID\":0},\"_acp_system_metadata\":{\"trackingId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0,\"rowVersion\":0,\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-1-2024-01-04/part-00000-26aa5864-10c1-48e7-a6df-6a294b350546-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-1-2024-01-04"},"size":8052,"modificationTime":1787122150001,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-1-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-1\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"rowId\":\"row-2024-01-04-batch-1-file-1-row-1\",\"rowVersion\":1,\"ingestTime\":20240104111,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-1-file-1-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-04-batch-1-file-1-row-1\",\"isDeleted\":false,\"commitBatchId\":\"commit-batch-1-2024-01-04\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-04-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-1-file-1-row-2\"},\"turnID\":\"turn-1-1-2\"},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-04-batch-1-file-1-row-2\",\"rowId\":\"row-2024-01-04-batch-1-file-1-row-2\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240104112,\"commitBatchId\":\"commit-batch-1-2024-01-04\",\"acp_sourceBatchId\":\"batch-1-2024-01-04\",\"rowVersion\":1}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"trackingId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"rowVersion\":0,\"rowId\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.checkpoint.parquet b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.checkpoint.parquet new file mode 100644 index 0000000000..d5373a63fd Binary files /dev/null and b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.checkpoint.parquet differ diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.json new file mode 100644 index 0000000000..21750bcdd0 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000007.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150006,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-92b1f2f5-21dc-4a6a-9e5d-25db1c63d6ff-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-04","_ACP_BATCHID":"batch-2-2024-01-04"},"size":8053,"modificationTime":1787122150006,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-2-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-1\",\"ingestTime\":20240104221,\"rowId\":\"row-2024-01-04-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-2\",\"turnID\":\"turn-2-2-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"trackingId\":\"tracking-2024-01-04-batch-2-file-2-row-2\",\"rowVersion\":1,\"ingestTime\":20240104222,\"isDeleted\":false,\"rowId\":\"row-2024-01-04-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-2-row-2@example.com\"}},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-2-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-04-batch-2-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-04-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"ingestTime\":0,\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0,\"commitBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-04/_ACP_BATCHID=batch-2-2024-01-04/part-00000-3a9c2844-ffe8-4e52-8bb0-4c16d11b3132-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-04","_ACP_DATE":"2024-01-04"},"size":8053,"modificationTime":1787122150006,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"ingestTime\":20240104211,\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-1\",\"isDeleted\":false,\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"rowVersion\":1,\"rowId\":\"row-2024-01-04-batch-2-file-1-row-1\",\"primaryIdentity\":{\"id\":\"user-2024-01-04-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-2-2024-01-04\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-04-1\",\"turnID\":\"turn-2-1-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"ingestTime\":20240104212,\"commitBatchId\":\"commit-batch-2-2024-01-04\",\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-04-batch-2-file-1-row-2\",\"rowVersion\":1,\"rowId\":\"row-2024-01-04-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-04\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-04-batch-2-file-1-row-2@example.com\"}},\"conversation\":{\"conversationID\":\"conversation-2024-01-04-1\",\"turnID\":\"turn-2-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-04-batch-2-file-1-row-2\",\"source\":\"generated\"}},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"rowId\":0,\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000008.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000008.json new file mode 100644 index 0000000000..c4c6e68bff --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000008.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150026,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-ecee6b33-3470-4bbf-986e-e08dfe777c7b-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-1-2024-01-05"},"size":8053,"modificationTime":1787122150026,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-1-2-1\",\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-2-row-1\",\"ingestTime\":20240105121,\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-2-row-1@example.com\"},\"rowVersion\":1},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-1-file-2-row-2\",\"ingestTime\":20240105122,\"trackingId\":\"tracking-2024-01-05-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-2-row-2@example.com\"},\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-05\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-1-file-2-row-2\"},\"conversationID\":\"conversation-2024-01-05-2\",\"turnID\":\"turn-1-2-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"commitBatchId\":0,\"trackingId\":0,\"rowId\":0,\"rowVersion\":0,\"ingestTime\":0,\"isDeleted\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-1-2024-01-05/part-00000-33f334a1-8c2c-41b2-9dbe-d66dd9ad8ae1-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-1-2024-01-05"},"size":8052,"modificationTime":1787122150026,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-1\",\"rowId\":\"row-2024-01-05-batch-1-file-1-row-1\",\"rowVersion\":1,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-1-file-1-row-1@example.com\"},\"ingestTime\":20240105111},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-1\"}}},\"maxValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-05-batch-1-file-1-row-2\",\"isDeleted\":false,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-05-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-05\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-1-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-1-2024-01-05\",\"ingestTime\":20240105112},\"conversation\":{\"turnID\":\"turn-1-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-1-file-1-row-2\"},\"conversationID\":\"conversation-2024-01-05-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"rowId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"commitBatchId\":0,\"ingestTime\":0,\"rowVersion\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000009.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000009.json new file mode 100644 index 0000000000..92dcc908fe --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000009.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150033,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-e7936629-0b64-4054-bb47-4206fe646f8b-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-05","_ACP_DATE":"2024-01-05"},"size":8053,"modificationTime":1787122150033,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-05-1\",\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"ingestTime\":20240105211,\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-1-row-1@example.com\"},\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-05-batch-2-file-1-row-1\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-2-1-2\",\"conversationID\":\"conversation-2024-01-05-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-05-batch-2-file-1-row-2\"}},\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-05-batch-2-file-1-row-2\",\"rowVersion\":1,\"isDeleted\":false,\"ingestTime\":20240105212,\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-05-batch-2-file-1-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\"}},\"nullCount\":{\"conversation\":{\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0},\"turnID\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"trackingId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0,\"isDeleted\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-05/_ACP_BATCHID=batch-2-2024-01-05/part-00000-3e56e624-1e0b-40c0-b7a0-4ead10a5517e-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-05","_ACP_BATCHID":"batch-2-2024-01-05"},"size":8053,"modificationTime":1787122150033,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-05-2\"},\"_acp_system_metadata\":{\"ingestTime\":20240105221,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-1\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"rowId\":\"row-2024-01-05-batch-2-file-2-row-1\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-05-batch-2-file-2-row-1@example.com\"}}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-05-batch-2-file-2-row-2\",\"acp_sourceBatchId\":\"batch-2-2024-01-05\",\"trackingId\":\"tracking-2024-01-05-batch-2-file-2-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-05\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-05-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"ingestTime\":20240105222,\"rowVersion\":1},\"conversation\":{\"conversationID\":\"conversation-2024-01-05-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-05-batch-2-file-2-row-2\",\"source\":\"generated\"},\"turnID\":\"turn-2-2-2\"}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"isDeleted\":0,\"ingestTime\":0,\"rowVersion\":0,\"trackingId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"rowId\":0},\"conversation\":{\"turnID\":0,\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000010.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000010.json new file mode 100644 index 0000000000..7dc16d426c --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000010.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150040,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-65f27845-2a8b-4172-a4f7-01e5d41ef9c2-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-1-2024-01-06"},"size":8053,"modificationTime":1787122150040,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-1-2-1\"},\"_acp_system_metadata\":{\"ingestTime\":20240106121,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"isDeleted\":false,\"rowId\":\"row-2024-01-06-batch-1-file-2-row-1\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-1\"}},\"maxValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-06-batch-1-file-2-row-2\",\"ingestTime\":20240106122,\"trackingId\":\"tracking-2024-01-06-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-2-row-2@example.com\"}},\"conversation\":{\"turnID\":\"turn-1-2-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-1-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-06-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"conversationID\":0,\"text\":{\"source\":0,\"raw\":0}},\"_acp_system_metadata\":{\"commitBatchId\":0,\"ingestTime\":0,\"acp_sourceBatchId\":0,\"isDeleted\":0,\"rowId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-1-2024-01-06/part-00000-0e5bbfac-6e04-4740-87e4-02b1403ab802-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-1-2024-01-06"},"size":8052,"modificationTime":1787122150040,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"ingestTime\":20240106111,\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-1-row-1@example.com\"},\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-1\",\"rowId\":\"row-2024-01-06-batch-1-file-1-row-1\",\"commitBatchId\":\"commit-batch-1-2024-01-06\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-1-1-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-1-file-1-row-2\"}},\"_acp_system_metadata\":{\"rowVersion\":1,\"ingestTime\":20240106112,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-1-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-1-file-1-row-2\",\"commitBatchId\":\"commit-batch-1-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-1-file-1-row-2@example.com\"},\"rowId\":\"row-2024-01-06-batch-1-file-1-row-2\"}},\"nullCount\":{\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0},\"_acp_system_metadata\":{\"rowVersion\":0,\"isDeleted\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"ingestTime\":0,\"trackingId\":0,\"acp_sourceBatchId\":0,\"rowId\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000011.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000011.json new file mode 100644 index 0000000000..ef44550d8b --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000011.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150047,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.6.2"}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-07f4b170-f275-4570-9f56-374968008e3a-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-06","_ACP_DATE":"2024-01-06"},"size":8053,"modificationTime":1787122150047,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-2\",\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-1\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-06-batch-2-file-2-row-1\",\"ingestTime\":20240106221,\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-1\",\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-2-row-1@example.com\"},\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"rowVersion\":1,\"isDeleted\":false}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-2-row-2\"},\"turnID\":\"turn-2-2-2\",\"conversationID\":\"conversation-2024-01-06-2\"},\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-2-row-2\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-06-batch-2-file-2-row-2@example.com\"},\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"isDeleted\":false,\"ingestTime\":20240106222,\"rowId\":\"row-2024-01-06-batch-2-file-2-row-2\",\"rowVersion\":1}},\"nullCount\":{\"_acp_system_metadata\":{\"ingestTime\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"isDeleted\":0,\"acp_sourceBatchId\":0,\"trackingId\":0,\"rowId\":0,\"commitBatchId\":0,\"rowVersion\":0},\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-06/_ACP_BATCHID=batch-2-2024-01-06/part-00000-77d00763-9ba2-4429-b8b2-ca8b3092ac40-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-06","_ACP_BATCHID":"batch-2-2024-01-06"},"size":8053,"modificationTime":1787122150047,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-06\",\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"ingestTime\":20240106211,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-1-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-06-batch-2-file-1-row-1\",\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-1\"},\"conversationID\":\"conversation-2024-01-06-1\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-06-batch-2-file-1-row-2\",\"ingestTime\":20240106212,\"primaryIdentity\":{\"id\":\"user-2024-01-06-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"isDeleted\":false,\"rowId\":\"row-2024-01-06-batch-2-file-1-row-2\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-06\",\"commitBatchId\":\"commit-batch-2-2024-01-06\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-06-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-06-batch-2-file-1-row-2\"},\"turnID\":\"turn-2-1-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"isDeleted\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"commitBatchId\":0,\"trackingId\":0,\"ingestTime\":0,\"rowId\":0,\"rowVersion\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000012.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000012.json new file mode 100644 index 0000000000..3ef9535355 --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000012.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150054,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-3c3ff685-8061-4b71-9d11-7ec5928b2545-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-1-2024-01-07"},"size":8052,"modificationTime":1787122150054,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-07-batch-1-file-1-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-1\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-1-file-1-row-1@example.com\"},\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"isDeleted\":false,\"ingestTime\":20240107111,\"acp_sourceBatchId\":\"batch-1-2024-01-07\"},\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-1\",\"source\":\"generated\"}}},\"maxValues\":{\"conversation\":{\"conversationID\":\"conversation-2024-01-07-1\",\"turnID\":\"turn-1-1-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-1-row-2\",\"source\":\"generated\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-1-file-1-row-2\",\"isDeleted\":false,\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-1-file-1-row-2@example.com\"},\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"rowVersion\":1,\"ingestTime\":20240107112,\"rowId\":\"row-2024-01-07-batch-1-file-1-row-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"rowVersion\":0,\"commitBatchId\":0,\"rowId\":0,\"isDeleted\":0,\"ingestTime\":0,\"trackingId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"acp_sourceBatchId\":0},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-1-2024-01-07/part-00000-00f3f17d-20f3-4772-9561-1d3b6e0ef2bd-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-1-2024-01-07"},"size":8053,"modificationTime":1787122150054,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"ingestTime\":20240107121,\"rowVersion\":1,\"isDeleted\":false,\"rowId\":\"row-2024-01-07-batch-1-file-2-row-1\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-1-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}},\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-1\"},\"conversation\":{\"conversationID\":\"conversation-2024-01-07-2\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-1\",\"source\":\"generated\"},\"turnID\":\"turn-1-2-1\"}},\"maxValues\":{\"conversation\":{\"turnID\":\"turn-1-2-2\",\"conversationID\":\"conversation-2024-01-07-2\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-1-file-2-row-2\"}},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-07-batch-1-file-2-row-2\",\"acp_sourceBatchId\":\"batch-1-2024-01-07\",\"isDeleted\":false,\"trackingId\":\"tracking-2024-01-07-batch-1-file-2-row-2\",\"rowVersion\":1,\"commitBatchId\":\"commit-batch-1-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-1-file-2-row-2@example.com\"},\"ingestTime\":20240107122},\"agenticExperience\":{\"version\":\"1.0\"}},\"nullCount\":{\"_acp_system_metadata\":{\"trackingId\":0,\"isDeleted\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"rowId\":0,\"commitBatchId\":0,\"rowVersion\":0,\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0}},\"agenticExperience\":{\"version\":0},\"conversation\":{\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0,\"turnID\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000013.json b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000013.json new file mode 100644 index 0000000000..bd2e866f3a --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/00000000000000000013.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1787122150061,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Append"},"engineInfo":"delta-rs:py-1.6.2","clientVersion":"delta-rs.py-1.6.2","operationMetrics":{"execution_time_ms":1,"num_added_files":2,"num_added_rows":4,"num_partitions":0,"num_removed_files":0}}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-9dca9883-8cd9-4950-87e5-9cd130fbdae7-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"batch-2-2024-01-07","_ACP_DATE":"2024-01-07"},"size":8053,"modificationTime":1787122150061,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"conversation\":{\"turnID\":\"turn-2-1-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-07-1\"},\"_acp_system_metadata\":{\"rowId\":\"row-2024-01-07-batch-2-file-1-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"namespace\":{\"code\":\"email\"},\"id\":\"user-2024-01-07-batch-2-file-1-row-1@example.com\"},\"ingestTime\":20240107211,\"isDeleted\":false},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"turnID\":\"turn-2-1-2\",\"conversationID\":\"conversation-2024-01-07-1\",\"text\":{\"source\":\"generated\",\"raw\":\"Conversation text for 2024-01-07-batch-2-file-1-row-2\"}},\"_acp_system_metadata\":{\"trackingId\":\"tracking-2024-01-07-batch-2-file-1-row-2\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"ingestTime\":20240107212,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"isDeleted\":false,\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-2-file-1-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-07-batch-2-file-1-row-2\",\"rowVersion\":1}},\"nullCount\":{\"agenticExperience\":{\"version\":0},\"conversation\":{\"turnID\":0,\"text\":{\"source\":0,\"raw\":0},\"conversationID\":0},\"_acp_system_metadata\":{\"rowId\":0,\"commitBatchId\":0,\"primaryIdentity\":{\"id\":0,\"namespace\":{\"code\":0}},\"trackingId\":0,\"acp_sourceBatchId\":0,\"ingestTime\":0,\"isDeleted\":0,\"rowVersion\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"add":{"path":"_ACP_DATE=2024-01-07/_ACP_BATCHID=batch-2-2024-01-07/part-00000-2ed0a638-c97e-4c96-bddf-351439da8c8f-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2024-01-07","_ACP_BATCHID":"batch-2-2024-01-07"},"size":8053,"modificationTime":1787122150061,"dataChange":true,"stats":"{\"numRecords\":2,\"minValues\":{\"_acp_system_metadata\":{\"rowVersion\":1,\"rowId\":\"row-2024-01-07-batch-2-file-2-row-1\",\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-1\",\"ingestTime\":20240107221,\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-2-file-2-row-1@example.com\",\"namespace\":{\"code\":\"email\"}}},\"conversation\":{\"turnID\":\"turn-2-2-1\",\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-1\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-07-2\"},\"agenticExperience\":{\"version\":\"1.0\"}},\"maxValues\":{\"agenticExperience\":{\"version\":\"1.0\"},\"conversation\":{\"text\":{\"raw\":\"Conversation text for 2024-01-07-batch-2-file-2-row-2\",\"source\":\"generated\"},\"conversationID\":\"conversation-2024-01-07-2\",\"turnID\":\"turn-2-2-2\"},\"_acp_system_metadata\":{\"commitBatchId\":\"commit-batch-2-2024-01-07\",\"primaryIdentity\":{\"id\":\"user-2024-01-07-batch-2-file-2-row-2@example.com\",\"namespace\":{\"code\":\"email\"}},\"rowId\":\"row-2024-01-07-batch-2-file-2-row-2\",\"isDeleted\":false,\"acp_sourceBatchId\":\"batch-2-2024-01-07\",\"ingestTime\":20240107222,\"rowVersion\":1,\"trackingId\":\"tracking-2024-01-07-batch-2-file-2-row-2\"}},\"nullCount\":{\"_acp_system_metadata\":{\"primaryIdentity\":{\"namespace\":{\"code\":0},\"id\":0},\"acp_sourceBatchId\":0,\"ingestTime\":0,\"rowVersion\":0,\"trackingId\":0,\"rowId\":0,\"commitBatchId\":0,\"isDeleted\":0},\"conversation\":{\"text\":{\"raw\":0,\"source\":0},\"conversationID\":0,\"turnID\":0},\"agenticExperience\":{\"version\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} diff --git a/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/_last_checkpoint b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/_last_checkpoint new file mode 100644 index 0000000000..571f53814a --- /dev/null +++ b/crates/core/tests/data/conversations_no_stats_parsed/delta/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"version":7,"size":18,"sizeInBytes":38166,"numOfAddFiles":16} \ No newline at end of file diff --git a/crates/core/tests/data/deep/_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet b/crates/core/tests/data/deep/_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet new file mode 100644 index 0000000000..13737d9a97 Binary files /dev/null and b/crates/core/tests/data/deep/_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet differ diff --git a/crates/core/tests/data/deep/_delta_log/00000000000000000000.json b/crates/core/tests/data/deep/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..78b334973d --- /dev/null +++ b/crates/core/tests/data/deep/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"3c5a399a-65d7-47ac-a792-e9525322f73e","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_acp_system_metadata\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"acp_sourceBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"commitBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"_id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"productListItems\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"SKU\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"quantity\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"priceTotal\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_experience\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"analytics\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"customDimensions\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"eVars\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"evar1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"evar2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"events\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"event1\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"event2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"value\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_DATE\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCHID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_ACP_DATE","_ACP_BATCHID"],"createdTime":1770389266405,"configuration":{}}} +{"add":{"path":"_ACP_DATE=2026-01-01/_ACP_BATCHID=b1/part-00000-dd885b7b-9bff-4a7d-b866-abb97c953ac9-c000.snappy.parquet","partitionValues":{"_ACP_BATCHID":"b1","_ACP_DATE":"2026-01-01"},"size":4305,"modificationTime":1770389266454,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"b1\",\"commitBatchId\":\"b1\"},\"_id\":\"id1\"},\"maxValues\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"b1\",\"commitBatchId\":\"b1\"},\"_id\":\"id1\"},\"nullCount\":{\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"commitBatchId\":0},\"_id\":0}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1770389266455,"operation":"WRITE","operationParameters":{"partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCHID\"]","mode":"Overwrite"},"engineInfo":"delta-rs:py-1.3.1","operationMetrics":{"execution_time_ms":52,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.3.1"}} \ No newline at end of file diff --git a/crates/core/tests/data/hstack_nullable_difference/_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet b/crates/core/tests/data/hstack_nullable_difference/_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet new file mode 100644 index 0000000000..386d831ccc Binary files /dev/null and b/crates/core/tests/data/hstack_nullable_difference/_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet differ diff --git a/crates/core/tests/data/hstack_nullable_difference/_delta_log/00000000000000000000.json b/crates/core/tests/data/hstack_nullable_difference/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..52c979fa68 --- /dev/null +++ b/crates/core/tests/data/hstack_nullable_difference/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz"],"writerFeatures":["timestampNtz"]}} +{"metaData":{"id":"898c9da5-4a43-48cb-ac42-3946be8a86cc","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"_acp_system_metadata\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"acp_sourceBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"commitBatchId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ingestTime\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"isDeleted\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowId\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"rowVersion\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"trackingId\",\"type\":\"string\",\"nullable\":false,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"orderData\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"productList\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"SKU\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"quantity\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"priceTotal\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"_id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCHID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_DATE\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_ACP_BATCH_ID\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["_ACP_DATE","_ACP_BATCH_ID"],"createdTime":1771242914810,"configuration":{}}} +{"add":{"path":"_ACP_DATE=2026-02-16/_ACP_BATCH_ID=b1/part-00000-26d1988d-7e8d-4cbb-8575-1008884b3df5-c000.snappy.parquet","partitionValues":{"_ACP_DATE":"2026-02-16","_ACP_BATCH_ID":"b1"},"size":4564,"modificationTime":1771242914852,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"orderData\":{\"date\":\"2025-01-01T13:00:00Z\"},\"_ACP_BATCHID\":\"batch1\",\"_acp_system_metadata\":{\"acp_sourceBatchId\":\"batch1\",\"commitBatchId\":\"batch1\",\"isDeleted\":false,\"ingestTime\":1,\"rowId\":\"row1\",\"rowVersion\":1,\"trackingId\":\"t1\"},\"_id\":\"id1\"},\"maxValues\":{\"_ACP_BATCHID\":\"batch1\",\"_acp_system_metadata\":{\"trackingId\":\"t1\",\"isDeleted\":false,\"commitBatchId\":\"batch1\",\"ingestTime\":1,\"rowId\":\"row1\",\"rowVersion\":1,\"acp_sourceBatchId\":\"batch1\"},\"orderData\":{\"date\":\"2025-01-01T13:00:00Z\"},\"_id\":\"id1\"},\"nullCount\":{\"_id\":0,\"_acp_system_metadata\":{\"acp_sourceBatchId\":0,\"rowId\":0,\"rowVersion\":0,\"trackingId\":0,\"isDeleted\":0,\"ingestTime\":0,\"commitBatchId\":0},\"_ACP_BATCHID\":0,\"orderData\":{\"date\":0}}}","tags":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1771242914853,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"_ACP_DATE\",\"_ACP_BATCH_ID\"]"},"engineInfo":"delta-rs:py-1.3.1","operationMetrics":{"execution_time_ms":44,"num_added_files":1,"num_added_rows":1,"num_partitions":0,"num_removed_files":0},"clientVersion":"delta-rs.py-1.3.1"}} diff --git a/crates/core/tests/it/eager_cache_schema_evolution.rs b/crates/core/tests/it/eager_cache_schema_evolution.rs new file mode 100644 index 0000000000..e6b0892911 --- /dev/null +++ b/crates/core/tests/it/eager_cache_schema_evolution.rs @@ -0,0 +1,158 @@ +use std::error::Error; + +use arrow_array::{Array, Int32Array, Int64Array, StringArray}; +use deltalake_core::kernel::transaction::CommitBuilder; +use deltalake_core::kernel::{Action, Add, DataType, MetadataExt, StructField, StructType}; +use deltalake_core::protocol::{DeltaOperation, SaveMode}; +use deltalake_core::{DeltaTable, DeltaTableBuilder}; +use url::Url; + +fn add(path: &str, stats: &str) -> Add { + Add { + path: path.to_string(), + size: 1, + modification_time: 1, + data_change: true, + stats: Some(stats.to_string()), + ..Default::default() + } +} + +fn write_operation() -> DeltaOperation { + DeltaOperation::Write { + mode: SaveMode::Append, + partition_by: None, + predicate: None, + } +} + +#[tokio::test] +async fn eager_cache_schema_evolution() -> Result<(), Box> { + let temp_dir = tempfile::tempdir()?; + let table_url = Url::from_directory_path(temp_dir.path()) + .map_err(|_| "temporary table path is not a directory")?; + let mut table = DeltaTable::try_from_url(table_url.clone()) + .await? + .create() + .with_columns([StructField::nullable("id", DataType::INTEGER)]) + .await?; + + CommitBuilder::default() + .with_actions(vec![Action::Add(add( + "old.parquet", + r#"{"numRecords":3,"nullCount":{"id":0},"minValues":{"id":1,"added":"not-an-integer"},"maxValues":{"id":3}}"#, + ))]) + .build( + Some(table.snapshot()?), + table.log_store(), + write_operation(), + ) + .await?; + table.update_state().await?; + + // Reopen version 1 so its active Add batch is eagerly materialized with the old stats schema. + let mut cached = DeltaTableBuilder::from_url(table_url)?.load().await?; + assert_eq!(cached.version(), Some(1)); + + let evolved_schema = StructType::try_new([ + StructField::nullable("id", DataType::INTEGER), + StructField::nullable("added", DataType::INTEGER), + ])?; + let metadata = cached + .snapshot()? + .metadata() + .clone() + .with_schema(&evolved_schema)?; + CommitBuilder::default() + .with_actions(vec![ + Action::Metadata(metadata), + Action::Add(add( + "new.parquet", + r#"{"numRecords":2,"nullCount":{"id":0,"added":0},"minValues":{"id":10,"added":100},"maxValues":{"id":20,"added":200}}"#, + )), + ]) + .build(Some(cached.snapshot()?), cached.log_store(), write_operation()) + .await?; + + // No checkpoint is created. The update must evolve the cached parsed struct and replay the + // new JSON commit rather than reparsing raw statistics for the old file. + cached.update_state().await?; + assert_eq!(cached.version(), Some(2)); + let actions = cached.snapshot()?.add_actions_table(true)?; + assert_eq!(actions.num_rows(), 2); + + let paths = actions + .column_by_name("path") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let num_records = actions + .column_by_name("num_records") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let null_id = actions + .column_by_name("null_count.id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let min_id = actions + .column_by_name("min.id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let max_id = actions + .column_by_name("max.id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let null_added = actions + .column_by_name("null_count.added") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let min_added = actions + .column_by_name("min.added") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let max_added = actions + .column_by_name("max.added") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + + for row in 0..actions.num_rows() { + match paths.value(row) { + "old.parquet" => { + assert_eq!(num_records.value(row), 3); + assert_eq!(null_id.value(row), 0); + assert_eq!(min_id.value(row), 1); + assert_eq!(max_id.value(row), 3); + assert!(null_added.is_null(row)); + assert!(min_added.is_null(row)); + assert!(max_added.is_null(row)); + } + "new.parquet" => { + assert_eq!(num_records.value(row), 2); + assert_eq!(null_id.value(row), 0); + assert_eq!(min_id.value(row), 10); + assert_eq!(max_id.value(row), 20); + assert_eq!(null_added.value(row), 0); + assert_eq!(min_added.value(row), 100); + assert_eq!(max_added.value(row), 200); + } + path => panic!("unexpected Add path {path}"), + } + } + + Ok(()) +} diff --git a/crates/core/tests/it/main.rs b/crates/core/tests/it/main.rs index a5e8d39d54..5b9041db3f 100644 --- a/crates/core/tests/it/main.rs +++ b/crates/core/tests/it/main.rs @@ -1,6 +1,7 @@ mod column_mapping_guardrails; mod command_update_table_metadata; mod dat; +mod eager_cache_schema_evolution; mod eager_snapshot; mod exotic_tables; mod time_travel; diff --git a/crates/core/tests/it_datafusion/integration_datafusion_deep.rs b/crates/core/tests/it_datafusion/integration_datafusion_deep.rs new file mode 100644 index 0000000000..c384684ae7 --- /dev/null +++ b/crates/core/tests/it_datafusion/integration_datafusion_deep.rs @@ -0,0 +1,275 @@ +use deltalake_core::delta_datafusion::DeltaNextPhysicalCodec; +use std::ops::Deref; +use std::sync::Arc; +use std::sync::Once; +use arrow_cast::display::FormatOptions; +use arrow_cast::pretty; +use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion::datasource::physical_plan::ParquetSource; +use datafusion::datasource::physical_plan::parquet::push_all_projection_hints::PushAllProjectionHints; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::physical_expr::projection::ProjectionExprs; +use datafusion::physical_plan::{collect, displayable, ExecutionPlan}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_datasource::source::DataSourceExec; +use datafusion_proto::physical_plan::{AsExecutionPlan, ComposedPhysicalExtensionCodec, DefaultPhysicalExtensionCodec}; +use datafusion_proto::protobuf::PhysicalPlanNode; +use prost::Message; +use tracing::info; +use deltalake_core::delta_datafusion::{DeltaPhysicalCodec, DeltaScanExec}; +use deltalake_core::delta_datafusion::udtf::register_delta_table_udtf; + +/// DF 53 stores deep-projection state on `ParquetSource` as +/// `projection_hints: ProjectionExprs` + `projection_hints_indices: Vec`, +/// populated post-planning by the `PushAllProjectionHints` physical optimizer rule. +/// `ProjectionExprs` derives `PartialEq`, sufficient for serde round-trip equality. +fn extract_projection_deep_from_plan( + plan: Arc, +) -> Vec<(ProjectionExprs, Vec)> { + let mut deep_projections: Vec<(ProjectionExprs, Vec)> = vec![]; + let _ = plan.apply(|pp| { + if let Some(dse) = pp.downcast_ref::() { + if let Some((_file_scan_conf, parquet_source)) = + dse.downcast_to_file_source::() + { + deep_projections.push(( + parquet_source.projection_hints.clone(), + parquet_source.projection_hints_indices.clone(), + )); + } + } + Ok(TreeNodeRecursion::Continue) + }); + deep_projections +} + +/// Setting `tracing`'s global default subscriber is a one-shot per process, +/// so multiple tests in the same binary must share a single init. +fn init_tracing() { + static INIT: Once = Once::new(); + INIT.call_once(|| { + let filter = tracing_subscriber::EnvFilter::from_default_env(); + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_env_filter(filter) + .finish(); + let _ = tracing::subscriber::set_global_default(subscriber); + let _ = pretty_env_logger::try_init(); + }); +} + +/// Build a `SessionContext` with the DF 53 `PushAllProjectionHints` physical +/// optimizer rule registered. Without this rule, `ParquetSource.projection_hints` +/// stays empty and the test's serde assertion is trivially true. +fn build_context(config: SessionConfig) -> SessionContext { + let state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(Arc::new(RuntimeEnv::default())) + .with_default_features() + .with_physical_optimizer_rule(Arc::new(PushAllProjectionHints {})) + .build(); + SessionContext::new_with_state(state) +} + +#[tokio::test] +async fn test_hstack_deep_column_pruning() -> datafusion::common::Result<()> { + init_tracing(); + + let config = SessionConfig::new() + .set_bool("datafusion.sql_parser.enable_ident_normalization", false); + + let ctx = build_context(config); + + register_delta_table_udtf(&ctx, None, None); + + let delta_path = format!( + "{}/tests/data/deep", + env!("CARGO_MANIFEST_DIR") + ); + + let query = format!(r#" + select + t1._id, t1.productListItems['SKU'], _ACP_DATE + from + delta_table('file://{}') as t1 + "#, delta_path); + + let plan = ctx.state().create_logical_plan(&query).await.expect("Error creating logical plan"); + let optimized_plan = ctx.state().optimize(&plan).expect("Error optimizing plan"); + let state = ctx.state(); + let query_planner = state.query_planner().clone(); + let physical_plan = query_planner + .create_physical_plan(&optimized_plan, &state) + .await.expect("Error creating physical plan"); + info!( + "Physical plan: {}", + displayable(physical_plan.deref()).set_show_schema(true).indent(true) + ); + let proj1 = extract_projection_deep_from_plan(physical_plan.clone()); + let batches1 = collect(physical_plan.clone(), ctx.state().task_ctx()).await?; + let results1 = pretty::pretty_format_batches_with_options(&batches1, &FormatOptions::default())?.to_string(); + println!("{}", results1); + + // codec + let codec = ComposedPhysicalExtensionCodec::new( + vec![ + Arc::new(DefaultPhysicalExtensionCodec {}), + Arc::new(DeltaPhysicalCodec{}) + ] + ); + let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec) + .unwrap(); + let bytes = proto.encode_to_vec(); + let plan_after_serde = PhysicalPlanNode::try_decode(&bytes) + .expect("Error try_decode") + .try_into_physical_plan(&ctx.task_ctx(), &codec) + .expect("try_into_physical_plan"); + info!( + "Physical plan after serde: {}", + displayable(plan_after_serde.deref()).set_show_schema(true).indent(true) + ); + let proj2 = extract_projection_deep_from_plan(plan_after_serde.clone()); + let batches2 = collect(plan_after_serde.clone(), ctx.state().task_ctx()).await?; + let results2 = pretty::pretty_format_batches_with_options(&batches2, &FormatOptions::default())?.to_string(); + println!("{}", results2); + + assert_eq!(results1, results2, "Batches not equal !"); + println!("proj1: {:?}", proj1); + println!("proj2: {:?}", proj2); + + assert_eq!(proj1, proj2, "Deep Projection not equal !"); + Ok(()) +} + + +#[tokio::test] +async fn test_hstack_nullable_new() -> datafusion::common::Result<()> { + init_tracing(); + + let config = SessionConfig::new() + .set_bool("datafusion.sql_parser.enable_ident_normalization", false) + .set_bool("datafusion.execution.parquet.schema_force_view_types", false); + + let ctx = build_context(config); + + register_delta_table_udtf(&ctx, None, None); + + let delta_path = format!( + "{}/tests/data/hstack_nullable_difference", + env!("CARGO_MANIFEST_DIR") + ); + + let query = format!(r#" + select + * + from + delta_table('file://{}') as t1 + "#, delta_path); + + let plan = ctx.state().create_logical_plan(&query).await.expect("Error creating logical plan"); + let optimized_plan = ctx.state().optimize(&plan).expect("Error optimizing plan"); + let state = ctx.state(); + let query_planner = state.query_planner().clone(); + let physical_plan = query_planner + .create_physical_plan(&optimized_plan, &state) + .await.expect("Error creating physical plan"); + info!( + "Physical plan: {}", + displayable(physical_plan.deref()).set_show_schema(true).indent(true) + ); + let proj1 = extract_projection_deep_from_plan(physical_plan.clone()); + let batches1 = collect(physical_plan.clone(), ctx.state().task_ctx()).await?; + let results1 = pretty::pretty_format_batches_with_options(&batches1, &FormatOptions::default())?.to_string(); + println!("{}", results1); + + Ok(()) +} + +#[tokio::test] +async fn test_hstack_deep_column_pruning_next_codec() -> datafusion::common::Result<()> { + let filter = tracing_subscriber::EnvFilter::from_default_env(); + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_env_filter(filter) + .finish(); + tracing::subscriber::set_global_default(subscriber).ok(); + pretty_env_logger::try_init().ok(); + + let config = SessionConfig::new() + .set_bool("datafusion.sql_parser.enable_ident_normalization", false); + + let ctx = SessionContext::new_with_config(config); + + register_delta_table_udtf(&ctx, None, None); + + let delta_path = format!( + "{}/tests/data/deep", + env!("CARGO_MANIFEST_DIR") + ); + + let query = format!(r#" + select + t1._id, t1.productListItems['SKU'], _ACP_DATE + from + delta_table_next('file://{}') as t1 + "#, delta_path); + + let plan = ctx.state().create_logical_plan(&query).await.expect("Error creating logical plan"); + let optimized_plan = ctx.state().optimize(&plan).expect("Error optimizing plan"); + let state = ctx.state(); + let query_planner = state.query_planner().clone(); + let physical_plan = query_planner + .create_physical_plan(&optimized_plan, &state) + .await.expect("Error creating physical plan"); + info!( + "Physical plan: {}", + displayable(physical_plan.deref()).set_show_schema(true).indent(true) + ); + let proj1 = extract_projection_deep_from_plan(physical_plan.clone()); + let batches1 = collect(physical_plan.clone(), ctx.state().task_ctx()).await?; + let results1 = pretty::pretty_format_batches_with_options(&batches1, &FormatOptions::default())?.to_string(); + println!("{}", results1); + + // codec + let codec = ComposedPhysicalExtensionCodec::new( + vec![ + Arc::new(DefaultPhysicalExtensionCodec {}), + Arc::new(DeltaNextPhysicalCodec{}) + ] + ); + + let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?; + let bytes = proto.encode_to_vec(); + let plan_after_serde = PhysicalPlanNode::try_decode(&bytes) + .expect("Error try_decode") + .try_into_physical_plan(&ctx.task_ctx(), &codec) + .expect("try_into_physical_plan"); + info!( + "Physical plan after serde: {}", + displayable(plan_after_serde.deref()).set_show_schema(true).indent(true) + ); + + let proj2 = extract_projection_deep_from_plan(plan_after_serde.clone()); + let batches2 = collect(plan_after_serde.clone(), ctx.state().task_ctx()).await?; + let results2 = pretty::pretty_format_batches_with_options(&batches2, &FormatOptions::default())?.to_string(); + println!("{}", results2); + + assert_eq!(results1, results2, "Batches not equal !"); + println!("proj1: {:?}", proj1); + println!("proj2: {:?}", proj2); + + assert_eq!(proj1, proj2, "Deep Projection not equal !"); + + Ok(()) +} + +fn find_exec_node(input: &Arc) -> Option<&T> { + if let Some(found) = input.downcast_ref::() { + Some(found) + } else { + input.children().iter() + .find_map(|child| find_exec_node(child)) + } +} + diff --git a/crates/core/tests/it_datafusion/integration_datafusion_predicate.rs b/crates/core/tests/it_datafusion/integration_datafusion_predicate.rs new file mode 100644 index 0000000000..6085acb4fe --- /dev/null +++ b/crates/core/tests/it_datafusion/integration_datafusion_predicate.rs @@ -0,0 +1,762 @@ +use async_trait::async_trait; +use bytes::Bytes; +use datafusion::common::{DataFusionError, Result as DataFusionResult}; +use datafusion::physical_plan::{ + collect, visit_execution_plan, ExecutionPlan, ExecutionPlanVisitor, +}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use delta_kernel::expressions::ColumnName; +use delta_kernel::{ + DeltaResult as KernelResult, DeltaResultIterator, DeltaResultIteratorStatic, Engine, + EngineData, EvaluationHandler, FileDataReadResultIterator, FileMeta, FileSlice, + FilteredEngineData, JsonHandler, ParquetFooter, ParquetHandler, PredicateRef, StorageHandler, +}; +use deltalake_core::delta_datafusion::DeltaScanExec; +use deltalake_core::kernel::DeltaResult; +use deltalake_core::{DeltaTableBuilder, SchemaRef}; +use futures::stream::BoxStream; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetRange, GetResult, ListResult, MultipartUpload, ObjectMeta, + ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; +use std::collections::HashMap; +use std::fmt; +use std::fmt::{Display, Formatter}; +use std::ops::Range; +use std::sync::{Arc, Mutex}; +use tracing::info; +use url::Url; + +/// Records every bounded byte-range read made against the inner object store, keyed by path. +/// +/// Intercepts `get_opts` (for single ranged reads) and `get_ranges` (for multi-range reads, +/// where the `ObjectStore` default impl coalesces ranges into one `get_opts` call — recording +/// at the `get_opts` level alone would hide the caller's per-column intent). `get_range` is +/// not overridden because its default impl routes through `get_opts` without coalescing. +#[derive(Debug)] +pub struct RangeRecordingObjectStore { + inner: Arc, + reads: Arc>>>>, +} + +impl Display for RangeRecordingObjectStore { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "RangeRecording({})", self.inner) + } +} + +#[allow(dead_code)] +impl RangeRecordingObjectStore { + pub fn new(inner: Arc) -> Self { + Self { + inner, + reads: Arc::new(Mutex::new(HashMap::new())), + } + } + + /// Returns all byte ranges recorded for any path whose string representation ends with `suffix`. + pub fn reads_for_path_suffix(&self, suffix: &str) -> Vec> { + self.reads + .lock() + .expect("lock not poisoned") + .iter() + .filter(|(k, _)| k.ends_with(suffix)) + .flat_map(|(_, v)| v.clone()) + .collect() + } + + fn clear_reads(&self) { + self.reads.lock().expect("lock not poisoned").clear(); + } + + fn record(&self, path: &Path, range: Range) { + self.reads + .lock() + .expect("lock not poisoned") + .entry(path.to_string()) + .or_default() + .push(range); + } + +} + +#[async_trait] +impl ObjectStore for RangeRecordingObjectStore { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOptions, + ) -> object_store::Result> { + Err(object_store::Error::NotImplemented { + operation: "put_multipart_opts".to_string(), + implementer: "RangeRecordingObjectStore".to_string(), + }) + } + + async fn get_opts( + &self, + location: &Path, + options: GetOptions, + ) -> object_store::Result { + // Only record bounded ranges. Parquet readers issue suffix reads for the footer + // (not column data) and don't issue unranged full reads for column access, so + // skipping those keeps the recorder focused on column-chunk traffic. + if let Some(GetRange::Bounded(r)) = &options.range { + self.record(location, r.clone()); + } + self.inner.get_opts(location, options).await + } + + async fn get_ranges( + &self, + location: &Path, + ranges: &[Range], + ) -> object_store::Result> { + for r in ranges { + self.record(location, r.clone()); + } + self.inner.get_ranges(location, ranges).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// Returns the byte ranges of parquet column chunks whose dotted path matches `predicate`. +fn parquet_column_ranges(parquet_path: &str, predicate: impl Fn(&str) -> bool) -> Vec> { + use parquet::file::reader::{FileReader, SerializedFileReader}; + #[allow(clippy::unwrap_used)] + let reader = SerializedFileReader::new(std::fs::File::open(parquet_path).unwrap()).unwrap(); + reader + .metadata() + .row_groups() + .iter() + .flat_map(|rg| rg.columns()) + .filter(|col| predicate(&col.column_descr().path().parts().join("."))) + .map(|col| { + // `file_offset` is deprecated in the Parquet spec and inconsistent across + // writers (often `0`); `data_page_offset` is the reliable fallback when + // there is no dictionary page. + let start = col + .dictionary_page_offset() + .unwrap_or(col.data_page_offset()) as u64; + start..start + col.compressed_size() as u64 + }) + .collect() +} + +fn ranges_overlap(a: &Range, b: &Range) -> bool { + a.start < b.end && b.start < a.end +} + +#[derive(Clone)] +struct ParquetReadCall { + files: Vec, + physical_schema: SchemaRef, + predicate: Option, +} + +struct TracingParquetHandler { + inner: Arc, + calls: Arc>>, +} + +impl ParquetHandler for TracingParquetHandler { + fn read_parquet_files( + &self, + files: &[FileMeta], + physical_schema: SchemaRef, + predicate: Option, + ) -> KernelResult { + info!( + "TracingEngine::TracingParquetHandler::read_parquet_files\n\tfiles: {:?}\n\tpredicate: {:?}", + &files, &predicate + ); + self.calls + .lock() + .expect("lock not poisoned") + .push(ParquetReadCall { + files: files.to_vec(), + physical_schema: physical_schema.clone(), + predicate: predicate.clone(), + }); + self.inner + .read_parquet_files(files, physical_schema, predicate) + } + + fn write_parquet_file( + &self, + location: Url, + data: DeltaResultIteratorStatic>, + ) -> KernelResult<()> { + self.inner.write_parquet_file(location, data) + } + + fn read_parquet_footer(&self, file: &FileMeta) -> KernelResult { + self.inner.read_parquet_footer(file) + } +} + +struct TracingJsonHandler { + inner: Arc, + predicates: Arc>>>, +} + +impl JsonHandler for TracingJsonHandler { + fn parse_json( + &self, + json_strings: Box, + output_schema: SchemaRef, + ) -> KernelResult> { + self.inner.parse_json(json_strings, output_schema) + } + + fn read_json_files( + &self, + files: &[FileMeta], + physical_schema: SchemaRef, + predicate: Option, + ) -> KernelResult { + info!( + "TracingEngine::TracingJsonHandler::read_json_files\n\tfiles: {:?}\n\tpredicate: {:?}", + &files, predicate + ); + self.predicates + .lock() + .expect("lock not poisoned") + .push(predicate.clone()); + self.inner + .read_json_files(files, physical_schema, predicate) + } + + fn write_json_file( + &self, + path: &Url, + data: DeltaResultIterator<'_, FilteredEngineData>, + overwrite: bool, + ) -> KernelResult<()> { + self.inner.write_json_file(path, data, overwrite) + } +} + +struct TracingStorageHandler { + inner: Arc, +} + +impl StorageHandler for TracingStorageHandler { + fn list_from( + &self, + path: &Url, + ) -> KernelResult>>> { + self.inner.list_from(path) + } + + fn read_files( + &self, + files: Vec, + ) -> KernelResult>>> { + self.inner.read_files(files) + } + + fn copy_atomic(&self, src: &Url, dest: &Url) -> KernelResult<()> { + self.inner.copy_atomic(src, dest) + } + + fn put(&self, path: &Url, data: Bytes, overwrite: bool) -> KernelResult<()> { + self.inner.put(path, data, overwrite) + } + + fn head(&self, path: &Url) -> KernelResult { + self.inner.head(path) + } + + fn delete(&self, path: &Url) -> KernelResult<()> { + self.inner.delete(path) + } +} + +pub struct TracingEngine { + inner: Arc, + parquet_handler: Arc, + storage_handler: Arc, + json_handler: Arc, + parquet_calls: Arc>>, + json_predicates: Arc>>>, +} + +impl TracingEngine { + fn new(inner: Arc) -> Self { + let parquet_calls = Arc::new(Mutex::new(Vec::new())); + let json_predicates = Arc::new(Mutex::new(Vec::new())); + Self { + inner: inner.clone(), + parquet_handler: Arc::new(TracingParquetHandler { + inner: inner.parquet_handler(), + calls: parquet_calls.clone(), + }), + storage_handler: Arc::new(TracingStorageHandler { + inner: inner.storage_handler(), + }), + json_handler: Arc::new(TracingJsonHandler { + inner: inner.json_handler(), + predicates: json_predicates.clone(), + }), + parquet_calls, + json_predicates, + } + } + + pub fn wrap(inner: Arc) -> Arc { + Arc::new(Self::new(inner)) + } + + fn take_parquet_calls(&self) -> Vec { + std::mem::take(&mut *self.parquet_calls.lock().expect("lock not poisoned")) + } + + fn take_json_predicates(&self) -> Vec> { + std::mem::take(&mut *self.json_predicates.lock().expect("lock not poisoned")) + } +} + +impl Engine for TracingEngine { + fn evaluation_handler(&self) -> Arc { + self.inner.evaluation_handler() + } + + fn storage_handler(&self) -> Arc { + self.storage_handler.clone() + } + + fn json_handler(&self) -> Arc { + self.json_handler.clone() + } + + fn parquet_handler(&self) -> Arc { + self.parquet_handler.clone() + } +} + +#[derive(Default)] +struct DeltaScanVisitor { + num_scanned: Option, +} + +impl ExecutionPlanVisitor for DeltaScanVisitor { + type Error = DataFusionError; + + fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> DataFusionResult { + if let Some(delta_scan) = plan.downcast_ref::() { + self.num_scanned = delta_scan + .metrics() + .and_then(|metrics| metrics.sum_by_name("count_files_scanned")) + .map(|value| value.as_usize()); + } + Ok(true) + } +} + +#[derive(Clone, Copy)] +enum QueryColumn { + BatchId, + ConversationId, +} + +struct PredicateQuery { + sql: &'static str, + selected_files: usize, + output_rows: usize, + column: QueryColumn, +} + +struct CheckpointColumnRanges { + stats_parsed: Vec>, + stats: Vec>, + partition_values: Vec>, + partition_values_parsed: Vec>, +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_stats_parsed_without_files() -> DeltaResult<()> { + run_engine_predicate("conversations", true, false).await +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_stats_parsed_with_files() -> DeltaResult<()> { + run_engine_predicate("conversations", true, true).await +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_no_stats_parsed_without_files() -> DeltaResult<()> { + run_engine_predicate("conversations_no_stats_parsed", false, false).await +} + +#[tokio::test(flavor = "multi_thread")] +pub async fn test_predicate_no_stats_parsed_with_files() -> DeltaResult<()> { + run_engine_predicate("conversations_no_stats_parsed", false, true).await +} + +async fn run_engine_predicate( + fixture: &str, + expect_parsed_checkpoint_fields: bool, + require_files: bool, +) -> DeltaResult<()> { + let filter = tracing_subscriber::EnvFilter::from_default_env(); + let subscriber = tracing_subscriber::fmt() + .pretty() + .with_env_filter(filter) + .finish(); + tracing::subscriber::set_global_default(subscriber).ok(); + pretty_env_logger::try_init().ok(); + + let config = + SessionConfig::new().set_bool("datafusion.sql_parser.enable_ident_normalization", false); + + let ctx = SessionContext::new_with_config(config); + + let recording_object_store = Arc::new(RangeRecordingObjectStore::new(Arc::new( + object_store::local::LocalFileSystem::new(), + ))); + + let path = format!( + "file://{}/tests/data/{fixture}/delta/", + env!("CARGO_MANIFEST_DIR") + ); + let url = Url::parse(&path)?; + let mut builder = DeltaTableBuilder::from_url(url.clone()) + .unwrap() + .with_storage_backend(recording_object_store.clone(), url) + .with_checkpoint_stats_json_fallback(true); + let engine = TracingEngine::wrap(builder.build_storage().unwrap().engine(None)); + builder = builder.with_engine(engine.clone()); + if !require_files { + builder = builder.without_files(); + } + + let table = builder.load().await.unwrap(); + table.update_datafusion_session(&ctx.state()).unwrap(); + let provider = table.table_provider().await.unwrap(); + + let construction_parquet_calls = engine.take_parquet_calls(); + let construction_json_predicates = engine.take_json_predicates(); + if require_files { + assert!( + construction_parquet_calls.iter().any(|call| { + call.files.iter().any(|file| { + let path = file.location.path(); + path.contains("/_delta_log/") && path.contains(".checkpoint.") + }) + }), + "eager table construction must read transaction-log parquet" + ); + assert!( + !construction_json_predicates.is_empty(), + "eager table construction must read transaction-log JSON" + ); + } + + ctx.register_table("conversations", provider).unwrap(); + engine.take_parquet_calls(); + engine.take_json_predicates(); + recording_object_store.clear_reads(); + + let checkpoint_name = "00000000000000000007.checkpoint.parquet"; + let checkpoint_path = format!( + "{}/tests/data/{fixture}/delta/_delta_log/{checkpoint_name}", + env!("CARGO_MANIFEST_DIR") + ); + let checkpoint_ranges = CheckpointColumnRanges { + stats_parsed: parquet_column_ranges(&checkpoint_path, |path| { + path.starts_with("add.stats_parsed.") + }), + stats: parquet_column_ranges(&checkpoint_path, |path| path == "add.stats"), + partition_values: parquet_column_ranges(&checkpoint_path, |path| { + path.starts_with("add.partitionValues.") + }), + partition_values_parsed: parquet_column_ranges(&checkpoint_path, |path| { + path.starts_with("add.partitionValues_parsed.") + }), + }; + + let queries = [ + PredicateQuery { + sql: "SELECT conversation['conversationID'] FROM conversations WHERE _ACP_BATCHID = 'batch-1-2024-01-01'", + selected_files: 2, + output_rows: 4, + column: QueryColumn::BatchId, + }, + PredicateQuery { + sql: "SELECT conversation['conversationID'] FROM conversations WHERE _ACP_BATCHID != 'batch-1-2024-01-01'", + selected_files: 26, + output_rows: 52, + column: QueryColumn::BatchId, + }, + PredicateQuery { + sql: "SELECT conversation['conversationID'] FROM conversations WHERE conversation['conversationID'] = 'conversation-2024-01-01-1'", + selected_files: 2, + output_rows: 4, + column: QueryColumn::ConversationId, + }, + ]; + + for query in queries { + run_predicate_query( + &ctx, + &engine, + &recording_object_store, + checkpoint_name, + &checkpoint_ranges, + expect_parsed_checkpoint_fields, + require_files, + &query, + ) + .await; + } + + Ok(()) +} + +async fn run_predicate_query( + ctx: &SessionContext, + engine: &TracingEngine, + recording_object_store: &RangeRecordingObjectStore, + checkpoint_name: &str, + checkpoint_ranges: &CheckpointColumnRanges, + expect_parsed_checkpoint_fields: bool, + require_files: bool, + query: &PredicateQuery, +) { + engine.take_parquet_calls(); + engine.take_json_predicates(); + recording_object_store.clear_reads(); + + let dataframe = ctx.sql(query.sql).await.unwrap(); + let physical_plan = dataframe.create_physical_plan().await.unwrap(); + let mut visitor = DeltaScanVisitor::default(); + visit_execution_plan(physical_plan.as_ref(), &mut visitor).unwrap(); + assert_eq!( + visitor.num_scanned, + Some(query.selected_files), + "unexpected selected file count for {}", + query.sql + ); + + let result = collect(physical_plan, ctx.task_ctx()).await.unwrap(); + let output_rows = result.iter().map(|batch| batch.num_rows()).sum::(); + assert_eq!( + output_rows, query.output_rows, + "unexpected output row count for {}", + query.sql + ); + + let parquet_calls = engine.take_parquet_calls(); + let json_predicates = engine.take_json_predicates(); + if require_files { + assert_eq!( + parquet_calls.len(), + 0, + "eager query must not make kernel parquet calls: {}", + query.sql + ); + assert_eq!( + json_predicates.len(), + 0, + "eager query must not make kernel JSON calls: {}", + query.sql + ); + } else { + // check we read parquet + json + { + assert_eq!( + json_predicates.len(), + 1, + "expected one kernel JSON call for {}", + query.sql + ); + let predicate = json_predicates[0] + .as_ref() + .expect("query JSON call must have a predicate"); + let query_column = match query.column { + QueryColumn::BatchId => ColumnName::new(["_ACP_BATCHID"]), + QueryColumn::ConversationId => ColumnName::new(["conversation", "conversationID"]), + }; + assert!( + predicate.references().contains(&query_column), + "JSON predicate must reference the query column for {}", + query.sql + ); + } + { + assert_eq!( + parquet_calls.len(), + 2, + "expected sidecar discovery then checkpoint action read for {}", + query.sql + ); + for call in &parquet_calls { + assert_eq!( + call.files.len(), + 1, + "each kernel parquet call must read one checkpoint file" + ); + assert!( + call.files[0].location.path().ends_with(checkpoint_name), + "kernel parquet call must read the version-7 checkpoint" + ); + } + + let sidecar_call = &parquet_calls[0]; + assert!(sidecar_call.physical_schema.field("sidecar").is_some()); + assert!(sidecar_call.physical_schema.field("add").is_none()); + assert!(sidecar_call.predicate.is_none()); + + let action_call = &parquet_calls[1]; + assert!(action_call.physical_schema.field("add").is_some()); + if !expect_parsed_checkpoint_fields { + assert!( + action_call.predicate.is_none(), + "checkpoint without parsed stats must not have an action predicate" + ); + return; + } + + let predicate = action_call + .predicate + .as_ref() + .expect("parsed checkpoint action read must have a predicate"); + let references = predicate.references(); + let partition_column = ColumnName::new(["add", "partitionValues_parsed", "_ACP_BATCHID"]); + match query.column { + QueryColumn::BatchId => assert!( + references.contains(&partition_column), + "checkpoint predicate must reference the parsed partition column" + ), + QueryColumn::ConversationId => { + assert!( + references.contains(&ColumnName::new([ + "add", + "stats_parsed", + "minValues", + "conversation", + "conversationID", + ])), + "checkpoint predicate must reference the nested minimum" + ); + assert!( + references.contains(&ColumnName::new([ + "add", + "stats_parsed", + "maxValues", + "conversation", + "conversationID", + ])), + "checkpoint predicate must reference the nested maximum" + ); + assert!( + !references.contains(&partition_column), + "nested checkpoint predicate must not retain the partition reference" + ); + } + } + } + } + + { + let checkpoint_reads = recording_object_store.reads_for_path_suffix(checkpoint_name); + if require_files { + assert!( + checkpoint_reads.is_empty(), + "eager query must not read checkpoint byte ranges" + ); + return; + } + + let ranges_were_read = |column_ranges: &[Range]| { + checkpoint_reads.iter().any(|read| { + column_ranges + .iter() + .any(|range| ranges_overlap(read, range)) + }) + }; + assert!( + !checkpoint_reads.is_empty(), + "expected checkpoint column reads" + ); + assert!( + !checkpoint_ranges.stats.is_empty(), + "checkpoint must contain add.stats" + ); + assert!( + ranges_were_read(&checkpoint_ranges.partition_values), + "expected add.partitionValues to be read" + ); + + if expect_parsed_checkpoint_fields { + assert!( + !checkpoint_ranges.stats_parsed.is_empty(), + "checkpoint must contain add.stats_parsed" + ); + assert!( + !checkpoint_ranges.partition_values_parsed.is_empty(), + "checkpoint must contain add.partitionValues_parsed" + ); + assert!( + ranges_were_read(&checkpoint_ranges.stats_parsed), + "expected add.stats_parsed to be read" + ); + assert!( + !ranges_were_read(&checkpoint_ranges.stats), + "expected add.stats not to be read" + ); + assert_eq!( + ranges_were_read(&checkpoint_ranges.partition_values_parsed), + matches!(query.column, QueryColumn::BatchId), + "parsed partition-value reads must be query-specific" + ); + } else { + assert!( + checkpoint_ranges.stats_parsed.is_empty(), + "checkpoint must not contain add.stats_parsed" + ); + assert!( + checkpoint_ranges.partition_values_parsed.is_empty(), + "checkpoint must not contain add.partitionValues_parsed" + ); + assert!( + ranges_were_read(&checkpoint_ranges.stats), + "expected add.stats to be read" + ); + } + } +} diff --git a/crates/core/tests/it_datafusion/main.rs b/crates/core/tests/it_datafusion/main.rs index 98953739c7..e26234101b 100644 --- a/crates/core/tests/it_datafusion/main.rs +++ b/crates/core/tests/it_datafusion/main.rs @@ -19,3 +19,6 @@ mod integration_datafusion; mod nanosecond_timestamps; mod read_delta_log_test; mod read_delta_partitions_test; +mod stats_parsed_checkpoint; +mod integration_datafusion_deep; +pub mod integration_datafusion_predicate; \ No newline at end of file diff --git a/crates/core/tests/it_datafusion/stats_parsed_checkpoint.rs b/crates/core/tests/it_datafusion/stats_parsed_checkpoint.rs new file mode 100644 index 0000000000..7dd37b0213 --- /dev/null +++ b/crates/core/tests/it_datafusion/stats_parsed_checkpoint.rs @@ -0,0 +1,447 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use arrow_array::{ArrayRef, Int32Array, RecordBatch, StringArray}; +use datafusion::assert_batches_sorted_eq; +use datafusion::common::{ScalarValue, stats::Precision}; +use datafusion::datasource::physical_plan::ParquetSource; +use datafusion::physical_plan::{ExecutionPlan, collect}; +use datafusion::prelude::{SessionContext, col, lit}; +use datafusion_datasource::source::DataSourceExec; +use delta_kernel::actions::STATS_PARSED; +use delta_kernel::schema::{DataType, SchemaRef, StructField}; +use delta_kernel::{ + DeltaResult as KernelResult, DeltaResultIteratorStatic, Engine, EngineData, EvaluationHandler, + FileDataReadResultIterator, FileMeta, JsonHandler, ParquetFooter, ParquetHandler, PredicateRef, + StorageHandler, +}; +use deltalake_core::checkpoints::create_checkpoint; +use deltalake_core::delta_datafusion::DeltaScanExec; +use deltalake_core::operations::write::SchemaMode; +use deltalake_core::protocol::SaveMode; +use deltalake_core::{DeltaTable, DeltaTableBuilder, TableProperty}; +use deltalake_test::TestResult; +use futures::TryStreamExt; +use parquet::arrow::async_reader::{ParquetObjectReader, ParquetRecordBatchStreamBuilder}; +use url::Url; + +struct StatsParsedParquetHandler { + inner: Arc, + checkpoint_add_reads: AtomicUsize, +} + +impl ParquetHandler for StatsParsedParquetHandler { + fn read_parquet_files( + &self, + files: &[FileMeta], + physical_schema: SchemaRef, + predicate: Option, + ) -> KernelResult { + let reads_checkpoint_adds = physical_schema.field("add").is_some() + && files + .iter() + .any(|file| file.location.path().contains(".checkpoint.")); + if reads_checkpoint_adds { + let add = physical_schema + .field("add") + .and_then(|field| match field.data_type() { + DataType::Struct(add) => Some(add), + _ => None, + }) + .expect("projected checkpoint Add action must be a struct"); + assert!( + add.field("stats").is_none(), + "compatibility reads must not project add.stats when add.stats_parsed is available" + ); + assert!( + add.field(STATS_PARSED).is_some(), + "compatibility reads must project add.stats_parsed" + ); + self.checkpoint_add_reads.fetch_add(1, Ordering::SeqCst); + } + + self.inner + .read_parquet_files(files, physical_schema, predicate) + } + + fn write_parquet_file( + &self, + location: Url, + data: DeltaResultIteratorStatic>, + ) -> KernelResult<()> { + self.inner.write_parquet_file(location, data) + } + + fn read_parquet_footer(&self, file: &FileMeta) -> KernelResult { + self.inner.read_parquet_footer(file) + } +} + +struct RecordingEngine { + inner: Arc, + parquet_handler: Arc, +} + +impl Engine for RecordingEngine { + fn evaluation_handler(&self) -> Arc { + self.inner.evaluation_handler() + } + + fn storage_handler(&self) -> Arc { + self.inner.storage_handler() + } + + fn json_handler(&self) -> Arc { + self.inner.json_handler() + } + + fn parquet_handler(&self) -> Arc { + self.parquet_handler.clone() + } +} + +#[tokio::test] +#[ignore = "Missing / broken fixture"] +async fn table_scan_prefers_checkpoint_stats_parsed() -> TestResult { + let fixture = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../delta-kernel-rs/kernel/tests/data/stats-parsed-partitioned"); + assert!(fixture.is_dir(), "missing fixture at {}", fixture.display()); + + let temp_dir = tempfile::tempdir()?; + let copy_options = fs_extra::dir::CopyOptions { + content_only: true, + ..Default::default() + }; + fs_extra::dir::copy(&fixture, temp_dir.path(), ©_options)?; + + let table_url = Url::from_directory_path(temp_dir.path()) + .map_err(|_| "temporary table path is not a directory")?; + + // Checkpoint version 2 so the latest scan combines checkpoint stats with version 3 JSON stats. + let checkpoint_table = DeltaTableBuilder::from_url(table_url.clone())? + .with_version(2) + .load() + .await?; + create_checkpoint(&checkpoint_table, None).await?; + + let base_engine = checkpoint_table.log_store().engine(None); + let checkpoint_url = table_url.join("_delta_log/00000000000000000002.checkpoint.parquet")?; + let checkpoint_meta = base_engine.storage_handler().head(&checkpoint_url)?; + let checkpoint_footer = base_engine + .parquet_handler() + .read_parquet_footer(&checkpoint_meta)?; + let checkpoint_add = checkpoint_footer + .schema + .field("add") + .and_then(|field| match field.data_type() { + DataType::Struct(add) => Some(add), + _ => None, + }) + .expect("checkpoint Add action must be a struct"); + assert!( + checkpoint_add.field("stats").is_some(), + "checkpoint must physically contain stats so the projection assertion is meaningful" + ); + assert!( + checkpoint_add.field(STATS_PARSED).is_some(), + "checkpoint must physically contain stats_parsed" + ); + + let parquet_handler = Arc::new(StatsParsedParquetHandler { + inner: base_engine.parquet_handler(), + checkpoint_add_reads: AtomicUsize::new(0), + }); + let engine: Arc = Arc::new(RecordingEngine { + inner: base_engine, + parquet_handler: parquet_handler.clone(), + }); + + // This is the builder equivalent of open_table, with an injected engine for observing the + // checkpoint projection. The DataFusion scan replays the opened table's materialized files. + let table = DeltaTableBuilder::from_url(table_url)? + .with_engine(engine) + .load() + .await?; + let checkpoint_add_reads_after_load = + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst); + assert_eq!( + checkpoint_add_reads_after_load, 1, + "the eager compatibility-cache read must read checkpoint Add actions exactly once" + ); + assert_eq!(table.version(), Some(3)); + + let context = SessionContext::new(); + let provider = table.table_provider().await?; + + // Keep all fixture files in the plan so their exact statistics remain observable. The + // narrower row predicate below correctly prunes the first file during kernel replay. + let stats_filter = col("user_id") + .gt_eq(lit(1_i64)) + .and(col("user_id").lt_eq(lit(9_i64))); + let stats_scan = provider + .scan(&context.state(), None, &[stats_filter], None) + .await?; + + let delta_scan = stats_scan + .downcast_ref::() + .expect("expected DeltaScanExec"); + let children = delta_scan.children(); + let data_source = children[0] + .downcast_ref::() + .expect("expected DataSourceExec child"); + let (file_scan, _) = data_source + .downcast_to_file_source::() + .expect("expected Parquet file source"); + let user_id_index = file_scan + .file_source + .table_schema() + .table_schema() + .index_of("user_id")?; + let mut parsed_file_stats = file_scan + .file_groups + .iter() + .flat_map(|group| group.iter()) + .map(|file| { + let stats = file + .statistics + .as_ref() + .expect("scan file must have parsed statistics"); + let user_id = &stats.column_statistics[user_id_index]; + let value = |precision: &Precision| match precision { + Precision::Exact(ScalarValue::Int64(Some(value))) => *value, + other => panic!("expected exact Int64 statistic, got {other:?}"), + }; + (value(&user_id.min_value), value(&user_id.max_value)) + }) + .collect::>(); + parsed_file_stats.sort_unstable(); + assert_eq!(parsed_file_stats, vec![(1, 3), (4, 6), (7, 9)]); + + let row_filter = col("user_id") + .gt_eq(lit(4_i64)) + .and(col("user_id").lt_eq(lit(8_i64))); + let row_scan = provider + .scan(&context.state(), None, &[row_filter], None) + .await?; + let batches = collect(row_scan, context.task_ctx()).await?; + + let expected = [ + "+------------+---------------+---------+", + "| _ACP_DATE | _ACP_BATCH_ID | user_id |", + "+------------+---------------+---------+", + "| 2024-01-01 | batch-002 | 4 |", + "| 2024-01-01 | batch-002 | 5 |", + "| 2024-01-01 | batch-002 | 6 |", + "| 2024-01-02 | batch-001 | 7 |", + "| 2024-01-02 | batch-001 | 8 |", + "+------------+---------------+---------+", + ]; + assert_batches_sorted_eq!(&expected, &batches); + assert_eq!( + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst), + checkpoint_add_reads_after_load, + "broad and narrow query scans must replay from the materialized seed instead of re-reading checkpoint Add actions" + ); + + Ok(()) +} + +#[tokio::test] +async fn table_scan_skips_old_physical_schema_file_from_checkpoint_stats_parsed() -> TestResult { + let temp_dir = tempfile::tempdir()?; + let table_url = Url::from_directory_path(temp_dir.path()) + .map_err(|_| "temporary table path is not a directory")?; + + let table = DeltaTable::try_from_url(table_url.clone()) + .await? + .create() + .with_columns([ + StructField::nullable("id", DataType::INTEGER), + StructField::nullable("part", DataType::STRING), + ]) + .with_partition_columns(["part"]) + .with_configuration_property(TableProperty::CheckpointWriteStatsAsStruct, Some("true")) + .with_configuration_property(TableProperty::CheckpointWriteStatsAsJson, Some("true")) + .await?; + + let old_batch = RecordBatch::try_from_iter_with_nullable(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + false, + ), + ( + "part", + Arc::new(StringArray::from(vec!["old", "old"])) as ArrayRef, + false, + ), + ])?; + let table = table + .write(vec![old_batch]) + .with_save_mode(SaveMode::Append) + .await?; + + let new_batch = RecordBatch::try_from_iter_with_nullable(vec![ + ( + "id", + Arc::new(Int32Array::from(vec![100, 101])) as ArrayRef, + false, + ), + ( + "part", + Arc::new(StringArray::from(vec!["new", "new"])) as ArrayRef, + false, + ), + ( + "added", + Arc::new(StringArray::from(vec![Some("present"), None])) as ArrayRef, + true, + ), + ])?; + let table = table + .write(vec![new_batch]) + .with_save_mode(SaveMode::Append) + .with_schema_mode(SchemaMode::Merge) + .await?; + assert_eq!(table.version(), Some(2)); + + let adds = table + .get_active_add_actions_by_partitions(&[]) + .try_collect::>() + .await?; + assert_eq!(adds.len(), 2, "expected one data file per partition"); + + let mut physical_files = Vec::with_capacity(2); + for add in adds { + let path = add.path().into_owned(); + let partition_values = add.partition_values_map(); + assert_eq!(partition_values.len(), 1); + let part = partition_values + .get("part") + .cloned() + .flatten() + .expect("data file must have a non-null part value"); + + let reader = ParquetObjectReader::new( + table.object_store(), + deltalake_core::Path::from(path.as_str()), + ); + let builder = ParquetRecordBatchStreamBuilder::new(reader).await?; + let physical_fields = builder + .schema() + .fields() + .iter() + .map(|field| field.name().to_string()) + .collect::>(); + physical_files.push((part, path, physical_fields)); + } + physical_files.sort_by(|left, right| left.0.cmp(&right.0)); + + let new_file = &physical_files[0]; + assert_eq!(new_file.0, "new"); + assert!(new_file.1.contains("part=new/"), "{:?}", new_file.1); + assert_eq!(new_file.2, vec!["id", "added"]); + + let old_file = &physical_files[1]; + assert_eq!(old_file.0, "old"); + assert!(old_file.1.contains("part=old/"), "{:?}", old_file.1); + assert_eq!(old_file.2, vec!["id"]); + + create_checkpoint(&table, None).await?; + + let base_engine = table.log_store().engine(None); + let checkpoint_url = table_url.join("_delta_log/00000000000000000002.checkpoint.parquet")?; + let checkpoint_meta = base_engine.storage_handler().head(&checkpoint_url)?; + let checkpoint_footer = base_engine + .parquet_handler() + .read_parquet_footer(&checkpoint_meta)?; + let checkpoint_add = checkpoint_footer + .schema + .field("add") + .and_then(|field| match field.data_type() { + DataType::Struct(add) => Some(add), + _ => None, + }) + .expect("checkpoint Add action must be a struct"); + assert!( + checkpoint_add.field("stats").is_some(), + "checkpoint must physically contain JSON stats" + ); + assert!( + checkpoint_add.field(STATS_PARSED).is_some(), + "checkpoint must physically contain stats_parsed" + ); + + let parquet_handler = Arc::new(StatsParsedParquetHandler { + inner: base_engine.parquet_handler(), + checkpoint_add_reads: AtomicUsize::new(0), + }); + let engine: Arc = Arc::new(RecordingEngine { + inner: base_engine, + parquet_handler: parquet_handler.clone(), + }); + let reopened = DeltaTableBuilder::from_url(table_url)? + .with_engine(engine) + .load() + .await?; + let checkpoint_add_reads_after_load = + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst); + assert_eq!(checkpoint_add_reads_after_load, 1); + + let context = SessionContext::new(); + let provider = reopened.table_provider().await?; + let id_filter = col("id").gt(lit(50_i32)); + let scan = provider + .scan(&context.state(), None, &[id_filter], None) + .await?; + + let delta_scan = scan + .downcast_ref::() + .expect("expected DeltaScanExec"); + let children = delta_scan.children(); + let data_source = children[0] + .downcast_ref::() + .expect("expected DataSourceExec child"); + let (file_scan, _) = data_source + .downcast_to_file_source::() + .expect("expected Parquet file source"); + let scanned_paths = file_scan + .file_groups + .iter() + .flat_map(|group| group.iter()) + .map(|file| file.object_meta.location.to_string()) + .collect::>(); + assert_eq!(scanned_paths.len(), 1, "{scanned_paths:?}"); + assert!(scanned_paths[0].ends_with(&new_file.1), "{scanned_paths:?}"); + assert!( + !scanned_paths[0].ends_with(&old_file.1), + "{scanned_paths:?}" + ); + + let batches = collect(scan.clone(), context.task_ctx()).await?; + assert_eq!( + delta_scan + .metrics() + .and_then(|metrics| metrics.sum_by_name("count_files_scanned")) + .map(|value| value.as_usize()), + Some(1) + ); + + let expected = [ + "+-----+------+---------+", + "| id | part | added |", + "+-----+------+---------+", + "| 100 | new | present |", + "| 101 | new | |", + "+-----+------+---------+", + ]; + assert_batches_sorted_eq!(&expected, &batches); + assert_eq!( + parquet_handler.checkpoint_add_reads.load(Ordering::SeqCst), + checkpoint_add_reads_after_load, + "query scan must not re-read checkpoint Add actions" + ); + + Ok(()) +}