Skip to content

[HSTACK] Delta-rs controls stats_parsed loading in the kernel #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
aditanase wants to merge 20 commits into
base: main-1.0
Choose a base branch
from
Draft

[HSTACK] Delta-rs controls stats_parsed loading in the kernel #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
97279e0
[HSTACK] chore: datafusion 54, buoyant_kernel 0.25.1 main Cargo.toml …
adragomir Jul 22, 2026
4524c07
[HSTACK] chore: expose extra methods: scan, DeltaScan struct properti…
adragomir Jul 7, 2026
ea3359c
[HSTACK] fix: optimize scan builder limit pushdown
aditanase Jul 7, 2026
b63e470
[HSTACK] fix: REEVALUATE, bring back old table provider
adragomir Jul 7, 2026
668a053
[HSTACK] feat: add delta table udtf and register function
adragomir Feb 6, 2026
162a269
[HSTACK] test: add support for deep projections TEST
adragomir Feb 6, 2026
f2624f8
[HSTACK] fix: schema mapping for record batches with an empty schema …
adragomir Jul 7, 2026
6611ae1
[HSTACK] feat: log segment size limiter
ccciudatu May 7, 2026
96edf4f
[HSTACK] feat: expose log metadata and total size to support weighted…
aditanase Feb 6, 2026
841c803
[HSTACK] feat: add support for log_store reconfigure
aditanase Feb 6, 2026
ebd223f
[HSTACK] fix: add table constructor that does not remove final `/`
adragomir Jul 7, 2026
c76cb33
[HSTACK] feat: implement physical codec for "next" version of Executi…
adragomir Aug 27, 2026
5d3e958
[HSTACK] feat: add option to modify the low-level Delta schema to mak…
adragomir Aug 13, 2026
6f088b6
[HSTACK] feat: expose snapshot load metrics in order to detect if log…
adragomir Aug 27, 2026
d92dabe
[HSTACK] feat: delta engine injection support
Jul 7, 2026
f740a79
[HSTACK] feat: add ability to construct Snapshot and EagerSnapshot ma…
adragomir Jul 7, 2026
419198a
[HSTACK] chore: downgrade time dep to overcome incompat with lancedb
adragomir Jul 23, 2026
1be32fe
[HSTACK] Delta-rs controls stats_parsed loading in the kernel
adragomir Aug 13, 2026
a5bdcf5
[HSTACK] feat: make to_datafusion_expr public
adragomir Aug 20, 2026
5a25b6e
[HSTACK][TEST] feat: Add predicate test
adragomir Aug 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 80 additions & 5 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,14 @@ documentation = "https://docs.rs/deltalake"
repository = "https://github.com/delta-io/delta.rs"

[workspace.dependencies]
#delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [
delta_kernel = { package = "buoyant_kernel", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = [
# delta_kernel = { package = "buoyant_kernel", path = "../delta-kernel-rs/kernel", features = [
delta_kernel = { package = "buoyant_kernel", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = [
#delta_kernel = { package = "buoyant_kernel", version = "0.25.0,<0.25.100", features = [
"arrow-58",
"internal-api",
] }
#delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false }
delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/buoyant-data/delta-kernel-rs", branch = "buoyant/main", features = ["arrow-58", "rustls"], default-features = false }
# delta_kernel_default_engine = { package = "buoyant_kernel_engine", path = "../delta-kernel-rs/default-engine", features = ["arrow-58", "rustls"], default-features = false }
delta_kernel_default_engine = { package = "buoyant_kernel_engine", git = "https://github.com/hstack/delta-kernel-rs", branch = "buoyant-v0.25.1-stats", features = ["arrow-58", "rustls"], default-features = false }
#delta_kernel_default_engine = { package = "buoyant_kernel_engine", version = "0.25.0,<0.25.100", features = ["arrow-58", "rustls"], default-features = false }


Expand Down Expand Up @@ -52,11 +52,13 @@ datafusion-proto = { version = "54.0.0" }
# serde
serde = { version = "1.0.194", features = ["derive"] }
serde_json = "1"
strum = { version = "0.28", features = ["derive"] }
prost = { version = "0.14.3" }

# "stdlib"
bytes = { version = "1" }
chrono = { version = "0.4.40", default-features = false, features = ["clock"] }
time = { version = "=0.3.53" }
time = { version = "=0.3.47" }
tracing = { version = "0.1", features = ["log"] }
regex = { version = "1" }
thiserror = { version = "2" }
Expand Down Expand Up @@ -160,3 +162,76 @@ inherits = "release"
opt-level = 3
codegen-units = 1
lto = "thin"

[patch.crates-io]
datafusion = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-catalog = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-catalog-listing = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-common-runtime = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-avro = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-csv = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-json = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-datasource-parquet = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-execution = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-ffi = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-aggregate = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-aggregate-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-nested = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-table = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-window = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-functions-window-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-expr = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-expr-adapter = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-expr-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-optimizer = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-pruning = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-physical-plan = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-proto = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-proto-common = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-session = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-spark = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-sql = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
datafusion-substrait = { git = 'https://github.com/hstack/datafusion.git', branch = 'main-54' }
#datafusion = { path = "../datafusion/datafusion/core" }
#datafusion-catalog = { path = "../datafusion/datafusion/catalog" }
#datafusion-catalog-listing = { path = "../datafusion/datafusion/catalog-listing" }
#datafusion-common = { path = "../datafusion/datafusion/common" }
#datafusion-common-runtime = { path = "../datafusion/datafusion/common-runtime" }
#datafusion-datasource = { path = "../datafusion/datafusion/datasource" }
#datafusion-datasource-arrow = { path = "../datafusion/datafusion/datasource-arrow" }
#datafusion-datasource-avro = { path = "../datafusion/datafusion/datasource-avro" }
#datafusion-datasource-csv = { path = "../datafusion/datafusion/datasource-csv" }
#datafusion-datasource-json = { path = "../datafusion/datafusion/datasource-json" }
#datafusion-datasource-parquet = { path = "../datafusion/datafusion/datasource-parquet" }
#datafusion-execution = { path = "../datafusion/datafusion/execution" }
#datafusion-expr = { path = "../datafusion/datafusion/expr" }
#datafusion-expr-common = { path = "../datafusion/datafusion/expr-common" }
#datafusion-ffi = { path = "../datafusion/datafusion/ffi" }
#datafusion-functions = { path = "../datafusion/datafusion/functions" }
#datafusion-functions-aggregate = { path = "../datafusion/datafusion/functions-aggregate" }
#datafusion-functions-aggregate-common = { path = "../datafusion/datafusion/functions-aggregate-common" }
#datafusion-functions-nested = { path = "../datafusion/datafusion/functions-nested" }
#datafusion-functions-table = { path = "../datafusion/datafusion/functions-table" }
#datafusion-functions-window = { path = "../datafusion/datafusion/functions-window" }
#datafusion-functions-window-common = { path = "../datafusion/datafusion/functions-window-common" }
#datafusion-optimizer = { path = "../datafusion/datafusion/optimizer" }
#datafusion-physical-expr = { path = "../datafusion/datafusion/physical-expr" }
#datafusion-physical-expr-adapter = { path = "../datafusion/datafusion/physical-expr-adapter" }
#datafusion-physical-expr-common = { path = "../datafusion/datafusion/physical-expr-common" }
#datafusion-physical-optimizer = { path = "../datafusion/datafusion/physical-optimizer" }
#datafusion-physical-plan = { path = "../datafusion/datafusion/physical-plan" }
#datafusion-proto = { path = "../datafusion/datafusion/proto" }
#datafusion-proto-common = { path = "../datafusion/datafusion/proto-common" }
#datafusion-pruning = { path = "../datafusion/datafusion/pruning" }
#datafusion-session = { path = "../datafusion/datafusion/session" }
#datafusion-spark = { path = "../datafusion/datafusion/spark" }
#datafusion-sql = { path = "../datafusion/datafusion/sql" }

#buoyant_kernel = { git = 'https://github.com/hstack/delta-kernel-rs.git', branch = 'buoyant-v0.25.1-stats' }
#buoyant_kernel = { path = "../delta-kernel-rs/kernel" }
3 changes: 3 additions & 0 deletions crates/core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ datafusion-proto = { workspace = true, optional = true }
# serde
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
strum = { workspace = true, features = ["derive"] }
prost = { workspace = true }

# "stdlib"
bytes = { workspace = true }
Expand Down Expand Up @@ -95,6 +97,7 @@ rand = "0.10"
sqlparser = { version = "0.61.0" }
humantime = { version = "2.1.0", optional = true }
validator = { version = "0.19", features = ["derive"] }
ctor = "0.10"

[dev-dependencies]
arrow = { workspace = true, features = ["prettyprint"]}
Expand Down
2 changes: 1 addition & 1 deletion crates/core/src/delta_datafusion/find_files.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ impl TreeNodeVisitor<'_> for FindFilesExprProperties {
/// rather than materializing every partition column — which avoids coercing values the
/// predicate never reads and prevents an unrelated/malformed partition value from
/// weakening pruning.
pub(crate) fn extract_partition_only_predicate(
pub fn extract_partition_only_predicate(
predicate: Expr,
partition_columns: &[String],
) -> DeltaResult<Option<(Expr, Vec<String>)>> {
Expand Down
22 changes: 20 additions & 2 deletions crates/core/src/delta_datafusion/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,25 @@ pub(crate) use column_mapping::ColumnMappingState;
pub(crate) use data_validation::{
DataValidationExec, constraints_to_exprs, generated_columns_to_exprs, validation_predicates,
};
pub use engine::DataFusionEngine;
pub(crate) use find_files::*;
pub(crate) use table_provider::next::normalize_path_as_file_id;
pub use table_provider::{
DeltaScanConfig, DeltaScanConfigBuilder, TableProviderBuilder, next::DeltaScanExec,
next::DeltaNextPhysicalCodec,
};
pub(crate) use table_provider::{
next::FILE_ID_COLUMN_DEFAULT, resolve_file_column_name, update_datafusion_session,
};

/// Converts a Delta Kernel expression to a DataFusion expression.
pub fn to_datafusion_expr(
expr: &delta_kernel::expressions::Expression,
output_type: &delta_kernel::schema::DataType,
) -> DataFusionResult<Expr> {
engine::to_datafusion_expr(expr, output_type)
}

pub(crate) const PATH_COLUMN: &str = "__delta_rs_path";

#[doc(hidden)]
Expand All @@ -100,8 +110,13 @@ pub mod planner;
mod session;
pub use session::SessionFallbackPolicy;
pub(crate) use session::{SessionResolveContext, resolve_session_state};
use crate::delta_datafusion::schema_null::rewrite_schema_with_nullable_fields;

mod table_provider;
pub mod udtf;
pub(crate) mod utils;
pub mod table_provider_old;
pub mod schema_null;

impl From<DeltaTableError> for DataFusionError {
fn from(err: DeltaTableError) -> Self {
Expand Down Expand Up @@ -232,7 +247,7 @@ fn _arrow_schema(
partition_columns: &[String],
wrap_partitions: bool,
) -> ArrowSchemaRef {
let fields = schema
let mut fields = schema
.fields()
.into_iter()
.filter(|f| !partition_columns.contains(&f.name().to_string()))
Expand Down Expand Up @@ -261,7 +276,10 @@ fn _arrow_schema(
}),
)
.collect::<Vec<_>>();
Arc::new(ArrowSchema::new(fields))
let mut schema = Arc::new(ArrowSchema::new(fields));
// @Hstack - add the option to have an env var that can nullify fields in the delta schema
schema = rewrite_schema_with_nullable_fields(schema);
schema
}

pub(crate) fn files_matching_predicate<'a>(
Expand Down
Loading
Loading