diff --git a/frame/agg_expr_test.mbt b/frame/agg_expr_test.mbt index 12cdf6c6..b389ab47 100644 --- a/frame/agg_expr_test.mbt +++ b/frame/agg_expr_test.mbt @@ -700,3 +700,336 @@ test "a deeply aliased aggregation takes the bare-column fast path without overf ) assert_eq(out.check_invariants(), Ok(())) } + +///| +/// Blackbox tests for the expression aggregation family added on the shared +/// reduction kernel — `std` / `variance` / `median` / `n_unique` / `first` / +/// `last` — driven through both reduction surfaces: a whole-frame `select` +/// (the per-scope evaluator) and a grouped `agg` (the single-pass fast path). +/// Covers each op's value, output dtype, and null / NaN rule: `std` / `var` +/// propagate NaN and reduce a `< 2`-cell scope to null, `median` skips NaN, +/// `n_unique` buckets NaN as one and excludes nulls, and `first` / `last` are +/// positional (a null leading / trailing cell, or an empty scope, is null). +/// Non-numeric `std` / `var` / `median` raise `TypeMismatch`. Construction and +/// rendering have their own coverage; this pins evaluation. + +///| +/// The single cell of the whole-frame reduction `e` over `df`, via `select`: +/// `select([agg])` collapses to one row, so cell `(0, "r")` is the reduction. +fn whole( + df : DataFrame, + e : @expr.Expr, +) -> @types.Scalar raise @types.DataError { + df.select([e.with_alias("r")]).item(0, "r") +} + +// ── std / variance: value, dtype, NaN propagation, < 2 cells ─────────── + +///| +test "std and variance reduce a whole frame to the sample statistics" { + // [2, 4, 6]: mean 4, squared deviations 4 + 0 + 4 = 8, sample variance + // (ddof = 1) 8 / 2 = 4.0, standard deviation its root, 2.0. Both are Float + // even though the source is Int. + let df = DataFrame::DataFrame([Series::from_ints("x", [2, 4, 6])]) + assert_eq(whole(df, @expr.col("x").std()), @types.Scalar::Float(2.0)) + assert_eq(whole(df, @expr.col("x").variance()), @types.Scalar::Float(4.0)) +} + +///| +test "std over a Float column keeps Float and propagates NaN" { + let df = DataFrame::DataFrame([Series::from_floats("x", [1.0, 2.0, 3.0])]) + // [1, 2, 3]: variance 1.0, std 1.0. + assert_eq(whole(df, @expr.col("x").std()), @types.Scalar::Float(1.0)) + // A non-null NaN is a value: it flows through the mean, so std / var + // propagate it (matching sum / mean), unlike min / max / median. + let nan = @double.not_a_number + let dfn = DataFrame::DataFrame([Series::from_floats("x", [1.0, 2.0, nan])]) + assert_true(whole(dfn, @expr.col("x").std()).as_float().is_nan()) + assert_true(whole(dfn, @expr.col("x").variance()).as_float().is_nan()) +} + +///| +test "variance and std of equal large-magnitude values are 0, not +inf" { + // Two equal cells near Double-max: the true variance is 0. A two-pass mean + // would sum them to +inf and report +inf; Welford never forms the raw sum, so + // the finite variance survives. + let df = DataFrame::DataFrame([Series::from_floats("x", [1.0e308, 1.0e308])]) + assert_eq(whole(df, @expr.col("x").variance()), @types.Scalar::Float(0.0)) + assert_eq(whole(df, @expr.col("x").std()), @types.Scalar::Float(0.0)) +} + +///| +test "variance and std of opposite-sign extremes saturate to +inf, never negative" { + // Finite cells more than Double-max apart: Welford's running `delta` + // overflows and used to poison `m2` to -inf — a mathematically impossible + // NEGATIVE variance, and sqrt(-inf) = NaN for std. The true variance + // (~2e616) exceeds Double's range, so the statistic saturates to +inf, + // as numpy (ddof=1) and Polars report. + let two = DataFrame::DataFrame([Series::from_floats("x", [-1.0e308, 1.0e308])]) + assert_eq( + whole(two, @expr.col("x").variance()), + @types.Scalar::Float(@double.infinity), + ) + assert_eq( + whole(two, @expr.col("x").std()), + @types.Scalar::Float(@double.infinity), + ) + // A longer alternating window drives the artefact to NaN instead of -inf; + // the finite-input rescue covers it the same way. + let three = DataFrame::DataFrame([ + Series::from_floats("x", [1.0e308, -1.0e308, 1.0e308]), + ]) + assert_eq( + whole(three, @expr.col("x").variance()), + @types.Scalar::Float(@double.infinity), + ) + // A genuine ±inf INPUT is not rescued: it propagates through the mean + // like sum / mean, so the statistic is NaN (inf - inf), not +inf. + let inf_in = DataFrame::DataFrame([ + Series::from_floats("x", [@double.infinity, 1.0]), + ]) + assert_true(whole(inf_in, @expr.col("x").variance()).as_float().is_nan()) + assert_true(whole(inf_in, @expr.col("x").std()).as_float().is_nan()) +} + +///| +test "std and variance are null for fewer than two non-null cells" { + // Sample statistics need ddof = 1 < cnt: one value, or none, has no + // sample variance, so the cell is null rather than 0.0. + let one = DataFrame::DataFrame([Series::from_ints("x", [5])]) + assert_eq(whole(one, @expr.col("x").std()), @types.Scalar::Null) + assert_eq(whole(one, @expr.col("x").variance()), @types.Scalar::Null) + let empty = DataFrame::DataFrame([Series::from_ints("x", [])]) + assert_eq(whole(empty, @expr.col("x").std()), @types.Scalar::Null) + // Nulls are skipped, so a lone non-null cell beside nulls is still < 2. + let sparse = DataFrame::DataFrame([ + Series::from_int_options("x", [Some(5), None, None]), + ]) + assert_eq(whole(sparse, @expr.col("x").variance()), @types.Scalar::Null) +} + +// ── median: value, even/odd, dtype, NaN skipped ─────────────────────── + +///| +test "median is the middle of the sorted non-null cells, always Float" { + // Odd count: the middle. Int widens, so the result is Float. + let odd = DataFrame::DataFrame([Series::from_ints("x", [3, 1, 2])]) + assert_eq(whole(odd, @expr.col("x").median()), @types.Scalar::Float(2.0)) + // Even count: the mean of the two middles. + let even = DataFrame::DataFrame([Series::from_ints("x", [1, 2, 3, 4])]) + assert_eq(whole(even, @expr.col("x").median()), @types.Scalar::Float(2.5)) +} + +///| +test "median propagates NaN, skips nulls, all-missing is null" { + let nan = @double.not_a_number + // NaN propagates — the sum / mean rule, not the min / max skip: a window + // that contains one has that value's order statistic. + let df = DataFrame::DataFrame([Series::from_floats("x", [1.0, 2.0, nan])]) + let got = whole(df, @expr.col("x").median()) + assert_true( + match got { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + // Nulls drop out; [10, null, 20, null, 30] medians to 20.0. + let sparse = DataFrame::DataFrame([ + Series::from_float_options("x", [ + Some(10.0), + None, + Some(20.0), + None, + Some(30.0), + ]), + ]) + assert_eq(whole(sparse, @expr.col("x").median()), @types.Scalar::Float(20.0)) + // Every cell missing → null. + let allnull = DataFrame::DataFrame([ + Series::from_float_options("x", [None, None]), + ]) + assert_eq(whole(allnull, @expr.col("x").median()), @types.Scalar::Null) +} + +///| +test "median of two near-Double-max middles stays finite (no overflow to Inf)" { + // The sum of the two middle cells overflows Double before the halving, yet the + // median lies between them and is finite. Regression guard: the naive + // `(lo + hi) / 2` returned +Inf here. + let big = DataFrame::DataFrame([Series::from_floats("x", [1.6e308, 1.7e308])]) + assert_eq( + whole(big, @expr.col("x").median()), + @types.Scalar::Float(1.6e308 / 2.0 + 1.7e308 / 2.0), + ) +} + +// ── n_unique: distinct non-null across every dtype ───────────────────── + +///| +test "n_unique counts distinct non-null values, an Int over any dtype" { + let ints = DataFrame::DataFrame([Series::from_ints("x", [1, 1, 2, 3, 3, 3])]) + assert_eq(whole(ints, @expr.col("x").n_unique()), @types.Scalar::Int(3)) + let bools = DataFrame::DataFrame([ + Series::from_bools("x", [true, true, false]), + ]) + assert_eq(whole(bools, @expr.col("x").n_unique()), @types.Scalar::Int(2)) + let strs = DataFrame::DataFrame([Series::from_strings("x", ["a", "a", "b"])]) + assert_eq(whole(strs, @expr.col("x").n_unique()), @types.Scalar::Int(2)) + // Empty scope → zero distinct values (never null). + let empty = DataFrame::DataFrame([Series::from_ints("x", [])]) + assert_eq(whole(empty, @expr.col("x").n_unique()), @types.Scalar::Int(0)) +} + +///| +test "n_unique buckets every NaN as one value and excludes nulls" { + let nan = @double.not_a_number + // Two distinct values — the finite 1.0 and the single NaN bucket — even + // though two NaN cells appear (NaN is one value, the n_unique / group rule). + let floats = DataFrame::DataFrame([ + Series::from_floats("x", [1.0, nan, 1.0, nan]), + ]) + assert_eq(whole(floats, @expr.col("x").n_unique()), @types.Scalar::Int(2)) + // Nulls are not a value here: [1, null, 1, 2] has two distinct values. + let sparse = DataFrame::DataFrame([ + Series::from_int_options("x", [Some(1), None, Some(1), Some(2)]), + ]) + assert_eq(whole(sparse, @expr.col("x").n_unique()), @types.Scalar::Int(2)) +} + +///| +test "n_unique agrees with the whole-column Series statistic" { + // The kernel reducer and `Series::n_unique` are separate implementations of + // the same count; pin that they agree so neither drifts. + let x = Series::from_int_options("x", [ + Some(1), + None, + Some(1), + Some(2), + Some(2), + ]) + let df = DataFrame::DataFrame([x]) + assert_eq( + whole(df, @expr.col("x").n_unique()), + @types.Scalar::Int(x.n_unique().to_int64()), + ) +} + +// ── first / last: positional, every dtype, null / empty ─────────────── + +///| +test "first and last take the positional cell, keeping the source dtype" { + let ints = DataFrame::DataFrame([Series::from_ints("x", [10, 20, 30])]) + assert_eq(whole(ints, @expr.col("x").first()), @types.Scalar::Int(10)) + assert_eq(whole(ints, @expr.col("x").last()), @types.Scalar::Int(30)) + let floats = DataFrame::DataFrame([Series::from_floats("x", [1.5, 2.5, 3.5])]) + assert_eq(whole(floats, @expr.col("x").last()), @types.Scalar::Float(3.5)) + let bools = DataFrame::DataFrame([ + Series::from_bools("x", [true, false, true]), + ]) + assert_eq(whole(bools, @expr.col("x").first()), @types.Scalar::Bool(true)) + let strs = DataFrame::DataFrame([Series::from_strings("x", ["a", "b", "c"])]) + assert_eq(whole(strs, @expr.col("x").first()), @types.Scalar::String("a")) +} + +///| +test "first and last are null on a null leading or trailing cell, or empty" { + // Positional: a null first / last cell is null, not the first / last + // *present* value. + let headnull = DataFrame::DataFrame([ + Series::from_int_options("x", [None, Some(2), Some(3)]), + ]) + assert_eq(whole(headnull, @expr.col("x").first()), @types.Scalar::Null) + let tailnull = DataFrame::DataFrame([ + Series::from_int_options("x", [Some(1), Some(2), None]), + ]) + assert_eq(whole(tailnull, @expr.col("x").last()), @types.Scalar::Null) + // An empty scope has no first / last cell → null. + let empty = DataFrame::DataFrame([Series::from_ints("x", [])]) + assert_eq(whole(empty, @expr.col("x").first()), @types.Scalar::Null) + assert_eq(whole(empty, @expr.col("x").last()), @types.Scalar::Null) +} + +// ── Non-numeric std / variance / median raise ───────────────────────── + +///| +test "std, variance, and median reject non-numeric columns" { + let strs = DataFrame::DataFrame([Series::from_strings("x", ["a", "b"])]) + assert_true( + (Ok(whole(strs, @expr.col("x").std())) catch { e => Err(e) }) + is Err( + @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) + ), + ) + assert_true( + (Ok(whole(strs, @expr.col("x").variance())) catch { e => Err(e) }) + is Err( + @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) + ), + ) + let bools = DataFrame::DataFrame([Series::from_bools("x", [true, false])]) + assert_true( + (Ok(whole(bools, @expr.col("x").median())) catch { e => Err(e) }) + is Err( + @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) + ), + ) +} + +// ── Grouped agg: per-group reductions over the fast path ─────────────── + +///| +/// Five-row sales fixture grouped by `region`: `east` = rows 0, 2, 3 +/// (quantity 10, 2, 6), `west` = rows 1, 4 (quantity 5, 4); `east` first. +fn grouped_sales() -> DataFrame raise @types.DataError { + DataFrame::DataFrame([ + Series::from_strings("region", ["east", "west", "east", "east", "west"]), + Series::from_ints("quantity", [10, 5, 2, 6, 4]), + ]) +} + +///| +test "grouped agg reduces each group with the new family" { + let out = grouped_sales() + .group_by([@expr.col("region")]) + .agg([ + @expr.col("quantity").std().with_alias("q_std"), + @expr.col("quantity").variance().with_alias("q_var"), + @expr.col("quantity").median().with_alias("q_med"), + @expr.col("quantity").n_unique().with_alias("q_nu"), + @expr.col("quantity").first().with_alias("q_first"), + @expr.col("quantity").last().with_alias("q_last"), + ]) + assert_eq(out.check_invariants(), Ok(())) + assert_eq(out.columns(), [ + "region", "q_std", "q_var", "q_med", "q_nu", "q_first", "q_last", + ]) + // east quantity = [10, 2, 6]: mean 6, var (16 + 16 + 0) / 2 = 16.0, + // std 4.0, median (sorted 2, 6, 10) 6.0, 3 distinct, first 10, last 6. + assert_eq(out.item(0, "q_std"), @types.Scalar::Float(4.0)) + assert_eq(out.item(0, "q_var"), @types.Scalar::Float(16.0)) + assert_eq(out.item(0, "q_med"), @types.Scalar::Float(6.0)) + assert_eq(out.item(0, "q_nu"), @types.Scalar::Int(3)) + assert_eq(out.item(0, "q_first"), @types.Scalar::Int(10)) + assert_eq(out.item(0, "q_last"), @types.Scalar::Int(6)) + // west quantity = [5, 4]: mean 4.5, var (0.25 + 0.25) / 1 = 0.5, std its + // root, median (sorted 4, 5) 4.5, 2 distinct, first 5, last 4. + assert_eq(out.item(1, "q_std"), @types.Scalar::Float(0.5.sqrt())) + assert_eq(out.item(1, "q_var"), @types.Scalar::Float(0.5)) + assert_eq(out.item(1, "q_med"), @types.Scalar::Float(4.5)) + assert_eq(out.item(1, "q_nu"), @types.Scalar::Int(2)) + assert_eq(out.item(1, "q_first"), @types.Scalar::Int(5)) + assert_eq(out.item(1, "q_last"), @types.Scalar::Int(4)) +} + +///| +test "grouped agg reduces a derived operand through the general path" { + // A non-bare operand bypasses the single-pass fast path and reduces through + // the general per-group evaluator — the same kernel, reached the other way. + let out = grouped_sales() + .group_by([@expr.col("region")]) + .agg([(@expr.col("quantity") + @expr.lit_int(0)).median().with_alias("m")]) + assert_eq(out.check_invariants(), Ok(())) + // east [10, 2, 6] → 6.0, west [5, 4] → 4.5, unchanged by the `+ 0`. + assert_eq(out.item(0, "m"), @types.Scalar::Float(6.0)) + assert_eq(out.item(1, "m"), @types.Scalar::Float(4.5)) +} diff --git a/frame/agg_family_test.mbt b/frame/agg_family_test.mbt deleted file mode 100644 index 86d38657..00000000 --- a/frame/agg_family_test.mbt +++ /dev/null @@ -1,332 +0,0 @@ -///| -/// Blackbox tests for the expression aggregation family added on the shared -/// reduction kernel — `std` / `variance` / `median` / `n_unique` / `first` / -/// `last` — driven through both reduction surfaces: a whole-frame `select` -/// (the per-scope evaluator) and a grouped `agg` (the single-pass fast path). -/// Covers each op's value, output dtype, and null / NaN rule: `std` / `var` -/// propagate NaN and reduce a `< 2`-cell scope to null, `median` skips NaN, -/// `n_unique` buckets NaN as one and excludes nulls, and `first` / `last` are -/// positional (a null leading / trailing cell, or an empty scope, is null). -/// Non-numeric `std` / `var` / `median` raise `TypeMismatch`. Construction and -/// rendering have their own coverage; this pins evaluation. - -///| -/// The single cell of the whole-frame reduction `e` over `df`, via `select`: -/// `select([agg])` collapses to one row, so cell `(0, "r")` is the reduction. -fn whole( - df : DataFrame, - e : @expr.Expr, -) -> @types.Scalar raise @types.DataError { - df.select([e.with_alias("r")]).item(0, "r") -} - -// ── std / variance: value, dtype, NaN propagation, < 2 cells ─────────── - -///| -test "std and variance reduce a whole frame to the sample statistics" { - // [2, 4, 6]: mean 4, squared deviations 4 + 0 + 4 = 8, sample variance - // (ddof = 1) 8 / 2 = 4.0, standard deviation its root, 2.0. Both are Float - // even though the source is Int. - let df = DataFrame::DataFrame([Series::from_ints("x", [2, 4, 6])]) - assert_eq(whole(df, @expr.col("x").std()), @types.Scalar::Float(2.0)) - assert_eq(whole(df, @expr.col("x").variance()), @types.Scalar::Float(4.0)) -} - -///| -test "std over a Float column keeps Float and propagates NaN" { - let df = DataFrame::DataFrame([Series::from_floats("x", [1.0, 2.0, 3.0])]) - // [1, 2, 3]: variance 1.0, std 1.0. - assert_eq(whole(df, @expr.col("x").std()), @types.Scalar::Float(1.0)) - // A non-null NaN is a value: it flows through the mean, so std / var - // propagate it (matching sum / mean), unlike min / max / median. - let nan = @double.not_a_number - let dfn = DataFrame::DataFrame([Series::from_floats("x", [1.0, 2.0, nan])]) - assert_true(whole(dfn, @expr.col("x").std()).as_float().is_nan()) - assert_true(whole(dfn, @expr.col("x").variance()).as_float().is_nan()) -} - -///| -test "variance and std of equal large-magnitude values are 0, not +inf" { - // Two equal cells near Double-max: the true variance is 0. A two-pass mean - // would sum them to +inf and report +inf; Welford never forms the raw sum, so - // the finite variance survives. - let df = DataFrame::DataFrame([Series::from_floats("x", [1.0e308, 1.0e308])]) - assert_eq(whole(df, @expr.col("x").variance()), @types.Scalar::Float(0.0)) - assert_eq(whole(df, @expr.col("x").std()), @types.Scalar::Float(0.0)) -} - -///| -test "variance and std of opposite-sign extremes saturate to +inf, never negative" { - // Finite cells more than Double-max apart: Welford's running `delta` - // overflows and used to poison `m2` to -inf — a mathematically impossible - // NEGATIVE variance, and sqrt(-inf) = NaN for std. The true variance - // (~2e616) exceeds Double's range, so the statistic saturates to +inf, - // as numpy (ddof=1) and Polars report. - let two = DataFrame::DataFrame([Series::from_floats("x", [-1.0e308, 1.0e308])]) - assert_eq( - whole(two, @expr.col("x").variance()), - @types.Scalar::Float(@double.infinity), - ) - assert_eq( - whole(two, @expr.col("x").std()), - @types.Scalar::Float(@double.infinity), - ) - // A longer alternating window drives the artefact to NaN instead of -inf; - // the finite-input rescue covers it the same way. - let three = DataFrame::DataFrame([ - Series::from_floats("x", [1.0e308, -1.0e308, 1.0e308]), - ]) - assert_eq( - whole(three, @expr.col("x").variance()), - @types.Scalar::Float(@double.infinity), - ) - // A genuine ±inf INPUT is not rescued: it propagates through the mean - // like sum / mean, so the statistic is NaN (inf - inf), not +inf. - let inf_in = DataFrame::DataFrame([ - Series::from_floats("x", [@double.infinity, 1.0]), - ]) - assert_true(whole(inf_in, @expr.col("x").variance()).as_float().is_nan()) - assert_true(whole(inf_in, @expr.col("x").std()).as_float().is_nan()) -} - -///| -test "std and variance are null for fewer than two non-null cells" { - // Sample statistics need ddof = 1 < cnt: one value, or none, has no - // sample variance, so the cell is null rather than 0.0. - let one = DataFrame::DataFrame([Series::from_ints("x", [5])]) - assert_eq(whole(one, @expr.col("x").std()), @types.Scalar::Null) - assert_eq(whole(one, @expr.col("x").variance()), @types.Scalar::Null) - let empty = DataFrame::DataFrame([Series::from_ints("x", [])]) - assert_eq(whole(empty, @expr.col("x").std()), @types.Scalar::Null) - // Nulls are skipped, so a lone non-null cell beside nulls is still < 2. - let sparse = DataFrame::DataFrame([ - Series::from_int_options("x", [Some(5), None, None]), - ]) - assert_eq(whole(sparse, @expr.col("x").variance()), @types.Scalar::Null) -} - -// ── median: value, even/odd, dtype, NaN skipped ─────────────────────── - -///| -test "median is the middle of the sorted non-null cells, always Float" { - // Odd count: the middle. Int widens, so the result is Float. - let odd = DataFrame::DataFrame([Series::from_ints("x", [3, 1, 2])]) - assert_eq(whole(odd, @expr.col("x").median()), @types.Scalar::Float(2.0)) - // Even count: the mean of the two middles. - let even = DataFrame::DataFrame([Series::from_ints("x", [1, 2, 3, 4])]) - assert_eq(whole(even, @expr.col("x").median()), @types.Scalar::Float(2.5)) -} - -///| -test "median propagates NaN, skips nulls, all-missing is null" { - let nan = @double.not_a_number - // NaN propagates — the sum / mean rule, not the min / max skip: a window - // that contains one has that value's order statistic. - let df = DataFrame::DataFrame([Series::from_floats("x", [1.0, 2.0, nan])]) - let got = whole(df, @expr.col("x").median()) - assert_true( - match got { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - // Nulls drop out; [10, null, 20, null, 30] medians to 20.0. - let sparse = DataFrame::DataFrame([ - Series::from_float_options("x", [ - Some(10.0), - None, - Some(20.0), - None, - Some(30.0), - ]), - ]) - assert_eq(whole(sparse, @expr.col("x").median()), @types.Scalar::Float(20.0)) - // Every cell missing → null. - let allnull = DataFrame::DataFrame([ - Series::from_float_options("x", [None, None]), - ]) - assert_eq(whole(allnull, @expr.col("x").median()), @types.Scalar::Null) -} - -///| -test "median of two near-Double-max middles stays finite (no overflow to Inf)" { - // The sum of the two middle cells overflows Double before the halving, yet the - // median lies between them and is finite. Regression guard: the naive - // `(lo + hi) / 2` returned +Inf here. - let big = DataFrame::DataFrame([Series::from_floats("x", [1.6e308, 1.7e308])]) - assert_eq( - whole(big, @expr.col("x").median()), - @types.Scalar::Float(1.6e308 / 2.0 + 1.7e308 / 2.0), - ) -} - -// ── n_unique: distinct non-null across every dtype ───────────────────── - -///| -test "n_unique counts distinct non-null values, an Int over any dtype" { - let ints = DataFrame::DataFrame([Series::from_ints("x", [1, 1, 2, 3, 3, 3])]) - assert_eq(whole(ints, @expr.col("x").n_unique()), @types.Scalar::Int(3)) - let bools = DataFrame::DataFrame([ - Series::from_bools("x", [true, true, false]), - ]) - assert_eq(whole(bools, @expr.col("x").n_unique()), @types.Scalar::Int(2)) - let strs = DataFrame::DataFrame([Series::from_strings("x", ["a", "a", "b"])]) - assert_eq(whole(strs, @expr.col("x").n_unique()), @types.Scalar::Int(2)) - // Empty scope → zero distinct values (never null). - let empty = DataFrame::DataFrame([Series::from_ints("x", [])]) - assert_eq(whole(empty, @expr.col("x").n_unique()), @types.Scalar::Int(0)) -} - -///| -test "n_unique buckets every NaN as one value and excludes nulls" { - let nan = @double.not_a_number - // Two distinct values — the finite 1.0 and the single NaN bucket — even - // though two NaN cells appear (NaN is one value, the n_unique / group rule). - let floats = DataFrame::DataFrame([ - Series::from_floats("x", [1.0, nan, 1.0, nan]), - ]) - assert_eq(whole(floats, @expr.col("x").n_unique()), @types.Scalar::Int(2)) - // Nulls are not a value here: [1, null, 1, 2] has two distinct values. - let sparse = DataFrame::DataFrame([ - Series::from_int_options("x", [Some(1), None, Some(1), Some(2)]), - ]) - assert_eq(whole(sparse, @expr.col("x").n_unique()), @types.Scalar::Int(2)) -} - -///| -test "n_unique agrees with the whole-column Series statistic" { - // The kernel reducer and `Series::n_unique` are separate implementations of - // the same count; pin that they agree so neither drifts. - let x = Series::from_int_options("x", [ - Some(1), - None, - Some(1), - Some(2), - Some(2), - ]) - let df = DataFrame::DataFrame([x]) - assert_eq( - whole(df, @expr.col("x").n_unique()), - @types.Scalar::Int(x.n_unique().to_int64()), - ) -} - -// ── first / last: positional, every dtype, null / empty ─────────────── - -///| -test "first and last take the positional cell, keeping the source dtype" { - let ints = DataFrame::DataFrame([Series::from_ints("x", [10, 20, 30])]) - assert_eq(whole(ints, @expr.col("x").first()), @types.Scalar::Int(10)) - assert_eq(whole(ints, @expr.col("x").last()), @types.Scalar::Int(30)) - let floats = DataFrame::DataFrame([Series::from_floats("x", [1.5, 2.5, 3.5])]) - assert_eq(whole(floats, @expr.col("x").last()), @types.Scalar::Float(3.5)) - let bools = DataFrame::DataFrame([ - Series::from_bools("x", [true, false, true]), - ]) - assert_eq(whole(bools, @expr.col("x").first()), @types.Scalar::Bool(true)) - let strs = DataFrame::DataFrame([Series::from_strings("x", ["a", "b", "c"])]) - assert_eq(whole(strs, @expr.col("x").first()), @types.Scalar::String("a")) -} - -///| -test "first and last are null on a null leading or trailing cell, or empty" { - // Positional: a null first / last cell is null, not the first / last - // *present* value. - let headnull = DataFrame::DataFrame([ - Series::from_int_options("x", [None, Some(2), Some(3)]), - ]) - assert_eq(whole(headnull, @expr.col("x").first()), @types.Scalar::Null) - let tailnull = DataFrame::DataFrame([ - Series::from_int_options("x", [Some(1), Some(2), None]), - ]) - assert_eq(whole(tailnull, @expr.col("x").last()), @types.Scalar::Null) - // An empty scope has no first / last cell → null. - let empty = DataFrame::DataFrame([Series::from_ints("x", [])]) - assert_eq(whole(empty, @expr.col("x").first()), @types.Scalar::Null) - assert_eq(whole(empty, @expr.col("x").last()), @types.Scalar::Null) -} - -// ── Non-numeric std / variance / median raise ───────────────────────── - -///| -test "std, variance, and median reject non-numeric columns" { - let strs = DataFrame::DataFrame([Series::from_strings("x", ["a", "b"])]) - assert_true( - (Ok(whole(strs, @expr.col("x").std())) catch { e => Err(e) }) - is Err( - @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) - ), - ) - assert_true( - (Ok(whole(strs, @expr.col("x").variance())) catch { e => Err(e) }) - is Err( - @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) - ), - ) - let bools = DataFrame::DataFrame([Series::from_bools("x", [true, false])]) - assert_true( - (Ok(whole(bools, @expr.col("x").median())) catch { e => Err(e) }) - is Err( - @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) - ), - ) -} - -// ── Grouped agg: per-group reductions over the fast path ─────────────── - -///| -/// Five-row sales fixture grouped by `region`: `east` = rows 0, 2, 3 -/// (quantity 10, 2, 6), `west` = rows 1, 4 (quantity 5, 4); `east` first. -fn grouped_sales() -> DataFrame raise @types.DataError { - DataFrame::DataFrame([ - Series::from_strings("region", ["east", "west", "east", "east", "west"]), - Series::from_ints("quantity", [10, 5, 2, 6, 4]), - ]) -} - -///| -test "grouped agg reduces each group with the new family" { - let out = grouped_sales() - .group_by([@expr.col("region")]) - .agg([ - @expr.col("quantity").std().with_alias("q_std"), - @expr.col("quantity").variance().with_alias("q_var"), - @expr.col("quantity").median().with_alias("q_med"), - @expr.col("quantity").n_unique().with_alias("q_nu"), - @expr.col("quantity").first().with_alias("q_first"), - @expr.col("quantity").last().with_alias("q_last"), - ]) - assert_eq(out.check_invariants(), Ok(())) - assert_eq(out.columns(), [ - "region", "q_std", "q_var", "q_med", "q_nu", "q_first", "q_last", - ]) - // east quantity = [10, 2, 6]: mean 6, var (16 + 16 + 0) / 2 = 16.0, - // std 4.0, median (sorted 2, 6, 10) 6.0, 3 distinct, first 10, last 6. - assert_eq(out.item(0, "q_std"), @types.Scalar::Float(4.0)) - assert_eq(out.item(0, "q_var"), @types.Scalar::Float(16.0)) - assert_eq(out.item(0, "q_med"), @types.Scalar::Float(6.0)) - assert_eq(out.item(0, "q_nu"), @types.Scalar::Int(3)) - assert_eq(out.item(0, "q_first"), @types.Scalar::Int(10)) - assert_eq(out.item(0, "q_last"), @types.Scalar::Int(6)) - // west quantity = [5, 4]: mean 4.5, var (0.25 + 0.25) / 1 = 0.5, std its - // root, median (sorted 4, 5) 4.5, 2 distinct, first 5, last 4. - assert_eq(out.item(1, "q_std"), @types.Scalar::Float(0.5.sqrt())) - assert_eq(out.item(1, "q_var"), @types.Scalar::Float(0.5)) - assert_eq(out.item(1, "q_med"), @types.Scalar::Float(4.5)) - assert_eq(out.item(1, "q_nu"), @types.Scalar::Int(2)) - assert_eq(out.item(1, "q_first"), @types.Scalar::Int(5)) - assert_eq(out.item(1, "q_last"), @types.Scalar::Int(4)) -} - -///| -test "grouped agg reduces a derived operand through the general path" { - // A non-bare operand bypasses the single-pass fast path and reduces through - // the general per-group evaluator — the same kernel, reached the other way. - let out = grouped_sales() - .group_by([@expr.col("region")]) - .agg([(@expr.col("quantity") + @expr.lit_int(0)).median().with_alias("m")]) - assert_eq(out.check_invariants(), Ok(())) - // east [10, 2, 6] → 6.0, west [5, 4] → 4.5, unchanged by the `+ 0`. - assert_eq(out.item(0, "m"), @types.Scalar::Float(6.0)) - assert_eq(out.item(1, "m"), @types.Scalar::Float(4.5)) -} diff --git a/frame/compare_exact_test.mbt b/frame/compare_exact_test.mbt deleted file mode 100644 index 4952ad36..00000000 --- a/frame/compare_exact_test.mbt +++ /dev/null @@ -1,84 +0,0 @@ -// Blackbox tests pinning the expr comparison kernel's exact `Int` vs `Float` -// behavior (no 2^53 promotion) — the authoritative home of that exactness, -// on both the Numeric fast arm and the Builtin nullable arm. - -///| -test "expr comparison: Int vs Float is exact past 2^53 (Numeric fast arm)" { - // i = [2^53+1, 2^53], f = [2^53, 2^53]; both columns null-free → Numeric. - let df = DataFrame::DataFrame([ - Series::from_ints("i", [9007199254740993L, 9007199254740992L]), - Series::from_floats("f", [9007199254740992.0, 9007199254740992.0]), - ]) - let t = @types.Scalar::Bool(true) - let f = @types.Scalar::Bool(false) - // Int-vs-Float, all six verbs. Row 0 (2^53+1 vs 2^53, distinct), row 1 equal. - let fwd = df.select([ - @expr.col("i").eq(@expr.col("f")).with_alias("eq"), - @expr.col("i").ne(@expr.col("f")).with_alias("ne"), - @expr.col("i").lt(@expr.col("f")).with_alias("lt"), - @expr.col("i").le(@expr.col("f")).with_alias("le"), - @expr.col("i").gt(@expr.col("f")).with_alias("gt"), - @expr.col("i").ge(@expr.col("f")).with_alias("ge"), - ]) - assert_eq(fwd.get_column("eq").to_scalars(), [f, t]) - assert_eq(fwd.get_column("ne").to_scalars(), [t, f]) - assert_eq(fwd.get_column("lt").to_scalars(), [f, f]) - assert_eq(fwd.get_column("le").to_scalars(), [f, t]) - assert_eq(fwd.get_column("gt").to_scalars(), [t, f]) - assert_eq(fwd.get_column("ge").to_scalars(), [t, t]) - // Float-vs-Int (the transposed verdict), all six verbs. Row 0 (2^53 vs - // 2^53+1), row 1 (2^53 vs 2^53). - let rev = df.select([ - @expr.col("f").eq(@expr.col("i")).with_alias("eq"), - @expr.col("f").ne(@expr.col("i")).with_alias("ne"), - @expr.col("f").lt(@expr.col("i")).with_alias("lt"), - @expr.col("f").le(@expr.col("i")).with_alias("le"), - @expr.col("f").gt(@expr.col("i")).with_alias("gt"), - @expr.col("f").ge(@expr.col("i")).with_alias("ge"), - ]) - assert_eq(rev.get_column("eq").to_scalars(), [f, t]) - assert_eq(rev.get_column("ne").to_scalars(), [t, f]) - assert_eq(rev.get_column("lt").to_scalars(), [t, f]) - assert_eq(rev.get_column("le").to_scalars(), [t, t]) - assert_eq(rev.get_column("gt").to_scalars(), [f, f]) - assert_eq(rev.get_column("ge").to_scalars(), [f, t]) -} - -///| -test "expr comparison: exact past 2^53 on the Builtin (nullable) arm" { - // A null forces the general Builtin arm; the exact verdict is unchanged and - // the null layer still nulls its row. - let df = DataFrame::DataFrame([ - Series::from_int_options("i", [Some(9007199254740993L), None]), - Series::from_float_options("f", [Some(9007199254740992.0), Some(1.0)]), - ]) - let out = df.select([@expr.col("i").eq(@expr.col("f")).with_alias("eq")]) - assert_eq(out.get_column("eq").to_scalars(), [ - @types.Scalar::Bool(false), - @types.Scalar::Null, - ]) -} - -///| -test "expr comparison: Int vs Float at the Int64 boundaries, ±Inf and NaN" { - // Past the Int64 range the helpers answer from the bounds alone: `MAX` is - // below the 2^63 `Double` it would round to, `MIN` is above `-Inf`, and - // `NaN` makes every verdict `false`. - let df = DataFrame::DataFrame([ - // MAX vs 2^63, MIN vs -Inf, 0 vs NaN, 0 vs +Inf — all null-free → Numeric. - Series::from_ints("i", [9223372036854775807L, -9223372036854775808L, 0L, 0L]), - Series::from_floats("f", [ - 9223372036854775808.0, @double.neg_infinity, @double.not_a_number, @double.infinity, - ]), - ]) - let t = @types.Scalar::Bool(true) - let f = @types.Scalar::Bool(false) - let out = df.select([ - @expr.col("i").eq(@expr.col("f")).with_alias("eq"), - @expr.col("i").lt(@expr.col("f")).with_alias("lt"), - @expr.col("i").gt(@expr.col("f")).with_alias("gt"), - ]) - assert_eq(out.get_column("eq").to_scalars(), [f, f, f, f]) - assert_eq(out.get_column("lt").to_scalars(), [t, f, f, t]) - assert_eq(out.get_column("gt").to_scalars(), [f, t, f, f]) -} diff --git a/frame/expr_eval_test.mbt b/frame/expr_eval_test.mbt index 7bec6d02..30ed10a2 100644 --- a/frame/expr_eval_test.mbt +++ b/frame/expr_eval_test.mbt @@ -1517,3 +1517,204 @@ test "comparison general arm: Float-led pairings through nullable operands" { @types.Scalar::Null, ]) } + +///| +/// Blackbox tests for the `LitSeries` literal-series node — `lit_series(s)` — +/// driven through the expression verbs. Covers the frame-tall append, the +/// in-place replace, the preserved nulls on the embedded series, the length-1 +/// broadcast and the length-mismatch error, use inside a compound expression +/// and as a precomputed filter mask, standalone projection with naming, the +/// preserved backend, an aggregation over the literal, the empty-frame edge, +/// and the not-reduction-shaped rejection under `agg`. + +///| +/// Three-row base frame for the literal-series tests. +fn lit_series_df() -> DataFrame raise @types.DataError { + DataFrame::DataFrame([ + Series::from_ints("a", [1, 2, 3]), + Series::from_strings("b", ["x", "y", "z"]), + ]) +} + +///| +test "lit_series: a frame-tall series appends as a new named column" { + // A ready-made column joins the frame, named after the series, at the + // rightmost position. + let extra = Series::from_floats("c", [ + (1.5 : Double), + (2.5 : Double), + (3.5 : Double), + ]) + let out = lit_series_df().with_columns([@expr.lit_series(extra)]) + assert_eq(out.columns(), ["a", "b", "c"]) + assert_eq(out.get_column("c").dtype(), @types.DataType::Float) + assert_eq(out.get_column("c").to_scalars(), [ + @types.Scalar::Float(1.5), + @types.Scalar::Float(2.5), + @types.Scalar::Float(3.5), + ]) +} + +///| +test "lit_series: the embedded series keeps its nulls" { + // The series is handed through verbatim, so a nullable column arrives with + // its validity bitmap intact. + let extra = Series::from_int_options("d", [Some(7), None, Some(9)]) + let out = lit_series_df().with_columns([@expr.lit_series(extra)]) + assert_eq(out.get_column("d").null_count(), 1) + assert_eq(out.get_column("d").to_scalars(), [ + @types.Scalar::Int(7), + @types.Scalar::Null, + @types.Scalar::Int(9), + ]) +} + +///| +test "lit_series: a frame-tall series replaces an existing column in place" { + // A same-named series replaces the existing column in place through + // `with_columns`, keeping its original position (a fresh name appends). + let replacement = Series::from_ints("a", [10, 20, 30]) + let out = lit_series_df().with_columns([@expr.lit_series(replacement)]) + assert_eq(out.columns(), ["a", "b"]) + assert_eq(out.get_column("a").to_scalars(), [ + @types.Scalar::Int(10), + @types.Scalar::Int(20), + @types.Scalar::Int(30), + ]) +} + +///| +test "lit_series: a renamed series swaps a column in place, dtype and all" { + // The retired `replace_column(name, s)` is `with_columns([lit_series(s.rename(name))])`: + // renaming a differently-named series onto an existing column swaps it in + // place — position and name preserved — with the dtype following the new + // data (a cross-dtype swap, here Int → Float). + let fresh = Series::from_floats("ignored", [ + (0.5 : Double), + (1.5 : Double), + (2.5 : Double), + ]) + let out = lit_series_df().with_columns([@expr.lit_series(fresh.rename("a"))]) + assert_eq(out.columns(), ["a", "b"]) + assert_eq(out.get_column("a").dtype(), @types.DataType::Float) + assert_eq(out.get_column("a").to_scalars(), [ + @types.Scalar::Float(0.5), + @types.Scalar::Float(1.5), + @types.Scalar::Float(2.5), + ]) +} + +///| +test "lit_series: a length-1 series broadcasts over the frame" { + // Like a scalar literal, a one-row series fills every row. + let one = Series::from_ints("k", [7]) + let out = lit_series_df().with_columns([@expr.lit_series(one)]) + assert_eq(out.get_column("k").to_scalars(), [ + @types.Scalar::Int(7), + @types.Scalar::Int(7), + @types.Scalar::Int(7), + ]) +} + +///| +test "lit_series: a length neither 1 nor the frame height raises LengthMismatch" { + let two = Series::from_ints("k", [1, 2]) + assert_eq( + Ok(lit_series_df().with_columns([@expr.lit_series(two)])) catch { + e => Err(e) + }, + Err(@types.DataError::LengthMismatch), + ) +} + +///| +// `select` agrees with `with_columns`: an off-length literal series is a +// LengthMismatch, not a silent change of the output row count. The frame is +// 3 rows; a 2-row literal series is neither frame-tall nor a length-1 +// broadcast, even when it is the only projected column. +test "lit_series: select rejects an off-length series instead of resizing the frame" { + let two = Series::from_ints("k", [1, 2]) + assert_eq( + Ok(lit_series_df().select([@expr.lit_series(two)])) catch { + e => Err(e) + }, + Err(@types.DataError::LengthMismatch), + ) +} + +///| +test "lit_series: combines positionally inside a compound expression" { + // A frame-tall literal column is row-aligned with the frame, so it adds to + // a column cell for cell. + let bonus = Series::from_ints("bonus", [100, 200, 300]) + let out = lit_series_df().with_columns([ + (@expr.col("a") + @expr.lit_series(bonus)).with_alias("total"), + ]) + assert_eq(out.get_column("total").to_scalars(), [ + @types.Scalar::Int(101), + @types.Scalar::Int(202), + @types.Scalar::Int(303), + ]) +} + +///| +test "lit_series: a Bool series is usable as a precomputed filter mask" { + let mask = Series::from_bools("keep", [true, false, true]) + let kept = lit_series_df().filter(@expr.lit_series(mask)) + assert_eq(kept.get_column("a").to_scalars(), [ + @types.Scalar::Int(1), + @types.Scalar::Int(3), + ]) +} + +///| +test "lit_series: projects standalone, named after the series unless aliased" { + let s = Series::from_ints("vals", [1, 2, 3]) + let out = lit_series_df().select([@expr.lit_series(s)]) + assert_eq(out.columns(), ["vals"]) + let aliased = lit_series_df().select([ + @expr.lit_series(s).with_alias("renamed"), + ]) + assert_eq(aliased.columns(), ["renamed"]) +} + +///| +test "lit_series: returned verbatim, so its backend is preserved" { + // A Numeric, all-valid series stays Numeric — `eval` hands it through, it + // is not re-derived through `try_column_to_numeric` like a computed column. + let num = Series::from_ints("n", [1, 2, 3]) + assert_eq(num.storage_kind(), @column.StorageKind::Numeric) + let out = lit_series_df().with_columns([@expr.lit_series(num)]) + assert_eq(out.get_column("n").storage_kind(), @column.StorageKind::Numeric) +} + +///| +test "lit_series: an aggregation over a literal series reduces it whole" { + // `lit_series(s).sum()` is reduction-shaped (the enclosing `Agg`): it sums + // the literal series itself — all four of its cells, independent of the + // base frame's three rows — to one cell. + let s = Series::from_ints("s", [10, 20, 30, 40]) + let out = lit_series_df().select([@expr.lit_series(s).sum().with_alias("t")]) + assert_eq(out.shape(), (1, 1)) + assert_eq(out.get_column("t").to_scalars(), [@types.Scalar::Int(100)]) +} + +///| +test "lit_series: a bare literal series is not reduction-shaped under agg" { + // Row-wise like a bare column, so `agg([lit_series(s)])` raises — the shape + // gate fires before any group work. + let s = Series::from_ints("s", [1, 2, 3]) + let grouped = lit_series_df().group_by([@expr.col("b")]) + assert_true( + (Ok(grouped.agg([@expr.lit_series(s)])) catch { e => Err(e) }) + is Err(@types.DataError::InvalidOperation(_)), + ) +} + +///| +test "lit_series on an empty frame accepts a 0-length series" { + let empty = DataFrame::DataFrame([Series::from_ints("i", [])]) + let out = empty.with_columns([@expr.lit_series(Series::from_ints("z", []))]) + assert_eq(out.shape(), (0, 2)) + assert_eq(out.columns(), ["i", "z"]) +} diff --git a/frame/expr_map_batches_test.mbt b/frame/expr_map_batches_test.mbt deleted file mode 100644 index 0272b979..00000000 --- a/frame/expr_map_batches_test.mbt +++ /dev/null @@ -1,211 +0,0 @@ -///| -/// Blackbox tests for the batched escape hatch `Expr::map_batches` — the -/// whole-`Series` UDF. Covers the row-wise form (`returns_scalar=false`) -/// through `with_columns` / `select`, backend canonicalisation, null handling, -/// closure error propagation, the length-1 broadcast, and the -/// `returns_scalar=true` form as a custom aggregation inside -/// `group_by(...).agg` (plus its rejection when the flag is omitted). - -///| -/// A grouping key and a nullable Int payload — exercises the batched map's -/// null-through path and its use as a per-group reduction. -fn mb_df() -> DataFrame raise @types.DataError { - DataFrame::DataFrame([ - Series::from_strings("region", ["west", "east", "west", "east", "west"]), - Series::from_int_options("q", [Some(1), None, Some(3), Some(4), Some(5)]), - ]) -} - -///| -test "map_batches: applies the closure to the whole column at once (Int out)" { - // Double every cell in one batched call; the null cell stays null. - let out = mb_df().with_columns([ - @expr.col("q").map_batches(label="double", s => { - Series::from_int_options( - s.name(), - s - .to_scalars() - .map(sc => { - match sc { - @types.Scalar::Int(v) => Some(v * 2) - _ => None - } - }), - ) - }), - ]) - assert_eq(out.get_column("q").dtype(), @types.DataType::Int) - assert_eq(out.get_column("q").to_scalars(), [ - @types.Scalar::Int(2), - @types.Scalar::Null, - @types.Scalar::Int(6), - @types.Scalar::Int(8), - @types.Scalar::Int(10), - ]) -} - -///| -test "map_batches: an all-valid numeric result converges onto Numeric" { - let df = DataFrame::DataFrame([Series::from_ints("base", [10, 20, 30])]) - let out = df.with_columns([ - @expr.col("base") - .map_batches(label="halved", s => { - Series::from_floats( - s.name(), - s.to_scalars().map(sc => sc.as_int().to_double() / 2.0), - ) - }) - .with_alias("half"), - ]) - assert_eq(out.get_column("half").dtype(), @types.DataType::Float) - assert_eq(out.get_column("half").to_scalars(), [ - @types.Scalar::Float(5.0), - @types.Scalar::Float(10.0), - @types.Scalar::Float(15.0), - ]) - // A computed, all-valid numeric column rides the `Numeric` fast path — the - // canonicalisation that keeps `collect ≡ execute` even though the closure - // built a plain `Builtin` series. - assert_eq(out.get_column("half").storage_kind(), @column.StorageKind::Numeric) -} - -///| -test "map_batches: used in select produces a standalone column" { - // The identity closure returns its input verbatim; the result is named - // after the leftmost input ("q"). - let out = mb_df().select([@expr.col("q").map_batches(label="id", s => s)]) - assert_eq(out.ncols(), 1) - assert_eq(out.get_column("q").to_scalars(), [ - @types.Scalar::Int(1), - @types.Scalar::Null, - @types.Scalar::Int(3), - @types.Scalar::Int(4), - @types.Scalar::Int(5), - ]) -} - -///| -test "map_batches: a closure raise propagates from the consuming verb" { - let result = Ok( - mb_df().with_columns([ - @expr.col("q").map_batches(label="boom", _s => { - raise @types.DataError::InvalidOperation("boom") - }), - ]), - ) catch { - e => Err(e) - } - assert_eq(result, Err(@types.DataError::InvalidOperation("boom"))) -} - -///| -test "map_batches: a length-1 result broadcasts over the frame in with_columns" { - // A reducing closure used outside `agg`: its length-1 result broadcasts to - // every row, exactly like a scalar literal. - let df = DataFrame::DataFrame([Series::from_ints("v", [10, 20, 30])]) - let out = df.with_columns([ - @expr.col("v") - .map_batches(label="total", returns_scalar=true, s => { - Series::from_ints("total", [s.sum().as_int()]) - }) - .with_alias("total"), - ]) - assert_eq(out.get_column("total").to_scalars(), [ - @types.Scalar::Int(60), - @types.Scalar::Int(60), - @types.Scalar::Int(60), - ]) -} - -///| -test "map_batches: returns_scalar acts as a custom aggregation in agg" { - // A "range" reduction (max - min) — not one of the built-in aggregations. - // Each group's rows are handed to the closure, which returns a length-1 - // series that becomes the group's cell. - let out = DataFrame::DataFrame([ - Series::from_strings("region", ["west", "east", "west", "east", "west"]), - Series::from_ints("v", [10, 20, 30, 40, 50]), - ]) - .group_by([@expr.col("region")]) - .agg([ - @expr.col("v") - .map_batches(label="range", returns_scalar=true, s => { - let hi = s.max() - let lo = s.min() - match (hi, lo) { - (@types.Scalar::Int(h), @types.Scalar::Int(l)) => - Series::from_ints("range", [h - l]) - _ => Series::from_ints("range", []) - } - }) - .with_alias("v_range"), - ]) - assert_eq(out.check_invariants(), Ok(())) - assert_eq(out.columns(), ["region", "v_range"]) - // west = rows [0,2,4] (10/30/50) → range 40; east = rows [1,3] (20/40) → 20. - assert_eq(out.get_column("v_range").to_scalars(), [ - @types.Scalar::Int(40), - @types.Scalar::Int(20), - ]) -} - -///| -test "map_batches: a returns_scalar closure that does not reduce raises LengthMismatch" { - // `returns_scalar` is a *declaration*, checked structurally before evaluation; - // whether the closure honours it is only knowable per group. Two cells is the - // silent-truncation case (cell 0 would have been kept) and zero cells the - // index-an-empty-array one, so both must reach the length contract every other - // expression consumer enforces. - let grouped = () => { - DataFrame::DataFrame([ - Series::from_strings("region", ["west", "east", "west"]), - Series::from_ints("q", [1, 2, 3]), - ]).group_by([@expr.col("region")]) - } - let wrong_lengths : Array[Array[Int64]] = [[7, 8], []] - for cells in wrong_lengths { - assert_eq( - Ok( - grouped().agg([ - @expr.col("q") - .map_batches(label="wrong", returns_scalar=true, _s => { - Series::from_ints("wrong", cells) - }) - .with_alias("q_wrong"), - ]), - ) catch { - e => Err(e) - }, - Err(@types.DataError::LengthMismatch), - ) - } -} - -///| -test "map_batches: a bare (non-scalar) map is rejected as an aggregation" { - // Without `returns_scalar`, the node is row-wise, so the reduction-shape - // gate rejects it before evaluation — the same gate a bare `col` trips. - assert_eq( - Ok( - mb_df() - .group_by([@expr.col("region")]) - .agg([@expr.col("q").map_batches(label="x", s => s)]), - ) catch { - e => Err(e) - }, - Err( - @types.DataError::InvalidOperation( - "aggregate expression must reduce each group to a single value: map_batches(\"x\", [col(q)])", - ), - ), - ) -} - -///| -test "map_batches over an empty frame yields an empty column" { - let empty = DataFrame::DataFrame([Series::from_ints("i", [])]) - let out = empty.with_columns([@expr.col("i").map_batches(label="id", s => s)]) - assert_eq(out.shape(), (0, 1)) - assert_eq(out.get_column("i").dtype(), @types.DataType::Int) - assert_eq(out.get_column("i").to_scalars(), []) -} diff --git a/frame/expr_map_test.mbt b/frame/expr_map_test.mbt index 3b065963..68411137 100644 --- a/frame/expr_map_test.mbt +++ b/frame/expr_map_test.mbt @@ -322,3 +322,215 @@ test "map_many as a filter predicate matches the native expression" { let by_native = df.filter(@expr.col("q").ge(@expr.lit_int(3))) assert_eq(by_map, by_native) } + +///| +/// Blackbox tests for the batched escape hatch `Expr::map_batches` — the +/// whole-`Series` UDF. Covers the row-wise form (`returns_scalar=false`) +/// through `with_columns` / `select`, backend canonicalisation, null handling, +/// closure error propagation, the length-1 broadcast, and the +/// `returns_scalar=true` form as a custom aggregation inside +/// `group_by(...).agg` (plus its rejection when the flag is omitted). + +///| +/// A grouping key and a nullable Int payload — exercises the batched map's +/// null-through path and its use as a per-group reduction. +fn mb_df() -> DataFrame raise @types.DataError { + DataFrame::DataFrame([ + Series::from_strings("region", ["west", "east", "west", "east", "west"]), + Series::from_int_options("q", [Some(1), None, Some(3), Some(4), Some(5)]), + ]) +} + +///| +test "map_batches: applies the closure to the whole column at once (Int out)" { + // Double every cell in one batched call; the null cell stays null. + let out = mb_df().with_columns([ + @expr.col("q").map_batches(label="double", s => { + Series::from_int_options( + s.name(), + s + .to_scalars() + .map(sc => { + match sc { + @types.Scalar::Int(v) => Some(v * 2) + _ => None + } + }), + ) + }), + ]) + assert_eq(out.get_column("q").dtype(), @types.DataType::Int) + assert_eq(out.get_column("q").to_scalars(), [ + @types.Scalar::Int(2), + @types.Scalar::Null, + @types.Scalar::Int(6), + @types.Scalar::Int(8), + @types.Scalar::Int(10), + ]) +} + +///| +test "map_batches: an all-valid numeric result converges onto Numeric" { + let df = DataFrame::DataFrame([Series::from_ints("base", [10, 20, 30])]) + let out = df.with_columns([ + @expr.col("base") + .map_batches(label="halved", s => { + Series::from_floats( + s.name(), + s.to_scalars().map(sc => sc.as_int().to_double() / 2.0), + ) + }) + .with_alias("half"), + ]) + assert_eq(out.get_column("half").dtype(), @types.DataType::Float) + assert_eq(out.get_column("half").to_scalars(), [ + @types.Scalar::Float(5.0), + @types.Scalar::Float(10.0), + @types.Scalar::Float(15.0), + ]) + // A computed, all-valid numeric column rides the `Numeric` fast path — the + // canonicalisation that keeps `collect ≡ execute` even though the closure + // built a plain `Builtin` series. + assert_eq(out.get_column("half").storage_kind(), @column.StorageKind::Numeric) +} + +///| +test "map_batches: used in select produces a standalone column" { + // The identity closure returns its input verbatim; the result is named + // after the leftmost input ("q"). + let out = mb_df().select([@expr.col("q").map_batches(label="id", s => s)]) + assert_eq(out.ncols(), 1) + assert_eq(out.get_column("q").to_scalars(), [ + @types.Scalar::Int(1), + @types.Scalar::Null, + @types.Scalar::Int(3), + @types.Scalar::Int(4), + @types.Scalar::Int(5), + ]) +} + +///| +test "map_batches: a closure raise propagates from the consuming verb" { + let result = Ok( + mb_df().with_columns([ + @expr.col("q").map_batches(label="boom", _s => { + raise @types.DataError::InvalidOperation("boom") + }), + ]), + ) catch { + e => Err(e) + } + assert_eq(result, Err(@types.DataError::InvalidOperation("boom"))) +} + +///| +test "map_batches: a length-1 result broadcasts over the frame in with_columns" { + // A reducing closure used outside `agg`: its length-1 result broadcasts to + // every row, exactly like a scalar literal. + let df = DataFrame::DataFrame([Series::from_ints("v", [10, 20, 30])]) + let out = df.with_columns([ + @expr.col("v") + .map_batches(label="total", returns_scalar=true, s => { + Series::from_ints("total", [s.sum().as_int()]) + }) + .with_alias("total"), + ]) + assert_eq(out.get_column("total").to_scalars(), [ + @types.Scalar::Int(60), + @types.Scalar::Int(60), + @types.Scalar::Int(60), + ]) +} + +///| +test "map_batches: returns_scalar acts as a custom aggregation in agg" { + // A "range" reduction (max - min) — not one of the built-in aggregations. + // Each group's rows are handed to the closure, which returns a length-1 + // series that becomes the group's cell. + let out = DataFrame::DataFrame([ + Series::from_strings("region", ["west", "east", "west", "east", "west"]), + Series::from_ints("v", [10, 20, 30, 40, 50]), + ]) + .group_by([@expr.col("region")]) + .agg([ + @expr.col("v") + .map_batches(label="range", returns_scalar=true, s => { + let hi = s.max() + let lo = s.min() + match (hi, lo) { + (@types.Scalar::Int(h), @types.Scalar::Int(l)) => + Series::from_ints("range", [h - l]) + _ => Series::from_ints("range", []) + } + }) + .with_alias("v_range"), + ]) + assert_eq(out.check_invariants(), Ok(())) + assert_eq(out.columns(), ["region", "v_range"]) + // west = rows [0,2,4] (10/30/50) → range 40; east = rows [1,3] (20/40) → 20. + assert_eq(out.get_column("v_range").to_scalars(), [ + @types.Scalar::Int(40), + @types.Scalar::Int(20), + ]) +} + +///| +test "map_batches: a returns_scalar closure that does not reduce raises LengthMismatch" { + // `returns_scalar` is a *declaration*, checked structurally before evaluation; + // whether the closure honours it is only knowable per group. Two cells is the + // silent-truncation case (cell 0 would have been kept) and zero cells the + // index-an-empty-array one, so both must reach the length contract every other + // expression consumer enforces. + let grouped = () => { + DataFrame::DataFrame([ + Series::from_strings("region", ["west", "east", "west"]), + Series::from_ints("q", [1, 2, 3]), + ]).group_by([@expr.col("region")]) + } + let wrong_lengths : Array[Array[Int64]] = [[7, 8], []] + for cells in wrong_lengths { + assert_eq( + Ok( + grouped().agg([ + @expr.col("q") + .map_batches(label="wrong", returns_scalar=true, _s => { + Series::from_ints("wrong", cells) + }) + .with_alias("q_wrong"), + ]), + ) catch { + e => Err(e) + }, + Err(@types.DataError::LengthMismatch), + ) + } +} + +///| +test "map_batches: a bare (non-scalar) map is rejected as an aggregation" { + // Without `returns_scalar`, the node is row-wise, so the reduction-shape + // gate rejects it before evaluation — the same gate a bare `col` trips. + assert_eq( + Ok( + mb_df() + .group_by([@expr.col("region")]) + .agg([@expr.col("q").map_batches(label="x", s => s)]), + ) catch { + e => Err(e) + }, + Err( + @types.DataError::InvalidOperation( + "aggregate expression must reduce each group to a single value: map_batches(\"x\", [col(q)])", + ), + ), + ) +} + +///| +test "map_batches over an empty frame yields an empty column" { + let empty = DataFrame::DataFrame([Series::from_ints("i", [])]) + let out = empty.with_columns([@expr.col("i").map_batches(label="id", s => s)]) + assert_eq(out.shape(), (0, 1)) + assert_eq(out.get_column("i").dtype(), @types.DataType::Int) + assert_eq(out.get_column("i").to_scalars(), []) +} diff --git a/frame/expr_numeric_test.mbt b/frame/expr_numeric_test.mbt new file mode 100644 index 00000000..08cf0e17 --- /dev/null +++ b/frame/expr_numeric_test.mbt @@ -0,0 +1,905 @@ +// Blackbox tests for the numeric expression operators, one family per +// section: the unary ops (`abs` / `floor` / `ceil` / `sign` / `round`), +// the binary arithmetic edges (`modulo` / `pow` / `floor_div`), the exact +// `Int` ↔ `Float` comparison the kernel promises, and the `Int64` extremes +// where 2^63 wraps rather than aborts. +// Blackbox tests for the unary numeric ops `Expr::abs` / `floor` / `ceil` / +// `sign` / `round`: per-dtype behavior on both backends, NaN pass-through, null +// propagation, and the non-numeric type error — plus, for `round`, the +// half-to-even rule, the `decimals~` places, and the non-finite edges. + +///| +test "abs / floor / ceil / sign on the Numeric fast arm" { + let df = DataFrame::DataFrame([ + Series::from_ints("i", [-3L, 3L, 0L]), + Series::from_floats("f", [-3.7, 3.2, 0.0]), + ]) + let out = df.select([ + @expr.col("i").abs().with_alias("i_abs"), + @expr.col("i").floor().with_alias("i_floor"), + @expr.col("i").ceil().with_alias("i_ceil"), + @expr.col("i").sign().with_alias("i_sign"), + @expr.col("f").abs().with_alias("f_abs"), + @expr.col("f").floor().with_alias("f_floor"), + @expr.col("f").ceil().with_alias("f_ceil"), + @expr.col("f").sign().with_alias("f_sign"), + ]) + // `Int`: floor/ceil are identity, abs and sign act. + assert_eq(out.get_column("i_abs").to_scalars(), [ + @types.Scalar::Int(3), + @types.Scalar::Int(3), + @types.Scalar::Int(0), + ]) + assert_eq(out.get_column("i_floor").to_scalars(), [ + @types.Scalar::Int(-3), + @types.Scalar::Int(3), + @types.Scalar::Int(0), + ]) + assert_eq(out.get_column("i_ceil").to_scalars(), [ + @types.Scalar::Int(-3), + @types.Scalar::Int(3), + @types.Scalar::Int(0), + ]) + assert_eq(out.get_column("i_sign").to_scalars(), [ + @types.Scalar::Int(-1), + @types.Scalar::Int(1), + @types.Scalar::Int(0), + ]) + // `Float`: floor toward −∞, ceil toward +∞. + assert_eq(out.get_column("f_abs").to_scalars(), [ + @types.Scalar::Float(3.7), + @types.Scalar::Float(3.2), + @types.Scalar::Float(0.0), + ]) + assert_eq(out.get_column("f_floor").to_scalars(), [ + @types.Scalar::Float(-4.0), + @types.Scalar::Float(3.0), + @types.Scalar::Float(0.0), + ]) + assert_eq(out.get_column("f_ceil").to_scalars(), [ + @types.Scalar::Float(-3.0), + @types.Scalar::Float(4.0), + @types.Scalar::Float(0.0), + ]) + assert_eq(out.get_column("f_sign").to_scalars(), [ + @types.Scalar::Float(-1.0), + @types.Scalar::Float(1.0), + @types.Scalar::Float(0.0), + ]) +} + +///| +test "unary numeric: Builtin arm, null propagation, NaN, non-numeric" { + let df = DataFrame::DataFrame([ + Series::from_int_options("i", [Some(-3L), None]), + Series::from_float_options("f", [Some(-3.7), None]), + ]) + let out = df.select([ + @expr.col("i").abs().with_alias("i_abs"), + @expr.col("f").floor().with_alias("f_floor"), + ]) + assert_eq(out.get_column("i_abs").to_scalars(), [ + @types.Scalar::Int(3), + @types.Scalar::Null, + ]) + assert_eq(out.get_column("f_floor").to_scalars(), [ + @types.Scalar::Float(-4.0), + @types.Scalar::Null, + ]) + // `NaN` passes through `abs` / `sign`. + let nans = DataFrame::DataFrame([ + Series::from_floats("f", [@double.not_a_number]), + ]) + let nout = nans.select([ + @expr.col("f").abs().with_alias("a"), + @expr.col("f").sign().with_alias("s"), + ]) + assert_true( + match nout.get_column("a").to_scalars()[0] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + assert_true( + match nout.get_column("s").to_scalars()[0] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + // A non-numeric operand raises. + let strs = DataFrame::DataFrame([Series::from_strings("s", ["x"])]) + assert_true( + (Ok(strs.select([@expr.col("s").abs()])) catch { e => Err(e) }) + is Err( + @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) + ), + ) +} + +///| +test "round: ties go to even (banker's rounding)" { + let df = DataFrame::DataFrame([ + Series::from_floats("x", [0.5, 1.5, 2.5, 3.5, -0.5, -1.5, -2.5, -3.5]), + ]) + let out = df.select([@expr.col("x").round().with_alias("r")]) + // Each exact half rounds to its even neighbour: 0.5→0, 1.5→2, 2.5→2, 3.5→4, + // and the negatives symmetrically. + assert_eq(out.get_column("r").to_scalars(), [ + @types.Scalar::Float(0.0), + @types.Scalar::Float(2.0), + @types.Scalar::Float(2.0), + @types.Scalar::Float(4.0), + @types.Scalar::Float(0.0), + @types.Scalar::Float(-2.0), + @types.Scalar::Float(-2.0), + @types.Scalar::Float(-4.0), + ]) + // `-0.5`'s even neighbour is `-0.0`: IEEE equality calls both zeros + // equal above, so the sign is asserted where IEEE makes it observable — + // under division, where +0.0 and -0.0 go to opposite infinities. + let cells = out.get_column("r").to_scalars() + assert_true(cells[0] is @types.Scalar::Float(v) && v == 0.0 && 1.0 / v > 0.0) + assert_true(cells[4] is @types.Scalar::Float(v) && v == 0.0 && 1.0 / v < 0.0) +} + +///| +test "round: non-ties go to the nearest integer; Int is identity" { + let df = DataFrame::DataFrame([ + Series::from_ints("i", [-3L, 4L, 0L]), + Series::from_floats("f", [3.2, 3.7, -3.7]), + ]) + let out = df.select([ + @expr.col("i").round().with_alias("ri"), + @expr.col("f").round().with_alias("rf"), + ]) + assert_eq(out.get_column("ri").to_scalars(), [ + @types.Scalar::Int(-3), + @types.Scalar::Int(4), + @types.Scalar::Int(0), + ]) + assert_eq(out.get_column("rf").to_scalars(), [ + @types.Scalar::Float(3.0), // 3.2 → 3 + @types.Scalar::Float(4.0), // 3.7 → 4 + @types.Scalar::Float(-4.0), // -3.7 → -4 + ]) +} + +///| +test "round: Builtin null, NaN / ±inf pass through, non-numeric raises" { + let df = DataFrame::DataFrame([ + Series::from_float_options("f", [Some(2.5), None]), + ]) + let out = df.select([@expr.col("f").round().with_alias("r")]) + // 2.5 rounds to the even 2; a null stays null. + assert_eq(out.get_column("r").to_scalars(), [ + @types.Scalar::Float(2.0), + @types.Scalar::Null, + ]) + // NaN and ±inf pass through unchanged. + let inf = 1.0 / 0.0 + let special = DataFrame::DataFrame([ + Series::from_floats("f", [@double.not_a_number, inf, -inf]), + ]) + let s = special + .select([@expr.col("f").round().with_alias("r")]) + .get_column("r") + .to_scalars() + assert_true( + match s[0] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + assert_eq(s[1], @types.Scalar::Float(inf)) + assert_eq(s[2], @types.Scalar::Float(-inf)) + // A non-numeric operand raises. + let strs = DataFrame::DataFrame([Series::from_strings("s", ["x"])]) + assert_true( + (Ok(strs.select([@expr.col("s").round()])) catch { e => Err(e) }) + is Err( + @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) + ), + ) +} + +///| +test "unary numeric: Expr::to_string renders the method calls" { + inspect(@expr.col("x").abs().to_string(), content="col(x).abs()") + inspect(@expr.col("x").floor().to_string(), content="col(x).floor()") + inspect(@expr.col("x").ceil().to_string(), content="col(x).ceil()") + inspect(@expr.col("x").sign().to_string(), content="col(x).sign()") + inspect(@expr.col("x").round().to_string(), content="col(x).round()") +} + +///| +test "round takes a decimal-place count" { + let df = DataFrame::DataFrame([ + Series::from_floats("f", [1.2345, 2.5, -1.2345, 0.125]), + Series::from_ints("i", [7, 8, 9, 10]), + ]) + let rounded = df.select([ + @expr.col("f").round(decimals=2).with_alias("two"), + @expr.col("f").round().with_alias("zero"), + @expr.col("i").round(decimals=3).with_alias("int"), + ]) + assert_eq(rounded.item(0, "two"), @types.Scalar::Float(1.23)) + assert_eq(rounded.item(2, "two"), @types.Scalar::Float(-1.23)) + // Ties still go to even, at the requested place: 0.125 → 0.12. + assert_eq(rounded.item(3, "two"), @types.Scalar::Float(0.12)) + // The default is unchanged: whole numbers, ties to even. + assert_eq(rounded.item(1, "zero"), @types.Scalar::Float(2.0)) + // An Int column is the identity at any setting. + assert_eq(rounded.item(0, "int"), @types.Scalar::Int(7)) + // A negative count clamps to 0 rather than raising. + assert_same_expr(@expr.col("f").round(decimals=-3), @expr.col("f").round()) + // A non-default count renders as the argument that builds it. + inspect(@expr.col("f").round(decimals=2), content="col(f).round(decimals=2)") +} + +///| +test "round(decimals) stays total at the numeric edges" { + let nan = 0.0 / 0.0 + let inf = 1.0 / 0.0 + let df = DataFrame::DataFrame([ + Series::from_floats("f", [nan, inf, -inf, 1.7976931348623157e308, 1.5]), + ]) + // Non-finite cells pass through untouched, as with the whole-number form. + let out = df.select([@expr.col("f").round(decimals=2).with_alias("r")]) + let cells = out.get_column("r").to_scalars() + assert_true( + match cells[0] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + assert_eq(cells[1], @types.Scalar::Float(inf)) + assert_eq(cells[2], @types.Scalar::Float(-inf)) + // Scaling the largest finite Double by 100 overflows, so the value is left + // as it is rather than round-tripping through infinity. + assert_eq(cells[3], @types.Scalar::Float(1.7976931348623157e308)) + // A decimal count so large that 10^decimals is infinite is likewise a no-op. + let huge = df.select([@expr.col("f").round(decimals=400).with_alias("r")]) + assert_eq(huge.get_column("r").to_scalars()[4], @types.Scalar::Float(1.5)) +} + +///| +test "round(decimals) is the identity past the value's own resolution" { + // Asking for more places than a `Double` can resolve must return the value + // unchanged. Scaling by `10^decimals` and dividing back does not: above + // `10^22` the scale is itself inexact, so the round trip moved the value by + // an ulp even though rounding could not have. + let df = DataFrame::DataFrame([ + Series::from_floats("f", [1.5, -1.5, 123456.789, 0.1, 1.0e15]), + ]) + for places in [17, 20, 30, 99, 300, 308, 309, 400] { + let out = df.select([@expr.col("f").round(decimals=places).with_alias("r")]) + assert_eq(out.get_column("r").to_scalars(), df.get_column("f").to_scalars()) + } + // A value that is already integral is untouched at any place, however few. + let integral = df.select([@expr.col("f").round(decimals=2).with_alias("r")]) + assert_eq(integral.item(4, "r"), @types.Scalar::Float(1.0e15)) + // The guard must not swallow the rounding that a fine place genuinely does: + // a magnitude small enough to still have a fraction at that place rounds. + let small = DataFrame::DataFrame([ + Series::from_floats("f", [4.0e-18, 6.0e-18, 1.0e-300]), + ]) + let rounded = small.select([@expr.col("f").round(decimals=17).with_alias("r")]) + assert_eq(rounded.item(0, "r"), @types.Scalar::Float(0.0)) + assert_eq(rounded.item(1, "r"), @types.Scalar::Float(1.0e-17)) + assert_eq(rounded.item(2, "r"), @types.Scalar::Float(0.0)) +} + +// Blackbox tests for `Expr::modulo` (`%`) and `Expr::pow` (`**`, always Float). + +///| +test "modulo: Int remainder carries the dividend's sign; every arm" { + let df = DataFrame::DataFrame([ + Series::from_ints("i", [7L, -7L, 6L]), + Series::from_ints("j", [2L, 2L, -4L]), + Series::from_floats("f", [7.5, -7.5, 8.0]), + Series::from_floats("g", [2.0, 2.0, 3.0]), + ]) + let out = df.select([ + @expr.col("i").modulo(@expr.col("j")).with_alias("ii"), + @expr.col("i").modulo(@expr.col("g")).with_alias("if"), + @expr.col("f").modulo(@expr.col("j")).with_alias("fi"), + @expr.col("f").modulo(@expr.col("g")).with_alias("ff"), + ]) + // ii: 7%2=1, -7%2=-1, 6%-4=2. + assert_eq(out.get_column("ii").to_scalars(), [ + @types.Scalar::Int(1), + @types.Scalar::Int(-1), + @types.Scalar::Int(2), + ]) + // if (Int%Float→Float): 7%2=1, -7%2=-1, 6%3=0. + assert_eq(out.get_column("if").to_scalars(), [ + @types.Scalar::Float(1.0), + @types.Scalar::Float(-1.0), + @types.Scalar::Float(0.0), + ]) + // fi (Float%Int): 7.5%2=1.5, -7.5%2=-1.5, 8%-4=0. + assert_eq(out.get_column("fi").to_scalars(), [ + @types.Scalar::Float(1.5), + @types.Scalar::Float(-1.5), + @types.Scalar::Float(0.0), + ]) + // ff: 7.5%2=1.5, -7.5%2=-1.5, 8%3=2. + assert_eq(out.get_column("ff").to_scalars(), [ + @types.Scalar::Float(1.5), + @types.Scalar::Float(-1.5), + @types.Scalar::Float(2.0), + ]) +} + +///| +test "modulo: Numeric Int-by-zero divisor yields null (fast arm)" { + let df = DataFrame::DataFrame([ + Series::from_ints("a", [10L, 5L]), + Series::from_ints("b", [0L, 2L]), + ]) + let out = df.select([@expr.col("a").modulo(@expr.col("b")).with_alias("m")]) + // 10%0=null, 5%2=1. + assert_eq(out.get_column("m").to_scalars(), [ + @types.Scalar::Null, + @types.Scalar::Int(1), + ]) +} + +///| +test "modulo: Int-by-zero null, Float-by-zero NaN, Builtin arms, non-numeric" { + let df = DataFrame::DataFrame([ + Series::from_int_options("i", [Some(10L), None, Some(5L)]), + Series::from_int_options("j", [Some(0L), Some(2L), Some(3L)]), + Series::from_float_options("f", [Some(1.0), Some(7.5), None]), + Series::from_float_options("g", [Some(0.0), Some(2.0), Some(2.0)]), + ]) + let out = df.select([ + @expr.col("i").modulo(@expr.col("j")).with_alias("ii"), + @expr.col("i").modulo(@expr.col("g")).with_alias("ig"), + @expr.col("f").modulo(@expr.col("j")).with_alias("fj"), + @expr.col("f").modulo(@expr.col("g")).with_alias("fg"), + ]) + // ii (Builtin Int/Int): 10%0=null, null, 5%3=2. + assert_eq(out.get_column("ii").to_scalars(), [ + @types.Scalar::Null, + @types.Scalar::Null, + @types.Scalar::Int(2), + ]) + // ig (Builtin Int/Float): 10%0.0=nan, null, 5%2.0=1.0. + let ig = out.get_column("ig").to_scalars() + assert_true( + match ig[0] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + assert_eq(ig[1], @types.Scalar::Null) + assert_eq(ig[2], @types.Scalar::Float(1.0)) + // fj (Builtin Float/Int): 1%0=nan, 7.5%2=1.5, null. + let fj = out.get_column("fj").to_scalars() + assert_true( + match fj[0] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + assert_eq(fj[1], @types.Scalar::Float(1.5)) + assert_eq(fj[2], @types.Scalar::Null) + // fg (Builtin Float/Float): 1%0.0=nan, 7.5%2.0=1.5, null. + let fg = out.get_column("fg").to_scalars() + assert_true( + match fg[0] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) + assert_eq(fg[1], @types.Scalar::Float(1.5)) + assert_eq(fg[2], @types.Scalar::Null) + // Non-numeric raises. + let strs = DataFrame::DataFrame([ + Series::from_strings("s", ["x"]), + Series::from_ints("n", [1L]), + ]) + assert_eq( + Ok(strs.select([@expr.col("s").modulo(@expr.col("n"))])) catch { + e => Err(e) + }, + Err( + @types.DataError::TypeMismatch( + @types.TypeMismatchDetail::Operation( + "take the remainder of", + @types.DataType::String, + @types.DataType::Int, + ), + ), + ), + ) +} + +///| +test "pow: always Float; every numeric arm" { + let df = DataFrame::DataFrame([ + Series::from_ints("i", [2L, 4L]), + Series::from_ints("j", [3L, 2L]), + Series::from_floats("f", [2.0, 9.0]), + Series::from_floats("g", [10.0, 0.5]), + ]) + let out = df.select([ + @expr.col("i").pow(@expr.col("j")).with_alias("ii"), + @expr.col("i").pow(@expr.col("g")).with_alias("ig"), + @expr.col("f").pow(@expr.col("j")).with_alias("fj"), + @expr.col("f").pow(@expr.col("g")).with_alias("fg"), + ]) + // ii: 2^3=8, 4^2=16 (Float). + assert_eq(out.get_column("ii").to_scalars(), [ + @types.Scalar::Float(8.0), + @types.Scalar::Float(16.0), + ]) + // ig (Int^Float): 2^10=1024, 4^0.5=2. + assert_eq(out.get_column("ig").to_scalars(), [ + @types.Scalar::Float(1024.0), + @types.Scalar::Float(2.0), + ]) + // fj (Float^Int): 2^3=8, 9^2=81. + assert_eq(out.get_column("fj").to_scalars(), [ + @types.Scalar::Float(8.0), + @types.Scalar::Float(81.0), + ]) + // fg (Float^Float): 2^10=1024, 9^0.5=3. + assert_eq(out.get_column("fg").to_scalars(), [ + @types.Scalar::Float(1024.0), + @types.Scalar::Float(3.0), + ]) +} + +///| +test "pow: negative / zero exponents, Builtin arms, null, non-numeric" { + let df = DataFrame::DataFrame([ + Series::from_float_options("b", [Some(2.0), Some(0.0), None]), + Series::from_float_options("e", [Some(-1.0), Some(0.0), Some(2.0)]), + Series::from_int_options("ib", [Some(2L), Some(3L), None]), + Series::from_int_options("ie", [Some(-2L), Some(0L), Some(2L)]), + ]) + let out = df.select([ + @expr.col("b").pow(@expr.col("e")).with_alias("ff"), + @expr.col("ib").pow(@expr.col("ie")).with_alias("ii"), + @expr.col("ib").pow(@expr.col("e")).with_alias("if"), + @expr.col("b").pow(@expr.col("ie")).with_alias("fi"), + ]) + // ff: 2^-1=0.5, 0^0=1, null. + assert_eq(out.get_column("ff").to_scalars(), [ + @types.Scalar::Float(0.5), + @types.Scalar::Float(1.0), + @types.Scalar::Null, + ]) + // ii (Builtin Int^Int, still Float): 2^-2=0.25, 3^0=1, null. + assert_eq(out.get_column("ii").to_scalars(), [ + @types.Scalar::Float(0.25), + @types.Scalar::Float(1.0), + @types.Scalar::Null, + ]) + // if / fi just exercise the remaining Builtin mixed arms. + assert_eq(out.get_column("if").to_scalars()[1], @types.Scalar::Float(1.0)) + assert_eq(out.get_column("fi").to_scalars()[1], @types.Scalar::Float(1.0)) + // Non-numeric raises. + let strs = DataFrame::DataFrame([ + Series::from_strings("s", ["x"]), + Series::from_ints("n", [2L]), + ]) + assert_eq( + Ok(strs.select([@expr.col("s").pow(@expr.col("n"))])) catch { + e => Err(e) + }, + Err( + @types.DataError::TypeMismatch( + @types.TypeMismatchDetail::Operation( + "exponentiate", + @types.DataType::String, + @types.DataType::Int, + ), + ), + ), + ) +} + +///| +test "mod / pow: Expr::to_string renders % and **" { + inspect( + @expr.col("a").modulo(@expr.col("b")).to_string(), + content="(col(a) % col(b))", + ) + inspect( + @expr.col("a").pow(@expr.col("b")).to_string(), + content="(col(a) ** col(b))", + ) +} + +// Blackbox tests for `Expr::floor_div` (Polars `//`): floor toward −∞, the +// Int-zero → null guard, Float IEEE zero division, mixed promotion, null +// propagation, and the non-numeric type error — on both backends. + +///| +test "floor_div: Int floors toward -inf, not truncation (Numeric arm)" { + let df = DataFrame::DataFrame([ + Series::from_ints("a", [7L, -7L, 7L, -7L, 6L]), + Series::from_ints("b", [2L, 2L, -2L, -2L, 3L]), + ]) + let out = df.select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) + // 7//2=3, -7//2=-4, 7//-2=-4, -7//-2=3, 6//3=2. + assert_eq(out.get_column("q").to_scalars(), [ + @types.Scalar::Int(3), + @types.Scalar::Int(-4), + @types.Scalar::Int(-4), + @types.Scalar::Int(3), + @types.Scalar::Int(2), + ]) +} + +///| +test "floor_div: Int division by zero is a null cell" { + let df = DataFrame::DataFrame([ + Series::from_ints("a", [10L, 5L]), + Series::from_ints("b", [0L, 2L]), + ]) + let out = df.select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) + assert_eq(out.get_column("q").to_scalars(), [ + @types.Scalar::Null, + @types.Scalar::Int(2), + ]) +} + +///| +test "floor_div: mixed Int/Float promotes to a floored Float" { + let df = DataFrame::DataFrame([ + Series::from_ints("a", [7L]), + Series::from_floats("b", [2.0]), + ]) + let out = df.select([ + @expr.col("a").floor_div(@expr.col("b")).with_alias("q"), + @expr.col("b").floor_div(@expr.col("a")).with_alias("r"), + ]) + // 7 // 2.0 = floor(3.5) = 3.0; 2.0 // 7 = floor(0.285…) = 0.0. + assert_eq(out.get_column("q").to_scalars(), [@types.Scalar::Float(3.0)]) + assert_eq(out.get_column("r").to_scalars(), [@types.Scalar::Float(0.0)]) +} + +///| +test "floor_div: Float floors the quotient; zero divisor is IEEE inf/nan" { + let df = DataFrame::DataFrame([ + Series::from_floats("a", [7.5, 1.0, 0.0]), + Series::from_floats("b", [2.0, 0.0, 0.0]), + ]) + let q = df + .select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) + .get_column("q") + .to_scalars() + assert_eq(q[0], @types.Scalar::Float(3.0)) // floor(3.75) + // 1.0 / 0.0 = +inf, floor(+inf) = +inf. + assert_true( + match q[1] { + @types.Scalar::Float(v) => v == @double.infinity + _ => false + }, + ) + // 0.0 / 0.0 = NaN, floor(NaN) = NaN. + assert_true( + match q[2] { + @types.Scalar::Float(v) => v.is_nan() + _ => false + }, + ) +} + +///| +test "floor_div: null propagates (Builtin arm) and non-numeric raises" { + let df = DataFrame::DataFrame([ + Series::from_int_options("a", [Some(7L), None]), + Series::from_float_options("b", [Some(2.0), Some(2.0)]), + ]) + let out = df.select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) + // 7 // 2.0 = 3.0; null // 2.0 = null. + assert_eq(out.get_column("q").to_scalars(), [ + @types.Scalar::Float(3.0), + @types.Scalar::Null, + ]) + let strs = DataFrame::DataFrame([ + Series::from_strings("s", ["x"]), + Series::from_ints("n", [1L]), + ]) + assert_eq( + Ok(strs.select([@expr.col("s").floor_div(@expr.col("n"))])) catch { + e => Err(e) + }, + Err( + @types.DataError::TypeMismatch( + @types.TypeMismatchDetail::Operation( + "floor-divide", + @types.DataType::String, + @types.DataType::Int, + ), + ), + ), + ) +} + +///| +test "floor_div: general (Builtin) arms for every numeric dtype pair" { + // Each operand column carries a null, so the pair takes the Builtin general + // arm rather than the `Numeric` fast path. + let df = DataFrame::DataFrame([ + Series::from_int_options("ia", [Some(7L), Some(10L), None]), + Series::from_int_options("ib", [Some(2L), Some(0L), Some(2L)]), + Series::from_float_options("fa", [Some(7.5), None, Some(1.0)]), + Series::from_float_options("fb", [Some(2.0), Some(2.0), Some(0.0)]), + ]) + let out = df.select([ + @expr.col("ia").floor_div(@expr.col("ib")).with_alias("ii"), + @expr.col("fa").floor_div(@expr.col("ib")).with_alias("fi"), + @expr.col("fa").floor_div(@expr.col("fb")).with_alias("ff"), + ]) + // ii: 7//2=3, 10//0=null (zero), null//2=null. + assert_eq(out.get_column("ii").to_scalars(), [ + @types.Scalar::Int(3), + @types.Scalar::Null, + @types.Scalar::Null, + ]) + // fi: 7.5//2=floor(3.75)=3.0, null//0=null, 1.0//2=floor(0.5)=0.0. + assert_eq(out.get_column("fi").to_scalars(), [ + @types.Scalar::Float(3.0), + @types.Scalar::Null, + @types.Scalar::Float(0.0), + ]) + // ff: 7.5//2.0=3.0, null//2.0=null, 1.0//0.0=floor(+inf)=+inf. + let ff = out.get_column("ff").to_scalars() + assert_eq(ff[0], @types.Scalar::Float(3.0)) + assert_eq(ff[1], @types.Scalar::Null) + assert_true( + match ff[2] { + @types.Scalar::Float(v) => v == @double.infinity + _ => false + }, + ) +} + +///| +test "floor_div: Expr::to_string renders it with //" { + let e = @expr.col("a").floor_div(@expr.col("b")) + inspect(e.to_string(), content="(col(a) // col(b))") +} + +// Integer arithmetic at the `Int64` edges. `Int64::MIN` has no positive +// counterpart, so negating, absolute-value-ing, multiplying or floor-dividing +// it by `-1` all produce `2^63`, which does not fit — every one of them wraps +// back to `Int64::MIN` rather than aborting, and must do so identically on all +// four backends. `//` is the one that needed help: wasm's `i64.div_s` traps on +// that pair by specification (`i64.rem_s` special-cases it to `0` instead), so +// `floor_div_i64` routes division by `-1` through negation. + +///| +let i64_min : Int64 = -9223372036854775808L + +///| +/// The extremes fixture on the `Numeric` backend (`from_ints` is all-valid) and +/// on `Builtin` (`from_int_options` carries a validity bitmap). Every kernel +/// has a fast path for the first and a nullable path for the second, so both +/// have to be exercised. +fn extremes(nullable : Bool) -> DataFrame raise @types.DataError { + let a = [i64_min, i64_min, -1L, 0L] + let b = [-1L, 1L, -1L, -1L] + DataFrame::DataFrame( + if nullable { + [ + Series::from_int_options("a", a.map(v => Some(v))), + Series::from_int_options("b", b.map(v => Some(v))), + ] + } else { + [Series::from_ints("a", a), Series::from_ints("b", b)] + }, + ) +} + +///| +/// Evaluate `build(col a, col b)` over both storage backends and return the +/// cells, asserting the two agree — the parity a backend-specific intrinsic +/// would break. +fn extremes_result( + build : (@expr.Expr, @expr.Expr) -> @expr.Expr, +) -> Array[@types.Scalar] raise { + let of = fn(nullable : Bool) -> Array[@types.Scalar] raise @types.DataError { + extremes(nullable) + .select([build(@expr.col("a"), @expr.col("b")).with_alias("r")]) + .get_column("r") + .to_scalars() + } + let numeric = of(false) + assert_eq(numeric, of(true)) + numeric +} + +///| +test "floor_div at Int64::MIN / -1 wraps instead of trapping" { + // Rows: MIN // -1, MIN // 1, -1 // -1, 0 // -1. + assert_eq(extremes_result((a, b) => a.floor_div(b)), [ + @types.Scalar::Int(i64_min), + @types.Scalar::Int(i64_min), + @types.Scalar::Int(1), + @types.Scalar::Int(0), + ]) +} + +///| +test "floor_div by -1 still floors: it agrees with negation everywhere" { + // The special case must not change any *representable* quotient. `// -1` is + // exact for every dividend, so it is negation — including the sign of zero + // and the odd/even cases a floor adjustment would disturb. + let df = DataFrame::DataFrame([ + Series::from_ints("a", [7L, -7L, 8L, -8L, 0L, 9223372036854775807L]), + Series::from_ints("b", [-1L, -1L, -1L, -1L, -1L, -1L]), + ]) + let out = df.select([ + @expr.col("a").floor_div(@expr.col("b")).with_alias("q"), + (-@expr.col("a")).with_alias("n"), + ]) + assert_eq(out.get_column("q").to_scalars(), out.get_column("n").to_scalars()) + assert_eq(out.get_column("q").to_scalars(), [ + @types.Scalar::Int(-7), + @types.Scalar::Int(7), + @types.Scalar::Int(-8), + @types.Scalar::Int(8), + @types.Scalar::Int(0), + @types.Scalar::Int(-9223372036854775807L), + ]) +} + +///| +test "modulo at Int64::MIN % -1 is zero on every backend" { + // `%` is evaluated as part of the same division `//` is, so it faults on the + // same pair — but only on some backends: wasm's `i64.rem_s` defines + // `Int64::MIN % -1` as `0`, while a native build lowers `%` to a hardware + // divide that raises the divide-by-zero fault. `0` is the answer either way + // (a remainder by ±1 is always 0), so `rem_i64` returns it without dividing. + assert_eq(extremes_result((a, b) => a.modulo(b)), [ + @types.Scalar::Int(0), + @types.Scalar::Int(0), + @types.Scalar::Int(0), + @types.Scalar::Int(0), + ]) +} + +///| +test "modulo by -1 is zero for every dividend, like modulo by 1" { + // The special case must agree with the general path wherever that path is + // defined: `a % -1` and `a % 1` are both `0` for every `a`. + let df = DataFrame::DataFrame([ + Series::from_ints("a", [7L, -7L, 0L, 9223372036854775807L, i64_min]), + ]) + let out = df.select([ + @expr.col("a").modulo(@expr.lit_int(-1)).with_alias("m"), + @expr.col("a").modulo(@expr.lit_int(1)).with_alias("p"), + ]) + let zeros = Array::make(5, @types.Scalar::Int(0)) + assert_eq(out.get_column("m").to_scalars(), zeros) + assert_eq(out.get_column("p").to_scalars(), zeros) +} + +///| +test "neg / abs / mul wrap at Int64::MIN" { + let wrapped = [ + @types.Scalar::Int(i64_min), + @types.Scalar::Int(i64_min), + @types.Scalar::Int(1), + @types.Scalar::Int(0), + ] + // -a and a * b (b = -1 except row 1) are the same wrap. + assert_eq(extremes_result((a, _) => -a), wrapped) + assert_eq(extremes_result((a, b) => a * b), [ + @types.Scalar::Int(i64_min), + @types.Scalar::Int(i64_min), + @types.Scalar::Int(1), + @types.Scalar::Int(0), + ]) + // abs leaves MIN alone (no positive counterpart) and is identity on the rest. + assert_eq(extremes_result((a, _) => a.abs()), [ + @types.Scalar::Int(i64_min), + @types.Scalar::Int(i64_min), + @types.Scalar::Int(1), + @types.Scalar::Int(0), + ]) +} + +///| +test "true division of Int64::MIN is Float, so it never overflows" { + // `/` is always Float in MoonFrame, so the pair that overflows in Int64 + // arithmetic is an ordinary IEEE quotient here. + assert_eq(extremes_result((a, b) => a / b), [ + @types.Scalar::Float(9223372036854775808.0), + @types.Scalar::Float(-9223372036854775808.0), + @types.Scalar::Float(1.0), + @types.Scalar::Float(-0.0), + ]) +} + +// Blackbox tests pinning the expr comparison kernel's exact `Int` vs `Float` +// behavior (no 2^53 promotion) — the authoritative home of that exactness, +// on both the Numeric fast arm and the Builtin nullable arm. + +///| +test "expr comparison: Int vs Float is exact past 2^53 (Numeric fast arm)" { + // i = [2^53+1, 2^53], f = [2^53, 2^53]; both columns null-free → Numeric. + let df = DataFrame::DataFrame([ + Series::from_ints("i", [9007199254740993L, 9007199254740992L]), + Series::from_floats("f", [9007199254740992.0, 9007199254740992.0]), + ]) + let t = @types.Scalar::Bool(true) + let f = @types.Scalar::Bool(false) + // Int-vs-Float, all six verbs. Row 0 (2^53+1 vs 2^53, distinct), row 1 equal. + let fwd = df.select([ + @expr.col("i").eq(@expr.col("f")).with_alias("eq"), + @expr.col("i").ne(@expr.col("f")).with_alias("ne"), + @expr.col("i").lt(@expr.col("f")).with_alias("lt"), + @expr.col("i").le(@expr.col("f")).with_alias("le"), + @expr.col("i").gt(@expr.col("f")).with_alias("gt"), + @expr.col("i").ge(@expr.col("f")).with_alias("ge"), + ]) + assert_eq(fwd.get_column("eq").to_scalars(), [f, t]) + assert_eq(fwd.get_column("ne").to_scalars(), [t, f]) + assert_eq(fwd.get_column("lt").to_scalars(), [f, f]) + assert_eq(fwd.get_column("le").to_scalars(), [f, t]) + assert_eq(fwd.get_column("gt").to_scalars(), [t, f]) + assert_eq(fwd.get_column("ge").to_scalars(), [t, t]) + // Float-vs-Int (the transposed verdict), all six verbs. Row 0 (2^53 vs + // 2^53+1), row 1 (2^53 vs 2^53). + let rev = df.select([ + @expr.col("f").eq(@expr.col("i")).with_alias("eq"), + @expr.col("f").ne(@expr.col("i")).with_alias("ne"), + @expr.col("f").lt(@expr.col("i")).with_alias("lt"), + @expr.col("f").le(@expr.col("i")).with_alias("le"), + @expr.col("f").gt(@expr.col("i")).with_alias("gt"), + @expr.col("f").ge(@expr.col("i")).with_alias("ge"), + ]) + assert_eq(rev.get_column("eq").to_scalars(), [f, t]) + assert_eq(rev.get_column("ne").to_scalars(), [t, f]) + assert_eq(rev.get_column("lt").to_scalars(), [t, f]) + assert_eq(rev.get_column("le").to_scalars(), [t, t]) + assert_eq(rev.get_column("gt").to_scalars(), [f, f]) + assert_eq(rev.get_column("ge").to_scalars(), [f, t]) +} + +///| +test "expr comparison: exact past 2^53 on the Builtin (nullable) arm" { + // A null forces the general Builtin arm; the exact verdict is unchanged and + // the null layer still nulls its row. + let df = DataFrame::DataFrame([ + Series::from_int_options("i", [Some(9007199254740993L), None]), + Series::from_float_options("f", [Some(9007199254740992.0), Some(1.0)]), + ]) + let out = df.select([@expr.col("i").eq(@expr.col("f")).with_alias("eq")]) + assert_eq(out.get_column("eq").to_scalars(), [ + @types.Scalar::Bool(false), + @types.Scalar::Null, + ]) +} + +///| +test "expr comparison: Int vs Float at the Int64 boundaries, ±Inf and NaN" { + // Past the Int64 range the helpers answer from the bounds alone: `MAX` is + // below the 2^63 `Double` it would round to, `MIN` is above `-Inf`, and + // `NaN` makes every verdict `false`. + let df = DataFrame::DataFrame([ + // MAX vs 2^63, MIN vs -Inf, 0 vs NaN, 0 vs +Inf — all null-free → Numeric. + Series::from_ints("i", [9223372036854775807L, -9223372036854775808L, 0L, 0L]), + Series::from_floats("f", [ + 9223372036854775808.0, @double.neg_infinity, @double.not_a_number, @double.infinity, + ]), + ]) + let t = @types.Scalar::Bool(true) + let f = @types.Scalar::Bool(false) + let out = df.select([ + @expr.col("i").eq(@expr.col("f")).with_alias("eq"), + @expr.col("i").lt(@expr.col("f")).with_alias("lt"), + @expr.col("i").gt(@expr.col("f")).with_alias("gt"), + ]) + assert_eq(out.get_column("eq").to_scalars(), [f, f, f, f]) + assert_eq(out.get_column("lt").to_scalars(), [t, f, f, t]) + assert_eq(out.get_column("gt").to_scalars(), [f, t, f, f]) +} diff --git a/frame/is_in_test.mbt b/frame/expr_predicates_test.mbt similarity index 54% rename from frame/is_in_test.mbt rename to frame/expr_predicates_test.mbt index d0aa7648..92b354c5 100644 --- a/frame/is_in_test.mbt +++ b/frame/expr_predicates_test.mbt @@ -1,3 +1,6 @@ +// Blackbox tests for the `is_*` expression predicates: `is_in` (membership +// as an OR of `eq`) and `is_between` (the range test whose `closed~` +// argument decides which endpoints count). // Blackbox tests for `Expr::is_in` — membership as an OR of `eq`. ///| @@ -268,3 +271,222 @@ test "is_in: reduces per group over an aggregation, rejects a bare column" { is Err(@types.DataError::InvalidOperation(_)), ) } + +// Blackbox tests for `Expr::is_between` — the range test `lo <= x <= hi`, whose +// `closed~` argument decides which endpoints are included (`Both` by default). + +///| +test "is_between: both endpoints are inclusive" { + let df = DataFrame::DataFrame([Series::from_ints("a", [0L, 1L, 5L, 10L, 11L])]) + let out = df.select([ + @expr.col("a") + .is_between(@expr.lit_int(1), @expr.lit_int(10)) + .with_alias("r"), + ]) + // 1 and 10 (the endpoints) are in range; 0 and 11 are out. + assert_eq(out.get_column("r").to_scalars(), [ + @types.Scalar::Bool(false), + @types.Scalar::Bool(true), + @types.Scalar::Bool(true), + @types.Scalar::Bool(true), + @types.Scalar::Bool(false), + ]) +} + +///| +test "is_between: Float, String, and Bool ordering" { + let df = DataFrame::DataFrame([ + Series::from_floats("f", [0.5, 2.0, 3.5]), + Series::from_strings("s", ["apple", "cherry", "zebra"]), + Series::from_bools("b", [true, false, true]), + ]) + let out = df.select([ + @expr.col("f") + .is_between(@expr.lit_float(1.0), @expr.lit_float(3.0)) + .with_alias("rf"), + @expr.col("s") + .is_between(@expr.lit_str("b"), @expr.lit_str("m")) + .with_alias("rs"), + @expr.col("b") + .is_between(@expr.lit_bool(false), @expr.lit_bool(true)) + .with_alias("rb"), + ]) + assert_eq(out.get_column("rf").to_scalars(), [ + @types.Scalar::Bool(false), // 0.5 < 1.0 + @types.Scalar::Bool(true), // 1.0 <= 2.0 <= 3.0 + @types.Scalar::Bool(false), // 3.5 > 3.0 + ]) + assert_eq(out.get_column("rs").to_scalars(), [ + @types.Scalar::Bool(false), // "apple" < "b" + @types.Scalar::Bool(true), // "b" <= "cherry" <= "m" + @types.Scalar::Bool(false), // "zebra" > "m" + ]) + // Every Bool is within [false, true]. + assert_eq(out.get_column("rb").to_scalars(), [ + @types.Scalar::Bool(true), + @types.Scalar::Bool(true), + @types.Scalar::Bool(true), + ]) +} + +///| +test "is_between: a null cell yields null" { + let df = DataFrame::DataFrame([ + Series::from_int_options("a", [Some(5L), None]), + ]) + let out = df.select([ + @expr.col("a") + .is_between(@expr.lit_int(0), @expr.lit_int(10)) + .with_alias("r"), + ]) + assert_eq(out.get_column("r").to_scalars(), [ + @types.Scalar::Bool(true), + @types.Scalar::Null, + ]) +} + +///| +test "is_between: column bounds propagate nulls under Kleene AND" { + let df = DataFrame::DataFrame([ + Series::from_ints("a", [5L, 20L]), + Series::from_int_options("lo", [None, None]), + Series::from_ints("hi", [10L, 10L]), + ]) + let out = df.select([ + @expr.col("a").is_between(@expr.col("lo"), @expr.col("hi")).with_alias("r"), + ]) + // row0: (5 >= null) & (5 <= 10) = null & true = null. + // row1: (20 >= null) & (20 <= 10) = null & false = false (a known false wins). + assert_eq(out.get_column("r").to_scalars(), [ + @types.Scalar::Null, + @types.Scalar::Bool(false), + ]) +} + +///| +test "is_between: equals ge().land(le()), operand shared" { + // A *computed* operand is evaluated once by the dedicated node; the result + // must still match the explicit `ge & le` lowering cell for cell. + let df = DataFrame::DataFrame([ + Series::from_int_options("x", [Some(2L), Some(7L), None]), + ]) + let operand = @expr.col("x") + @expr.lit_int(1) + let via_between = df + .select([ + operand.is_between(@expr.lit_int(3), @expr.lit_int(8)).with_alias("r"), + ]) + .get_column("r") + .to_scalars() + let via_manual = df + .select([ + operand + .ge(@expr.lit_int(3)) + .land(operand.le(@expr.lit_int(8))) + .with_alias("r"), + ]) + .get_column("r") + .to_scalars() + assert_eq(via_between, via_manual) +} + +///| +test "is_between: a non-orderable operand raises structured TypeMismatch" { + let df = DataFrame::DataFrame([Series::from_strings("s", ["x"])]) + assert_eq( + Ok( + df.select([@expr.col("s").is_between(@expr.lit_int(0), @expr.lit_int(10))]), + ) catch { + e => Err(e) + }, + Err( + @types.DataError::TypeMismatch( + @types.TypeMismatchDetail::Operation( + "compare", + @types.DataType::String, + @types.DataType::Int, + ), + ), + ), + ) +} + +///| +test "is_between: reduces per group over an aggregation" { + let df = DataFrame::DataFrame([ + Series::from_strings("k", ["x", "x", "y"]), + Series::from_ints("v", [10L, 20L, 5L]), + ]) + let out = df + .group_by([@expr.col("k")]) + .agg([ + @expr.col("v") + .sum() + .is_between(@expr.lit_int(0), @expr.lit_int(25)) + .with_alias("in_range"), + ]) + assert_eq(out.get_column("in_range").to_scalars(), [ + @types.Scalar::Bool(false), // x: 30 > 25 + @types.Scalar::Bool(true), // y: 5 + ]) +} + +///| +test "is_between honours each closed setting" { + let df = DataFrame::DataFrame([Series::from_ints("n", [1, 2, 3, 4])]) + let bounds = (@expr.lit_int(2), @expr.lit_int(3)) + let (lo, hi) = bounds + let mask = fn( + closed : @types.ClosedInterval, + ) -> Array[@types.Scalar] raise @types.DataError { + df + .select([@expr.col("n").is_between(lo, hi, closed~).with_alias("in")]) + .get_column("in") + .to_scalars() + } + // Both endpoints (the default) keeps 2 and 3. + assert_eq(mask(@types.ClosedInterval::Both), [ + @types.Scalar::Bool(false), + @types.Scalar::Bool(true), + @types.Scalar::Bool(true), + @types.Scalar::Bool(false), + ]) + // Left keeps the lower bound only, Right the upper, None neither. + assert_eq(mask(@types.ClosedInterval::Left), [ + @types.Scalar::Bool(false), + @types.Scalar::Bool(true), + @types.Scalar::Bool(false), + @types.Scalar::Bool(false), + ]) + assert_eq(mask(@types.ClosedInterval::Right), [ + @types.Scalar::Bool(false), + @types.Scalar::Bool(false), + @types.Scalar::Bool(true), + @types.Scalar::Bool(false), + ]) + assert_eq(mask(@types.ClosedInterval::None), [ + @types.Scalar::Bool(false), + @types.Scalar::Bool(false), + @types.Scalar::Bool(false), + @types.Scalar::Bool(false), + ]) + // The endpoint rule is part of the expression's identity, and renders as the + // argument that builds it. + assert_diff_expr( + @expr.col("n").is_between(lo, hi), + @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::Left), + ) + inspect( + @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::None), + content="col(n).is_between(2, 3, closed=None)", + ) + inspect( + @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::Left), + content="col(n).is_between(2, 3, closed=Left)", + ) + inspect( + @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::Right), + content="col(n).is_between(2, 3, closed=Right)", + ) + // The default renders bare, so existing plan snapshots are unchanged. + inspect(@expr.col("n").is_between(lo, hi), content="col(n).is_between(2, 3)") +} diff --git a/frame/floor_div_test.mbt b/frame/floor_div_test.mbt deleted file mode 100644 index b88edbf6..00000000 --- a/frame/floor_div_test.mbt +++ /dev/null @@ -1,152 +0,0 @@ -// Blackbox tests for `Expr::floor_div` (Polars `//`): floor toward −∞, the -// Int-zero → null guard, Float IEEE zero division, mixed promotion, null -// propagation, and the non-numeric type error — on both backends. - -///| -test "floor_div: Int floors toward -inf, not truncation (Numeric arm)" { - let df = DataFrame::DataFrame([ - Series::from_ints("a", [7L, -7L, 7L, -7L, 6L]), - Series::from_ints("b", [2L, 2L, -2L, -2L, 3L]), - ]) - let out = df.select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) - // 7//2=3, -7//2=-4, 7//-2=-4, -7//-2=3, 6//3=2. - assert_eq(out.get_column("q").to_scalars(), [ - @types.Scalar::Int(3), - @types.Scalar::Int(-4), - @types.Scalar::Int(-4), - @types.Scalar::Int(3), - @types.Scalar::Int(2), - ]) -} - -///| -test "floor_div: Int division by zero is a null cell" { - let df = DataFrame::DataFrame([ - Series::from_ints("a", [10L, 5L]), - Series::from_ints("b", [0L, 2L]), - ]) - let out = df.select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) - assert_eq(out.get_column("q").to_scalars(), [ - @types.Scalar::Null, - @types.Scalar::Int(2), - ]) -} - -///| -test "floor_div: mixed Int/Float promotes to a floored Float" { - let df = DataFrame::DataFrame([ - Series::from_ints("a", [7L]), - Series::from_floats("b", [2.0]), - ]) - let out = df.select([ - @expr.col("a").floor_div(@expr.col("b")).with_alias("q"), - @expr.col("b").floor_div(@expr.col("a")).with_alias("r"), - ]) - // 7 // 2.0 = floor(3.5) = 3.0; 2.0 // 7 = floor(0.285…) = 0.0. - assert_eq(out.get_column("q").to_scalars(), [@types.Scalar::Float(3.0)]) - assert_eq(out.get_column("r").to_scalars(), [@types.Scalar::Float(0.0)]) -} - -///| -test "floor_div: Float floors the quotient; zero divisor is IEEE inf/nan" { - let df = DataFrame::DataFrame([ - Series::from_floats("a", [7.5, 1.0, 0.0]), - Series::from_floats("b", [2.0, 0.0, 0.0]), - ]) - let q = df - .select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) - .get_column("q") - .to_scalars() - assert_eq(q[0], @types.Scalar::Float(3.0)) // floor(3.75) - // 1.0 / 0.0 = +inf, floor(+inf) = +inf. - assert_true( - match q[1] { - @types.Scalar::Float(v) => v == @double.infinity - _ => false - }, - ) - // 0.0 / 0.0 = NaN, floor(NaN) = NaN. - assert_true( - match q[2] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) -} - -///| -test "floor_div: null propagates (Builtin arm) and non-numeric raises" { - let df = DataFrame::DataFrame([ - Series::from_int_options("a", [Some(7L), None]), - Series::from_float_options("b", [Some(2.0), Some(2.0)]), - ]) - let out = df.select([@expr.col("a").floor_div(@expr.col("b")).with_alias("q")]) - // 7 // 2.0 = 3.0; null // 2.0 = null. - assert_eq(out.get_column("q").to_scalars(), [ - @types.Scalar::Float(3.0), - @types.Scalar::Null, - ]) - let strs = DataFrame::DataFrame([ - Series::from_strings("s", ["x"]), - Series::from_ints("n", [1L]), - ]) - assert_eq( - Ok(strs.select([@expr.col("s").floor_div(@expr.col("n"))])) catch { - e => Err(e) - }, - Err( - @types.DataError::TypeMismatch( - @types.TypeMismatchDetail::Operation( - "floor-divide", - @types.DataType::String, - @types.DataType::Int, - ), - ), - ), - ) -} - -///| -test "floor_div: general (Builtin) arms for every numeric dtype pair" { - // Each operand column carries a null, so the pair takes the Builtin general - // arm rather than the `Numeric` fast path. - let df = DataFrame::DataFrame([ - Series::from_int_options("ia", [Some(7L), Some(10L), None]), - Series::from_int_options("ib", [Some(2L), Some(0L), Some(2L)]), - Series::from_float_options("fa", [Some(7.5), None, Some(1.0)]), - Series::from_float_options("fb", [Some(2.0), Some(2.0), Some(0.0)]), - ]) - let out = df.select([ - @expr.col("ia").floor_div(@expr.col("ib")).with_alias("ii"), - @expr.col("fa").floor_div(@expr.col("ib")).with_alias("fi"), - @expr.col("fa").floor_div(@expr.col("fb")).with_alias("ff"), - ]) - // ii: 7//2=3, 10//0=null (zero), null//2=null. - assert_eq(out.get_column("ii").to_scalars(), [ - @types.Scalar::Int(3), - @types.Scalar::Null, - @types.Scalar::Null, - ]) - // fi: 7.5//2=floor(3.75)=3.0, null//0=null, 1.0//2=floor(0.5)=0.0. - assert_eq(out.get_column("fi").to_scalars(), [ - @types.Scalar::Float(3.0), - @types.Scalar::Null, - @types.Scalar::Float(0.0), - ]) - // ff: 7.5//2.0=3.0, null//2.0=null, 1.0//0.0=floor(+inf)=+inf. - let ff = out.get_column("ff").to_scalars() - assert_eq(ff[0], @types.Scalar::Float(3.0)) - assert_eq(ff[1], @types.Scalar::Null) - assert_true( - match ff[2] { - @types.Scalar::Float(v) => v == @double.infinity - _ => false - }, - ) -} - -///| -test "floor_div: Expr::to_string renders it with //" { - let e = @expr.col("a").floor_div(@expr.col("b")) - inspect(e.to_string(), content="(col(a) // col(b))") -} diff --git a/frame/int64_extremes_test.mbt b/frame/int64_extremes_test.mbt deleted file mode 100644 index 6647194e..00000000 --- a/frame/int64_extremes_test.mbt +++ /dev/null @@ -1,151 +0,0 @@ -// Integer arithmetic at the `Int64` edges. `Int64::MIN` has no positive -// counterpart, so negating, absolute-value-ing, multiplying or floor-dividing -// it by `-1` all produce `2^63`, which does not fit — every one of them wraps -// back to `Int64::MIN` rather than aborting, and must do so identically on all -// four backends. `//` is the one that needed help: wasm's `i64.div_s` traps on -// that pair by specification (`i64.rem_s` special-cases it to `0` instead), so -// `floor_div_i64` routes division by `-1` through negation. - -///| -let i64_min : Int64 = -9223372036854775808L - -///| -/// The extremes fixture on the `Numeric` backend (`from_ints` is all-valid) and -/// on `Builtin` (`from_int_options` carries a validity bitmap). Every kernel -/// has a fast path for the first and a nullable path for the second, so both -/// have to be exercised. -fn extremes(nullable : Bool) -> DataFrame raise @types.DataError { - let a = [i64_min, i64_min, -1L, 0L] - let b = [-1L, 1L, -1L, -1L] - DataFrame::DataFrame( - if nullable { - [ - Series::from_int_options("a", a.map(v => Some(v))), - Series::from_int_options("b", b.map(v => Some(v))), - ] - } else { - [Series::from_ints("a", a), Series::from_ints("b", b)] - }, - ) -} - -///| -/// Evaluate `build(col a, col b)` over both storage backends and return the -/// cells, asserting the two agree — the parity a backend-specific intrinsic -/// would break. -fn extremes_result( - build : (@expr.Expr, @expr.Expr) -> @expr.Expr, -) -> Array[@types.Scalar] raise { - let of = fn(nullable : Bool) -> Array[@types.Scalar] raise @types.DataError { - extremes(nullable) - .select([build(@expr.col("a"), @expr.col("b")).with_alias("r")]) - .get_column("r") - .to_scalars() - } - let numeric = of(false) - assert_eq(numeric, of(true)) - numeric -} - -///| -test "floor_div at Int64::MIN / -1 wraps instead of trapping" { - // Rows: MIN // -1, MIN // 1, -1 // -1, 0 // -1. - assert_eq(extremes_result((a, b) => a.floor_div(b)), [ - @types.Scalar::Int(i64_min), - @types.Scalar::Int(i64_min), - @types.Scalar::Int(1), - @types.Scalar::Int(0), - ]) -} - -///| -test "floor_div by -1 still floors: it agrees with negation everywhere" { - // The special case must not change any *representable* quotient. `// -1` is - // exact for every dividend, so it is negation — including the sign of zero - // and the odd/even cases a floor adjustment would disturb. - let df = DataFrame::DataFrame([ - Series::from_ints("a", [7L, -7L, 8L, -8L, 0L, 9223372036854775807L]), - Series::from_ints("b", [-1L, -1L, -1L, -1L, -1L, -1L]), - ]) - let out = df.select([ - @expr.col("a").floor_div(@expr.col("b")).with_alias("q"), - (-@expr.col("a")).with_alias("n"), - ]) - assert_eq(out.get_column("q").to_scalars(), out.get_column("n").to_scalars()) - assert_eq(out.get_column("q").to_scalars(), [ - @types.Scalar::Int(-7), - @types.Scalar::Int(7), - @types.Scalar::Int(-8), - @types.Scalar::Int(8), - @types.Scalar::Int(0), - @types.Scalar::Int(-9223372036854775807L), - ]) -} - -///| -test "modulo at Int64::MIN % -1 is zero on every backend" { - // `%` is evaluated as part of the same division `//` is, so it faults on the - // same pair — but only on some backends: wasm's `i64.rem_s` defines - // `Int64::MIN % -1` as `0`, while a native build lowers `%` to a hardware - // divide that raises the divide-by-zero fault. `0` is the answer either way - // (a remainder by ±1 is always 0), so `rem_i64` returns it without dividing. - assert_eq(extremes_result((a, b) => a.modulo(b)), [ - @types.Scalar::Int(0), - @types.Scalar::Int(0), - @types.Scalar::Int(0), - @types.Scalar::Int(0), - ]) -} - -///| -test "modulo by -1 is zero for every dividend, like modulo by 1" { - // The special case must agree with the general path wherever that path is - // defined: `a % -1` and `a % 1` are both `0` for every `a`. - let df = DataFrame::DataFrame([ - Series::from_ints("a", [7L, -7L, 0L, 9223372036854775807L, i64_min]), - ]) - let out = df.select([ - @expr.col("a").modulo(@expr.lit_int(-1)).with_alias("m"), - @expr.col("a").modulo(@expr.lit_int(1)).with_alias("p"), - ]) - let zeros = Array::make(5, @types.Scalar::Int(0)) - assert_eq(out.get_column("m").to_scalars(), zeros) - assert_eq(out.get_column("p").to_scalars(), zeros) -} - -///| -test "neg / abs / mul wrap at Int64::MIN" { - let wrapped = [ - @types.Scalar::Int(i64_min), - @types.Scalar::Int(i64_min), - @types.Scalar::Int(1), - @types.Scalar::Int(0), - ] - // -a and a * b (b = -1 except row 1) are the same wrap. - assert_eq(extremes_result((a, _) => -a), wrapped) - assert_eq(extremes_result((a, b) => a * b), [ - @types.Scalar::Int(i64_min), - @types.Scalar::Int(i64_min), - @types.Scalar::Int(1), - @types.Scalar::Int(0), - ]) - // abs leaves MIN alone (no positive counterpart) and is identity on the rest. - assert_eq(extremes_result((a, _) => a.abs()), [ - @types.Scalar::Int(i64_min), - @types.Scalar::Int(i64_min), - @types.Scalar::Int(1), - @types.Scalar::Int(0), - ]) -} - -///| -test "true division of Int64::MIN is Float, so it never overflows" { - // `/` is always Float in MoonFrame, so the pair that overflows in Int64 - // arithmetic is an ordinary IEEE quotient here. - assert_eq(extremes_result((a, b) => a / b), [ - @types.Scalar::Float(9223372036854775808.0), - @types.Scalar::Float(-9223372036854775808.0), - @types.Scalar::Float(1.0), - @types.Scalar::Float(-0.0), - ]) -} diff --git a/frame/is_between_test.mbt b/frame/is_between_test.mbt deleted file mode 100644 index 7f33d76e..00000000 --- a/frame/is_between_test.mbt +++ /dev/null @@ -1,218 +0,0 @@ -// Blackbox tests for `Expr::is_between` — the range test `lo <= x <= hi`, whose -// `closed~` argument decides which endpoints are included (`Both` by default). - -///| -test "is_between: both endpoints are inclusive" { - let df = DataFrame::DataFrame([Series::from_ints("a", [0L, 1L, 5L, 10L, 11L])]) - let out = df.select([ - @expr.col("a") - .is_between(@expr.lit_int(1), @expr.lit_int(10)) - .with_alias("r"), - ]) - // 1 and 10 (the endpoints) are in range; 0 and 11 are out. - assert_eq(out.get_column("r").to_scalars(), [ - @types.Scalar::Bool(false), - @types.Scalar::Bool(true), - @types.Scalar::Bool(true), - @types.Scalar::Bool(true), - @types.Scalar::Bool(false), - ]) -} - -///| -test "is_between: Float, String, and Bool ordering" { - let df = DataFrame::DataFrame([ - Series::from_floats("f", [0.5, 2.0, 3.5]), - Series::from_strings("s", ["apple", "cherry", "zebra"]), - Series::from_bools("b", [true, false, true]), - ]) - let out = df.select([ - @expr.col("f") - .is_between(@expr.lit_float(1.0), @expr.lit_float(3.0)) - .with_alias("rf"), - @expr.col("s") - .is_between(@expr.lit_str("b"), @expr.lit_str("m")) - .with_alias("rs"), - @expr.col("b") - .is_between(@expr.lit_bool(false), @expr.lit_bool(true)) - .with_alias("rb"), - ]) - assert_eq(out.get_column("rf").to_scalars(), [ - @types.Scalar::Bool(false), // 0.5 < 1.0 - @types.Scalar::Bool(true), // 1.0 <= 2.0 <= 3.0 - @types.Scalar::Bool(false), // 3.5 > 3.0 - ]) - assert_eq(out.get_column("rs").to_scalars(), [ - @types.Scalar::Bool(false), // "apple" < "b" - @types.Scalar::Bool(true), // "b" <= "cherry" <= "m" - @types.Scalar::Bool(false), // "zebra" > "m" - ]) - // Every Bool is within [false, true]. - assert_eq(out.get_column("rb").to_scalars(), [ - @types.Scalar::Bool(true), - @types.Scalar::Bool(true), - @types.Scalar::Bool(true), - ]) -} - -///| -test "is_between: a null cell yields null" { - let df = DataFrame::DataFrame([ - Series::from_int_options("a", [Some(5L), None]), - ]) - let out = df.select([ - @expr.col("a") - .is_between(@expr.lit_int(0), @expr.lit_int(10)) - .with_alias("r"), - ]) - assert_eq(out.get_column("r").to_scalars(), [ - @types.Scalar::Bool(true), - @types.Scalar::Null, - ]) -} - -///| -test "is_between: column bounds propagate nulls under Kleene AND" { - let df = DataFrame::DataFrame([ - Series::from_ints("a", [5L, 20L]), - Series::from_int_options("lo", [None, None]), - Series::from_ints("hi", [10L, 10L]), - ]) - let out = df.select([ - @expr.col("a").is_between(@expr.col("lo"), @expr.col("hi")).with_alias("r"), - ]) - // row0: (5 >= null) & (5 <= 10) = null & true = null. - // row1: (20 >= null) & (20 <= 10) = null & false = false (a known false wins). - assert_eq(out.get_column("r").to_scalars(), [ - @types.Scalar::Null, - @types.Scalar::Bool(false), - ]) -} - -///| -test "is_between: equals ge().land(le()), operand shared" { - // A *computed* operand is evaluated once by the dedicated node; the result - // must still match the explicit `ge & le` lowering cell for cell. - let df = DataFrame::DataFrame([ - Series::from_int_options("x", [Some(2L), Some(7L), None]), - ]) - let operand = @expr.col("x") + @expr.lit_int(1) - let via_between = df - .select([ - operand.is_between(@expr.lit_int(3), @expr.lit_int(8)).with_alias("r"), - ]) - .get_column("r") - .to_scalars() - let via_manual = df - .select([ - operand - .ge(@expr.lit_int(3)) - .land(operand.le(@expr.lit_int(8))) - .with_alias("r"), - ]) - .get_column("r") - .to_scalars() - assert_eq(via_between, via_manual) -} - -///| -test "is_between: a non-orderable operand raises structured TypeMismatch" { - let df = DataFrame::DataFrame([Series::from_strings("s", ["x"])]) - assert_eq( - Ok( - df.select([@expr.col("s").is_between(@expr.lit_int(0), @expr.lit_int(10))]), - ) catch { - e => Err(e) - }, - Err( - @types.DataError::TypeMismatch( - @types.TypeMismatchDetail::Operation( - "compare", - @types.DataType::String, - @types.DataType::Int, - ), - ), - ), - ) -} - -///| -test "is_between: reduces per group over an aggregation" { - let df = DataFrame::DataFrame([ - Series::from_strings("k", ["x", "x", "y"]), - Series::from_ints("v", [10L, 20L, 5L]), - ]) - let out = df - .group_by([@expr.col("k")]) - .agg([ - @expr.col("v") - .sum() - .is_between(@expr.lit_int(0), @expr.lit_int(25)) - .with_alias("in_range"), - ]) - assert_eq(out.get_column("in_range").to_scalars(), [ - @types.Scalar::Bool(false), // x: 30 > 25 - @types.Scalar::Bool(true), // y: 5 - ]) -} - -///| -test "is_between honours each closed setting" { - let df = DataFrame::DataFrame([Series::from_ints("n", [1, 2, 3, 4])]) - let bounds = (@expr.lit_int(2), @expr.lit_int(3)) - let (lo, hi) = bounds - let mask = fn( - closed : @types.ClosedInterval, - ) -> Array[@types.Scalar] raise @types.DataError { - df - .select([@expr.col("n").is_between(lo, hi, closed~).with_alias("in")]) - .get_column("in") - .to_scalars() - } - // Both endpoints (the default) keeps 2 and 3. - assert_eq(mask(@types.ClosedInterval::Both), [ - @types.Scalar::Bool(false), - @types.Scalar::Bool(true), - @types.Scalar::Bool(true), - @types.Scalar::Bool(false), - ]) - // Left keeps the lower bound only, Right the upper, None neither. - assert_eq(mask(@types.ClosedInterval::Left), [ - @types.Scalar::Bool(false), - @types.Scalar::Bool(true), - @types.Scalar::Bool(false), - @types.Scalar::Bool(false), - ]) - assert_eq(mask(@types.ClosedInterval::Right), [ - @types.Scalar::Bool(false), - @types.Scalar::Bool(false), - @types.Scalar::Bool(true), - @types.Scalar::Bool(false), - ]) - assert_eq(mask(@types.ClosedInterval::None), [ - @types.Scalar::Bool(false), - @types.Scalar::Bool(false), - @types.Scalar::Bool(false), - @types.Scalar::Bool(false), - ]) - // The endpoint rule is part of the expression's identity, and renders as the - // argument that builds it. - assert_diff_expr( - @expr.col("n").is_between(lo, hi), - @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::Left), - ) - inspect( - @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::None), - content="col(n).is_between(2, 3, closed=None)", - ) - inspect( - @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::Left), - content="col(n).is_between(2, 3, closed=Left)", - ) - inspect( - @expr.col("n").is_between(lo, hi, closed=@types.ClosedInterval::Right), - content="col(n).is_between(2, 3, closed=Right)", - ) - // The default renders bare, so existing plan snapshots are unchanged. - inspect(@expr.col("n").is_between(lo, hi), content="col(n).is_between(2, 3)") -} diff --git a/frame/lit_series_test.mbt b/frame/lit_series_test.mbt deleted file mode 100644 index 4d455550..00000000 --- a/frame/lit_series_test.mbt +++ /dev/null @@ -1,200 +0,0 @@ -///| -/// Blackbox tests for the `LitSeries` literal-series node — `lit_series(s)` — -/// driven through the expression verbs. Covers the frame-tall append, the -/// in-place replace, the preserved nulls on the embedded series, the length-1 -/// broadcast and the length-mismatch error, use inside a compound expression -/// and as a precomputed filter mask, standalone projection with naming, the -/// preserved backend, an aggregation over the literal, the empty-frame edge, -/// and the not-reduction-shaped rejection under `agg`. - -///| -/// Three-row base frame for the literal-series tests. -fn lit_series_df() -> DataFrame raise @types.DataError { - DataFrame::DataFrame([ - Series::from_ints("a", [1, 2, 3]), - Series::from_strings("b", ["x", "y", "z"]), - ]) -} - -///| -test "lit_series: a frame-tall series appends as a new named column" { - // A ready-made column joins the frame, named after the series, at the - // rightmost position. - let extra = Series::from_floats("c", [ - (1.5 : Double), - (2.5 : Double), - (3.5 : Double), - ]) - let out = lit_series_df().with_columns([@expr.lit_series(extra)]) - assert_eq(out.columns(), ["a", "b", "c"]) - assert_eq(out.get_column("c").dtype(), @types.DataType::Float) - assert_eq(out.get_column("c").to_scalars(), [ - @types.Scalar::Float(1.5), - @types.Scalar::Float(2.5), - @types.Scalar::Float(3.5), - ]) -} - -///| -test "lit_series: the embedded series keeps its nulls" { - // The series is handed through verbatim, so a nullable column arrives with - // its validity bitmap intact. - let extra = Series::from_int_options("d", [Some(7), None, Some(9)]) - let out = lit_series_df().with_columns([@expr.lit_series(extra)]) - assert_eq(out.get_column("d").null_count(), 1) - assert_eq(out.get_column("d").to_scalars(), [ - @types.Scalar::Int(7), - @types.Scalar::Null, - @types.Scalar::Int(9), - ]) -} - -///| -test "lit_series: a frame-tall series replaces an existing column in place" { - // A same-named series replaces the existing column in place through - // `with_columns`, keeping its original position (a fresh name appends). - let replacement = Series::from_ints("a", [10, 20, 30]) - let out = lit_series_df().with_columns([@expr.lit_series(replacement)]) - assert_eq(out.columns(), ["a", "b"]) - assert_eq(out.get_column("a").to_scalars(), [ - @types.Scalar::Int(10), - @types.Scalar::Int(20), - @types.Scalar::Int(30), - ]) -} - -///| -test "lit_series: a renamed series swaps a column in place, dtype and all" { - // The retired `replace_column(name, s)` is `with_columns([lit_series(s.rename(name))])`: - // renaming a differently-named series onto an existing column swaps it in - // place — position and name preserved — with the dtype following the new - // data (a cross-dtype swap, here Int → Float). - let fresh = Series::from_floats("ignored", [ - (0.5 : Double), - (1.5 : Double), - (2.5 : Double), - ]) - let out = lit_series_df().with_columns([@expr.lit_series(fresh.rename("a"))]) - assert_eq(out.columns(), ["a", "b"]) - assert_eq(out.get_column("a").dtype(), @types.DataType::Float) - assert_eq(out.get_column("a").to_scalars(), [ - @types.Scalar::Float(0.5), - @types.Scalar::Float(1.5), - @types.Scalar::Float(2.5), - ]) -} - -///| -test "lit_series: a length-1 series broadcasts over the frame" { - // Like a scalar literal, a one-row series fills every row. - let one = Series::from_ints("k", [7]) - let out = lit_series_df().with_columns([@expr.lit_series(one)]) - assert_eq(out.get_column("k").to_scalars(), [ - @types.Scalar::Int(7), - @types.Scalar::Int(7), - @types.Scalar::Int(7), - ]) -} - -///| -test "lit_series: a length neither 1 nor the frame height raises LengthMismatch" { - let two = Series::from_ints("k", [1, 2]) - assert_eq( - Ok(lit_series_df().with_columns([@expr.lit_series(two)])) catch { - e => Err(e) - }, - Err(@types.DataError::LengthMismatch), - ) -} - -///| -// `select` agrees with `with_columns`: an off-length literal series is a -// LengthMismatch, not a silent change of the output row count. The frame is -// 3 rows; a 2-row literal series is neither frame-tall nor a length-1 -// broadcast, even when it is the only projected column. -test "lit_series: select rejects an off-length series instead of resizing the frame" { - let two = Series::from_ints("k", [1, 2]) - assert_eq( - Ok(lit_series_df().select([@expr.lit_series(two)])) catch { - e => Err(e) - }, - Err(@types.DataError::LengthMismatch), - ) -} - -///| -test "lit_series: combines positionally inside a compound expression" { - // A frame-tall literal column is row-aligned with the frame, so it adds to - // a column cell for cell. - let bonus = Series::from_ints("bonus", [100, 200, 300]) - let out = lit_series_df().with_columns([ - (@expr.col("a") + @expr.lit_series(bonus)).with_alias("total"), - ]) - assert_eq(out.get_column("total").to_scalars(), [ - @types.Scalar::Int(101), - @types.Scalar::Int(202), - @types.Scalar::Int(303), - ]) -} - -///| -test "lit_series: a Bool series is usable as a precomputed filter mask" { - let mask = Series::from_bools("keep", [true, false, true]) - let kept = lit_series_df().filter(@expr.lit_series(mask)) - assert_eq(kept.get_column("a").to_scalars(), [ - @types.Scalar::Int(1), - @types.Scalar::Int(3), - ]) -} - -///| -test "lit_series: projects standalone, named after the series unless aliased" { - let s = Series::from_ints("vals", [1, 2, 3]) - let out = lit_series_df().select([@expr.lit_series(s)]) - assert_eq(out.columns(), ["vals"]) - let aliased = lit_series_df().select([ - @expr.lit_series(s).with_alias("renamed"), - ]) - assert_eq(aliased.columns(), ["renamed"]) -} - -///| -test "lit_series: returned verbatim, so its backend is preserved" { - // A Numeric, all-valid series stays Numeric — `eval` hands it through, it - // is not re-derived through `try_column_to_numeric` like a computed column. - let num = Series::from_ints("n", [1, 2, 3]) - assert_eq(num.storage_kind(), @column.StorageKind::Numeric) - let out = lit_series_df().with_columns([@expr.lit_series(num)]) - assert_eq(out.get_column("n").storage_kind(), @column.StorageKind::Numeric) -} - -///| -test "lit_series: an aggregation over a literal series reduces it whole" { - // `lit_series(s).sum()` is reduction-shaped (the enclosing `Agg`): it sums - // the literal series itself — all four of its cells, independent of the - // base frame's three rows — to one cell. - let s = Series::from_ints("s", [10, 20, 30, 40]) - let out = lit_series_df().select([@expr.lit_series(s).sum().with_alias("t")]) - assert_eq(out.shape(), (1, 1)) - assert_eq(out.get_column("t").to_scalars(), [@types.Scalar::Int(100)]) -} - -///| -test "lit_series: a bare literal series is not reduction-shaped under agg" { - // Row-wise like a bare column, so `agg([lit_series(s)])` raises — the shape - // gate fires before any group work. - let s = Series::from_ints("s", [1, 2, 3]) - let grouped = lit_series_df().group_by([@expr.col("b")]) - assert_true( - (Ok(grouped.agg([@expr.lit_series(s)])) catch { e => Err(e) }) - is Err(@types.DataError::InvalidOperation(_)), - ) -} - -///| -test "lit_series on an empty frame accepts a 0-length series" { - let empty = DataFrame::DataFrame([Series::from_ints("i", [])]) - let out = empty.with_columns([@expr.lit_series(Series::from_ints("z", []))]) - assert_eq(out.shape(), (0, 2)) - assert_eq(out.columns(), ["i", "z"]) -} diff --git a/frame/mod_pow_test.mbt b/frame/mod_pow_test.mbt deleted file mode 100644 index 1900e2c4..00000000 --- a/frame/mod_pow_test.mbt +++ /dev/null @@ -1,224 +0,0 @@ -// Blackbox tests for `Expr::modulo` (`%`) and `Expr::pow` (`**`, always Float). - -///| -test "modulo: Int remainder carries the dividend's sign; every arm" { - let df = DataFrame::DataFrame([ - Series::from_ints("i", [7L, -7L, 6L]), - Series::from_ints("j", [2L, 2L, -4L]), - Series::from_floats("f", [7.5, -7.5, 8.0]), - Series::from_floats("g", [2.0, 2.0, 3.0]), - ]) - let out = df.select([ - @expr.col("i").modulo(@expr.col("j")).with_alias("ii"), - @expr.col("i").modulo(@expr.col("g")).with_alias("if"), - @expr.col("f").modulo(@expr.col("j")).with_alias("fi"), - @expr.col("f").modulo(@expr.col("g")).with_alias("ff"), - ]) - // ii: 7%2=1, -7%2=-1, 6%-4=2. - assert_eq(out.get_column("ii").to_scalars(), [ - @types.Scalar::Int(1), - @types.Scalar::Int(-1), - @types.Scalar::Int(2), - ]) - // if (Int%Float→Float): 7%2=1, -7%2=-1, 6%3=0. - assert_eq(out.get_column("if").to_scalars(), [ - @types.Scalar::Float(1.0), - @types.Scalar::Float(-1.0), - @types.Scalar::Float(0.0), - ]) - // fi (Float%Int): 7.5%2=1.5, -7.5%2=-1.5, 8%-4=0. - assert_eq(out.get_column("fi").to_scalars(), [ - @types.Scalar::Float(1.5), - @types.Scalar::Float(-1.5), - @types.Scalar::Float(0.0), - ]) - // ff: 7.5%2=1.5, -7.5%2=-1.5, 8%3=2. - assert_eq(out.get_column("ff").to_scalars(), [ - @types.Scalar::Float(1.5), - @types.Scalar::Float(-1.5), - @types.Scalar::Float(2.0), - ]) -} - -///| -test "modulo: Numeric Int-by-zero divisor yields null (fast arm)" { - let df = DataFrame::DataFrame([ - Series::from_ints("a", [10L, 5L]), - Series::from_ints("b", [0L, 2L]), - ]) - let out = df.select([@expr.col("a").modulo(@expr.col("b")).with_alias("m")]) - // 10%0=null, 5%2=1. - assert_eq(out.get_column("m").to_scalars(), [ - @types.Scalar::Null, - @types.Scalar::Int(1), - ]) -} - -///| -test "modulo: Int-by-zero null, Float-by-zero NaN, Builtin arms, non-numeric" { - let df = DataFrame::DataFrame([ - Series::from_int_options("i", [Some(10L), None, Some(5L)]), - Series::from_int_options("j", [Some(0L), Some(2L), Some(3L)]), - Series::from_float_options("f", [Some(1.0), Some(7.5), None]), - Series::from_float_options("g", [Some(0.0), Some(2.0), Some(2.0)]), - ]) - let out = df.select([ - @expr.col("i").modulo(@expr.col("j")).with_alias("ii"), - @expr.col("i").modulo(@expr.col("g")).with_alias("ig"), - @expr.col("f").modulo(@expr.col("j")).with_alias("fj"), - @expr.col("f").modulo(@expr.col("g")).with_alias("fg"), - ]) - // ii (Builtin Int/Int): 10%0=null, null, 5%3=2. - assert_eq(out.get_column("ii").to_scalars(), [ - @types.Scalar::Null, - @types.Scalar::Null, - @types.Scalar::Int(2), - ]) - // ig (Builtin Int/Float): 10%0.0=nan, null, 5%2.0=1.0. - let ig = out.get_column("ig").to_scalars() - assert_true( - match ig[0] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - assert_eq(ig[1], @types.Scalar::Null) - assert_eq(ig[2], @types.Scalar::Float(1.0)) - // fj (Builtin Float/Int): 1%0=nan, 7.5%2=1.5, null. - let fj = out.get_column("fj").to_scalars() - assert_true( - match fj[0] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - assert_eq(fj[1], @types.Scalar::Float(1.5)) - assert_eq(fj[2], @types.Scalar::Null) - // fg (Builtin Float/Float): 1%0.0=nan, 7.5%2.0=1.5, null. - let fg = out.get_column("fg").to_scalars() - assert_true( - match fg[0] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - assert_eq(fg[1], @types.Scalar::Float(1.5)) - assert_eq(fg[2], @types.Scalar::Null) - // Non-numeric raises. - let strs = DataFrame::DataFrame([ - Series::from_strings("s", ["x"]), - Series::from_ints("n", [1L]), - ]) - assert_eq( - Ok(strs.select([@expr.col("s").modulo(@expr.col("n"))])) catch { - e => Err(e) - }, - Err( - @types.DataError::TypeMismatch( - @types.TypeMismatchDetail::Operation( - "take the remainder of", - @types.DataType::String, - @types.DataType::Int, - ), - ), - ), - ) -} - -///| -test "pow: always Float; every numeric arm" { - let df = DataFrame::DataFrame([ - Series::from_ints("i", [2L, 4L]), - Series::from_ints("j", [3L, 2L]), - Series::from_floats("f", [2.0, 9.0]), - Series::from_floats("g", [10.0, 0.5]), - ]) - let out = df.select([ - @expr.col("i").pow(@expr.col("j")).with_alias("ii"), - @expr.col("i").pow(@expr.col("g")).with_alias("ig"), - @expr.col("f").pow(@expr.col("j")).with_alias("fj"), - @expr.col("f").pow(@expr.col("g")).with_alias("fg"), - ]) - // ii: 2^3=8, 4^2=16 (Float). - assert_eq(out.get_column("ii").to_scalars(), [ - @types.Scalar::Float(8.0), - @types.Scalar::Float(16.0), - ]) - // ig (Int^Float): 2^10=1024, 4^0.5=2. - assert_eq(out.get_column("ig").to_scalars(), [ - @types.Scalar::Float(1024.0), - @types.Scalar::Float(2.0), - ]) - // fj (Float^Int): 2^3=8, 9^2=81. - assert_eq(out.get_column("fj").to_scalars(), [ - @types.Scalar::Float(8.0), - @types.Scalar::Float(81.0), - ]) - // fg (Float^Float): 2^10=1024, 9^0.5=3. - assert_eq(out.get_column("fg").to_scalars(), [ - @types.Scalar::Float(1024.0), - @types.Scalar::Float(3.0), - ]) -} - -///| -test "pow: negative / zero exponents, Builtin arms, null, non-numeric" { - let df = DataFrame::DataFrame([ - Series::from_float_options("b", [Some(2.0), Some(0.0), None]), - Series::from_float_options("e", [Some(-1.0), Some(0.0), Some(2.0)]), - Series::from_int_options("ib", [Some(2L), Some(3L), None]), - Series::from_int_options("ie", [Some(-2L), Some(0L), Some(2L)]), - ]) - let out = df.select([ - @expr.col("b").pow(@expr.col("e")).with_alias("ff"), - @expr.col("ib").pow(@expr.col("ie")).with_alias("ii"), - @expr.col("ib").pow(@expr.col("e")).with_alias("if"), - @expr.col("b").pow(@expr.col("ie")).with_alias("fi"), - ]) - // ff: 2^-1=0.5, 0^0=1, null. - assert_eq(out.get_column("ff").to_scalars(), [ - @types.Scalar::Float(0.5), - @types.Scalar::Float(1.0), - @types.Scalar::Null, - ]) - // ii (Builtin Int^Int, still Float): 2^-2=0.25, 3^0=1, null. - assert_eq(out.get_column("ii").to_scalars(), [ - @types.Scalar::Float(0.25), - @types.Scalar::Float(1.0), - @types.Scalar::Null, - ]) - // if / fi just exercise the remaining Builtin mixed arms. - assert_eq(out.get_column("if").to_scalars()[1], @types.Scalar::Float(1.0)) - assert_eq(out.get_column("fi").to_scalars()[1], @types.Scalar::Float(1.0)) - // Non-numeric raises. - let strs = DataFrame::DataFrame([ - Series::from_strings("s", ["x"]), - Series::from_ints("n", [2L]), - ]) - assert_eq( - Ok(strs.select([@expr.col("s").pow(@expr.col("n"))])) catch { - e => Err(e) - }, - Err( - @types.DataError::TypeMismatch( - @types.TypeMismatchDetail::Operation( - "exponentiate", - @types.DataType::String, - @types.DataType::Int, - ), - ), - ), - ) -} - -///| -test "mod / pow: Expr::to_string renders % and **" { - inspect( - @expr.col("a").modulo(@expr.col("b")).to_string(), - content="(col(a) % col(b))", - ) - inspect( - @expr.col("a").pow(@expr.col("b")).to_string(), - content="(col(a) ** col(b))", - ) -} diff --git a/frame/unary_numeric_test.mbt b/frame/unary_numeric_test.mbt deleted file mode 100644 index 5ecb47cc..00000000 --- a/frame/unary_numeric_test.mbt +++ /dev/null @@ -1,285 +0,0 @@ -// Blackbox tests for the unary numeric ops `Expr::abs` / `floor` / `ceil` / -// `sign` / `round`: per-dtype behavior on both backends, NaN pass-through, null -// propagation, and the non-numeric type error — plus, for `round`, the -// half-to-even rule, the `decimals~` places, and the non-finite edges. - -///| -test "abs / floor / ceil / sign on the Numeric fast arm" { - let df = DataFrame::DataFrame([ - Series::from_ints("i", [-3L, 3L, 0L]), - Series::from_floats("f", [-3.7, 3.2, 0.0]), - ]) - let out = df.select([ - @expr.col("i").abs().with_alias("i_abs"), - @expr.col("i").floor().with_alias("i_floor"), - @expr.col("i").ceil().with_alias("i_ceil"), - @expr.col("i").sign().with_alias("i_sign"), - @expr.col("f").abs().with_alias("f_abs"), - @expr.col("f").floor().with_alias("f_floor"), - @expr.col("f").ceil().with_alias("f_ceil"), - @expr.col("f").sign().with_alias("f_sign"), - ]) - // `Int`: floor/ceil are identity, abs and sign act. - assert_eq(out.get_column("i_abs").to_scalars(), [ - @types.Scalar::Int(3), - @types.Scalar::Int(3), - @types.Scalar::Int(0), - ]) - assert_eq(out.get_column("i_floor").to_scalars(), [ - @types.Scalar::Int(-3), - @types.Scalar::Int(3), - @types.Scalar::Int(0), - ]) - assert_eq(out.get_column("i_ceil").to_scalars(), [ - @types.Scalar::Int(-3), - @types.Scalar::Int(3), - @types.Scalar::Int(0), - ]) - assert_eq(out.get_column("i_sign").to_scalars(), [ - @types.Scalar::Int(-1), - @types.Scalar::Int(1), - @types.Scalar::Int(0), - ]) - // `Float`: floor toward −∞, ceil toward +∞. - assert_eq(out.get_column("f_abs").to_scalars(), [ - @types.Scalar::Float(3.7), - @types.Scalar::Float(3.2), - @types.Scalar::Float(0.0), - ]) - assert_eq(out.get_column("f_floor").to_scalars(), [ - @types.Scalar::Float(-4.0), - @types.Scalar::Float(3.0), - @types.Scalar::Float(0.0), - ]) - assert_eq(out.get_column("f_ceil").to_scalars(), [ - @types.Scalar::Float(-3.0), - @types.Scalar::Float(4.0), - @types.Scalar::Float(0.0), - ]) - assert_eq(out.get_column("f_sign").to_scalars(), [ - @types.Scalar::Float(-1.0), - @types.Scalar::Float(1.0), - @types.Scalar::Float(0.0), - ]) -} - -///| -test "unary numeric: Builtin arm, null propagation, NaN, non-numeric" { - let df = DataFrame::DataFrame([ - Series::from_int_options("i", [Some(-3L), None]), - Series::from_float_options("f", [Some(-3.7), None]), - ]) - let out = df.select([ - @expr.col("i").abs().with_alias("i_abs"), - @expr.col("f").floor().with_alias("f_floor"), - ]) - assert_eq(out.get_column("i_abs").to_scalars(), [ - @types.Scalar::Int(3), - @types.Scalar::Null, - ]) - assert_eq(out.get_column("f_floor").to_scalars(), [ - @types.Scalar::Float(-4.0), - @types.Scalar::Null, - ]) - // `NaN` passes through `abs` / `sign`. - let nans = DataFrame::DataFrame([ - Series::from_floats("f", [@double.not_a_number]), - ]) - let nout = nans.select([ - @expr.col("f").abs().with_alias("a"), - @expr.col("f").sign().with_alias("s"), - ]) - assert_true( - match nout.get_column("a").to_scalars()[0] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - assert_true( - match nout.get_column("s").to_scalars()[0] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - // A non-numeric operand raises. - let strs = DataFrame::DataFrame([Series::from_strings("s", ["x"])]) - assert_true( - (Ok(strs.select([@expr.col("s").abs()])) catch { e => Err(e) }) - is Err( - @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) - ), - ) -} - -///| -test "round: ties go to even (banker's rounding)" { - let df = DataFrame::DataFrame([ - Series::from_floats("x", [0.5, 1.5, 2.5, 3.5, -0.5, -1.5, -2.5, -3.5]), - ]) - let out = df.select([@expr.col("x").round().with_alias("r")]) - // Each exact half rounds to its even neighbour: 0.5→0, 1.5→2, 2.5→2, 3.5→4, - // and the negatives symmetrically. - assert_eq(out.get_column("r").to_scalars(), [ - @types.Scalar::Float(0.0), - @types.Scalar::Float(2.0), - @types.Scalar::Float(2.0), - @types.Scalar::Float(4.0), - @types.Scalar::Float(0.0), - @types.Scalar::Float(-2.0), - @types.Scalar::Float(-2.0), - @types.Scalar::Float(-4.0), - ]) - // `-0.5`'s even neighbour is `-0.0`: IEEE equality calls both zeros - // equal above, so the sign is asserted where IEEE makes it observable — - // under division, where +0.0 and -0.0 go to opposite infinities. - let cells = out.get_column("r").to_scalars() - assert_true(cells[0] is @types.Scalar::Float(v) && v == 0.0 && 1.0 / v > 0.0) - assert_true(cells[4] is @types.Scalar::Float(v) && v == 0.0 && 1.0 / v < 0.0) -} - -///| -test "round: non-ties go to the nearest integer; Int is identity" { - let df = DataFrame::DataFrame([ - Series::from_ints("i", [-3L, 4L, 0L]), - Series::from_floats("f", [3.2, 3.7, -3.7]), - ]) - let out = df.select([ - @expr.col("i").round().with_alias("ri"), - @expr.col("f").round().with_alias("rf"), - ]) - assert_eq(out.get_column("ri").to_scalars(), [ - @types.Scalar::Int(-3), - @types.Scalar::Int(4), - @types.Scalar::Int(0), - ]) - assert_eq(out.get_column("rf").to_scalars(), [ - @types.Scalar::Float(3.0), // 3.2 → 3 - @types.Scalar::Float(4.0), // 3.7 → 4 - @types.Scalar::Float(-4.0), // -3.7 → -4 - ]) -} - -///| -test "round: Builtin null, NaN / ±inf pass through, non-numeric raises" { - let df = DataFrame::DataFrame([ - Series::from_float_options("f", [Some(2.5), None]), - ]) - let out = df.select([@expr.col("f").round().with_alias("r")]) - // 2.5 rounds to the even 2; a null stays null. - assert_eq(out.get_column("r").to_scalars(), [ - @types.Scalar::Float(2.0), - @types.Scalar::Null, - ]) - // NaN and ±inf pass through unchanged. - let inf = 1.0 / 0.0 - let special = DataFrame::DataFrame([ - Series::from_floats("f", [@double.not_a_number, inf, -inf]), - ]) - let s = special - .select([@expr.col("f").round().with_alias("r")]) - .get_column("r") - .to_scalars() - assert_true( - match s[0] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - assert_eq(s[1], @types.Scalar::Float(inf)) - assert_eq(s[2], @types.Scalar::Float(-inf)) - // A non-numeric operand raises. - let strs = DataFrame::DataFrame([Series::from_strings("s", ["x"])]) - assert_true( - (Ok(strs.select([@expr.col("s").round()])) catch { e => Err(e) }) - is Err( - @types.DataError::TypeMismatch(@types.TypeMismatchDetail::Message(_)) - ), - ) -} - -///| -test "unary numeric: Expr::to_string renders the method calls" { - inspect(@expr.col("x").abs().to_string(), content="col(x).abs()") - inspect(@expr.col("x").floor().to_string(), content="col(x).floor()") - inspect(@expr.col("x").ceil().to_string(), content="col(x).ceil()") - inspect(@expr.col("x").sign().to_string(), content="col(x).sign()") - inspect(@expr.col("x").round().to_string(), content="col(x).round()") -} - -///| -test "round takes a decimal-place count" { - let df = DataFrame::DataFrame([ - Series::from_floats("f", [1.2345, 2.5, -1.2345, 0.125]), - Series::from_ints("i", [7, 8, 9, 10]), - ]) - let rounded = df.select([ - @expr.col("f").round(decimals=2).with_alias("two"), - @expr.col("f").round().with_alias("zero"), - @expr.col("i").round(decimals=3).with_alias("int"), - ]) - assert_eq(rounded.item(0, "two"), @types.Scalar::Float(1.23)) - assert_eq(rounded.item(2, "two"), @types.Scalar::Float(-1.23)) - // Ties still go to even, at the requested place: 0.125 → 0.12. - assert_eq(rounded.item(3, "two"), @types.Scalar::Float(0.12)) - // The default is unchanged: whole numbers, ties to even. - assert_eq(rounded.item(1, "zero"), @types.Scalar::Float(2.0)) - // An Int column is the identity at any setting. - assert_eq(rounded.item(0, "int"), @types.Scalar::Int(7)) - // A negative count clamps to 0 rather than raising. - assert_same_expr(@expr.col("f").round(decimals=-3), @expr.col("f").round()) - // A non-default count renders as the argument that builds it. - inspect(@expr.col("f").round(decimals=2), content="col(f).round(decimals=2)") -} - -///| -test "round(decimals) stays total at the numeric edges" { - let nan = 0.0 / 0.0 - let inf = 1.0 / 0.0 - let df = DataFrame::DataFrame([ - Series::from_floats("f", [nan, inf, -inf, 1.7976931348623157e308, 1.5]), - ]) - // Non-finite cells pass through untouched, as with the whole-number form. - let out = df.select([@expr.col("f").round(decimals=2).with_alias("r")]) - let cells = out.get_column("r").to_scalars() - assert_true( - match cells[0] { - @types.Scalar::Float(v) => v.is_nan() - _ => false - }, - ) - assert_eq(cells[1], @types.Scalar::Float(inf)) - assert_eq(cells[2], @types.Scalar::Float(-inf)) - // Scaling the largest finite Double by 100 overflows, so the value is left - // as it is rather than round-tripping through infinity. - assert_eq(cells[3], @types.Scalar::Float(1.7976931348623157e308)) - // A decimal count so large that 10^decimals is infinite is likewise a no-op. - let huge = df.select([@expr.col("f").round(decimals=400).with_alias("r")]) - assert_eq(huge.get_column("r").to_scalars()[4], @types.Scalar::Float(1.5)) -} - -///| -test "round(decimals) is the identity past the value's own resolution" { - // Asking for more places than a `Double` can resolve must return the value - // unchanged. Scaling by `10^decimals` and dividing back does not: above - // `10^22` the scale is itself inexact, so the round trip moved the value by - // an ulp even though rounding could not have. - let df = DataFrame::DataFrame([ - Series::from_floats("f", [1.5, -1.5, 123456.789, 0.1, 1.0e15]), - ]) - for places in [17, 20, 30, 99, 300, 308, 309, 400] { - let out = df.select([@expr.col("f").round(decimals=places).with_alias("r")]) - assert_eq(out.get_column("r").to_scalars(), df.get_column("f").to_scalars()) - } - // A value that is already integral is untouched at any place, however few. - let integral = df.select([@expr.col("f").round(decimals=2).with_alias("r")]) - assert_eq(integral.item(4, "r"), @types.Scalar::Float(1.0e15)) - // The guard must not swallow the rounding that a fine place genuinely does: - // a magnitude small enough to still have a fraction at that place rounds. - let small = DataFrame::DataFrame([ - Series::from_floats("f", [4.0e-18, 6.0e-18, 1.0e-300]), - ]) - let rounded = small.select([@expr.col("f").round(decimals=17).with_alias("r")]) - assert_eq(rounded.item(0, "r"), @types.Scalar::Float(0.0)) - assert_eq(rounded.item(1, "r"), @types.Scalar::Float(1.0e-17)) - assert_eq(rounded.item(2, "r"), @types.Scalar::Float(0.0)) -} diff --git a/lazy/date_test.mbt b/lazy/date_test.mbt deleted file mode 100644 index 1d4354ad..00000000 --- a/lazy/date_test.mbt +++ /dev/null @@ -1,51 +0,0 @@ -// The `Date` dtype through the lazy engine: a date literal renders its ISO -// spelling in the plan (the shared literal formatter), and a predicate -// carrying one crosses an aggregate under the same group-cell-stability rule -// as any other literal — the Date arm of that rule is what the registry's -// Date row exercises here. - -///| -fn date_df() -> LazyFrame raise @types.DataError { - let df = @frame.DataFrame::from_rows( - @types.Schema::Schema([ - @types.Field::Field("d", @types.DataType::Date), - @types.Field::Field("v", @types.DataType::Int), - ]), - [ - [@types.Scalar::Date(19723L), @types.Scalar::Int(1)], - [@types.Scalar::Date(19753L), @types.Scalar::Int(2)], - ], - ) - LazyFrame::LazyFrame(df) -} - -///| -test "a Date literal renders ISO in the plan" { - let lf = date_df().filter( - @expr.col("d").gt(@expr.lit(@types.Scalar::Date(19723L))), - ) - let rendered = lf.explain() - assert_true(rendered.contains("1970-01-01") == false) - assert_true(rendered.contains("2024-01-01")) -} - -///| -test "a Date literal crosses an aggregate like any other literal" { - // `lit(Date) < col(d)` reads no column on its literal side and compares - // date-wise: row-stable and group-cell-stable, so the filter sinks into - // the aggregate and the collected rows agree with the eager equivalent. - let plan = date_df() - .group_by([@expr.col("v")]) - .agg([@expr.col("v").sum().with_alias("total")]) - .filter( - @expr.lit(@types.Scalar::Date(0L)).lt(@expr.lit(@types.Scalar::Date(9L))), - ) - let eager = date_df() - .collect() - .group_by([@expr.col("v")]) - .agg([@expr.col("v").sum().with_alias("total")]) - .filter( - @expr.lit(@types.Scalar::Date(0L)).lt(@expr.lit(@types.Scalar::Date(9L))), - ) - assert_eq(plan.collect().nrows(), eager.nrows()) -} diff --git a/lazy/optimize_predicates_test.mbt b/lazy/optimize_predicates_test.mbt index 3ed8d95d..e24218fa 100644 --- a/lazy/optimize_predicates_test.mbt +++ b/lazy/optimize_predicates_test.mbt @@ -1089,3 +1089,54 @@ test "scan_csv: a column-less predicate is absorbed like any other" { "SCAN_NDJSON \"events.ndjson\" WHERE false", ) } +// The `Date` dtype through the lazy engine: a date literal renders its ISO +// spelling in the plan (the shared literal formatter), and a predicate +// carrying one crosses an aggregate under the same group-cell-stability rule +// as any other literal — the Date arm of that rule is what the registry's +// Date row exercises here. + +///| +fn date_df() -> LazyFrame raise @types.DataError { + let df = @frame.DataFrame::from_rows( + @types.Schema::Schema([ + @types.Field::Field("d", @types.DataType::Date), + @types.Field::Field("v", @types.DataType::Int), + ]), + [ + [@types.Scalar::Date(19723L), @types.Scalar::Int(1)], + [@types.Scalar::Date(19753L), @types.Scalar::Int(2)], + ], + ) + LazyFrame::LazyFrame(df) +} + +///| +test "a Date literal renders ISO in the plan" { + let lf = date_df().filter( + @expr.col("d").gt(@expr.lit(@types.Scalar::Date(19723L))), + ) + let rendered = lf.explain() + assert_true(rendered.contains("1970-01-01") == false) + assert_true(rendered.contains("2024-01-01")) +} + +///| +test "a Date literal crosses an aggregate like any other literal" { + // `lit(Date) < col(d)` reads no column on its literal side and compares + // date-wise: row-stable and group-cell-stable, so the filter sinks into + // the aggregate and the collected rows agree with the eager equivalent. + let plan = date_df() + .group_by([@expr.col("v")]) + .agg([@expr.col("v").sum().with_alias("total")]) + .filter( + @expr.lit(@types.Scalar::Date(0L)).lt(@expr.lit(@types.Scalar::Date(9L))), + ) + let eager = date_df() + .collect() + .group_by([@expr.col("v")]) + .agg([@expr.col("v").sum().with_alias("total")]) + .filter( + @expr.lit(@types.Scalar::Date(0L)).lt(@expr.lit(@types.Scalar::Date(9L))), + ) + assert_eq(plan.collect().nrows(), eager.nrows()) +} diff --git a/lazy/optimize_str_test.mbt b/lazy/optimize_str_test.mbt deleted file mode 100644 index bd1f3448..00000000 --- a/lazy/optimize_str_test.mbt +++ /dev/null @@ -1,79 +0,0 @@ -///| -/// Optimizer interaction with the string namespace. A `Str` node raises only on -/// scope-independent facts — the dtype-level `TypeMismatch` of a non-String -/// operand, and for the regex ops the `InvalidOperation` of a pattern that does -/// not compile, hoisted out of the per-cell map — never on a value, so it is -/// `row_stable` — a filter sinks past a `with_columns` -/// that derives a string column, exactly as it does past arithmetic. And a -/// `Str` wrapping an aggregation inherits that aggregation's -/// group-cell-stability, so an `str_*` over `first` keeps a group filter -/// above the aggregate. Both pin the `Str` arms added to `row_stable` / -/// `group_cell_stable`; the shared fixtures are the package's own. - -///| -test "a trailing filter sinks below a string-namespace with_columns stage" { - let df = sales_df() - // `upper` is derived from `region` by a row-stable `str_*` op, but the - // predicate reads `qty`, so the filter crosses the stage with no - // string-specific rule — the same swap arithmetic earns. - let derived = [@expr.col("region").str_to_uppercase().with_alias("upper")] - let predicate = @expr.col("qty").gt(@expr.lit_int(50)) - let plan = LazyFrame::LazyFrame(df).with_columns(derived).filter(predicate) - assert_eq( - plan.explain(optimized=true), - "WITH_COLUMNS [col(region).str_to_uppercase() as upper]\n" + - " FILTER (col(qty) > 50)\n" + - " SCAN [4×3]", - ) - assert_eq(plan.collect(), df.with_columns(derived).filter(predicate)) -} - -///| -test "a string predicate sinks below a selection passing its bare column" { - let df = sales_df() - // The predicate `region.str_starts_with("w")` is row-stable and reads only - // the bare `region` the selection passes through, so it crosses the SELECT; - // the projection pass then narrows the scan below the sunk filter (`bonus` - // is dead). - let projection = [@expr.col("region"), @expr.col("qty")] - let predicate = @expr.col("region").str_starts_with("w") - let plan = LazyFrame::LazyFrame(df).select(projection).filter(predicate) - assert_eq( - plan.explain(optimized=true), - "SELECT [col(region), col(qty)]\n" + - " FILTER col(region).str_starts_with(\"w\")\n" + - " SELECT [col(qty), col(region)]\n" + - " SCAN [4×3]", - ) - assert_eq(plan.collect(), df.select(projection).filter(predicate)) -} - -///| -test "fence: a string op over an aggregation keeps a group filter above" { - let df = sales_df() - // The predicate reads the bare key `region` and is row-stable, so - // group-cell-stability decides: `first(region)` can be null for an all-null - // group, and the `str_*` wrapper inherits that — its cell is not - // group-cell-stable, so the filter stays above the aggregate (the scan - // still narrows below it, `bonus` being dead). - let aggs = [ - @expr.col("qty").sum().with_alias("total"), - @expr.col("region").first().str_to_uppercase().with_alias("u"), - ] - let predicate = @expr.col("region").eq(@expr.lit_str("west")) - let plan = LazyFrame::LazyFrame(df) - .group_by([@expr.col("region")]) - .agg(aggs) - .filter(predicate) - assert_eq( - plan.explain(optimized=true), - "FILTER (col(region) == \"west\")\n" + - " AGGREGATE [col(qty).sum() as total, col(region).first().str_to_uppercase() as u] BY [col(region)]\n" + - " SELECT [col(qty), col(region)]\n" + - " SCAN [4×3]", - ) - assert_eq( - plan.collect(), - df.group_by([@expr.col("region")]).agg(aggs).filter(predicate), - ) -} diff --git a/lazy/optimize_test.mbt b/lazy/optimize_test.mbt index 1f4cba50..53c09877 100644 --- a/lazy/optimize_test.mbt +++ b/lazy/optimize_test.mbt @@ -76,3 +76,83 @@ test "the flagship pipeline sinks its filter and narrows its scan" { assert_eq(out, df.with_columns(derived).select(projection).filter(predicate)) assert_eq(out.check_invariants(), Ok(())) } + +///| +/// Optimizer interaction with the string namespace. A `Str` node raises only on +/// scope-independent facts — the dtype-level `TypeMismatch` of a non-String +/// operand, and for the regex ops the `InvalidOperation` of a pattern that does +/// not compile, hoisted out of the per-cell map — never on a value, so it is +/// `row_stable` — a filter sinks past a `with_columns` +/// that derives a string column, exactly as it does past arithmetic. And a +/// `Str` wrapping an aggregation inherits that aggregation's +/// group-cell-stability, so an `str_*` over `first` keeps a group filter +/// above the aggregate. Both pin the `Str` arms added to `row_stable` / +/// `group_cell_stable`; the shared fixtures are the package's own. + +///| +test "a trailing filter sinks below a string-namespace with_columns stage" { + let df = sales_df() + // `upper` is derived from `region` by a row-stable `str_*` op, but the + // predicate reads `qty`, so the filter crosses the stage with no + // string-specific rule — the same swap arithmetic earns. + let derived = [@expr.col("region").str_to_uppercase().with_alias("upper")] + let predicate = @expr.col("qty").gt(@expr.lit_int(50)) + let plan = LazyFrame::LazyFrame(df).with_columns(derived).filter(predicate) + assert_eq( + plan.explain(optimized=true), + "WITH_COLUMNS [col(region).str_to_uppercase() as upper]\n" + + " FILTER (col(qty) > 50)\n" + + " SCAN [4×3]", + ) + assert_eq(plan.collect(), df.with_columns(derived).filter(predicate)) +} + +///| +test "a string predicate sinks below a selection passing its bare column" { + let df = sales_df() + // The predicate `region.str_starts_with("w")` is row-stable and reads only + // the bare `region` the selection passes through, so it crosses the SELECT; + // the projection pass then narrows the scan below the sunk filter (`bonus` + // is dead). + let projection = [@expr.col("region"), @expr.col("qty")] + let predicate = @expr.col("region").str_starts_with("w") + let plan = LazyFrame::LazyFrame(df).select(projection).filter(predicate) + assert_eq( + plan.explain(optimized=true), + "SELECT [col(region), col(qty)]\n" + + " FILTER col(region).str_starts_with(\"w\")\n" + + " SELECT [col(qty), col(region)]\n" + + " SCAN [4×3]", + ) + assert_eq(plan.collect(), df.select(projection).filter(predicate)) +} + +///| +test "fence: a string op over an aggregation keeps a group filter above" { + let df = sales_df() + // The predicate reads the bare key `region` and is row-stable, so + // group-cell-stability decides: `first(region)` can be null for an all-null + // group, and the `str_*` wrapper inherits that — its cell is not + // group-cell-stable, so the filter stays above the aggregate (the scan + // still narrows below it, `bonus` being dead). + let aggs = [ + @expr.col("qty").sum().with_alias("total"), + @expr.col("region").first().str_to_uppercase().with_alias("u"), + ] + let predicate = @expr.col("region").eq(@expr.lit_str("west")) + let plan = LazyFrame::LazyFrame(df) + .group_by([@expr.col("region")]) + .agg(aggs) + .filter(predicate) + assert_eq( + plan.explain(optimized=true), + "FILTER (col(region) == \"west\")\n" + + " AGGREGATE [col(qty).sum() as total, col(region).first().str_to_uppercase() as u] BY [col(region)]\n" + + " SELECT [col(qty), col(region)]\n" + + " SCAN [4×3]", + ) + assert_eq( + plan.collect(), + df.group_by([@expr.col("region")]).agg(aggs).filter(predicate), + ) +} diff --git a/series/series_stats_test.mbt b/series/stats_test.mbt similarity index 100% rename from series/series_stats_test.mbt rename to series/stats_test.mbt