From c8cf88e66cd118e7ea0f946d095152215efc07bb Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Sat, 26 Sep 2026 11:30:31 +0100 Subject: [PATCH 1/6] test(r13): pin method calls before they move onto the registry; file P56, D3 (R13 slice 5) Three matrices through both evaluators (default, --case-insensitive, numeric receiver), expectations from DuckDB's contains/starts_with/ends_with/length/ instr/trim family. 33 divergences recorded: P56 (NULL receiver answers FALSE in both evaluators), D3 (WHERE's search methods always ignore case; the registry ignores the switch), and the registry rejecting a numeric receiver (D2 extended to methods). Three DIFFER corpus cases via DuckDB's dot-call syntax; parity contract holds at 212. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/SQL_PARITY.md | 45 ++++ tests/comparison/corpus/02_where.toml | 31 +++ tests/evaluator_matrix_tests.rs | 317 ++++++++++++++++++++++++++ 3 files changed, 393 insertions(+) diff --git a/docs/SQL_PARITY.md b/docs/SQL_PARITY.md index 4fbb0426..0fcc94bd 100644 --- a/docs/SQL_PARITY.md +++ b/docs/SQL_PARITY.md @@ -2542,6 +2542,26 @@ out of date. `'price (usd)%'`. - **Found:** 2026-09-19, pinning LIKE for R13 slice 4. +### P56 — A method call over a NULL receiver answers FALSE, not NULL +- **Status:** 🔴 OPEN — pinned 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) + slice 5; fixed by that slice. +- **Corpus:** `02_where.toml :: where_not_method_null_receiver`, + `select_method_null_receiver` (both `expect = "DIFFER"`). The methods DuckDB + has no twin for are pinned in the evaluator matrix (`EXPECTED_METHODS`). +- **Observed:** `WHERE NOT label.contains('e')` returns every NULL label along + with the labels lacking an `e`; `SELECT label.contains('e')` shows `false` for + a NULL label. In WHERE, `s.Length()`, `s.IndexOf(..)` and `s.Trim*()` on the + left of a comparison also give a non-NULL value for a NULL `s`: + `WHERE s.IndexOf('z') = -1` selects the NULL rows. +- **Cause:** both evaluators. WHERE's hand-rolled `Contains` / `StartsWith` / + `EndsWith` return `Trilean::False` for anything not a value, and its + `evaluate_length` / `evaluate_indexof` / `evaluate_trim` read NULL as empty + text. The registry's `CONTAINS` / `STARTSWITH` / `ENDSWITH` return + `Boolean(false)` for a NULL argument. +- **Why it matters:** the same NULL rule as [P18](#p18) — a function of NULL is + NULL, so `NOT` over it does not select the row. +- **Found:** 2026-09-26, pinning method calls for R13 slice 5. + --- ## Deferred / won't fix (intentional) @@ -2575,6 +2595,31 @@ out of date. - **Considered:** erroring like DuckDB (strict, but breaks existing SELECTs that rely on it), and FALSE everywhere (a silent non-match is the least useful answer). +- **Extended 2026-09-26 (R13 slice 5)** to the string methods: `n.Contains('5')`, + `n.StartsWith('1')` and `n.Length()` read a number as its displayed text. + WHERE already did; the registry functions behind the value evaluator errored. + Pinned in `EXPECTED_NUMBER_METHODS`. + +### D3 — String methods follow `--case-insensitive`, like every comparison +- **Status:** ⚪ DELIBERATE (decided 2026-09-26, R13 slice 5). +- **Corpus:** `02_where.toml :: where_method_contains_case_sensitive` + (`expect = "DIFFER"` until the slice lands). The switch itself has no DuckDB + counterpart; its expected answers are DuckDB's over `lower(s)`, in + `EXPECTED_METHODS_CASE_INSENSITIVE`. +- **Behaviour:** `.Contains()`, `.StartsWith()`, `.EndsWith()` and `.IndexOf()` + search case-sensitively by default and case-insensitively under + `--case-insensitive` — the same rule as `=`, `IN` and `LIKE`. In every clause. +- **Before:** WHERE's own versions always ignored case (an early TUI choice), + while the registry functions the same methods mean in SELECT never did, even + under the switch. So `'ABC'.Contains('b')` was TRUE in WHERE and FALSE in + SELECT. +- **Rationale:** one rule, tied to the one switch, rather than per-method + decisions made at different times. Queries that relied on WHERE's + `.Contains()` ignoring case change answer; they get it back with + `--case-insensitive`. Consistent with DuckDB's `contains`, which is + case-sensitive. +- **Considered:** keeping WHERE's behaviour and making the registry match it — + rejected: it makes one family of predicates disagree with `=` and LIKE. _When we consciously diverge from the reference engine on results (rather than simply not implementing a feature), record it here with the rationale so the diff --git a/tests/comparison/corpus/02_where.toml b/tests/comparison/corpus/02_where.toml index bd06ad1a..0c27433e 100644 --- a/tests/comparison/corpus/02_where.toml +++ b/tests/comparison/corpus/02_where.toml @@ -322,3 +322,34 @@ data = "pattern_matching.csv" sql = "SELECT id, s FROM pattern_matching WHERE s LIKE 'a(b%' ORDER BY id" # P55. `(` opened a regex group, so the query failed: "Invalid LIKE pattern # 'a(b%': regex parse error ... unclosed group". + +# --- R13 slice 5: method calls (P56, D3) --- +# DuckDB accepts the dot-call form, so `label.contains('e')` is contains(label, +# 'e') on both sides. The other methods (StartsWith, IndexOf, Trim*) have no +# DuckDB twin under the same name and are pinned in the evaluator matrix only. + +[[case]] +id = "where_not_method_null_receiver" +data = "null_edges.csv" +sql = "SELECT id FROM null_edges WHERE NOT label.contains('e') ORDER BY id" +expect = "DIFFER" +# P56. A NULL label made `.contains()` FALSE, not NULL, so NOT selected it: +# ids 2, 5, 10, 11, 12 came back alongside 4, 6, 8, 9. + +[[case]] +id = "where_method_contains_case_sensitive" +data = "null_edges.csv" +sql = "SELECT id FROM null_edges WHERE label IS NOT NULL AND NOT label.contains('E') ORDER BY id" +expect = "DIFFER" +# D3. WHERE's `.contains()` ignored case whatever the switch said, so `'E'` +# found delta, charlie, echo and foxtrot and NOT dropped them; `contains` is +# case-sensitive, as `=` and LIKE are here outside --case-insensitive, so every +# non-NULL label is kept. (Shaped to return rows on both sides: an empty +# reference result is reported as a column mismatch, not a row diff.) + +[[case]] +id = "select_method_null_receiver" +data = "null_edges.csv" +sql = "SELECT id, label.contains('e') AS c FROM null_edges ORDER BY id" +expect = "DIFFER" +# P56, in SELECT: the registry's CONTAINS answered false for a NULL label. diff --git a/tests/evaluator_matrix_tests.rs b/tests/evaluator_matrix_tests.rs index 221ac617..0774776f 100644 --- a/tests/evaluator_matrix_tests.rs +++ b/tests/evaluator_matrix_tests.rs @@ -302,6 +302,298 @@ const KNOWN_NUMBER_LIKE: &[Known] = &[]; // newline). All six went FIXED when LIKE delegated to `sql_like`. const KNOWN_LIKE: &[Known] = &[]; +/// Method calls (R13 slice 5). WHERE has its own `Contains` / `StartsWith` / +/// `EndsWith` and its own left-side `Length` / `IndexOf` / `Trim*`; the value +/// evaluator maps the same methods onto registry functions. The methods are +/// ours, not SQL, so each expected answer comes from DuckDB running the +/// function it corresponds to: `contains`, `starts_with`, `ends_with`, +/// `length`, `instr(s, x) - 1` (IndexOf is 0-based, -1 when absent), `trim`, +/// `ltrim`, `rtrim`. +/// +/// | row | s | n | +/// |-----|--------|------| +/// | 0 | 'abc' | 5 | +/// | 1 | 'ABC' | 12 | +/// | 2 | NULL | NULL | +/// | 3 | ' aB ' | 7.5 | +fn method_table() -> DataTable { + let mut t = DataTable::new("t"); + t.add_column(DataColumn::new("s")); + t.add_column(DataColumn::new("n")); + let text = |v: Option<&str>| v.map_or(DataValue::Null, |s| DataValue::String(s.to_string())); + for (s, n) in [ + (Some("abc"), DataValue::Integer(5)), + (Some("ABC"), DataValue::Integer(12)), + (None, DataValue::Null), + (Some(" aB "), DataValue::Float(7.5)), + ] { + t.add_row(DataRow::new(vec![text(s), n])).unwrap(); + } + t +} + +/// Generated by DuckDB from the same table, as for `EXPECTED`. +const EXPECTED_METHODS: &[(&str, &str)] = &[ + // Case-sensitive outside --case-insensitive, like every other comparison + // (D3), and NULL for a NULL receiver - so NOT does not select it (P56). + ("s.Contains('b')", "TFUF"), + ("s.Contains('B')", "FTUT"), + ("NOT s.Contains('b')", "FTUT"), + ("s.StartsWith('a')", "TFUF"), + ("s.EndsWith('C')", "FTUF"), + // Value-returning methods, on the left of a comparison. + ("s.Length() > 3", "FFUT"), + ("s.IndexOf('b') = 1", "TFUF"), + ("s.IndexOf('z') = -1", "TTUT"), + ("s.Trim() = 'aB'", "FFUT"), + ("s.TrimStart() = 'aB '", "FFUT"), + ("s.TrimEnd() = ' aB'", "FFUT"), +]; + +/// The same methods under `--case-insensitive`. DuckDB has no such mode; the +/// answers are DuckDB's over `lower(s)` (and `COLLATE NOCASE` for `=`), the +/// same derivation as `EXPECTED_CASE_INSENSITIVE`. D3: a method's search +/// follows the switch exactly as `=` and LIKE do. +const EXPECTED_METHODS_CASE_INSENSITIVE: &[(&str, &str)] = &[ + ("s.Contains('B')", "TTUT"), + ("NOT s.Contains('B')", "FFUF"), + ("s.StartsWith('A')", "TTUF"), + ("s.EndsWith('c')", "TTUF"), + ("s.IndexOf('B') = 1", "TTUF"), + ("s.Trim() = 'AB'", "FFUT"), +]; + +/// A method over a number. DuckDB refuses these (no implicit cast to VARCHAR), +/// so the expected answers are the D2 rule extended to methods: a number is +/// read as the text sql-cli displays for it (`5`, `12`, `7.5`). +const EXPECTED_NUMBER_METHODS: &[(&str, &str)] = &[ + ("n.Contains('5')", "TFUT"), + ("n.StartsWith('1')", "FTUF"), + ("n.Length() > 1", "FTUT"), +]; + +// Recorded by R13 slice 5's pin (2026-09-26), grouped by fault. The observed +// strings combine faults, so an entry may carry two. +const NULL_RECEIVER: &str = "P56: a method over a NULL receiver answers FALSE (WHERE's Length / \ + IndexOf / Trim* a non-NULL value) instead of NULL"; +const WHERE_ALWAYS_NOCASE: &str = "D3: WHERE's own Contains / StartsWith / EndsWith / IndexOf \ + ignore case whatever the switch says"; +const IGNORES_SWITCH: &str = "D3: the registry functions a method maps to compare \ + case-sensitively even under --case-insensitive"; +const NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE: &str = + "P56 (NULL receiver answers FALSE) and D3 (WHERE's search methods always ignore case)"; +const NULL_RECEIVER_AND_IGNORES_SWITCH: &str = + "P56 (NULL receiver answers FALSE) and D3 (registry functions ignore --case-insensitive)"; +const NUMBER_ERRORS: &str = "D2 extended to methods: the registry functions reject a numeric \ + receiver instead of reading its displayed text"; + +const KNOWN_METHODS: &[Known] = &[ + Known { + evaluator: Evaluator::Where, + predicate: "s.Contains('b')", + observed: "TTFT", + why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.Contains('B')", + observed: "TTFT", + why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, + }, + Known { + evaluator: Evaluator::Where, + predicate: "NOT s.Contains('b')", + observed: "FFTF", + why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.StartsWith('a')", + observed: "TTFF", + why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.EndsWith('C')", + observed: "TTFF", + why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.Length() > 3", + observed: "FFFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.IndexOf('b') = 1", + observed: "TTFF", + why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.IndexOf('z') = -1", + observed: "TTTT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.Trim() = 'aB'", + observed: "FFFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.TrimStart() = 'aB '", + observed: "FFFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.TrimEnd() = ' aB'", + observed: "FFFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.Contains('b')", + observed: "TFFF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.Contains('B')", + observed: "FTFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "NOT s.Contains('b')", + observed: "FTTT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.StartsWith('a')", + observed: "TFFF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.EndsWith('C')", + observed: "FTFF", + why: NULL_RECEIVER, + }, +]; + +const KNOWN_METHODS_CASE_INSENSITIVE: &[Known] = &[ + Known { + evaluator: Evaluator::Where, + predicate: "s.Contains('B')", + observed: "TTFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "NOT s.Contains('B')", + observed: "FFTF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.StartsWith('A')", + observed: "TTFF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.EndsWith('c')", + observed: "TTFF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.IndexOf('B') = 1", + observed: "TTFF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.Trim() = 'AB'", + observed: "FFFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.Contains('B')", + observed: "FTFT", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Value, + predicate: "NOT s.Contains('B')", + observed: "TFTF", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.StartsWith('A')", + observed: "FTFF", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.EndsWith('c')", + observed: "TFFF", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Value, + predicate: "s.IndexOf('B') = 1", + observed: "FTUF", + why: IGNORES_SWITCH, + }, +]; + +const KNOWN_NUMBER_METHODS: &[Known] = &[ + Known { + evaluator: Evaluator::Where, + predicate: "n.Contains('5')", + observed: "TFFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "n.StartsWith('1')", + observed: "FTFF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "n.Length() > 1", + observed: "FTFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "n.Contains('5')", + observed: "EEFE", + why: NUMBER_ERRORS, + }, + Known { + evaluator: Evaluator::Value, + predicate: "n.StartsWith('1')", + observed: "EEFE", + why: NUMBER_ERRORS, + }, + Known { + evaluator: Evaluator::Value, + predicate: "n.Length() > 1", + observed: "EEUE", + why: NUMBER_ERRORS, + }, +]; + fn trilean_char(t: Trilean) -> char { match t { Trilean::True => 'T', @@ -383,6 +675,31 @@ fn like_over_a_number_through_both_evaluators() { ); } +#[test] +fn every_method_call_through_both_evaluators() { + check_matrix(&method_table(), EXPECTED_METHODS, KNOWN_METHODS, false); +} + +#[test] +fn every_method_call_through_both_evaluators_case_insensitive() { + check_matrix( + &method_table(), + EXPECTED_METHODS_CASE_INSENSITIVE, + KNOWN_METHODS_CASE_INSENSITIVE, + true, + ); +} + +#[test] +fn method_calls_over_a_number_through_both_evaluators() { + check_matrix( + &method_table(), + EXPECTED_NUMBER_METHODS, + KNOWN_NUMBER_METHODS, + false, + ); +} + fn check_matrix( t: &DataTable, expected_answers: &[(&str, &str)], From 9b8d857a4be8b94ff35ea9c6b5c87cce8d412001 Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Sat, 26 Sep 2026 11:36:51 +0100 Subject: [PATCH 2/6] test(r13): pin the search functions' function forms under D3 (R13 slice 5) CONTAINS / INSTR / ENDSWITH as functions share the methods' implementation, so D3 (follow --case-insensitive), P56 (NULL gives NULL) and D2 (a number is its displayed text) apply to them too. 14 entries recorded; WHERE already delegates function calls, so both columns agree. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/evaluator_matrix_tests.rs | 93 +++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/evaluator_matrix_tests.rs b/tests/evaluator_matrix_tests.rs index 0774776f..b18888a5 100644 --- a/tests/evaluator_matrix_tests.rs +++ b/tests/evaluator_matrix_tests.rs @@ -348,6 +348,10 @@ const EXPECTED_METHODS: &[(&str, &str)] = &[ ("s.Trim() = 'aB'", "FFUT"), ("s.TrimStart() = 'aB '", "FFUT"), ("s.TrimEnd() = ' aB'", "FFUT"), + // The function forms share the method's implementation and its rules. + ("CONTAINS(s, 'B')", "FTUT"), + ("INSTR(s, 'b') = 2", "TFUF"), + ("CONTAINS(s, NULL)", "UUUU"), ]; /// The same methods under `--case-insensitive`. DuckDB has no such mode; the @@ -361,6 +365,9 @@ const EXPECTED_METHODS_CASE_INSENSITIVE: &[(&str, &str)] = &[ ("s.EndsWith('c')", "TTUF"), ("s.IndexOf('B') = 1", "TTUF"), ("s.Trim() = 'AB'", "FFUT"), + ("CONTAINS(s, 'b')", "TTUT"), + ("INSTR(s, 'b') = 2", "TTUF"), + ("ENDSWITH(s, 'c')", "TTUF"), ]; /// A method over a number. DuckDB refuses these (no implicit cast to VARCHAR), @@ -370,6 +377,8 @@ const EXPECTED_NUMBER_METHODS: &[(&str, &str)] = &[ ("n.Contains('5')", "TFUT"), ("n.StartsWith('1')", "FTUF"), ("n.Length() > 1", "FTUT"), + ("INSTR(n, '5') = 1", "TFUF"), + ("CONTAINS(n, '.')", "FFUT"), ]; // Recorded by R13 slice 5's pin (2026-09-26), grouped by fault. The observed @@ -484,6 +493,30 @@ const KNOWN_METHODS: &[Known] = &[ observed: "FTFF", why: NULL_RECEIVER, }, + Known { + evaluator: Evaluator::Where, + predicate: "CONTAINS(s, 'B')", + observed: "FTFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "CONTAINS(s, 'B')", + observed: "FTFT", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Where, + predicate: "CONTAINS(s, NULL)", + observed: "FFFF", + why: NULL_RECEIVER, + }, + Known { + evaluator: Evaluator::Value, + predicate: "CONTAINS(s, NULL)", + observed: "FFFF", + why: NULL_RECEIVER, + }, ]; const KNOWN_METHODS_CASE_INSENSITIVE: &[Known] = &[ @@ -553,6 +586,42 @@ const KNOWN_METHODS_CASE_INSENSITIVE: &[Known] = &[ observed: "FTUF", why: IGNORES_SWITCH, }, + Known { + evaluator: Evaluator::Where, + predicate: "CONTAINS(s, 'b')", + observed: "TFFF", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Value, + predicate: "CONTAINS(s, 'b')", + observed: "TFFF", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Where, + predicate: "INSTR(s, 'b') = 2", + observed: "TFUF", + why: IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Value, + predicate: "INSTR(s, 'b') = 2", + observed: "TFUF", + why: IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Where, + predicate: "ENDSWITH(s, 'c')", + observed: "TFFF", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, + Known { + evaluator: Evaluator::Value, + predicate: "ENDSWITH(s, 'c')", + observed: "TFFF", + why: NULL_RECEIVER_AND_IGNORES_SWITCH, + }, ]; const KNOWN_NUMBER_METHODS: &[Known] = &[ @@ -592,6 +661,30 @@ const KNOWN_NUMBER_METHODS: &[Known] = &[ observed: "EEUE", why: NUMBER_ERRORS, }, + Known { + evaluator: Evaluator::Where, + predicate: "INSTR(n, '5') = 1", + observed: "EEUE", + why: NUMBER_ERRORS, + }, + Known { + evaluator: Evaluator::Value, + predicate: "INSTR(n, '5') = 1", + observed: "EEUE", + why: NUMBER_ERRORS, + }, + Known { + evaluator: Evaluator::Where, + predicate: "CONTAINS(n, '.')", + observed: "EEFE", + why: NUMBER_ERRORS, + }, + Known { + evaluator: Evaluator::Value, + predicate: "CONTAINS(n, '.')", + observed: "EEFE", + why: NUMBER_ERRORS, + }, ]; fn trilean_char(t: Trilean) -> char { From f45f49408a29460846e55f62894ddb97af9f98de Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Sat, 26 Sep 2026 11:48:35 +0100 Subject: [PATCH 3/6] fix(r13): search functions read NULL as NULL, a number as its text, and follow --case-insensitive (R13 slice 5) CONTAINS / STARTSWITH / ENDSWITH / INDEXOF / INSTR (and their method forms) read both operands through one `search_text`: NULL gives NULL (P56), any other value is its displayed text (D2), lowercased under --case-insensitive (D3). The mode reaches a function through a new default-implemented `SqlFunction::evaluate_with_case`, and `MethodFunction::evaluate_method` takes it too; the value evaluator's three registry call sites pass it in. LENGTH reads a number as its text as well. Exactly the 27 value-evaluator and function-form matrix entries went FIXED; WHERE's own method arms did not move. Parity moved by exactly one case, select_method_null_receiver (179 -> 180 AGREE / 212). FORMAL examples pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/SQL_PARITY.md | 23 +++- src/data/arithmetic_evaluator.rs | 6 +- src/sql/functions/mod.rs | 9 ++ src/sql/functions/string_methods.rs | 161 ++++++++++++----------- tests/comparison/corpus/02_where.toml | 2 +- tests/evaluator_matrix_tests.rs | 181 +------------------------- 6 files changed, 124 insertions(+), 258 deletions(-) diff --git a/docs/SQL_PARITY.md b/docs/SQL_PARITY.md index 0fcc94bd..3ff9e221 100644 --- a/docs/SQL_PARITY.md +++ b/docs/SQL_PARITY.md @@ -2543,10 +2543,12 @@ out of date. - **Found:** 2026-09-19, pinning LIKE for R13 slice 4. ### P56 — A method call over a NULL receiver answers FALSE, not NULL -- **Status:** 🔴 OPEN — pinned 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) - slice 5; fixed by that slice. -- **Corpus:** `02_where.toml :: where_not_method_null_receiver`, - `select_method_null_receiver` (both `expect = "DIFFER"`). The methods DuckDB +- **Status:** 🟡 HALF FIXED — pinned 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) + slice 5. The registry's search functions (SELECT, HAVING, every clause but + WHERE's own method arms, and every function form) fixed the same day — + 179 → **180 AGREE**. WHERE's method calls go when WHERE delegates them. +- **Corpus:** `02_where.toml :: where_not_method_null_receiver` + (`expect = "DIFFER"`), `select_method_null_receiver` (AGREE). The methods DuckDB has no twin for are pinned in the evaluator matrix (`EXPECTED_METHODS`). - **Observed:** `WHERE NOT label.contains('e')` returns every NULL label along with the labels lacking an `e`; `SELECT label.contains('e')` shows `false` for @@ -2607,8 +2609,17 @@ out of date. counterpart; its expected answers are DuckDB's over `lower(s)`, in `EXPECTED_METHODS_CASE_INSENSITIVE`. - **Behaviour:** `.Contains()`, `.StartsWith()`, `.EndsWith()` and `.IndexOf()` - search case-sensitively by default and case-insensitively under - `--case-insensitive` — the same rule as `=`, `IN` and `LIKE`. In every clause. + — and their function forms `CONTAINS`, `STARTSWITH`, `ENDSWITH`, `INDEXOF`, + `INSTR` — search case-sensitively by default and case-insensitively under + `--case-insensitive`: the same rule as `=`, `IN` and `LIKE`, in every clause. + The mode reaches a function through `SqlFunction::evaluate_with_case`; a + function that compares text overrides it, every other one ignores it. +- **Still to follow the switch:** the functions that *match* text while + transforming it — `REPLACE`, `FREQUENCY`, `SUBSTRING_BEFORE` / + `SUBSTRING_AFTER`, `SPLIT_PART`, `LEFT` / `RIGHT` with a delimiter. Each to be + pinned then moved onto `evaluate_with_case` (user decision 2026-09-26: the + switch applies across the whole range, even where captured smoke output has + to be regenerated). - **Before:** WHERE's own versions always ignored case (an early TUI choice), while the registry functions the same methods mean in SELECT never did, even under the switch. So `'ABC'.Contains('b')` was TRUE in WHERE and FALSE in diff --git a/src/data/arithmetic_evaluator.rs b/src/data/arithmetic_evaluator.rs index c05b0c32..6ff6ddf5 100644 --- a/src/data/arithmetic_evaluator.rs +++ b/src/data/arithmetic_evaluator.rs @@ -753,7 +753,7 @@ impl<'a> ArithmeticEvaluator<'a> { // Get the function and call it let func = self.function_registry.get(name).unwrap(); - return func.evaluate(&evaluated_args); + return func.evaluate_with_case(&evaluated_args, self.case_insensitive); } // If not in registry, return error for unknown function @@ -1297,7 +1297,7 @@ impl<'a> ArithmeticEvaluator<'a> { for arg in args { method_args.push(self.evaluate(arg, row_index)?); } - return method_fn.evaluate_method(value, &method_args); + return method_fn.evaluate_method(value, &method_args, self.case_insensitive); } // Otherwise, proxy the method through the function registry. @@ -1328,7 +1328,7 @@ impl<'a> ArithmeticEvaluator<'a> { // Get the function and call it let func = self.function_registry.get(function_name).unwrap(); - return func.evaluate(&func_args); + return func.evaluate_with_case(&func_args, self.case_insensitive); } // If not in registry, the method is not supported diff --git a/src/sql/functions/mod.rs b/src/sql/functions/mod.rs index 8d67badf..3363a99c 100644 --- a/src/sql/functions/mod.rs +++ b/src/sql/functions/mod.rs @@ -137,6 +137,15 @@ pub trait SqlFunction: Send + Sync { /// Evaluate the function with the given arguments fn evaluate(&self, args: &[DataValue]) -> Result; + /// Evaluate under the engine's string-comparison mode (`--case-insensitive`). + /// A function whose answer depends on comparing text overrides this, so it + /// follows the switch exactly as `=` and LIKE do (D3 in docs/SQL_PARITY.md); + /// every other function ignores the mode. + fn evaluate_with_case(&self, args: &[DataValue], case_insensitive: bool) -> Result { + let _ = case_insensitive; + self.evaluate(args) + } + /// Validate arguments before evaluation (default implementation checks count) fn validate_args(&self, args: &[DataValue]) -> Result<()> { let sig = self.signature(); diff --git a/src/sql/functions/string_methods.rs b/src/sql/functions/string_methods.rs index 77aa49f1..0e262b5a 100644 --- a/src/sql/functions/string_methods.rs +++ b/src/sql/functions/string_methods.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result}; +use std::borrow::Cow; use super::{ArgCount, FunctionCategory, FunctionSignature, SqlFunction}; use crate::data::datatable::DataValue; @@ -12,15 +13,39 @@ pub trait MethodFunction: SqlFunction { /// Get the method name this function handles fn method_name(&self) -> &'static str; - /// Evaluate as a method (first arg is implicit 'self') - fn evaluate_method(&self, receiver: &DataValue, args: &[DataValue]) -> Result { + /// Evaluate as a method (first arg is implicit 'self'), under the engine's + /// string-comparison mode (see `SqlFunction::evaluate_with_case`). + fn evaluate_method( + &self, + receiver: &DataValue, + args: &[DataValue], + case_insensitive: bool, + ) -> Result { // Default implementation: prepend receiver to args and call evaluate let mut full_args = vec![receiver.clone()]; full_args.extend_from_slice(args); - self.evaluate(&full_args) + self.evaluate_with_case(&full_args, case_insensitive) } } +/// The text a search function reads an operand as (R13 slice 5): `None` for +/// NULL, whose answer is then NULL (P56); any other value as the text sql-cli +/// displays for it, so a number is searched as it looks (D2); lowercased under +/// `--case-insensitive`, as `=` and LIKE compare (D3). +fn search_text(value: &DataValue, case_insensitive: bool) -> Option> { + let text = match value { + DataValue::Null => return None, + DataValue::String(s) => Cow::Borrowed(s.as_str()), + DataValue::InternedString(s) => Cow::Borrowed(s.as_str()), + other => Cow::Owned(other.to_string()), + }; + Some(if case_insensitive { + Cow::Owned(text.to_lowercase()) + } else { + text + }) +} + /// `ToUpper` method function pub struct ToUpperMethod; @@ -244,11 +269,9 @@ impl SqlFunction for LengthMethod { fn evaluate(&self, args: &[DataValue]) -> Result { self.validate_args(args)?; - match &args[0] { - DataValue::String(s) => Ok(DataValue::Integer(s.len() as i64)), - DataValue::InternedString(s) => Ok(DataValue::Integer(s.len() as i64)), - DataValue::Null => Ok(DataValue::Null), - _ => Err(anyhow!("Length expects a string argument")), + match search_text(&args[0], false) { + Some(text) => Ok(DataValue::Integer(text.len() as i64)), + None => Ok(DataValue::Null), } } } @@ -282,23 +305,20 @@ impl SqlFunction for ContainsMethod { } fn evaluate(&self, args: &[DataValue]) -> Result { - self.validate_args(args)?; + self.evaluate_with_case(args, false) + } - let haystack = match &args[0] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Boolean(false)), - _ => return Err(anyhow!("Contains expects string arguments")), - }; + fn evaluate_with_case(&self, args: &[DataValue], case_insensitive: bool) -> Result { + self.validate_args(args)?; - let needle = match &args[1] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Boolean(false)), - _ => return Err(anyhow!("Contains expects string arguments")), + let (Some(haystack), Some(needle)) = ( + search_text(&args[0], case_insensitive), + search_text(&args[1], case_insensitive), + ) else { + return Ok(DataValue::Null); }; - Ok(DataValue::Boolean(haystack.contains(needle))) + Ok(DataValue::Boolean(haystack.contains(needle.as_ref()))) } } @@ -331,23 +351,20 @@ impl SqlFunction for StartsWithMethod { } fn evaluate(&self, args: &[DataValue]) -> Result { - self.validate_args(args)?; + self.evaluate_with_case(args, false) + } - let string = match &args[0] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Boolean(false)), - _ => return Err(anyhow!("StartsWith expects string arguments")), - }; + fn evaluate_with_case(&self, args: &[DataValue], case_insensitive: bool) -> Result { + self.validate_args(args)?; - let prefix = match &args[1] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Boolean(false)), - _ => return Err(anyhow!("StartsWith expects string arguments")), + let (Some(string), Some(prefix)) = ( + search_text(&args[0], case_insensitive), + search_text(&args[1], case_insensitive), + ) else { + return Ok(DataValue::Null); }; - Ok(DataValue::Boolean(string.starts_with(prefix))) + Ok(DataValue::Boolean(string.starts_with(prefix.as_ref()))) } } @@ -380,23 +397,20 @@ impl SqlFunction for EndsWithMethod { } fn evaluate(&self, args: &[DataValue]) -> Result { - self.validate_args(args)?; + self.evaluate_with_case(args, false) + } - let string = match &args[0] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Boolean(false)), - _ => return Err(anyhow!("EndsWith expects string arguments")), - }; + fn evaluate_with_case(&self, args: &[DataValue], case_insensitive: bool) -> Result { + self.validate_args(args)?; - let suffix = match &args[1] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Boolean(false)), - _ => return Err(anyhow!("EndsWith expects string arguments")), + let (Some(string), Some(suffix)) = ( + search_text(&args[0], case_insensitive), + search_text(&args[1], case_insensitive), + ) else { + return Ok(DataValue::Null); }; - Ok(DataValue::Boolean(string.ends_with(suffix))) + Ok(DataValue::Boolean(string.ends_with(suffix.as_ref()))) } } @@ -490,7 +504,12 @@ impl MethodFunction for SubstringMethod { "Substring" } - fn evaluate_method(&self, receiver: &DataValue, args: &[DataValue]) -> Result { + fn evaluate_method( + &self, + receiver: &DataValue, + args: &[DataValue], + _case_insensitive: bool, + ) -> Result { // C# `.Substring()` form keeps 0-based indexing (.NET semantics). let mut full_args = vec![receiver.clone()]; full_args.extend_from_slice(args); @@ -937,23 +956,20 @@ impl SqlFunction for IndexOfMethod { } fn evaluate(&self, args: &[DataValue]) -> Result { - self.validate_args(args)?; + self.evaluate_with_case(args, false) + } - let string = match &args[0] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Null), - _ => return Err(anyhow!("IndexOf expects string arguments")), - }; + fn evaluate_with_case(&self, args: &[DataValue], case_insensitive: bool) -> Result { + self.validate_args(args)?; - let substring = match &args[1] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Null), - _ => return Err(anyhow!("IndexOf expects string arguments")), + let (Some(string), Some(substring)) = ( + search_text(&args[0], case_insensitive), + search_text(&args[1], case_insensitive), + ) else { + return Ok(DataValue::Null); }; - match string.find(substring) { + match string.find(substring.as_ref()) { Some(pos) => Ok(DataValue::Integer(pos as i64)), None => Ok(DataValue::Integer(-1)), // Return -1 if not found } @@ -990,23 +1006,20 @@ impl SqlFunction for InstrFunction { } fn evaluate(&self, args: &[DataValue]) -> Result { - self.validate_args(args)?; + self.evaluate_with_case(args, false) + } - let string = match &args[0] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Null), - _ => return Err(anyhow!("INSTR expects string arguments")), - }; + fn evaluate_with_case(&self, args: &[DataValue], case_insensitive: bool) -> Result { + self.validate_args(args)?; - let substring = match &args[1] { - DataValue::String(s) => s.as_str(), - DataValue::InternedString(s) => s.as_str(), - DataValue::Null => return Ok(DataValue::Null), - _ => return Err(anyhow!("INSTR expects string arguments")), + let (Some(string), Some(substring)) = ( + search_text(&args[0], case_insensitive), + search_text(&args[1], case_insensitive), + ) else { + return Ok(DataValue::Null); }; - match string.find(substring) { + match string.find(substring.as_ref()) { Some(pos) => Ok(DataValue::Integer((pos + 1) as i64)), // 1-based for SQL None => Ok(DataValue::Integer(0)), // Return 0 if not found (SQL standard) } diff --git a/tests/comparison/corpus/02_where.toml b/tests/comparison/corpus/02_where.toml index 0c27433e..2f9af4ec 100644 --- a/tests/comparison/corpus/02_where.toml +++ b/tests/comparison/corpus/02_where.toml @@ -351,5 +351,5 @@ expect = "DIFFER" id = "select_method_null_receiver" data = "null_edges.csv" sql = "SELECT id, label.contains('e') AS c FROM null_edges ORDER BY id" -expect = "DIFFER" # P56, in SELECT: the registry's CONTAINS answered false for a NULL label. +# AGREE since the search functions read NULL as NULL (R13 slice 5). diff --git a/tests/evaluator_matrix_tests.rs b/tests/evaluator_matrix_tests.rs index b18888a5..899f2d5f 100644 --- a/tests/evaluator_matrix_tests.rs +++ b/tests/evaluator_matrix_tests.rs @@ -381,20 +381,15 @@ const EXPECTED_NUMBER_METHODS: &[(&str, &str)] = &[ ("CONTAINS(n, '.')", "FFUT"), ]; -// Recorded by R13 slice 5's pin (2026-09-26), grouped by fault. The observed -// strings combine faults, so an entry may carry two. -const NULL_RECEIVER: &str = "P56: a method over a NULL receiver answers FALSE (WHERE's Length / \ - IndexOf / Trim* a non-NULL value) instead of NULL"; -const WHERE_ALWAYS_NOCASE: &str = "D3: WHERE's own Contains / StartsWith / EndsWith / IndexOf \ - ignore case whatever the switch says"; -const IGNORES_SWITCH: &str = "D3: the registry functions a method maps to compare \ - case-sensitively even under --case-insensitive"; +// Recorded by R13 slice 5's pin (2026-09-26), grouped by fault; an observed +// string can combine two. The pin also recorded 27 value-evaluator and +// function-form entries - the registry's search functions answered FALSE for +// NULL, ignored --case-insensitive and rejected a number - which all went FIXED +// when those functions moved onto one `search_text` reader. What is left is +// WHERE's own method-call arms, which go when WHERE delegates method calls. +const NULL_RECEIVER: &str = "P56: WHERE's method over a NULL receiver answers FALSE (its Length / IndexOf / Trim* a non-NULL value) instead of NULL"; const NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE: &str = "P56 (NULL receiver answers FALSE) and D3 (WHERE's search methods always ignore case)"; -const NULL_RECEIVER_AND_IGNORES_SWITCH: &str = - "P56 (NULL receiver answers FALSE) and D3 (registry functions ignore --case-insensitive)"; -const NUMBER_ERRORS: &str = "D2 extended to methods: the registry functions reject a numeric \ - receiver instead of reading its displayed text"; const KNOWN_METHODS: &[Known] = &[ Known { @@ -463,60 +458,6 @@ const KNOWN_METHODS: &[Known] = &[ observed: "FFFT", why: NULL_RECEIVER, }, - Known { - evaluator: Evaluator::Value, - predicate: "s.Contains('b')", - observed: "TFFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Value, - predicate: "s.Contains('B')", - observed: "FTFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Value, - predicate: "NOT s.Contains('b')", - observed: "FTTT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Value, - predicate: "s.StartsWith('a')", - observed: "TFFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Value, - predicate: "s.EndsWith('C')", - observed: "FTFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "CONTAINS(s, 'B')", - observed: "FTFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Value, - predicate: "CONTAINS(s, 'B')", - observed: "FTFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "CONTAINS(s, NULL)", - observed: "FFFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Value, - predicate: "CONTAINS(s, NULL)", - observed: "FFFF", - why: NULL_RECEIVER, - }, ]; const KNOWN_METHODS_CASE_INSENSITIVE: &[Known] = &[ @@ -556,72 +497,6 @@ const KNOWN_METHODS_CASE_INSENSITIVE: &[Known] = &[ observed: "FFFT", why: NULL_RECEIVER, }, - Known { - evaluator: Evaluator::Value, - predicate: "s.Contains('B')", - observed: "FTFT", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Value, - predicate: "NOT s.Contains('B')", - observed: "TFTF", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Value, - predicate: "s.StartsWith('A')", - observed: "FTFF", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Value, - predicate: "s.EndsWith('c')", - observed: "TFFF", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Value, - predicate: "s.IndexOf('B') = 1", - observed: "FTUF", - why: IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Where, - predicate: "CONTAINS(s, 'b')", - observed: "TFFF", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Value, - predicate: "CONTAINS(s, 'b')", - observed: "TFFF", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Where, - predicate: "INSTR(s, 'b') = 2", - observed: "TFUF", - why: IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Value, - predicate: "INSTR(s, 'b') = 2", - observed: "TFUF", - why: IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Where, - predicate: "ENDSWITH(s, 'c')", - observed: "TFFF", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, - Known { - evaluator: Evaluator::Value, - predicate: "ENDSWITH(s, 'c')", - observed: "TFFF", - why: NULL_RECEIVER_AND_IGNORES_SWITCH, - }, ]; const KNOWN_NUMBER_METHODS: &[Known] = &[ @@ -643,48 +518,6 @@ const KNOWN_NUMBER_METHODS: &[Known] = &[ observed: "FTFT", why: NULL_RECEIVER, }, - Known { - evaluator: Evaluator::Value, - predicate: "n.Contains('5')", - observed: "EEFE", - why: NUMBER_ERRORS, - }, - Known { - evaluator: Evaluator::Value, - predicate: "n.StartsWith('1')", - observed: "EEFE", - why: NUMBER_ERRORS, - }, - Known { - evaluator: Evaluator::Value, - predicate: "n.Length() > 1", - observed: "EEUE", - why: NUMBER_ERRORS, - }, - Known { - evaluator: Evaluator::Where, - predicate: "INSTR(n, '5') = 1", - observed: "EEUE", - why: NUMBER_ERRORS, - }, - Known { - evaluator: Evaluator::Value, - predicate: "INSTR(n, '5') = 1", - observed: "EEUE", - why: NUMBER_ERRORS, - }, - Known { - evaluator: Evaluator::Where, - predicate: "CONTAINS(n, '.')", - observed: "EEFE", - why: NUMBER_ERRORS, - }, - Known { - evaluator: Evaluator::Value, - predicate: "CONTAINS(n, '.')", - observed: "EEFE", - why: NUMBER_ERRORS, - }, ]; fn trilean_char(t: Trilean) -> char { From d6ac5aacc786c33896a5b5d35febc2433da23841 Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Sat, 26 Sep 2026 13:08:11 +0100 Subject: [PATCH 4/6] test(r13): pin methods WHERE rejects on the left of a comparison; file P57 (R13 slice 5) `WHERE s.ToUpper() = 'ABC'` fails the query: WHERE's left-side method arm knows only Length / IndexOf / Trim*. Two matrix rows, a qualified-receiver row, and a GAP corpus case via DuckDB's `label.upper()`. Parity contract holds (213). Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/SQL_PARITY.md | 14 ++++++++++++++ tests/comparison/corpus/02_where.toml | 9 +++++++++ tests/evaluator_matrix_tests.rs | 23 +++++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/docs/SQL_PARITY.md b/docs/SQL_PARITY.md index 3ff9e221..eb4b0105 100644 --- a/docs/SQL_PARITY.md +++ b/docs/SQL_PARITY.md @@ -2564,6 +2564,20 @@ out of date. NULL, so `NOT` over it does not select the row. - **Found:** 2026-09-26, pinning method calls for R13 slice 5. +### P57 — Only five methods may sit on the left of a WHERE comparison +- **Status:** 🔴 OPEN — pinned 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) + slice 5; fixed by that slice. +- **Corpus:** `02_where.toml :: where_method_other_than_legacy_on_left` + (`expect = "GAP"`); matrix rows `s.ToUpper() = 'ABC'`, + `s.Replace('a', 'z') = 'zbc'`. +- **Observed:** `WHERE label.upper() = 'ECHO'` fails the query — "Method + 'upper' cannot be used in comparisons". The same expression works in SELECT, + and chained forms such as `s.Trim().Length() = 2` already work in WHERE. +- **Cause:** a method call on the left of a comparison is the one shape WHERE + did not delegate to the value evaluator; its own arm knows `Length`, + `IndexOf`, `Trim`, `TrimStart` and `TrimEnd` and rejects everything else. +- **Found:** 2026-09-26, pinning method calls for R13 slice 5. + --- ## Deferred / won't fix (intentional) diff --git a/tests/comparison/corpus/02_where.toml b/tests/comparison/corpus/02_where.toml index 2f9af4ec..8abaebcb 100644 --- a/tests/comparison/corpus/02_where.toml +++ b/tests/comparison/corpus/02_where.toml @@ -353,3 +353,12 @@ data = "null_edges.csv" sql = "SELECT id, label.contains('e') AS c FROM null_edges ORDER BY id" # P56, in SELECT: the registry's CONTAINS answered false for a NULL label. # AGREE since the search functions read NULL as NULL (R13 slice 5). + +[[case]] +id = "where_method_other_than_legacy_on_left" +data = "null_edges.csv" +sql = "SELECT id FROM null_edges WHERE label.upper() = 'ECHO' ORDER BY id" +expect = "GAP" +# P57. WHERE accepted only Length / IndexOf / Trim* on the left of a +# comparison, so any other method failed the query: "Method 'upper' cannot be +# used in comparisons". SELECT has always evaluated it. diff --git a/tests/evaluator_matrix_tests.rs b/tests/evaluator_matrix_tests.rs index 899f2d5f..05064490 100644 --- a/tests/evaluator_matrix_tests.rs +++ b/tests/evaluator_matrix_tests.rs @@ -348,6 +348,10 @@ const EXPECTED_METHODS: &[(&str, &str)] = &[ ("s.Trim() = 'aB'", "FFUT"), ("s.TrimStart() = 'aB '", "FFUT"), ("s.TrimEnd() = ' aB'", "FFUT"), + // Any other method on the left of a comparison, and a qualified receiver. + ("s.ToUpper() = 'ABC'", "TTUF"), + ("s.Replace('a', 'z') = 'zbc'", "TFUF"), + ("t.s.Contains('b')", "TFUF"), // The function forms share the method's implementation and its rules. ("CONTAINS(s, 'B')", "FTUT"), ("INSTR(s, 'b') = 2", "TFUF"), @@ -390,8 +394,27 @@ const EXPECTED_NUMBER_METHODS: &[(&str, &str)] = &[ const NULL_RECEIVER: &str = "P56: WHERE's method over a NULL receiver answers FALSE (its Length / IndexOf / Trim* a non-NULL value) instead of NULL"; const NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE: &str = "P56 (NULL receiver answers FALSE) and D3 (WHERE's search methods always ignore case)"; +const WHERE_METHOD_ALLOWLIST: &str = "P57: on the left of a WHERE comparison only Length / IndexOf / Trim* are accepted; any other method fails the query"; const KNOWN_METHODS: &[Known] = &[ + Known { + evaluator: Evaluator::Where, + predicate: "s.ToUpper() = 'ABC'", + observed: "EEEE", + why: WHERE_METHOD_ALLOWLIST, + }, + Known { + evaluator: Evaluator::Where, + predicate: "s.Replace('a', 'z') = 'zbc'", + observed: "EEEE", + why: WHERE_METHOD_ALLOWLIST, + }, + Known { + evaluator: Evaluator::Where, + predicate: "t.s.Contains('b')", + observed: "TTFT", + why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, + }, Known { evaluator: Evaluator::Where, predicate: "s.Contains('b')", From 81f890d9f2cbb8dae26ce1131ee4f028ef21fbaa Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Sat, 26 Sep 2026 13:35:20 +0100 Subject: [PATCH 5/6] refactor(r13): WHERE's method calls delegate; its method arms and readers go (R13 slice 5) A method call as a predicate, and a comparison whatever its left operand, now evaluate through the value evaluator and the registry's method functions. evaluate_binary_op's legacy tail and everything reachable only from it are deleted: the Contains/StartsWith/EndsWith and Length/IndexOf/Trim* arms, WHERE's own column resolver, literal readers, compare_trilean and did-you-mean suggester (~700 lines). The remaining 28 matrix entries went FIXED. Parity moved by exactly the three WHERE cases (P56, P57, D3): 180 -> 183 AGREE / 213. Not caught by the pin: a numeric Trim() receiver. WHERE trimmed its text; the registry's TRIM errored, failing the presidents FORMAL example. TRIM / TRIMSTART / TRIMEND now read a number as its displayed text (D2), pinned. Three tests asserted the old always-case-insensitive rule; rewritten to assert D3 in both modes. No captured example output changed. Contains / Length / StartsWith over trades_100000.csv run at load time, level with LIKE. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/ENGINE_REFACTORING.md | 53 +- docs/SQL_PARITY.md | 29 +- src/data/query_engine.rs | 7 +- src/data/recursive_where_evaluator.rs | 713 +----------------- src/sql/functions/string_methods.rs | 34 +- tests/comparison/corpus/02_where.toml | 3 - tests/evaluator_matrix_tests.rs | 168 +---- tests/method_evaluation_test.rs | 34 +- .../test_string_methods_comprehensive.py | 29 +- 9 files changed, 146 insertions(+), 924 deletions(-) diff --git a/docs/ENGINE_REFACTORING.md b/docs/ENGINE_REFACTORING.md index 4d816f41..9b73b653 100644 --- a/docs/ENGINE_REFACTORING.md +++ b/docs/ENGINE_REFACTORING.md @@ -475,8 +475,9 @@ feature work**, and so we can tell the difference between "this is awkward" and **slices 1 and 2 done 2026-09-13, slice 3 (with 3b) done 2026-09-14**, slice 4 in progress (`BETWEEN`, `IN`, `NOT IN` delegated 2026-09-15; comparisons 2026-09-18, closing P54; IS NULL and LIKE 2026-09-19, closing P55). Every - WHERE *operator* now delegates; what remains of slice 4 (AND / OR / NOT / - CASE) lands after slice 5. + WHERE *operator* now delegates. **Slice 5 (method calls) done 2026-09-26**, + closing P56 and P57 and deciding D3. What remains of slice 4 (AND / OR / + NOT / CASE) is next, then slice 6. **The active workstream:** parity fixes that touch expression evaluation land as slices of this entry, not as patches. - **Where:** `src/data/arithmetic_evaluator.rs` (`ArithmeticEvaluator`, value → @@ -704,6 +705,49 @@ feature work**, and so we can tell the difference between "this is awkward" and left-side method-call arm, all waiting on slice 5. WHERE's readers (`number_literal_value`, the `DATETIME` arms, `compare_trilean`) are now reached only through that arm. +- **Slice 5 (2026-09-26) — method calls.** Five commits; one user decision. + - *Pin.* Three matrices (`EXPECTED_METHODS`, `…_CASE_INSENSITIVE`, + `EXPECTED_NUMBER_METHODS`; DuckDB's `contains` / `starts_with` / + `ends_with` / `length` / `instr` / `trim` family as the reference, with + `IndexOf` as `instr − 1`), then the function forms and the methods WHERE + refused. **55 divergences, both evaluators**: a NULL receiver answered + FALSE (**P56**) — so `WHERE NOT label.contains('e')` selected every NULL + label; WHERE's search methods always ignored case while the registry's + ignored the switch; the registry rejected a number; and WHERE accepted + only `Length` / `IndexOf` / `Trim*` on the left of a comparison, failing + `WHERE s.ToUpper() = 'ABC'` (**P57**). DuckDB's dot-call syntax + (`label.contains('e')`) made four of them real corpus cases. + - *D3, decided by the user:* the search methods and functions follow + `--case-insensitive` like `=` and LIKE, case-sensitive by default — "one + rule tied to the switch rather than decisions made at different times", + accepting that queries relying on WHERE's old behaviour change answer. + - *Registry first.* `SqlFunction::evaluate_with_case` (default: ignore the + mode) carries the switch into a function; `MethodFunction::evaluate_method` + takes it too; the value evaluator's three registry call sites pass it. The + five search functions (and LENGTH / TRIM*) read operands through one + `search_text`: NULL → NULL, a number as its displayed text (D2), lowercased + under the switch. Exactly the 27 value-evaluator and function-form entries + went FIXED; 179 → 180 AGREE. + - *Delegate.* WHERE's method-call predicate and its comparison arm delegate + unconditionally; `evaluate_binary_op`'s legacy tail, the method arms, and + everything reachable only from them went — WHERE's own column resolver, + literal readers, `compare_trilean` and "did you mean" suggester (~700 + lines). The remaining 28 WHERE entries went FIXED; parity moved by exactly + the three WHERE cases, 180 → **183 AGREE** / 213. + - *What the pin missed.* A numeric `Trim()` receiver: WHERE trimmed its text, + the registry's TRIM errored, and the `presidents` FORMAL example failed on + `r.Greatness.Trim() <> 'NA'`. Fixed by the same reader and pinned. The + matrix covers what someone thought to put in it; the examples are a + second net. + - *Two tests stated the old rule* (`test_case_sensitivity`, and two Python + tests whose docstrings said "appears to be case-insensitive based on + logs"); rewritten to assert D3 in both modes. No captured output changed. + - *Cost:* nil — `Contains`, `Length() >`, `NOT StartsWith` over + `trades_100000.csv` all run at load time (~2.5 s), level with LIKE. + - *Left:* the functions that match text while transforming it (REPLACE, + FREQUENCY, SUBSTRING_BEFORE/AFTER, SPLIT_PART, LEFT/RIGHT with a + delimiter) do not yet follow the switch — listed under D3. LENGTH / + INDEXOF / INSTR count bytes, not characters (unpinned). - **Slices, in the R10 pattern — no-op slices kept apart from the one that changes answers:** 1. ✅ **Pin the divergences, no engine change.** *(Done 2026-09-13.)* A per-operator matrix run through @@ -727,7 +771,7 @@ feature work**, and so we can tell the difference between "this is awkward" and the hand-written arm is deleted. After slice 2 the rules already agree, so each commit is a provable no-op — parity exactly unchanged, and the slice-1 matrix still green. - 5. **Method calls:** WHERE's hand-rolled `Contains`/`StartsWith`/`Trim`… move + 5. ✅ **Method calls:** *(Done 2026-09-26 — see result above.)* WHERE's hand-rolled `Contains`/`StartsWith`/`Trim`… move onto the registry mapping the value evaluator already uses. 6. **Collapse sites converge:** HAVING's `is_truthy` and JOIN's bare comparison go through the same collapse. JOIN NULL keys need care on the *hash* path @@ -769,7 +813,7 @@ R8 legacy WHERE ──── independent; stage 2 is self-contained, do it in a R10 Trilean ──────── DONE; closed P18/P19 (parity 125 → 129) R11 ORDER BY resolver ─ independent; small, but a behaviour change — wants its own parity run R12 aggregate registries ─ independent; step 1 is a provable no-op, do it before the next aggregate fix -R13 one evaluator ─── ACTIVE from 2026-09-13; slices 1 (pin), 2 (3VL), 3 (construction + case mode) DONE; 4 operators DONE (BETWEEN/IN, comparisons, IS NULL, LIKE) → 5 (method calls) → 4 tail (AND/OR/NOT/CASE) → 6 +R13 one evaluator ─── ACTIVE from 2026-09-13; slices 1 (pin), 2 (3VL), 3 (construction + case mode) DONE; 4 operators DONE (BETWEEN/IN, comparisons, IS NULL, LIKE); 5 (method calls) DONE → 4 tail (AND/OR/NOT/CASE) → 6 ``` **A note on ordering, from the P18/P19 work being next.** The WHERE evaluator @@ -813,3 +857,4 @@ AGREE count — which makes it safe to land well before the semantics change. | 2026-09-15 | R13 slice 4 (first part): WHERE's `BETWEEN`, `IN`, `NOT IN` delegate to the value evaluator. Latent alias-resolution fault pinned and fixed first; per-node TRACE logging found to be most of the value evaluator's cost and removed (SELECT/GROUP BY expressions ~2 s faster over 100k rows). Parity byte-identical at every commit | — | | 2026-09-18 | R13 slice 4 (second part): WHERE comparisons delegate. Operand readers pinned first (third matrix); P53 and P54 filed; value evaluator's `DATETIME()` moved to local time before delegating. Closes P54 — 175 → **176 AGREE**. Part 1 found to have fixed P54 for IN/BETWEEN unnoticed | — | | 2026-09-19 | R13 slice 4 (third part): IS NULL and LIKE delegate; one `sql_like` for both evaluators. P55 (LIKE compiled as an unescaped regex) filed and closed — 176 → **179 AGREE**; the value evaluator's exponential LIKE matcher replaced first; D2 (LIKE over a number matches its displayed text) decided; `EvaluationContext` removed | — | +| 2026-09-26 | R13 slice 5: method calls. Pinned 55 divergences across both evaluators; P56 (NULL receiver → FALSE) and P57 (WHERE refused all but five methods on a comparison's left) filed and closed; D3 decided (search methods follow `--case-insensitive`). Registry search functions fixed first (180), then WHERE delegated and ~700 lines of its arms and readers deleted — 179 → **183 AGREE** / 213 | — | diff --git a/docs/SQL_PARITY.md b/docs/SQL_PARITY.md index eb4b0105..e3331009 100644 --- a/docs/SQL_PARITY.md +++ b/docs/SQL_PARITY.md @@ -2543,12 +2543,12 @@ out of date. - **Found:** 2026-09-19, pinning LIKE for R13 slice 4. ### P56 — A method call over a NULL receiver answers FALSE, not NULL -- **Status:** 🟡 HALF FIXED — pinned 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) - slice 5. The registry's search functions (SELECT, HAVING, every clause but - WHERE's own method arms, and every function form) fixed the same day — - 179 → **180 AGREE**. WHERE's method calls go when WHERE delegates them. -- **Corpus:** `02_where.toml :: where_not_method_null_receiver` - (`expect = "DIFFER"`), `select_method_null_receiver` (AGREE). The methods DuckDB +- **Status:** ✅ FIXED 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) slice 5, + in two steps: the registry's search functions read NULL as NULL + (179 → 180 AGREE), then WHERE delegated its method calls to them and its own + arms were deleted (with P57 and D3, 180 → **183 AGREE**). +- **Corpus:** `02_where.toml :: where_not_method_null_receiver`, + `select_method_null_receiver` (both AGREE). The methods DuckDB has no twin for are pinned in the evaluator matrix (`EXPECTED_METHODS`). - **Observed:** `WHERE NOT label.contains('e')` returns every NULL label along with the labels lacking an `e`; `SELECT label.contains('e')` shows `false` for @@ -2565,10 +2565,11 @@ out of date. - **Found:** 2026-09-26, pinning method calls for R13 slice 5. ### P57 — Only five methods may sit on the left of a WHERE comparison -- **Status:** 🔴 OPEN — pinned 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) - slice 5; fixed by that slice. +- **Status:** ✅ FIXED 2026-09-26 by [R13](ENGINE_REFACTORING.md#r13) slice 5: + WHERE's comparisons delegate whatever the left operand is, so any registry + method works there. - **Corpus:** `02_where.toml :: where_method_other_than_legacy_on_left` - (`expect = "GAP"`); matrix rows `s.ToUpper() = 'ABC'`, + (AGREE); matrix rows `s.ToUpper() = 'ABC'`, `s.Replace('a', 'z') = 'zbc'`. - **Observed:** `WHERE label.upper() = 'ECHO'` fails the query — "Method 'upper' cannot be used in comparisons". The same expression works in SELECT, @@ -2614,12 +2615,16 @@ out of date. - **Extended 2026-09-26 (R13 slice 5)** to the string methods: `n.Contains('5')`, `n.StartsWith('1')` and `n.Length()` read a number as its displayed text. WHERE already did; the registry functions behind the value evaluator errored. - Pinned in `EXPECTED_NUMBER_METHODS`. + Pinned in `EXPECTED_NUMBER_METHODS`. `Trim()` / `TrimStart()` / `TrimEnd()` + likewise: WHERE trimmed a number's text, and once WHERE delegated, the + registry's TRIM rejecting a number failed the `presidents` example + (`r.Greatness.Trim() <> 'NA'` over a mostly numeric column) — fixed the same + way and pinned (`n.Trim() = '7.5'`). ### D3 — String methods follow `--case-insensitive`, like every comparison - **Status:** ⚪ DELIBERATE (decided 2026-09-26, R13 slice 5). -- **Corpus:** `02_where.toml :: where_method_contains_case_sensitive` - (`expect = "DIFFER"` until the slice lands). The switch itself has no DuckDB +- **Corpus:** `02_where.toml :: where_method_contains_case_sensitive` (AGREE + since 2026-09-26). The switch itself has no DuckDB counterpart; its expected answers are DuckDB's over `lower(s)`, in `EXPECTED_METHODS_CASE_INSENSITIVE`. - **Behaviour:** `.Contains()`, `.StartsWith()`, `.EndsWith()` and `.IndexOf()` diff --git a/src/data/query_engine.rs b/src/data/query_engine.rs index a4eef90d..43a60dd3 100644 --- a/src/data/query_engine.rs +++ b/src/data/query_engine.rs @@ -3474,11 +3474,12 @@ mod tests { println!("Row {i}: status = {status:?}"); } - // Test 1: Basic string contains (should work) - println!("\n--- Test 1: status.Contains('pend') ---"); + // Test 1: Basic string contains (should work). Case-sensitive outside + // --case-insensitive (D3), so the search is spelled as the data is. + println!("\n--- Test 1: status.Contains('Pend') ---"); let result = engine.execute( table.clone(), - "SELECT * FROM test WHERE status.Contains('pend')", + "SELECT * FROM test WHERE status.Contains('Pend')", ); match result { Ok(view) => { diff --git a/src/data/recursive_where_evaluator.rs b/src/data/recursive_where_evaluator.rs index 4e2e2bea..49082c45 100644 --- a/src/data/recursive_where_evaluator.rs +++ b/src/data/recursive_where_evaluator.rs @@ -1,12 +1,9 @@ -use crate::data::arithmetic_evaluator::{sql_like, ArithmeticEvaluator}; +use crate::data::arithmetic_evaluator::ArithmeticEvaluator; use crate::data::datatable::{DataTable, DataValue}; use crate::data::query_engine::ExecutionContext; use crate::data::trilean::Trilean; -use crate::data::value_comparisons::compare_with_op; -use crate::sql::parser::ast::ColumnRef; use crate::sql::recursive_parser::{Condition, LogicalOp, SqlExpression, WhereClause}; use anyhow::{anyhow, Result}; -use chrono::{Local, NaiveDate, NaiveDateTime, NaiveTime}; use std::cell::RefCell; use tracing::debug; @@ -55,303 +52,6 @@ impl<'a, 'exec> RecursiveWhereEvaluator<'a, 'exec> { } } - /// Find a column name similar to the given name using edit distance - fn find_similar_column(&self, name: &str) -> Option { - let columns = self.table.column_names(); - let mut best_match: Option<(String, usize)> = None; - - for col in columns { - let distance = self.edit_distance(&col.to_lowercase(), &name.to_lowercase()); - // Only suggest if distance is small (likely a typo) - // Allow up to 3 edits for longer names - let max_distance = if name.len() > 10 { 3 } else { 2 }; - if distance <= max_distance { - match &best_match { - None => best_match = Some((col, distance)), - Some((_, best_dist)) if distance < *best_dist => { - best_match = Some((col, distance)); - } - _ => {} - } - } - } - - best_match.map(|(name, _)| name) - } - - /// Calculate Levenshtein edit distance between two strings - fn edit_distance(&self, s1: &str, s2: &str) -> usize { - let len1 = s1.len(); - let len2 = s2.len(); - let mut matrix = vec![vec![0; len2 + 1]; len1 + 1]; - - for i in 0..=len1 { - matrix[i][0] = i; - } - for j in 0..=len2 { - matrix[0][j] = j; - } - - for (i, c1) in s1.chars().enumerate() { - for (j, c2) in s2.chars().enumerate() { - let cost = usize::from(c1 != c2); - matrix[i + 1][j + 1] = std::cmp::min( - matrix[i][j + 1] + 1, // deletion - std::cmp::min( - matrix[i + 1][j] + 1, // insertion - matrix[i][j] + cost, // substitution - ), - ); - } - } - - matrix[len1][len2] - } - - /// Compare two values under SQL three-valued logic: if **either** operand - /// is NULL the answer is UNKNOWN, never TRUE or FALSE. - /// - /// This test has to live here, at the predicate layer, and must not be - /// pushed down into `compare_with_op`. That function answers in `bool`, and - /// the `compare_values` beneath it deliberately reports `NULL = NULL` as - /// `Ordering::Equal` because **`ORDER BY` needs NULLs to group together**. - /// Correct for sorting, wrong for predicates — and using the one answer for - /// both is what made `WHERE score = NULL` return the NULL rows (P18). - /// - /// `IS NULL` / `IS NOT NULL` / `IS [NOT] NULL` are handled by their own - /// match arms before reaching any caller of this, and stay two-valued — - /// they are the sanctioned way to test for NULL, which is why nothing is - /// lost by making `=` never match one. - fn compare_trilean(&self, left: &DataValue, right: &DataValue, op: &str) -> Trilean { - if matches!(left, DataValue::Null) || matches!(right, DataValue::Null) { - return Trilean::Unknown; - } - Trilean::from_bool(compare_with_op(left, right, op, self.case_insensitive)) - } - - /// Evaluate the `Length()` method on a column value - fn evaluate_length( - &self, - object: &str, - row_index: usize, - ) -> Result<(Option, String)> { - // Handle qualified column names (table.column or alias.column) - let resolved_column = if object.contains('.') { - if let Some(dot_pos) = object.rfind('.') { - let col_name = &object[dot_pos + 1..]; - col_name - } else { - object - } - } else { - object - }; - - let col_index = if let Some(idx) = self.table.get_column_index(resolved_column) { - idx - } else if resolved_column != object { - // If not found, try the original name - if let Some(idx) = self.table.get_column_index(object) { - idx - } else { - let suggestion = self.find_similar_column(resolved_column); - return Err(match suggestion { - Some(similar) => { - anyhow!("Column '{}' not found. Did you mean '{}'?", object, similar) - } - None => anyhow!("Column '{}' not found", object), - }); - } - } else { - let suggestion = self.find_similar_column(resolved_column); - return Err(match suggestion { - Some(similar) => { - anyhow!("Column '{}' not found. Did you mean '{}'?", object, similar) - } - None => anyhow!("Column '{}' not found", object), - }); - }; - - let value = self.table.get_value(row_index, col_index); - let length_value = match value { - Some(DataValue::String(s)) => Some(DataValue::Integer(s.len() as i64)), - Some(DataValue::InternedString(s)) => Some(DataValue::Integer(s.len() as i64)), - Some(DataValue::Integer(n)) => Some(DataValue::Integer(n.to_string().len() as i64)), - Some(DataValue::Float(f)) => Some(DataValue::Integer(f.to_string().len() as i64)), - _ => Some(DataValue::Integer(0)), - }; - Ok((length_value, format!("{object}.Length()"))) - } - - /// Evaluate the `IndexOf()` method on a column value - fn evaluate_indexof( - &self, - object: &str, - search_str: &str, - row_index: usize, - ) -> Result<(Option, String)> { - // Handle qualified column names (table.column or alias.column) - let resolved_column = if object.contains('.') { - if let Some(dot_pos) = object.rfind('.') { - let col_name = &object[dot_pos + 1..]; - col_name - } else { - object - } - } else { - object - }; - - let col_index = if let Some(idx) = self.table.get_column_index(resolved_column) { - idx - } else if resolved_column != object { - // If not found, try the original name - if let Some(idx) = self.table.get_column_index(object) { - idx - } else { - let suggestion = self.find_similar_column(resolved_column); - return Err(match suggestion { - Some(similar) => { - anyhow!("Column '{}' not found. Did you mean '{}'?", object, similar) - } - None => anyhow!("Column '{}' not found", object), - }); - } - } else { - let suggestion = self.find_similar_column(resolved_column); - return Err(match suggestion { - Some(similar) => { - anyhow!("Column '{}' not found. Did you mean '{}'?", object, similar) - } - None => anyhow!("Column '{}' not found", object), - }); - }; - - let value = self.table.get_value(row_index, col_index); - let index_value = match value { - Some(DataValue::String(s)) => { - // Case-insensitive search by default, following Contains behavior - let pos = s - .to_lowercase() - .find(&search_str.to_lowercase()) - .map_or(-1, |idx| idx as i64); - Some(DataValue::Integer(pos)) - } - Some(DataValue::InternedString(s)) => { - let pos = s - .to_lowercase() - .find(&search_str.to_lowercase()) - .map_or(-1, |idx| idx as i64); - Some(DataValue::Integer(pos)) - } - Some(DataValue::Integer(n)) => { - let str_val = n.to_string(); - let pos = str_val.find(search_str).map_or(-1, |idx| idx as i64); - Some(DataValue::Integer(pos)) - } - Some(DataValue::Float(f)) => { - let str_val = f.to_string(); - let pos = str_val.find(search_str).map_or(-1, |idx| idx as i64); - Some(DataValue::Integer(pos)) - } - _ => Some(DataValue::Integer(-1)), // Return -1 for not found - }; - - if row_index < 3 { - debug!( - "RecursiveWhereEvaluator: Row {} IndexOf('{}') = {:?}", - row_index, search_str, index_value - ); - } - Ok((index_value, format!("{object}.IndexOf('{search_str}')"))) - } - - /// Apply the appropriate trim operation based on `trim_type` - fn apply_trim<'b>(s: &'b str, trim_type: &str) -> &'b str { - match trim_type { - "trim" => s.trim(), - "trimstart" => s.trim_start(), - "trimend" => s.trim_end(), - _ => s, - } - } - - /// Evaluate trim methods (Trim, `TrimStart`, `TrimEnd`) on a column value - fn evaluate_trim( - &self, - object: &str, - row_index: usize, - trim_type: &str, - ) -> Result<(Option, String)> { - // Handle qualified column names (table.column or alias.column) - let resolved_column = if object.contains('.') { - if let Some(dot_pos) = object.rfind('.') { - let col_name = &object[dot_pos + 1..]; - col_name - } else { - object - } - } else { - object - }; - - let col_index = if let Some(idx) = self.table.get_column_index(resolved_column) { - idx - } else if resolved_column != object { - // If not found, try the original name - if let Some(idx) = self.table.get_column_index(object) { - idx - } else { - let suggestion = self.find_similar_column(resolved_column); - return Err(match suggestion { - Some(similar) => { - anyhow!("Column '{}' not found. Did you mean '{}'?", object, similar) - } - None => anyhow!("Column '{}' not found", object), - }); - } - } else { - let suggestion = self.find_similar_column(resolved_column); - return Err(match suggestion { - Some(similar) => { - anyhow!("Column '{}' not found. Did you mean '{}'?", object, similar) - } - None => anyhow!("Column '{}' not found", object), - }); - }; - - let value = self.table.get_value(row_index, col_index); - let trimmed_value = match value { - Some(DataValue::String(s)) => Some(DataValue::String( - Self::apply_trim(s, trim_type).to_string(), - )), - Some(DataValue::InternedString(s)) => Some(DataValue::String( - Self::apply_trim(s, trim_type).to_string(), - )), - Some(DataValue::Integer(n)) => { - let str_val = n.to_string(); - Some(DataValue::String( - Self::apply_trim(&str_val, trim_type).to_string(), - )) - } - Some(DataValue::Float(f)) => { - let str_val = f.to_string(); - Some(DataValue::String( - Self::apply_trim(&str_val, trim_type).to_string(), - )) - } - _ => Some(DataValue::String(String::new())), - }; - - let method_name = match trim_type { - "trim" => "Trim", - "trimstart" => "TrimStart", - "trimend" => "TrimEnd", - _ => "Trim", - }; - Ok((trimmed_value, format!("{object}.{method_name}()"))) - } - /// Evaluate a WHERE clause for a specific row pub fn evaluate(&mut self, where_clause: &WhereClause, row_index: usize) -> Result { // Only log for first few rows to avoid performance impact @@ -472,15 +172,11 @@ impl<'a, 'exec> RecursiveWhereEvaluator<'a, 'exec> { // `NOT UNKNOWN` is UNKNOWN, not TRUE — see P18/P19. Ok(inner_result.negate()) } - SqlExpression::MethodCall { - object, - method, - args, - } => { - if row_index < 3 { - debug!("RecursiveWhereEvaluator: evaluate_expression() - found MethodCall, delegating to evaluate_method_call"); - } - self.evaluate_method_call(object, method, args, row_index) + // `WHERE s.Contains('x')`: the registry's method function, through + // the value evaluator (R13 slice 5). + SqlExpression::MethodCall { .. } => { + reject_unlifted_window(expr)?; + self.evaluate_delegated(expr, row_index) } SqlExpression::CaseExpression { when_branches, @@ -554,179 +250,12 @@ impl<'a, 'exec> RecursiveWhereEvaluator<'a, 'exec> { } // Comparisons, NULL tests and LIKE are the value evaluator's (R13 - // slice 4): its operand readers, three-valued `compare_trilean` and - // `sql_like` are the one implementation. One shape stays below for - // now - a legacy method call on the left (`Length()`, `IndexOf()`, - // `Trim*()`), whose registry twins are slice 5's to check. - if delegates_to_value_evaluator(&op_upper) - && !matches!(left, SqlExpression::MethodCall { .. }) - { - reject_unlifted_window(left)?; - reject_unlifted_window(right)?; - return self.evaluate_delegated(expr, row_index); - } - - // Both operands are resolved to values here and compared below, so the - // NULL -> UNKNOWN rule in `compare_trilean` applies whatever shape the - // operands take. Comparisons with an arithmetic operand used to be - // handed whole to the ArithmeticEvaluator, whose comparison answers - // `false` for a NULL operand -- which `NOT` then flipped to TRUE. - // The left side can also be one of the legacy method calls. - let cell_value = match left { - SqlExpression::MethodCall { - object, - method, - args, - } => { - // Handle method calls that return values (like Length(), IndexOf()) - let (value, _label) = match method.to_lowercase().as_str() { - "length" => { - if !args.is_empty() { - return Err(anyhow::anyhow!("Length() takes no arguments")); - } - self.evaluate_length(object, row_index)? - } - "indexof" => { - if args.len() != 1 { - return Err(anyhow::anyhow!("IndexOf() requires exactly 1 argument")); - } - let search_str = self.extract_string_value(&args[0])?; - self.evaluate_indexof(object, &search_str, row_index)? - } - "trim" => { - if !args.is_empty() { - return Err(anyhow::anyhow!("Trim() takes no arguments")); - } - self.evaluate_trim(object, row_index, "trim")? - } - "trimstart" => { - if !args.is_empty() { - return Err(anyhow::anyhow!("TrimStart() takes no arguments")); - } - self.evaluate_trim(object, row_index, "trimstart")? - } - "trimend" => { - if !args.is_empty() { - return Err(anyhow::anyhow!("TrimEnd() takes no arguments")); - } - self.evaluate_trim(object, row_index, "trimend")? - } - _ => { - return Err(anyhow::anyhow!( - "Method '{}' cannot be used in comparisons", - method - )); - } - }; - value - } - _ => self.evaluate_operand_value(left, row_index)?, - }; - - let compare_value = self - .evaluate_operand_value(right, row_index)? - .unwrap_or(DataValue::Null); - - if row_index < 3 { - debug!( - "RecursiveWhereEvaluator: evaluate_binary_op() - row {} {:?} {} {:?}", - row_index, cell_value, op, compare_value - ); - } - - // Handle special operators that aren't standard comparisons - match op_upper.as_str() { - // LIKE operator - handle specially - // Reached only with a legacy method call on the left; every other - // LIKE delegated above. Same function, so the same rules. - "LIKE" => Ok(sql_like( - &cell_value.unwrap_or(DataValue::Null), - &compare_value, - self.case_insensitive, - )), - - // IS NULL / IS NOT NULL - "IS NULL" => Ok(Trilean::from_bool( - cell_value.is_none() || matches!(cell_value, Some(DataValue::Null)), - )), - "IS NOT NULL" => Ok(Trilean::from_bool( - cell_value.is_some() && !matches!(cell_value, Some(DataValue::Null)), - )), - - // Standard comparison operators - use centralized logic - _ => { - let table_value = cell_value.unwrap_or(DataValue::Null); - Ok(self.compare_trilean(&table_value, &compare_value, op)) - } - } - } - - /// Resolve the value of any WHERE operand -- either side of a comparison, - /// the probe and items of an IN list, the value and bounds of BETWEEN. - /// - /// Columns and literals are read directly; anything else (arithmetic, a - /// function, a CASE) is evaluated by the `ArithmeticEvaluator`. The direct - /// arms are a fast path, not a semantic difference: constructing an - /// `ArithmeticEvaluator` builds the function and aggregate registries, and - /// `WHERE x = 5` evaluates its literal once per row. - /// - /// This replaced a literal-only reader that answered NULL for every other - /// shape, which made `WHERE a < b` UNKNOWN on every row (P46). There is - /// deliberately no catch-all that invents a value: an operand the - /// evaluator cannot handle is an error. - fn evaluate_operand_value( - &self, - expr: &SqlExpression, - row_index: usize, - ) -> Result> { - match expr { - SqlExpression::Column(column_ref) => { - let col_index = self.resolve_column_index(column_ref)?; - Ok(self.table.get_value(row_index, col_index).cloned()) - } - SqlExpression::StringLiteral(s) => Ok(Some(DataValue::String(s.clone()))), - SqlExpression::BooleanLiteral(b) => Ok(Some(DataValue::Boolean(*b))), - SqlExpression::Null => Ok(Some(DataValue::Null)), - SqlExpression::NumberLiteral(n) => Ok(Some(number_literal_value(n))), - SqlExpression::DateTimeConstructor { - year, - month, - day, - hour, - minute, - second, - } => { - let naive_date = NaiveDate::from_ymd_opt(*year, *month, *day) - .ok_or_else(|| anyhow::anyhow!("Invalid date: {}-{}-{}", year, month, day))?; - let naive_time = NaiveTime::from_hms_opt( - hour.unwrap_or(0), - minute.unwrap_or(0), - second.unwrap_or(0), - ) - .ok_or_else(|| anyhow::anyhow!("Invalid time"))?; - Ok(Some(datetime_value(NaiveDateTime::new( - naive_date, naive_time, - )))) - } - SqlExpression::DateTimeToday { - hour, - minute, - second, - } => { - let today = Local::now().date_naive(); - let time = NaiveTime::from_hms_opt( - hour.unwrap_or(0), - minute.unwrap_or(0), - second.unwrap_or(0), - ) - .ok_or_else(|| anyhow::anyhow!("Invalid time"))?; - Ok(Some(datetime_value(NaiveDateTime::new(today, time)))) - } - _ => { - reject_unlifted_window(expr)?; - Ok(Some(self.evaluate_arithmetic(expr, row_index)?)) - } - } + // slices 4 and 5): its operand readers, three-valued `compare_trilean`, + // `sql_like` and the registry's method functions are the one + // implementation, whatever shape either operand takes. + reject_unlifted_window(left)?; + reject_unlifted_window(right)?; + self.evaluate_delegated(expr, row_index) } /// Evaluate a value expression with the evaluation's shared @@ -757,202 +286,6 @@ impl<'a, 'exec> RecursiveWhereEvaluator<'a, 'exec> { .ok_or_else(|| anyhow!("Predicate did not evaluate to a truth value: {value:?}")) } - fn evaluate_method_call( - &self, - object: &str, - method: &str, - args: &[SqlExpression], - row_index: usize, - ) -> Result { - if row_index < 3 { - debug!( - "RecursiveWhereEvaluator: evaluate_method_call - object='{}', method='{}', row={}", - object, method, row_index - ); - } - - // Get column value - let col_index = self.table.get_column_index(object).ok_or_else(|| { - let suggestion = self.find_similar_column(object); - match suggestion { - Some(similar) => { - anyhow!("Column '{}' not found. Did you mean '{}'?", object, similar) - } - None => anyhow!("Column '{}' not found", object), - } - })?; - - let cell_value = self.table.get_value(row_index, col_index).cloned(); - if row_index < 3 { - debug!( - "RecursiveWhereEvaluator: Row {} column '{}' value = {:?}", - row_index, object, cell_value - ); - } - - match method.to_lowercase().as_str() { - "contains" => { - if args.len() != 1 { - return Err(anyhow::anyhow!("Contains requires exactly 1 argument")); - } - let search_str = self.extract_string_value(&args[0])?; - // Pre-compute lowercase once instead of for every row - let search_lower = search_str.to_lowercase(); - - // Type coercion: convert numeric values to strings for string methods - match cell_value { - Some(DataValue::String(ref s)) => { - let result = s.to_lowercase().contains(&search_lower); - // Only log first few rows to avoid performance impact - if row_index < 3 { - debug!("RecursiveWhereEvaluator: Row {} contains('{}') on '{}' = {} (case-insensitive)", row_index, search_str, s, result); - } - Ok(Trilean::from_bool(result)) - } - Some(DataValue::InternedString(ref s)) => { - let result = s.to_lowercase().contains(&search_lower); - // Only log first few rows to avoid performance impact - if row_index < 3 { - debug!("RecursiveWhereEvaluator: Row {} contains('{}') on interned '{}' = {} (case-insensitive)", row_index, search_str, s, result); - } - Ok(Trilean::from_bool(result)) - } - Some(DataValue::Integer(n)) => { - let str_val = n.to_string(); - let result = str_val.contains(&search_str); - if row_index < 3 { - debug!("RecursiveWhereEvaluator: Row {} contains('{}') on integer '{}' = {}", row_index, search_str, str_val, result); - } - Ok(Trilean::from_bool(result)) - } - Some(DataValue::Float(f)) => { - let str_val = f.to_string(); - let result = str_val.contains(&search_str); - if row_index < 3 { - debug!( - "RecursiveWhereEvaluator: Row {} contains('{}') on float '{}' = {}", - row_index, search_str, str_val, result - ); - } - Ok(Trilean::from_bool(result)) - } - Some(DataValue::Boolean(b)) => { - let str_val = b.to_string(); - let result = str_val.contains(&search_str); - if row_index < 3 { - debug!("RecursiveWhereEvaluator: Row {} contains('{}') on boolean '{}' = {}", row_index, search_str, str_val, result); - } - Ok(Trilean::from_bool(result)) - } - Some(DataValue::DateTime(dt)) => { - // DateTime columns can use string methods via coercion - let result = dt.contains(&search_str); - if row_index < 3 { - debug!("RecursiveWhereEvaluator: Row {} contains('{}') on datetime '{}' = {}", row_index, search_str, dt, result); - } - Ok(Trilean::from_bool(result)) - } - _ => { - if row_index < 3 { - debug!("RecursiveWhereEvaluator: Row {} contains('{}') on null/empty value = false", row_index, search_str); - } - Ok(Trilean::False) - } - } - } - "startswith" => { - if args.len() != 1 { - return Err(anyhow::anyhow!("StartsWith requires exactly 1 argument")); - } - let prefix = self.extract_string_value(&args[0])?; - - // Type coercion: convert numeric values to strings for string methods - match cell_value { - Some(DataValue::String(ref s)) => Ok(Trilean::from_bool( - s.to_lowercase().starts_with(&prefix.to_lowercase()), - )), - Some(DataValue::InternedString(ref s)) => Ok(Trilean::from_bool( - s.to_lowercase().starts_with(&prefix.to_lowercase()), - )), - Some(DataValue::Integer(n)) => { - Ok(Trilean::from_bool(n.to_string().starts_with(&prefix))) - } - Some(DataValue::Float(f)) => { - Ok(Trilean::from_bool(f.to_string().starts_with(&prefix))) - } - Some(DataValue::Boolean(b)) => { - Ok(Trilean::from_bool(b.to_string().starts_with(&prefix))) - } - Some(DataValue::DateTime(dt)) => { - Ok(Trilean::from_bool(dt.starts_with(&prefix))) - } - _ => Ok(Trilean::False), - } - } - "endswith" => { - if args.len() != 1 { - return Err(anyhow::anyhow!("EndsWith requires exactly 1 argument")); - } - let suffix = self.extract_string_value(&args[0])?; - - // Type coercion: convert numeric values to strings for string methods - match cell_value { - Some(DataValue::String(ref s)) => Ok(Trilean::from_bool( - s.to_lowercase().ends_with(&suffix.to_lowercase()), - )), - Some(DataValue::InternedString(ref s)) => Ok(Trilean::from_bool( - s.to_lowercase().ends_with(&suffix.to_lowercase()), - )), - Some(DataValue::Integer(n)) => { - Ok(Trilean::from_bool(n.to_string().ends_with(&suffix))) - } - Some(DataValue::Float(f)) => { - Ok(Trilean::from_bool(f.to_string().ends_with(&suffix))) - } - Some(DataValue::Boolean(b)) => { - Ok(Trilean::from_bool(b.to_string().ends_with(&suffix))) - } - Some(DataValue::DateTime(dt)) => Ok(Trilean::from_bool(dt.ends_with(&suffix))), - _ => Ok(Trilean::False), - } - } - _ => Err(anyhow::anyhow!("Unsupported method: {}", method)), - } - } - - /// The index of a column operand. With an `ExecutionContext` this is the - /// unified alias-aware resolution, and its answer is the index itself: it - /// used to be turned back into a bare name and looked up again, which - /// finds the first column of that name rather than the one the qualifier - /// picked. - fn resolve_column_index(&self, column_ref: &ColumnRef) -> Result { - if let Some(exec_ctx) = self.exec_context { - return exec_ctx.resolve_column_index(self.table, column_ref); - } - // Without a context: a dotted name is read as its last segment. - let column_name = match column_ref.name.rfind('.') { - Some(dot_pos) => &column_ref.name[dot_pos + 1..], - None => column_ref.name.as_str(), - }; - self.table.get_column_index(column_name).ok_or_else(|| { - match self.find_similar_column(column_name) { - Some(similar) => anyhow!( - "Column '{}' not found. Did you mean '{}'?", - column_name, - similar - ), - None => anyhow!("Column '{}' not found", column_name), - } - }) - } - - fn extract_string_value(&self, expr: &SqlExpression) -> Result { - match expr { - SqlExpression::StringLiteral(s) => Ok(s.clone()), - _ => Err(anyhow::anyhow!("Expected string literal, got: {:?}", expr)), - } - } - /// Evaluate a CASE expression as a boolean (for WHERE clauses) fn evaluate_case_expression_as_bool( &mut self, @@ -1074,12 +407,6 @@ fn reject_unlifted_window(expr: &SqlExpression) -> Result<()> { /// The binary predicates WHERE hands whole to the value evaluator: the /// comparisons and the NULL tests. Expects the operator already upper-cased. -fn delegates_to_value_evaluator(op_upper: &str) -> bool { - matches!( - op_upper, - "=" | "!=" | "<>" | "<" | "<=" | ">" | ">=" | "IS NULL" | "IS NOT NULL" | "LIKE" - ) -} /// Operators whose result is a truth value, handled by `evaluate_binary_op`'s /// comparison arms. Expects the operator already upper-cased. @@ -1090,22 +417,6 @@ fn is_predicate_operator(op_upper: &str) -> bool { ) } -/// A numeric literal as the WHERE path has always read it: whole numbers as -/// `Integer`, others as `Float`, and text that does not parse as a string. -fn number_literal_value(n: &str) -> DataValue { - match n.parse::() { - Ok(num) if num.fract() == 0.0 && num >= i64::MIN as f64 && num <= i64::MAX as f64 => { - DataValue::Integer(num as i64) - } - Ok(num) => DataValue::Float(num), - Err(_) => DataValue::String(n.to_string()), - } -} - -fn datetime_value(naive: NaiveDateTime) -> DataValue { - DataValue::DateTime(naive.format("%Y-%m-%d %H:%M:%S%.3f").to_string()) -} - #[cfg(test)] mod three_valued_logic_tests { //! Regression tests for P18/P19 — SQL three-valued logic in `WHERE`. diff --git a/src/sql/functions/string_methods.rs b/src/sql/functions/string_methods.rs index 0e262b5a..3e09c8e2 100644 --- a/src/sql/functions/string_methods.rs +++ b/src/sql/functions/string_methods.rs @@ -28,7 +28,7 @@ pub trait MethodFunction: SqlFunction { } } -/// The text a search function reads an operand as (R13 slice 5): `None` for +/// The text a string function reads an operand as (R13 slice 5): `None` for /// NULL, whose answer is then NULL (P56); any other value as the text sql-cli /// displays for it, so a number is searched as it looks (D2); lowercased under /// `--case-insensitive`, as `=` and LIKE compare (D3). @@ -149,11 +149,9 @@ impl SqlFunction for TrimMethod { fn evaluate(&self, args: &[DataValue]) -> Result { self.validate_args(args)?; - match &args[0] { - DataValue::String(s) => Ok(DataValue::String(s.trim().to_string())), - DataValue::InternedString(s) => Ok(DataValue::String(s.trim().to_string())), - DataValue::Null => Ok(DataValue::Null), - _ => Err(anyhow!("Trim expects a string argument")), + match search_text(&args[0], false) { + Some(text) => Ok(DataValue::String(text.trim().to_string())), + None => Ok(DataValue::Null), } } } @@ -189,11 +187,9 @@ impl SqlFunction for TrimStartMethod { fn evaluate(&self, args: &[DataValue]) -> Result { self.validate_args(args)?; - match &args[0] { - DataValue::String(s) => Ok(DataValue::String(s.trim_start().to_string())), - DataValue::InternedString(s) => Ok(DataValue::String(s.trim_start().to_string())), - DataValue::Null => Ok(DataValue::Null), - _ => Err(anyhow!("TrimStart expects a string argument")), + match search_text(&args[0], false) { + Some(text) => Ok(DataValue::String(text.trim_start().to_string())), + None => Ok(DataValue::Null), } } } @@ -229,11 +225,9 @@ impl SqlFunction for TrimEndMethod { fn evaluate(&self, args: &[DataValue]) -> Result { self.validate_args(args)?; - match &args[0] { - DataValue::String(s) => Ok(DataValue::String(s.trim_end().to_string())), - DataValue::InternedString(s) => Ok(DataValue::String(s.trim_end().to_string())), - DataValue::Null => Ok(DataValue::Null), - _ => Err(anyhow!("TrimEnd expects a string argument")), + match search_text(&args[0], false) { + Some(text) => Ok(DataValue::String(text.trim_end().to_string())), + None => Ok(DataValue::Null), } } } @@ -723,11 +717,9 @@ impl SqlFunction for TrimFunction { fn evaluate(&self, args: &[DataValue]) -> Result { self.validate_args(args)?; - match &args[0] { - DataValue::String(s) => Ok(DataValue::String(s.trim().to_string())), - DataValue::InternedString(s) => Ok(DataValue::String(s.trim().to_string())), - DataValue::Null => Ok(DataValue::Null), - _ => Err(anyhow!("TRIM expects a string argument")), + match search_text(&args[0], false) { + Some(text) => Ok(DataValue::String(text.trim().to_string())), + None => Ok(DataValue::Null), } } } diff --git a/tests/comparison/corpus/02_where.toml b/tests/comparison/corpus/02_where.toml index 8abaebcb..0bbf4abc 100644 --- a/tests/comparison/corpus/02_where.toml +++ b/tests/comparison/corpus/02_where.toml @@ -332,7 +332,6 @@ sql = "SELECT id, s FROM pattern_matching WHERE s LIKE 'a(b%' ORDER BY id" id = "where_not_method_null_receiver" data = "null_edges.csv" sql = "SELECT id FROM null_edges WHERE NOT label.contains('e') ORDER BY id" -expect = "DIFFER" # P56. A NULL label made `.contains()` FALSE, not NULL, so NOT selected it: # ids 2, 5, 10, 11, 12 came back alongside 4, 6, 8, 9. @@ -340,7 +339,6 @@ expect = "DIFFER" id = "where_method_contains_case_sensitive" data = "null_edges.csv" sql = "SELECT id FROM null_edges WHERE label IS NOT NULL AND NOT label.contains('E') ORDER BY id" -expect = "DIFFER" # D3. WHERE's `.contains()` ignored case whatever the switch said, so `'E'` # found delta, charlie, echo and foxtrot and NOT dropped them; `contains` is # case-sensitive, as `=` and LIKE are here outside --case-insensitive, so every @@ -358,7 +356,6 @@ sql = "SELECT id, label.contains('e') AS c FROM null_edges ORDER BY id" id = "where_method_other_than_legacy_on_left" data = "null_edges.csv" sql = "SELECT id FROM null_edges WHERE label.upper() = 'ECHO' ORDER BY id" -expect = "GAP" # P57. WHERE accepted only Length / IndexOf / Trim* on the left of a # comparison, so any other method failed the query: "Method 'upper' cannot be # used in comparisons". SELECT has always evaluated it. diff --git a/tests/evaluator_matrix_tests.rs b/tests/evaluator_matrix_tests.rs index 05064490..69ab3eeb 100644 --- a/tests/evaluator_matrix_tests.rs +++ b/tests/evaluator_matrix_tests.rs @@ -381,167 +381,21 @@ const EXPECTED_NUMBER_METHODS: &[(&str, &str)] = &[ ("n.Contains('5')", "TFUT"), ("n.StartsWith('1')", "FTUF"), ("n.Length() > 1", "FTUT"), + ("n.Trim() = '7.5'", "FFUT"), ("INSTR(n, '5') = 1", "TFUF"), ("CONTAINS(n, '.')", "FFUT"), ]; -// Recorded by R13 slice 5's pin (2026-09-26), grouped by fault; an observed -// string can combine two. The pin also recorded 27 value-evaluator and -// function-form entries - the registry's search functions answered FALSE for -// NULL, ignored --case-insensitive and rejected a number - which all went FIXED -// when those functions moved onto one `search_text` reader. What is left is -// WHERE's own method-call arms, which go when WHERE delegates method calls. -const NULL_RECEIVER: &str = "P56: WHERE's method over a NULL receiver answers FALSE (its Length / IndexOf / Trim* a non-NULL value) instead of NULL"; -const NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE: &str = - "P56 (NULL receiver answers FALSE) and D3 (WHERE's search methods always ignore case)"; -const WHERE_METHOD_ALLOWLIST: &str = "P57: on the left of a WHERE comparison only Length / IndexOf / Trim* are accepted; any other method fails the query"; - -const KNOWN_METHODS: &[Known] = &[ - Known { - evaluator: Evaluator::Where, - predicate: "s.ToUpper() = 'ABC'", - observed: "EEEE", - why: WHERE_METHOD_ALLOWLIST, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.Replace('a', 'z') = 'zbc'", - observed: "EEEE", - why: WHERE_METHOD_ALLOWLIST, - }, - Known { - evaluator: Evaluator::Where, - predicate: "t.s.Contains('b')", - observed: "TTFT", - why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.Contains('b')", - observed: "TTFT", - why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.Contains('B')", - observed: "TTFT", - why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, - }, - Known { - evaluator: Evaluator::Where, - predicate: "NOT s.Contains('b')", - observed: "FFTF", - why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.StartsWith('a')", - observed: "TTFF", - why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.EndsWith('C')", - observed: "TTFF", - why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.Length() > 3", - observed: "FFFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.IndexOf('b') = 1", - observed: "TTFF", - why: NULL_RECEIVER_AND_WHERE_ALWAYS_NOCASE, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.IndexOf('z') = -1", - observed: "TTTT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.Trim() = 'aB'", - observed: "FFFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.TrimStart() = 'aB '", - observed: "FFFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.TrimEnd() = ' aB'", - observed: "FFFT", - why: NULL_RECEIVER, - }, -]; - -const KNOWN_METHODS_CASE_INSENSITIVE: &[Known] = &[ - Known { - evaluator: Evaluator::Where, - predicate: "s.Contains('B')", - observed: "TTFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "NOT s.Contains('B')", - observed: "FFTF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.StartsWith('A')", - observed: "TTFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.EndsWith('c')", - observed: "TTFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.IndexOf('B') = 1", - observed: "TTFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "s.Trim() = 'AB'", - observed: "FFFT", - why: NULL_RECEIVER, - }, -]; - -const KNOWN_NUMBER_METHODS: &[Known] = &[ - Known { - evaluator: Evaluator::Where, - predicate: "n.Contains('5')", - observed: "TFFT", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "n.StartsWith('1')", - observed: "FTFF", - why: NULL_RECEIVER, - }, - Known { - evaluator: Evaluator::Where, - predicate: "n.Length() > 1", - observed: "FTFT", - why: NULL_RECEIVER, - }, -]; +// R13 slice 5 recorded 55 entries across these three lists (2026-09-26): P56 +// (a NULL receiver answered FALSE, in both evaluators), D3 (WHERE's search +// methods always ignored case; the registry's ignored the switch), the +// registry rejecting a number (D2), and P57 (WHERE refused any other method on +// the left of a comparison). The registry's went FIXED when its search +// functions moved onto one `search_text` reader; WHERE's when it delegated +// method calls to the value evaluator and its own arms were deleted. +const KNOWN_METHODS: &[Known] = &[]; +const KNOWN_METHODS_CASE_INSENSITIVE: &[Known] = &[]; +const KNOWN_NUMBER_METHODS: &[Known] = &[]; fn trilean_char(t: Trilean) -> char { match t { diff --git a/tests/method_evaluation_test.rs b/tests/method_evaluation_test.rs index bc453c9e..d3394395 100644 --- a/tests/method_evaluation_test.rs +++ b/tests/method_evaluation_test.rs @@ -247,16 +247,28 @@ fn test_indexof_with_greater_than() { #[test] fn test_case_sensitivity() { + // D3 (docs/SQL_PARITY.md): a method's search follows --case-insensitive, + // exactly as `=` and LIKE do. It used to ignore case in WHERE regardless. let table = create_test_table(); - let mut evaluator = RecursiveWhereEvaluator::new(&table); - - // Test that methods are case-insensitive for search strings - let where_clause1 = extract_where_clause("SELECT * FROM test WHERE name.Contains('WIDGET')"); - let where_clause2 = extract_where_clause("SELECT * FROM test WHERE name.Contains('widget')"); - let where_clause3 = extract_where_clause("SELECT * FROM test WHERE name.Contains('WiDgEt')"); - - // All should match the same row (Widget) - assert!(evaluator.evaluate(&where_clause1, 0).unwrap().is_true()); - assert!(evaluator.evaluate(&where_clause2, 0).unwrap().is_true()); - assert!(evaluator.evaluate(&where_clause3, 0).unwrap().is_true()); + let spellings = ["Widget", "widget", "WIDGET", "WiDgEt"]; + + let mut sensitive = RecursiveWhereEvaluator::new(&table); + for search in spellings { + let clause = extract_where_clause(&format!( + "SELECT * FROM test WHERE name.Contains('{search}')" + )); + let matched = sensitive.evaluate(&clause, 0).unwrap().is_true(); + assert_eq!(matched, search == "Widget", "default mode, '{search}'"); + } + + let mut insensitive = RecursiveWhereEvaluator::with_case_insensitive(&table, true); + for search in spellings { + let clause = extract_where_clause(&format!( + "SELECT * FROM test WHERE name.Contains('{search}')" + )); + assert!( + insensitive.evaluate(&clause, 0).unwrap().is_true(), + "--case-insensitive, '{search}'" + ); + } } diff --git a/tests/python_tests/test_string_methods_comprehensive.py b/tests/python_tests/test_string_methods_comprehensive.py index 4041f847..d3f4c838 100644 --- a/tests/python_tests/test_string_methods_comprehensive.py +++ b/tests/python_tests/test_string_methods_comprehensive.py @@ -23,13 +23,14 @@ def setup_class(cls): subprocess.run(["cargo", "build", "--release"], cwd=cls.project_root, check=True) - def run_query(self, csv_file: str, query: str): + def run_query(self, csv_file: str, query: str, extra_args=()): """Helper to run a SQL query""" cmd = [ self.sql_cli, str(self.project_root / "data" / csv_file), "-q", query, - "-o", "csv" + "-o", "csv", + *extra_args, ] result = subprocess.run(cmd, capture_output=True, text=True, timeout=5) @@ -112,11 +113,13 @@ def test_contains_finds_substring(self): assert set(df['id'].tolist()) == {1, 4, 7, 10} def test_contains_case_sensitive(self): - """Test Contains() - appears to be case-insensitive based on logs""" - df, _ = self.run_query("test_simple_strings.csv", - "SELECT id FROM test_simple_strings WHERE status.Contains('active')") - # Contains appears to be case-insensitive, so it matches Active and Inactive - assert len(df) == 7 # Active (1,3,5,7,9) + Inactive (2,8) + """Contains() follows --case-insensitive, like = and LIKE (D3)""" + query = "SELECT id FROM test_simple_strings WHERE status.Contains('active')" + df, _ = self.run_query("test_simple_strings.csv", query) + # Case-sensitive by default: only Inactive contains 'active' + assert set(df['id'].tolist()) == {2, 8} + df, _ = self.run_query("test_simple_strings.csv", query, ["--case-insensitive"]) + # Active (1,3,5,7,9) + Inactive (2,8) assert set(df['id'].tolist()) == {1, 2, 3, 5, 7, 8, 9} # STARTSWITH METHOD @@ -128,11 +131,13 @@ def test_startswith_prefix_check(self): assert df.iloc[0]['id'] == 1 def test_startswith_case_sensitive(self): - """Test StartsWith() - appears to be case-insensitive""" - df, _ = self.run_query("test_simple_strings.csv", - "SELECT id FROM test_simple_strings WHERE status.StartsWith('a')") - # StartsWith appears to be case-insensitive like Contains - assert len(df) == 6 # Active and Archived both start with 'a' or 'A' + """StartsWith() follows --case-insensitive, like = and LIKE (D3)""" + query = "SELECT id FROM test_simple_strings WHERE status.StartsWith('a')" + df, _ = self.run_query("test_simple_strings.csv", query) + # Case-sensitive by default: every status is capitalised + assert len(df) == 0 + df, _ = self.run_query("test_simple_strings.csv", query, ["--case-insensitive"]) + # Active and Archived assert set(df['id'].tolist()) == {1, 3, 5, 6, 7, 9} # ENDSWITH METHOD From 89032d87036ed85b11967fa8e48dceb60d9ce684 Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Sat, 26 Sep 2026 13:35:43 +0100 Subject: [PATCH 6/6] docs(r13): merge an orphaned doc comment left by the deletion Co-Authored-By: Claude Opus 5.5 (1M context) --- src/data/recursive_where_evaluator.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/data/recursive_where_evaluator.rs b/src/data/recursive_where_evaluator.rs index 49082c45..f9d81fcb 100644 --- a/src/data/recursive_where_evaluator.rs +++ b/src/data/recursive_where_evaluator.rs @@ -405,11 +405,9 @@ fn reject_unlifted_window(expr: &SqlExpression) -> Result<()> { } } -/// The binary predicates WHERE hands whole to the value evaluator: the -/// comparisons and the NULL tests. Expects the operator already upper-cased. - -/// Operators whose result is a truth value, handled by `evaluate_binary_op`'s -/// comparison arms. Expects the operator already upper-cased. +/// Operators whose result is a truth value - the comparisons, LIKE and the NULL +/// tests - which WHERE hands whole to the value evaluator. Expects the operator +/// already upper-cased. fn is_predicate_operator(op_upper: &str) -> bool { matches!( op_upper,