diff --git a/crates/core/src/expression/evaluator.rs b/crates/core/src/expression/evaluator.rs index e023af96e..cee1186ea 100755 --- a/crates/core/src/expression/evaluator.rs +++ b/crates/core/src/expression/evaluator.rs @@ -141,6 +141,23 @@ fn placeholder_numeric<'a>( } } +/// Order two values of the same orderable type: `S` and `B` bytewise, `N` +/// numerically. Returns `None` for any other pair (different types, or a type +/// with no order). +#[must_use] +pub fn compare_orderable(left: &AttributeValue, right: &AttributeValue) -> Option { + match (left, right) { + (AttributeValue::S(l), AttributeValue::S(r)) => Some(l.cmp(r)), + (AttributeValue::B(l), AttributeValue::B(r)) => Some(l.cmp(r)), + (AttributeValue::N(l), AttributeValue::N(r)) => { + let l = l.parse::().ok()?; + let r = r.parse::().ok()?; + Some(l.cmp(&r)) + } + _ => None, + } +} + /// Compare two `AttributeValue`s using the given operator. /// /// `DynamoDB` comparison rules: diff --git a/crates/core/src/expression/key_condition.rs b/crates/core/src/expression/key_condition.rs index edaa1f8e5..d8fb7122d 100755 --- a/crates/core/src/expression/key_condition.rs +++ b/crates/core/src/expression/key_condition.rs @@ -807,12 +807,19 @@ fn parse_key_clause_inner(tokens: &[Token], pos: &mut usize) -> Result Result { - // size(path) is a function that returns a value, usable as an operand - if name.eq_ignore_ascii_case("size") - && *pos + 1 < tokens.len() - && tokens[*pos + 1] == Token::LParen - { + Token::Ident(_) => { + if *pos + 1 < tokens.len() && tokens[*pos + 1] == Token::LParen { return parse_function_call(tokens, pos); } let elements = parser_common::parse_path(tokens, pos)?; @@ -230,7 +227,7 @@ fn parse_operand(tokens: &[Token], pos: &mut usize) -> Result Result { let name = match &tokens[*pos] { - Token::Ident(n) => n.to_ascii_lowercase(), + Token::Ident(n) => n.clone(), _ => { return Err(validation_err( "Invalid ConditionExpression: expected function name", @@ -241,13 +238,22 @@ fn parse_function_call(tokens: &[Token], pos: &mut usize) -> Result Option { } } -fn is_function_name(name: &str) -> bool { - matches!( - name, - "attribute_exists" | "attribute_not_exists" | "attribute_type" | "begins_with" | "contains" - ) -} - fn validation_err(msg: &str) -> DynamoDbError { DynamoDbError::ValidationException(msg.to_owned()) } diff --git a/crates/core/src/expression/projection.rs b/crates/core/src/expression/projection.rs index fd84f4c5f..ee90d5201 100755 --- a/crates/core/src/expression/projection.rs +++ b/crates/core/src/expression/projection.rs @@ -125,13 +125,13 @@ fn prefix_projection_error(err: DynamoDbError) -> DynamoDbError { /// A path element after `#name` resolution. #[derive(PartialEq, Eq)] -enum ResolvedElement { +pub(crate) enum ResolvedElement { Attr(String), Index(usize), } /// Resolve `#name` references in parsed paths, rejecting index-start paths. -fn resolve_paths( +pub(crate) fn resolve_paths( paths: &[Vec], maps: &ExpressionMaps, ) -> Result>, DynamoDbError> { @@ -166,11 +166,20 @@ fn resolve_paths( /// Reject paths where one is a prefix of another (DynamoDB's overlap rule), /// reporting the first offending pair in expression order. Siblings are fine. fn check_overlap(resolved: &[Vec]) -> Result<(), DynamoDbError> { + check_overlap_for(resolved, super::ExpressionKind::Projection) +} + +/// The overlap rule for any expression kind that names document paths; the +/// message carries the kind's `Invalid :` prefix. +pub(crate) fn check_overlap_for( + resolved: &[Vec], + kind: super::ExpressionKind, +) -> Result<(), DynamoDbError> { for i in 0..resolved.len() { for j in (i + 1)..resolved.len() { if is_prefix(&resolved[i], &resolved[j]) || is_prefix(&resolved[j], &resolved[i]) { return Err(DynamoDbError::ValidationException(format!( - "Invalid ProjectionExpression: Two document paths overlap with each other; \ + "Invalid {kind}: Two document paths overlap with each other; \ must remove or rewrite one of these paths; path one: {}, path two: {}", render_path(&resolved[i]), render_path(&resolved[j]), diff --git a/crates/core/src/expression/resolver.rs b/crates/core/src/expression/resolver.rs index fa14081ec..c7a5f9141 100755 --- a/crates/core/src/expression/resolver.rs +++ b/crates/core/src/expression/resolver.rs @@ -273,33 +273,79 @@ pub fn validate_unused_attributes( collect_action_refs(action, &mut used_names, &mut used_values); } - for key in names.keys() { - if !used_names.contains(key.strip_prefix('#').unwrap_or(key)) { - let display_key = if key.starts_with('#') { + // The service lists every unused key in one set (`keys: {:x, :y, :z}`). + // Its order is a hash artifact no client can depend on; the keys are + // sorted here so the message is the same on every backend and process. + let mut unused_names: Vec = names + .keys() + .filter(|key| !used_names.contains(key.strip_prefix('#').unwrap_or(key))) + .map(|key| { + if key.starts_with('#') { key.clone() } else { format!("#{key}") - }; - return Err(DynamoDbError::ValidationException(format!( - "Value provided in ExpressionAttributeNames unused in expressions: keys: {{{display_key}}}" - ))); - } + } + }) + .collect(); + if !unused_names.is_empty() { + unused_names.sort(); + return Err(DynamoDbError::ValidationException(format!( + "Value provided in ExpressionAttributeNames unused in expressions: keys: {{{}}}", + unused_names.join(", ") + ))); } - for key in values.keys() { - if !used_values.contains(key.strip_prefix(':').unwrap_or(key)) { - let display_key = if key.starts_with(':') { + let mut unused_values: Vec = values + .keys() + .filter(|key| !used_values.contains(key.strip_prefix(':').unwrap_or(key))) + .map(|key| { + if key.starts_with(':') { key.clone() } else { format!(":{key}") - }; - return Err(DynamoDbError::ValidationException(format!( - "Value provided in ExpressionAttributeValues unused in expressions: keys: {{{display_key}}}" - ))); - } + } + }) + .collect(); + if !unused_values.is_empty() { + unused_values.sort(); + return Err(DynamoDbError::ValidationException(format!( + "Value provided in ExpressionAttributeValues unused in expressions: keys: {{{}}}", + unused_values.join(", ") + ))); } Ok(()) } +/// Reject an update expression whose actions name overlapping document paths. +/// +/// Amazon DynamoDB refuses two actions whose paths are equal or where one is a +/// prefix of the other, across all four sections, after `#name` resolution: +/// `SET a = :v, a = :w`, `SET m = :v, m.x = :w`, `SET a.b = :v REMOVE a`, and +/// `SET #l[1] = :v, #l[1].x = :w` with `#l` bound to `lst` all fail. Siblings +/// (`m.x` and `m.y`) are fine. The first offending pair in expression order is +/// named, rendered as the service renders it: `[lst, [1]]`. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the `Invalid +/// UpdateExpression:` prefix. +pub fn validate_update_paths_disjoint( + actions: &[super::ast::UpdateAction], + maps: &ExpressionMaps, +) -> Result<(), DynamoDbError> { + use super::ast::UpdateAction; + let paths: Vec> = actions + .iter() + .map(|action| match action { + UpdateAction::Set { path, .. } + | UpdateAction::Remove { path } + | UpdateAction::Add { path, .. } + | UpdateAction::Delete { path, .. } => path.clone(), + }) + .collect(); + let resolved = super::projection::resolve_paths(&paths, maps)?; + super::projection::check_overlap_for(&resolved, super::ExpressionKind::Update) +} + fn collect_expr_refs( expr: &super::ast::Expr, names: &mut std::collections::HashSet, @@ -490,7 +536,7 @@ pub fn validate_begins_with_operands( /// Return the DynamoDB type code for an AttributeValue (S, N, B, BOOL, NULL, /// L, M, SS, NS, BS). Used in operand-type validation messages. -pub(crate) fn attribute_type_code(val: &AttributeValue) -> &'static str { +pub fn attribute_type_code(val: &AttributeValue) -> &'static str { match val { AttributeValue::S(_) => "S", AttributeValue::N(_) => "N", @@ -848,4 +894,38 @@ mod tests { collect_value_placeholders(&expr, &mut out); assert!(out.is_empty()); } + + // The service lists every unused key in one set, in an order that is a hash + // artifact. ExtendDB sorts the keys so the message is the same on every + // backend and process; this is ExtendDB's own choice, not a measurement. + // Each round builds fresh maps, which iterate in a fresh random order, so a + // missing sort cannot pass by luck. + #[test] + fn unused_keys_are_listed_sorted() { + let none = std::collections::HashSet::new(); + let unused = |names: &HashMap, values: &HashMap| { + match validate_unused_attributes(names, values, &[], &[], &none, &none) { + Err(DynamoDbError::ValidationException(msg)) => msg, + other => panic!("unexpected {other:?}"), + } + }; + for _ in 0..20 { + let names: HashMap = ["#z", "#a", "#m"] + .iter() + .map(|k| ((*k).to_owned(), "x".to_owned())) + .collect(); + let values: HashMap = [":z", ":a", ":m"] + .iter() + .map(|k| ((*k).to_owned(), AttributeValue::N("1".to_owned()))) + .collect(); + assert_eq!( + unused(&HashMap::new(), &values), + "Value provided in ExpressionAttributeValues unused in expressions: keys: {:a, :m, :z}" + ); + assert_eq!( + unused(&names, &HashMap::new()), + "Value provided in ExpressionAttributeNames unused in expressions: keys: {#a, #m, #z}" + ); + } + } } diff --git a/crates/core/src/expression/semantics/condition_rules.rs b/crates/core/src/expression/semantics/condition_rules.rs new file mode 100644 index 000000000..a7a55ce93 --- /dev/null +++ b/crates/core/src/expression/semantics/condition_rules.rs @@ -0,0 +1,159 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The condition-expression rules of the semantic pass. See the module +//! documentation in the parent module for the order the service checks them in. + +use super::super::ast::Expr; +use super::super::kind::ExpressionKind; +use super::super::resolver::{ + ExpressionMaps, attribute_type_code, validate_ordering_operand_types, +}; +use super::errors::{ + invalid_function_name, not_allowed_this_way, operand_count_err, operand_type_err, + requires_document_path, validation, with_kind, +}; +use super::refs::{check_path_refs, check_value_defined}; +use super::{CONDITION_FUNCTIONS, UPDATE_FUNCTIONS}; +use crate::error::DynamoDbError; +use crate::types::AttributeValue; + +/// Validate a parsed condition, filter, or key-condition tree against the +/// service's semantic rules, in the service's order. +/// +/// `kind` names the parameter in the messages (`Invalid ConditionExpression:`, +/// `Invalid FilterExpression:`). +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` for the first rule the +/// expression breaks in the order described in the module documentation. +pub fn validate_condition_semantics( + expr: &Expr, + maps: &ExpressionMaps, + kind: ExpressionKind, +) -> Result<(), DynamoDbError> { + walk_condition(expr, maps, kind) +} + +/// A node in boolean position: the root, or an operand of AND, OR, NOT. +fn walk_condition( + expr: &Expr, + maps: &ExpressionMaps, + kind: ExpressionKind, +) -> Result<(), DynamoDbError> { + match expr { + Expr::And(l, r) | Expr::Or(l, r) => { + walk_condition(l, maps, kind)?; + walk_condition(r, maps, kind) + } + Expr::Not(inner) => walk_condition(inner, maps, kind), + Expr::Compare { left, right, .. } => { + walk_condition_operand(left, maps, kind)?; + walk_condition_operand(right, maps, kind)?; + validate_ordering_operand_types(expr, maps).map_err(|e| with_kind(e, kind)) + } + Expr::Between { operand, low, high } => { + walk_condition_operand(operand, maps, kind)?; + walk_condition_operand(low, maps, kind)?; + walk_condition_operand(high, maps, kind)?; + validate_ordering_operand_types(expr, maps).map_err(|e| with_kind(e, kind)) + } + Expr::In { operand, list } => { + walk_condition_operand(operand, maps, kind)?; + for item in list { + walk_condition_operand(item, maps, kind)?; + } + Ok(()) + } + Expr::Function { name, args } => walk_condition_function(name, args, maps, kind, false), + // A bare path or placeholder in condition position is a syntax error + // the grammar pass already reported; `Expected` desugaring never + // produces one either. + Expr::Path(_) | Expr::Placeholder(_) | Expr::Arithmetic { .. } => Ok(()), + } +} + +/// A node in operand position: a side of a comparison, a BETWEEN bound, an +/// IN member, or a function operand. +fn walk_condition_operand( + expr: &Expr, + maps: &ExpressionMaps, + kind: ExpressionKind, +) -> Result<(), DynamoDbError> { + match expr { + Expr::Path(path) => check_path_refs(path, maps, kind), + Expr::Placeholder(name) => check_value_defined(name, maps, kind), + Expr::Function { name, args } => walk_condition_function(name, args, maps, kind, true), + _ => Ok(()), + } +} + +/// The checks on a condition function call, after its operands: the name, +/// then its position (`size` returns a value and is refused in condition +/// position; the five boolean functions are refused in operand position: +/// `attribute_type(pk, :v) = :v` reports the position, not the operand type), +/// then the operand count, the document-path rule, and the operand types. +fn walk_condition_function( + name: &str, + args: &[Expr], + maps: &ExpressionMaps, + kind: ExpressionKind, + operand_position: bool, +) -> Result<(), DynamoDbError> { + for arg in args { + walk_condition_operand(arg, maps, kind)?; + } + if !CONDITION_FUNCTIONS.contains(&name) { + if UPDATE_FUNCTIONS.contains(&name) { + return Err(validation( + kind, + &format!("The function is not allowed in a condition expression; function: {name}"), + )); + } + return Err(invalid_function_name(kind, name)); + } + if operand_position != (name == "size") { + return Err(not_allowed_this_way(kind, name)); + } + let expected = match name { + "attribute_exists" | "attribute_not_exists" | "size" => 1, + _ => 2, + }; + if args.len() != expected { + return Err(operand_count_err(kind, name, args.len())); + } + if matches!(name, "attribute_exists" | "attribute_not_exists") + && !matches!(args[0], Expr::Path(_)) + { + return Err(requires_document_path(kind, name)); + } + if name == "begins_with" + && let Expr::Placeholder(p) = &args[1] + && let Some(val) = maps.values.get(p) + && !matches!(val, AttributeValue::S(_) | AttributeValue::B(_)) + { + return Err(operand_type_err(kind, name, attribute_type_code(val))); + } + if name == "attribute_type" + && let Expr::Placeholder(p) = &args[1] + && let Some(val) = maps.values.get(p) + { + match val { + AttributeValue::S(type_name) => { + const VALID: [&str; 10] = + ["S", "SS", "N", "NS", "B", "BS", "BOOL", "NULL", "L", "M"]; + if !VALID.contains(&type_name.as_str()) { + return Err(validation( + kind, + &format!( + "Invalid attribute type name found; type: {type_name}, valid types: {{S,SS,N,NS,B,BS,BOOL,NULL,L,M}}" + ), + )); + } + } + other => return Err(operand_type_err(kind, name, attribute_type_code(other))), + } + } + Ok(()) +} diff --git a/crates/core/src/expression/semantics/errors.rs b/crates/core/src/expression/semantics/errors.rs new file mode 100644 index 000000000..230a26ea5 --- /dev/null +++ b/crates/core/src/expression/semantics/errors.rs @@ -0,0 +1,59 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The message builders for the semantic rules. Each one carries the +//! service's text for one rule, with the parameter name `kind` supplies. + +use super::super::kind::ExpressionKind; +use crate::error::DynamoDbError; + +pub(super) fn validation(kind: ExpressionKind, msg: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!("Invalid {kind}: {msg}")) +} + +pub(super) fn with_kind(err: DynamoDbError, kind: ExpressionKind) -> DynamoDbError { + match err { + DynamoDbError::ValidationException(msg) if !msg.starts_with("Invalid ") => { + validation(kind, &msg) + } + other => other, + } +} + +pub(super) fn invalid_function_name(kind: ExpressionKind, name: &str) -> DynamoDbError { + validation(kind, &format!("Invalid function name; function: {name}")) +} + +pub(super) fn not_allowed_this_way(kind: ExpressionKind, name: &str) -> DynamoDbError { + validation( + kind, + &format!( + "The function is not allowed to be used this way in an expression; function: {name}" + ), + ) +} + +pub(super) fn operand_count_err(kind: ExpressionKind, name: &str, count: usize) -> DynamoDbError { + validation( + kind, + &format!( + "Incorrect number of operands for operator or function; operator or function: {name}, number of operands: {count}" + ), + ) +} + +pub(super) fn requires_document_path(kind: ExpressionKind, name: &str) -> DynamoDbError { + validation( + kind, + &format!("Operator or function requires a document path; operator or function: {name}"), + ) +} + +pub(super) fn operand_type_err(kind: ExpressionKind, name: &str, type_code: &str) -> DynamoDbError { + validation( + kind, + &format!( + "Incorrect operand type for operator or function; operator or function: {name}, operand type: {type_code}" + ), + ) +} diff --git a/crates/core/src/expression/semantics/mod.rs b/crates/core/src/expression/semantics/mod.rs new file mode 100644 index 000000000..07b4fd5aa --- /dev/null +++ b/crates/core/src/expression/semantics/mod.rs @@ -0,0 +1,79 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Semantic validation of parsed `UpdateExpression` and `ConditionExpression` +//! trees, run after the grammar pass and before any table is read. +//! +//! Amazon DynamoDB reports every syntax error first, then walks the parsed +//! tree once, in expression order, checking each node after its operands +//! (a post-order walk), and only then checks overlapping paths and unused +//! names and values. The order of the messages below was measured against +//! the service on 2026-10-14; each rule names the request that pinned it. +//! +//! Update expression, per node: +//! - a document path: every `#name` element is defined; +//! - a `:value` placeholder: defined; +//! - `+` and `-`: after both operands, a placeholder operand must be a +//! number (`SET a = :s + :n` with a string `:s`); +//! - a function call: after its operands, `list_append` may not be an +//! operand of a function or of `+`/`-` (`if_not_exists(list_append(:a, +//! :b), :v)`); the name must be `if_not_exists` or `list_append` in that +//! case (`IF_NOT_EXISTS(x, :v)` is refused); a condition function is +//! refused by name (`SET x = size(x)`); both take two operands; the first +//! operand of `if_not_exists` is a document path; a placeholder operand of +//! `list_append` is a list; +//! - an `ADD` or `DELETE` action: after the path and the placeholder, the +//! value is a number or a set for `ADD` and a set for `DELETE`; +//! - a section: after its actions, the keyword may not have appeared before +//! (`SET a = :v SET b = :q` reports `:q`, `SET a = :v SET b = :v ADD n :s` +//! reports the repeated `SET`). +//! +//! Condition expression, per node: +//! - operands of a comparison, `BETWEEN`, or `IN` are walked first; a +//! function in operand position must be `size`, and `size` in condition +//! position is refused (`attribute_exists(pk) = :v`, `size(pk)`); +//! - a function call: after its operands, the name must be one of the six +//! condition functions as written (`ATTRIBUTE_EXISTS(pk)` is refused), +//! `if_not_exists` and `list_append` are refused by name, the operand count +//! is fixed per function, `attribute_exists` and `attribute_not_exists` +//! take a document path (`attribute_type(:v, :t)` is accepted), the second operand of +//! `begins_with` is a string or binary, and the second operand of +//! `attribute_type` is a string naming a type; +//! - a comparison or `BETWEEN` with an ordering operator: a placeholder +//! operand is a string, number, or binary. + +const CONDITION_FUNCTIONS: [&str; 6] = [ + "attribute_exists", + "attribute_not_exists", + "attribute_type", + "begins_with", + "contains", + "size", +]; + +const UPDATE_FUNCTIONS: [&str; 2] = ["if_not_exists", "list_append"]; + +/// The service's refusal of `name(` in a `KeyConditionExpression` clause +/// (measured 2026-10-27). `size` is a nested operation, the other condition +/// functions are not key condition operators (the message names no function), +/// and any other name, in any case, is not a function (`Begins_With`, `foo`). +/// `begins_with` itself is accepted and never reaches this. +pub(super) fn key_condition_function_refusal(name: &str) -> String { + if name == "size" { + "KeyConditionExpressions cannot contain nested operations".to_owned() + } else if CONDITION_FUNCTIONS.contains(&name) { + "Invalid operator used in KeyConditionExpression".to_owned() + } else { + format!("Invalid function name; function: {name}") + } +} + +mod condition_rules; +mod errors; +mod refs; +#[cfg(test)] +mod tests; +mod update_rules; + +pub use condition_rules::validate_condition_semantics; +pub use update_rules::validate_update_semantics; diff --git a/crates/core/src/expression/semantics/refs.rs b/crates/core/src/expression/semantics/refs.rs new file mode 100644 index 000000000..157e60573 --- /dev/null +++ b/crates/core/src/expression/semantics/refs.rs @@ -0,0 +1,49 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The name and value usage checks shared by both passes: every `#name` +//! element of a document path, and every `:value` placeholder, is defined. + +use super::super::ast::PathElement; +use super::super::kind::ExpressionKind; +use super::super::resolver::ExpressionMaps; +use super::errors::validation; +use crate::error::DynamoDbError; + +pub(super) fn check_path_refs( + path: &[PathElement], + maps: &ExpressionMaps, + kind: ExpressionKind, +) -> Result<(), DynamoDbError> { + for element in path { + if let PathElement::Attribute(name) = element + && let Some(ref_name) = name.strip_prefix('#') + && !maps.names.contains_key(ref_name) + { + return Err(validation( + kind, + &format!( + "An expression attribute name used in the document path is not defined; attribute name: #{ref_name}" + ), + )); + } + } + Ok(()) +} + +pub(super) fn check_value_defined( + name: &str, + maps: &ExpressionMaps, + kind: ExpressionKind, +) -> Result<(), DynamoDbError> { + if maps.values.contains_key(name) { + Ok(()) + } else { + Err(validation( + kind, + &format!( + "An expression attribute value used in expression is not defined; attribute value: :{name}" + ), + )) + } +} diff --git a/crates/core/src/expression/semantics/tests.rs b/crates/core/src/expression/semantics/tests.rs new file mode 100644 index 000000000..52aae7b06 --- /dev/null +++ b/crates/core/src/expression/semantics/tests.rs @@ -0,0 +1,434 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The semantic-pass tests. Each one names the request that pinned it. + +use super::{validate_condition_semantics, validate_update_semantics}; +use crate::error::DynamoDbError; +use crate::expression::kind::ExpressionKind; +use crate::expression::parser::parse_condition; +use crate::expression::resolver::ExpressionMaps; +use crate::expression::tokenizer::tokenize; +use crate::expression::update_parser::parse_update_sections; +use crate::types::AttributeValue; +use std::collections::HashMap; + +const U: &str = "Invalid UpdateExpression: "; +const C: &str = "Invalid ConditionExpression: "; + +fn maps(names: &[(&str, &str)], values: &[(&str, AttributeValue)]) -> ExpressionMaps { + ExpressionMaps::new( + names + .iter() + .map(|(k, v)| (k.trim_start_matches('#').to_owned(), (*v).to_owned())) + .collect::>(), + values + .iter() + .map(|(k, v)| (k.trim_start_matches(':').to_owned(), v.clone())) + .collect::>(), + ) +} + +fn n() -> AttributeValue { + AttributeValue::N("1".to_owned()) +} +fn s() -> AttributeValue { + AttributeValue::S("x".to_owned()) +} +fn l() -> AttributeValue { + AttributeValue::L(vec![]) +} + +fn update_err(expr: &str, m: &ExpressionMaps) -> String { + let tokens = tokenize(expr).unwrap(); + let parsed = parse_update_sections(&tokens, Some(expr)).unwrap(); + match validate_update_semantics(&parsed, m) { + Ok(()) => "OK".to_owned(), + Err(DynamoDbError::ValidationException(msg)) => msg, + Err(other) => panic!("unexpected {other:?}"), + } +} + +fn condition_err(expr: &str, m: &ExpressionMaps) -> String { + let tokens = tokenize(expr).unwrap(); + let parsed = parse_condition(&tokens).unwrap(); + match validate_condition_semantics(&parsed, m, ExpressionKind::Condition) { + Ok(()) => "OK".to_owned(), + Err(DynamoDbError::ValidationException(msg)) => msg, + Err(other) => panic!("unexpected {other:?}"), + } +} + +// Every expectation below is a response captured from Amazon DynamoDB +// on 2026-10-14 (issue #108 notes, probe-fix-round-*.py). + +#[test] +fn update_refs_in_expression_order() { + let m = maps(&[], &[(":v", n())]); + assert_eq!( + update_err("SET a = :q, #n = :v", &m), + format!( + "{U}An expression attribute value used in expression is not defined; attribute value: :q" + ) + ); + assert_eq!( + update_err("SET #n = :q", &m), + format!( + "{U}An expression attribute name used in the document path is not defined; attribute name: #n" + ) + ); +} + +#[test] +fn repeated_section_after_its_own_actions() { + let m = maps(&[], &[(":v", n()), (":s", s()), (":n", n())]); + assert_eq!( + update_err("SET a = :v SET b = :q", &m), + format!( + "{U}An expression attribute value used in expression is not defined; attribute value: :q" + ) + ); + assert_eq!( + update_err("SET a = :v SET b = :v ADD n :s", &m), + format!("{U}The \"SET\" section can only be used once in an update expression;") + ); + assert_eq!( + update_err("ADD n :s ADD m :n", &m), + format!( + "{U}Incorrect operand type for operator or function; operator: ADD, operand type: STRING, typeSet: ALLOWED_FOR_ADD_OPERAND" + ) + ); + assert_eq!( + update_err("SET a = :n SET b = :s + :n", &m), + format!( + "{U}Incorrect operand type for operator or function; operator or function: +, operand type: S" + ) + ); +} + +#[test] +fn function_names_are_case_sensitive() { + let m = maps(&[], &[(":v", n())]); + assert_eq!( + update_err("SET x = IF_NOT_EXISTS(x, :v)", &m), + format!("{U}Invalid function name; function: IF_NOT_EXISTS") + ); + assert_eq!( + update_err("SET x = foo(x, :v)", &m), + format!("{U}Invalid function name; function: foo") + ); + assert_eq!( + update_err("SET x = attribute_exists(x)", &m), + format!( + "{U}The function is not allowed in an update expression; function: attribute_exists" + ) + ); + assert_eq!( + update_err("SET x = IF_NOT_EXISTS(x, :q)", &m), + format!( + "{U}An expression attribute value used in expression is not defined; attribute value: :q" + ) + ); + assert_eq!(update_err("SET x = if_not_exists(x, :v)", &m), "OK"); +} + +#[test] +fn nested_list_append_before_its_siblings() { + let m = maps(&[], &[(":a", l()), (":b", l()), (":n", n())]); + assert_eq!( + update_err("SET l = if_not_exists(list_append(:a, :b), :q)", &m), + format!( + "{U}The function is not allowed to be used this way in an expression; function: list_append" + ) + ); + assert_eq!( + update_err("SET l = if_not_exists(list_append(:q, :b), :a)", &m), + format!( + "{U}An expression attribute value used in expression is not defined; attribute value: :q" + ) + ); + assert_eq!( + update_err("SET l = list_append(:a, :a) + :n", &m), + format!( + "{U}The function is not allowed to be used this way in an expression; function: list_append" + ) + ); + assert_eq!( + update_err("SET l = list_append(if_not_exists(l, :a), :b)", &m), + "OK" + ); +} + +#[test] +fn operand_counts_and_document_path() { + let m = maps(&[], &[(":a", l()), (":s", s())]); + assert_eq!( + update_err("SET a = if_not_exists(:a)", &m), + format!( + "{U}Incorrect number of operands for operator or function; operator or function: if_not_exists, number of operands: 1" + ) + ); + assert_eq!( + update_err("SET a = if_not_exists(a, :q, :q2)", &m), + format!( + "{U}An expression attribute value used in expression is not defined; attribute value: :q" + ) + ); + assert_eq!( + update_err("SET l = if_not_exists(:a, :a)", &m), + format!( + "{U}Operator or function requires a document path; operator or function: if_not_exists" + ) + ); + assert_eq!( + update_err("SET l = list_append(:s, :a)", &m), + format!( + "{U}Incorrect operand type for operator or function; operator or function: list_append, operand type: S" + ) + ); +} + +#[test] +fn add_and_delete_operand_types() { + let m = maps( + &[], + &[ + (":s", s()), + (":n", n()), + (":b", AttributeValue::B(vec![1])), + ( + ":ss", + AttributeValue::SS(std::iter::once("a".to_owned()).collect()), + ), + (":l", l()), + ], + ); + assert_eq!( + update_err("ADD n :s", &m), + format!( + "{U}Incorrect operand type for operator or function; operator: ADD, operand type: STRING, typeSet: ALLOWED_FOR_ADD_OPERAND" + ) + ); + assert_eq!( + update_err("ADD n :b", &m), + format!( + "{U}Incorrect operand type for operator or function; operator: ADD, operand type: Binary, typeSet: ALLOWED_FOR_ADD_OPERAND" + ) + ); + assert_eq!( + update_err("DELETE s :n", &m), + format!( + "{U}Incorrect operand type for operator or function; operator: DELETE, operand type: NUMBER, typeSet: ALLOWED_FOR_ADD_OPERAND" + ) + ); + assert_eq!( + update_err("DELETE s :l", &m), + format!( + "{U}Incorrect operand type for operator or function; operator: DELETE, operand type: LIST, typeSet: ALLOWED_FOR_ADD_OPERAND" + ) + ); + assert_eq!(update_err("ADD n :n, s :ss DELETE t :ss", &m), "OK"); + assert_eq!( + update_err("ADD n :q", &m), + format!( + "{U}An expression attribute value used in expression is not defined; attribute value: :q" + ) + ); +} + +#[test] +fn overlap_after_the_walk() { + let m = maps(&[], &[(":v", n()), (":s", s())]); + assert_eq!( + update_err("SET a = :v, a.b = :v ADD n :s", &m), + format!( + "{U}Incorrect operand type for operator or function; operator: ADD, operand type: STRING, typeSet: ALLOWED_FOR_ADD_OPERAND" + ) + ); + assert_eq!( + update_err("SET a = :v, a.b = :v", &m), + format!( + "{U}Two document paths overlap with each other; must remove or rewrite one of these paths; path one: [a], path two: [a, b]" + ) + ); +} + +#[test] +fn condition_function_names_and_positions() { + let m = maps(&[], &[(":v", s()), (":n", n())]); + assert_eq!( + condition_err("ATTRIBUTE_EXISTS(pk)", &m), + format!("{C}Invalid function name; function: ATTRIBUTE_EXISTS") + ); + assert_eq!( + condition_err("foo(pk) AND bar(pk)", &m), + format!("{C}Invalid function name; function: foo") + ); + assert_eq!( + condition_err("if_not_exists(a, :v) = :v", &m), + format!( + "{C}The function is not allowed in a condition expression; function: if_not_exists" + ) + ); + assert_eq!( + condition_err("attribute_exists(pk) = :v", &m), + format!( + "{C}The function is not allowed to be used this way in an expression; function: attribute_exists" + ) + ); + assert_eq!( + condition_err("size(pk)", &m), + format!( + "{C}The function is not allowed to be used this way in an expression; function: size" + ) + ); + assert_eq!( + condition_err("NOT size(pk)", &m), + format!( + "{C}The function is not allowed to be used this way in an expression; function: size" + ) + ); + assert_eq!( + condition_err("size(attribute_exists(pk)) > :n", &m), + format!( + "{C}The function is not allowed to be used this way in an expression; function: attribute_exists" + ) + ); + assert_eq!( + condition_err("attribute_exists(size(pk))", &m), + format!( + "{C}Operator or function requires a document path; operator or function: attribute_exists" + ) + ); + assert_eq!( + condition_err("attribute_exists(:v)", &m), + format!( + "{C}Operator or function requires a document path; operator or function: attribute_exists" + ) + ); + assert_eq!( + condition_err("foo(#n)", &m), + format!( + "{C}An expression attribute name used in the document path is not defined; attribute name: #n" + ) + ); + assert_eq!( + condition_err(":q = #n", &m), + format!( + "{C}An expression attribute value used in expression is not defined; attribute value: :q" + ) + ); + assert_eq!( + condition_err("attribute_type(pk, :v) = :v", &m), + format!( + "{C}The function is not allowed to be used this way in an expression; function: attribute_type" + ) + ); + assert_eq!(condition_err("size(pk) = size(pk)", &m), "OK"); + assert_eq!(condition_err("begins_with(:v, pk)", &m), "OK"); + assert_eq!( + condition_err("size(pk) > :n AND attribute_exists(pk)", &m), + "OK" + ); +} + +#[test] +fn condition_operand_counts_and_types() { + let m = maps( + &[], + &[ + (":n", n()), + (":t", AttributeValue::S("S".to_owned())), + (":xx", AttributeValue::S("XX".to_owned())), + (":l", l()), + ], + ); + assert_eq!( + condition_err("attribute_exists(pk, pk)", &m), + format!( + "{C}Incorrect number of operands for operator or function; operator or function: attribute_exists, number of operands: 2" + ) + ); + assert_eq!( + condition_err("begins_with(pk)", &m), + format!( + "{C}Incorrect number of operands for operator or function; operator or function: begins_with, number of operands: 1" + ) + ); + assert_eq!( + condition_err("size(pk, pk) > :n", &m), + format!( + "{C}Incorrect number of operands for operator or function; operator or function: size, number of operands: 2" + ) + ); + assert_eq!( + condition_err("begins_with(pk, :n)", &m), + format!( + "{C}Incorrect operand type for operator or function; operator or function: begins_with, operand type: N" + ) + ); + assert_eq!( + condition_err("attribute_type(pk, :n)", &m), + format!( + "{C}Incorrect operand type for operator or function; operator or function: attribute_type, operand type: N" + ) + ); + assert_eq!( + condition_err("attribute_type(pk, :xx)", &m), + format!( + "{C}Invalid attribute type name found; type: XX, valid types: {{S,SS,N,NS,B,BS,BOOL,NULL,L,M}}" + ) + ); + assert_eq!( + condition_err("pk > :l", &m), + format!( + "{C}Incorrect operand type for operator or function; operator or function: >, operand type: L" + ) + ); + assert_eq!(condition_err("attribute_type(pk, :t)", &m), "OK"); +} + +fn key_condition_err(expr: &str) -> String { + let tokens = tokenize(expr).unwrap(); + match crate::expression::parse_key_condition(&tokens) { + Ok(_) => "OK".to_owned(), + Err(DynamoDbError::ValidationException(msg)) => msg, + Err(other) => panic!("unexpected {other:?}"), + } +} + +// Captured from Amazon DynamoDB on 2026-10-27: a condition function other than +// `size` and `begins_with` is not a key condition operator, `size` is a nested +// operation, and any other name (or `begins_with` in another case) is not a +// function. +#[test] +fn key_condition_function_names() { + const K: &str = "Invalid KeyConditionExpression: "; + for call in [ + "contains(sk, :v)", + "attribute_exists(sk)", + "attribute_not_exists(sk)", + "attribute_type(sk, :t)", + ] { + assert_eq!( + key_condition_err(&format!("pk = :p AND {call}")), + format!("{K}Invalid operator used in KeyConditionExpression"), + "{call}" + ); + } + assert_eq!( + key_condition_err("contains(pk, :v) AND sk = :s"), + format!("{K}Invalid operator used in KeyConditionExpression") + ); + assert_eq!( + key_condition_err("pk = :p AND size(sk) > :n"), + format!("{K}KeyConditionExpressions cannot contain nested operations") + ); + for name in ["foo", "Begins_With", "Contains"] { + assert_eq!( + key_condition_err(&format!("pk = :p AND {name}(sk, :v)")), + format!("{K}Invalid function name; function: {name}"), + ); + } + assert_eq!(key_condition_err("pk = :p AND begins_with(sk, :v)"), "OK"); +} diff --git a/crates/core/src/expression/semantics/update_rules.rs b/crates/core/src/expression/semantics/update_rules.rs new file mode 100644 index 000000000..3a63bc47e --- /dev/null +++ b/crates/core/src/expression/semantics/update_rules.rs @@ -0,0 +1,162 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The update-expression rules of the semantic pass. See the module +//! documentation in the parent module for the order the service checks them in. + +use super::super::ast::{ArithOp, Expr, UpdateAction}; +use super::super::kind::ExpressionKind; +use super::super::resolver::{ExpressionMaps, attribute_type_code}; +use super::super::update_parser::{ParsedUpdate, repeated_section_err}; +use super::errors::{ + invalid_function_name, operand_count_err, operand_type_err, requires_document_path, validation, +}; +use super::refs::{check_path_refs, check_value_defined}; +use super::{CONDITION_FUNCTIONS, UPDATE_FUNCTIONS}; +use crate::error::DynamoDbError; +use crate::types::AttributeValue; + +/// Validate a parsed update expression against the service's semantic rules, +/// in the service's order, then reject overlapping paths. +/// +/// Messages carry the `Invalid UpdateExpression:` prefix. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` for the first rule the +/// expression breaks in the order described in the module documentation. +pub fn validate_update_semantics( + parsed: &ParsedUpdate, + maps: &ExpressionMaps, +) -> Result<(), DynamoDbError> { + let kind = ExpressionKind::Update; + let mut seen: Vec<&'static str> = Vec::new(); + for section in &parsed.sections { + for action in &parsed.actions[section.actions.clone()] { + match action { + UpdateAction::Set { path, value } => { + check_path_refs(path, maps, kind)?; + walk_set_value(value, maps, false)?; + } + UpdateAction::Remove { path } => check_path_refs(path, maps, kind)?, + UpdateAction::Add { path, value } => { + check_path_refs(path, maps, kind)?; + check_add_delete_value("ADD", value, maps)?; + } + UpdateAction::Delete { path, value } => { + check_path_refs(path, maps, kind)?; + check_add_delete_value("DELETE", value, maps)?; + } + } + } + if seen.contains(§ion.keyword) { + return Err(repeated_section_err(section.keyword)); + } + seen.push(section.keyword); + } + super::super::resolver::validate_update_paths_disjoint(&parsed.actions, maps) +} + +/// Walk a SET value. `nested` is true inside a function's operands or an +/// arithmetic operand, where `list_append` is refused. +fn walk_set_value(expr: &Expr, maps: &ExpressionMaps, nested: bool) -> Result<(), DynamoDbError> { + let kind = ExpressionKind::Update; + match expr { + Expr::Path(path) => check_path_refs(path, maps, kind), + Expr::Placeholder(name) => check_value_defined(name, maps, kind), + Expr::Arithmetic { left, op, right } => { + walk_set_value(left, maps, true)?; + walk_set_value(right, maps, true)?; + let symbol = match op { + ArithOp::Add => "+", + ArithOp::Sub => "-", + }; + for operand in [left.as_ref(), right.as_ref()] { + if let Expr::Placeholder(name) = operand + && let Some(val) = maps.values.get(name) + && !matches!(val, AttributeValue::N(_)) + { + return Err(operand_type_err(kind, symbol, attribute_type_code(val))); + } + } + Ok(()) + } + Expr::Function { name, args } => { + for arg in args { + walk_set_value(arg, maps, true)?; + } + if nested && name == "list_append" { + return Err(validation( + kind, + "The function is not allowed to be used this way in an expression; function: list_append", + )); + } + if !UPDATE_FUNCTIONS.contains(&name.as_str()) { + if CONDITION_FUNCTIONS.contains(&name.as_str()) { + return Err(validation( + kind, + &format!( + "The function is not allowed in an update expression; function: {name}" + ), + )); + } + return Err(invalid_function_name(kind, name)); + } + if args.len() != 2 { + return Err(operand_count_err(kind, name, args.len())); + } + if name == "if_not_exists" && !matches!(args[0], Expr::Path(_)) { + return Err(requires_document_path(kind, name)); + } + if name == "list_append" { + for arg in args { + if let Expr::Placeholder(p) = arg + && let Some(val) = maps.values.get(p) + && !matches!(val, AttributeValue::L(_)) + { + return Err(operand_type_err(kind, name, attribute_type_code(val))); + } + } + } + Ok(()) + } + // The update grammar produces no other node kinds in a SET value. + _ => Ok(()), + } +} + +/// `ADD` takes a number or a set; `DELETE` takes a set. The service names +/// the offending type in its own vocabulary and the same allowed set for +/// both operators (measured: `DELETE s :n` reports `NUMBER`, `typeSet: +/// ALLOWED_FOR_ADD_OPERAND`). +fn check_add_delete_value( + operator: &str, + value: &Expr, + maps: &ExpressionMaps, +) -> Result<(), DynamoDbError> { + let kind = ExpressionKind::Update; + let Expr::Placeholder(name) = value else { + return Ok(()); + }; + check_value_defined(name, maps, kind)?; + let Some(val) = maps.values.get(name) else { + return Ok(()); + }; + let type_name = match val { + AttributeValue::SS(_) | AttributeValue::NS(_) | AttributeValue::BS(_) => return Ok(()), + AttributeValue::N(_) if operator == "ADD" => return Ok(()), + AttributeValue::N(_) => "NUMBER", + AttributeValue::S(_) => "STRING", + AttributeValue::B(_) => "Binary", + AttributeValue::Bool(_) => "BOOL", + AttributeValue::Null => "NULL", + AttributeValue::L(_) => "LIST", + AttributeValue::M(_) => "MAP", + }; + Err(validation( + kind, + &format!( + "Incorrect operand type for operator or function; operator: {operator}, operand type: {type_name}, typeSet: ALLOWED_FOR_ADD_OPERAND" + ), + )) +} diff --git a/crates/core/src/expression/update_parser.rs b/crates/core/src/expression/update_parser.rs deleted file mode 100755 index a67c22e6c..000000000 --- a/crates/core/src/expression/update_parser.rs +++ /dev/null @@ -1,571 +0,0 @@ -// Copyright 2026 ExtendDB contributors -// SPDX-License-Identifier: Apache-2.0 - -//! Update expression parser. -//! -//! Parses `UpdateExpression` strings into a list of `UpdateAction`s. -//! Phase 3 supports SET and REMOVE actions. -//! -//! Grammar: -//! ```text -//! update_expr → action_clause ( action_clause )* -//! action_clause → SET set_action ( ',' set_action )* -//! | REMOVE remove_action ( ',' remove_action )* -//! set_action → path '=' value_expr -//! value_expr → operand ( ('+' | '-') operand )? -//! remove_action → path -//! ``` - -use super::ast::{ArithOp, Expr, UpdateAction}; -use super::parser_common; -use super::tokenizer::Token; -use crate::error::DynamoDbError; - -/// Parse an update expression token stream into a list of actions. -/// -/// # Errors -/// -/// Returns `ValidationException` for syntax errors. -pub fn parse_update(tokens: &[Token]) -> Result, DynamoDbError> { - parser_common::check_redundant_parens(tokens).map_err(|msg| { - DynamoDbError::ValidationException(format!("Invalid UpdateExpression: {msg}")) - })?; - - let mut pos = 0; - let mut actions = Vec::new(); - - while pos < tokens.len() { - match &tokens[pos] { - Token::Set => { - pos += 1; - parse_set_actions(tokens, &mut pos, &mut actions)?; - } - Token::Remove => { - pos += 1; - parse_remove_actions(tokens, &mut pos, &mut actions)?; - } - Token::Add => { - pos += 1; - parse_add_actions(tokens, &mut pos, &mut actions)?; - } - Token::Delete => { - pos += 1; - parse_delete_actions(tokens, &mut pos, &mut actions)?; - } - _ => { - return Err(validation_err( - "Invalid UpdateExpression: expected SET, REMOVE, ADD, or DELETE", - )); - } - } - } - - if actions.is_empty() { - return Err(validation_err( - "Invalid UpdateExpression: expression must contain at least one action", - )); - } - - Ok(actions) -} - -/// Parse an update expression with DynamoDB-compatible syntax error messages. -pub fn parse_update_from( - tokens: &[Token], - source: &str, -) -> Result, DynamoDbError> { - parser_common::check_redundant_parens(tokens).map_err(|msg| { - DynamoDbError::ValidationException(format!("Invalid UpdateExpression: {msg}")) - })?; - - let mut pos = 0; - let mut actions = Vec::new(); - - while pos < tokens.len() { - match &tokens[pos] { - Token::Set => { - pos += 1; - parse_set_actions(tokens, &mut pos, &mut actions)?; - } - Token::Remove => { - pos += 1; - parse_remove_actions(tokens, &mut pos, &mut actions)?; - } - Token::Add => { - pos += 1; - parse_add_actions(tokens, &mut pos, &mut actions)?; - } - Token::Delete => { - pos += 1; - parse_delete_actions(tokens, &mut pos, &mut actions)?; - } - token => { - let token_text = token_display_text(token); - let near = build_near_context(source, &token_text); - return Err(validation_err(&format!( - "Invalid UpdateExpression: Syntax error; token: \"{token_text}\", near: \"{near}\"" - ))); - } - } - } - - if actions.is_empty() { - return Err(validation_err( - "Invalid UpdateExpression: expression must contain at least one action", - )); - } - - Ok(actions) -} - -fn token_display_text(token: &Token) -> String { - match token { - Token::Ident(s) => s.clone(), - Token::NameRef(s) => format!("#{s}"), - Token::Placeholder(s) => format!(":{s}"), - Token::Eq => "=".to_owned(), - Token::Ne => "<>".to_owned(), - Token::Lt => "<".to_owned(), - Token::Le => "<=".to_owned(), - Token::Gt => ">".to_owned(), - Token::Ge => ">=".to_owned(), - Token::Plus => "+".to_owned(), - Token::Minus => "-".to_owned(), - Token::Comma => ",".to_owned(), - Token::Dot => ".".to_owned(), - Token::LBracket => "[".to_owned(), - Token::RBracket => "]".to_owned(), - Token::LParen => "(".to_owned(), - Token::RParen => ")".to_owned(), - Token::And => "AND".to_owned(), - Token::Or => "OR".to_owned(), - Token::Not => "NOT".to_owned(), - Token::Between => "BETWEEN".to_owned(), - Token::In => "IN".to_owned(), - Token::Set => "SET".to_owned(), - Token::Remove => "REMOVE".to_owned(), - Token::Add => "ADD".to_owned(), - Token::Delete => "DELETE".to_owned(), - } -} - -fn build_near_context(source: &str, token_text: &str) -> String { - if source.is_empty() { - return token_text.to_owned(); - } - if let Some(start) = source.find(token_text) { - let end = std::cmp::min(start + token_text.len() + 7, source.len()); - source[start..end].trim_end().to_owned() - } else { - token_text.to_owned() - } -} - -fn parse_set_actions( - tokens: &[Token], - pos: &mut usize, - actions: &mut Vec, -) -> Result<(), DynamoDbError> { - loop { - let path = parser_common::parse_path(tokens, pos)?; - parser_common::expect_token(tokens, pos, &Token::Eq, "=", "UpdateExpression")?; - let value = parse_value_expr(tokens, pos)?; - actions.push(UpdateAction::Set { path, value }); - - if *pos < tokens.len() && tokens[*pos] == Token::Comma { - *pos += 1; - } else { - break; - } - } - Ok(()) -} - -fn parse_remove_actions( - tokens: &[Token], - pos: &mut usize, - actions: &mut Vec, -) -> Result<(), DynamoDbError> { - loop { - let path = parser_common::parse_path(tokens, pos)?; - actions.push(UpdateAction::Remove { path }); - - if *pos < tokens.len() && tokens[*pos] == Token::Comma { - *pos += 1; - } else { - break; - } - } - Ok(()) -} - -/// Parse a SET value expression: `operand` or `operand +/- operand`. -fn parse_value_expr(tokens: &[Token], pos: &mut usize) -> Result { - let left = parse_set_operand(tokens, pos)?; - - if *pos < tokens.len() { - let arith_op = match &tokens[*pos] { - Token::Plus => Some(ArithOp::Add), - Token::Minus => Some(ArithOp::Sub), - _ => None, - }; - if let Some(op) = arith_op { - *pos += 1; - let right = parse_set_operand(tokens, pos)?; - return Ok(Expr::Arithmetic { - left: Box::new(left), - op, - right: Box::new(right), - }); - } - } - - Ok(left) -} - -/// Parse a SET operand: path, placeholder, or function call (`if_not_exists`, `list_append`). -fn parse_set_operand(tokens: &[Token], pos: &mut usize) -> Result { - if *pos >= tokens.len() { - return Err(validation_err("Invalid UpdateExpression: expected operand")); - } - - match &tokens[*pos] { - Token::Placeholder(name) => { - let expr = Expr::Placeholder(name.clone()); - *pos += 1; - Ok(expr) - } - Token::Ident(name) => { - let fn_lower = name.to_ascii_lowercase(); - if is_set_function(&fn_lower) - && *pos + 1 < tokens.len() - && tokens[*pos + 1] == Token::LParen - { - parse_set_function_call(tokens, pos) - } else { - parser_common::parse_path(tokens, pos).map(Expr::Path) - } - } - Token::NameRef(_) => parser_common::parse_path(tokens, pos).map(Expr::Path), - Token::LParen => { - *pos += 1; // consume '(' - let expr = parse_value_expr(tokens, pos)?; - if *pos >= tokens.len() || tokens[*pos] != Token::RParen { - return Err(validation_err( - "Invalid UpdateExpression: expected closing ')'", - )); - } - *pos += 1; // consume ')' - Ok(expr) - } - _ => Err(validation_err( - "Invalid UpdateExpression: expected path, placeholder, or function", - )), - } -} - -fn parse_set_function_call(tokens: &[Token], pos: &mut usize) -> Result { - let name = match &tokens[*pos] { - Token::Ident(n) => n.to_ascii_lowercase(), - _ => { - return Err(validation_err( - "Invalid UpdateExpression: expected function name", - )); - } - }; - *pos += 1; - parser_common::expect_token(tokens, pos, &Token::LParen, "(", "UpdateExpression")?; - - let mut args = Vec::new(); - if *pos < tokens.len() && tokens[*pos] != Token::RParen { - args.push(parse_set_operand(tokens, pos)?); - while *pos < tokens.len() && tokens[*pos] == Token::Comma { - *pos += 1; - args.push(parse_set_operand(tokens, pos)?); - } - } - - parser_common::expect_token(tokens, pos, &Token::RParen, ")", "UpdateExpression")?; - Ok(Expr::Function { name, args }) -} - -fn is_set_function(name: &str) -> bool { - matches!(name, "if_not_exists" | "list_append") -} - -/// Parse ADD actions: `ADD path value (, path value)*` -fn parse_add_actions( - tokens: &[Token], - pos: &mut usize, - actions: &mut Vec, -) -> Result<(), DynamoDbError> { - loop { - let path = parser_common::parse_path(tokens, pos)?; - let value = parse_add_delete_value(tokens, pos)?; - actions.push(UpdateAction::Add { path, value }); - if *pos < tokens.len() && tokens[*pos] == Token::Comma { - *pos += 1; - } else { - break; - } - } - Ok(()) -} - -/// Parse DELETE actions: `DELETE path value (, path value)*` -fn parse_delete_actions( - tokens: &[Token], - pos: &mut usize, - actions: &mut Vec, -) -> Result<(), DynamoDbError> { - loop { - let path = parser_common::parse_path(tokens, pos)?; - let value = parse_add_delete_value(tokens, pos)?; - actions.push(UpdateAction::Delete { path, value }); - if *pos < tokens.len() && tokens[*pos] == Token::Comma { - *pos += 1; - } else { - break; - } - } - Ok(()) -} - -/// Parse a value for ADD/DELETE — only placeholders are valid. -fn parse_add_delete_value(tokens: &[Token], pos: &mut usize) -> Result { - if *pos >= tokens.len() { - return Err(validation_err( - "Invalid UpdateExpression: expected value for ADD/DELETE action", - )); - } - match &tokens[*pos] { - Token::Placeholder(name) => { - let expr = Expr::Placeholder(name.clone()); - *pos += 1; - Ok(expr) - } - _ => Err(validation_err( - "Invalid UpdateExpression: ADD/DELETE value must be a placeholder", - )), - } -} - -fn validation_err(msg: &str) -> DynamoDbError { - DynamoDbError::ValidationException(msg.to_owned()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::expression::ast::{ArithOp, Expr, PathElement, UpdateAction}; - use crate::expression::tokenizer::tokenize; - - fn parse(input: &str) -> Result, DynamoDbError> { - let tokens = tokenize(input)?; - parse_update(&tokens) - } - - #[test] - fn parse_simple_set() { - let actions = parse("SET price = :p").unwrap(); - assert_eq!(actions.len(), 1); - assert!(matches!(&actions[0], UpdateAction::Set { path, .. } if path.len() == 1)); - } - - #[test] - fn parse_multiple_set_actions() { - let actions = parse("SET a = :v1, b = :v2, c = :v3").unwrap(); - assert_eq!(actions.len(), 3); - assert!( - actions - .iter() - .all(|a| matches!(a, UpdateAction::Set { .. })) - ); - } - - #[test] - fn parse_set_with_arithmetic_add() { - let actions = parse("SET price = price + :inc").unwrap(); - assert_eq!(actions.len(), 1); - if let UpdateAction::Set { value, .. } = &actions[0] { - assert!(matches!( - value, - Expr::Arithmetic { - op: ArithOp::Add, - .. - } - )); - } else { - panic!("Expected Set action"); - } - } - - #[test] - fn parse_set_with_arithmetic_sub() { - let actions = parse("SET stock = stock - :dec").unwrap(); - if let UpdateAction::Set { value, .. } = &actions[0] { - assert!(matches!( - value, - Expr::Arithmetic { - op: ArithOp::Sub, - .. - } - )); - } else { - panic!("Expected Set action"); - } - } - - #[test] - fn parse_set_with_if_not_exists() { - let actions = parse("SET price = if_not_exists(price, :default)").unwrap(); - if let UpdateAction::Set { value, .. } = &actions[0] { - if let Expr::Function { name, args } = value { - assert_eq!(name, "if_not_exists"); - assert_eq!(args.len(), 2); - } else { - panic!("Expected Function expression"); - } - } else { - panic!("Expected Set action"); - } - } - - #[test] - fn parse_set_with_list_append() { - let actions = parse("SET tags = list_append(tags, :newtags)").unwrap(); - if let UpdateAction::Set { value, .. } = &actions[0] { - assert!(matches!(value, Expr::Function { name, .. } if name == "list_append")); - } else { - panic!("Expected Set action"); - } - } - - #[test] - fn parse_simple_remove() { - let actions = parse("REMOVE oldattr").unwrap(); - assert_eq!(actions.len(), 1); - assert!(matches!(&actions[0], UpdateAction::Remove { .. })); - } - - #[test] - fn parse_multiple_remove() { - let actions = parse("REMOVE a, b, c").unwrap(); - assert_eq!(actions.len(), 3); - } - - #[test] - fn parse_set_and_remove_combined() { - let actions = parse("SET a = :v REMOVE b").unwrap(); - assert_eq!(actions.len(), 2); - assert!(matches!(&actions[0], UpdateAction::Set { .. })); - assert!(matches!(&actions[1], UpdateAction::Remove { .. })); - } - - #[test] - fn parse_add_action() { - let actions = parse("ADD counter :inc").unwrap(); - assert_eq!(actions.len(), 1); - assert!(matches!(&actions[0], UpdateAction::Add { .. })); - } - - #[test] - fn parse_delete_action() { - let actions = parse("DELETE colors :remove_colors").unwrap(); - assert_eq!(actions.len(), 1); - assert!(matches!(&actions[0], UpdateAction::Delete { .. })); - } - - #[test] - fn parse_all_four_action_types() { - let actions = parse("SET a = :v ADD b :inc DELETE c :rm REMOVE d").unwrap(); - assert_eq!(actions.len(), 4); - } - - #[test] - fn parse_nested_path_in_set() { - let actions = parse("SET address.city = :city").unwrap(); - if let UpdateAction::Set { path, .. } = &actions[0] { - assert_eq!(path.len(), 2); - assert_eq!(path[0], PathElement::Attribute("address".into())); - assert_eq!(path[1], PathElement::Attribute("city".into())); - } else { - panic!("Expected Set action"); - } - } - - #[test] - fn parse_list_index_in_set() { - let actions = parse("SET items[0] = :v").unwrap(); - if let UpdateAction::Set { path, .. } = &actions[0] { - assert_eq!(path.len(), 2); - assert_eq!(path[1], PathElement::Index(0)); - } else { - panic!("Expected Set action"); - } - } - - #[test] - fn parse_empty_expression_rejected() { - let tokens = tokenize("").unwrap(); - // Empty tokens — parse_update should fail - let err = parse_update(&tokens).unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("at least one action")) - ); - } - - #[test] - fn parse_invalid_action_keyword_rejected() { - let err = parse("price = :v").unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("expected SET")) - ); - } - - #[test] - fn parse_name_ref_in_path() { - let actions = parse("SET #n = :v").unwrap(); - if let UpdateAction::Set { path, .. } = &actions[0] { - assert_eq!(path[0], PathElement::Attribute("#n".into())); - } else { - panic!("Expected Set action"); - } - } - - #[test] - fn redundant_parens_rejected() { - let err = parse("SET c = ((c - :v))").unwrap_err(); - assert!( - matches!(&err, DynamoDbError::ValidationException(msg) if msg.contains("redundant parentheses")), - "Expected redundant parens error, got: {err:?}" - ); - } - - #[test] - fn single_parens_accepted() { - let actions = parse("SET c = (c - :v)").unwrap(); - assert!(matches!(&actions[0], UpdateAction::Set { .. })); - } - - #[test] - fn unmatched_paren_rejected() { - // Opening paren with no matching close must be a ValidationException, - // not a panic or a silently-accepted expression. - let err = parse("SET c = (c - :v").unwrap_err(); - assert!( - matches!(&err, DynamoDbError::ValidationException(msg) if msg.contains("closing")), - "Expected unmatched-paren error, got: {err:?}" - ); - } - - #[test] - fn unmatched_close_paren_rejected() { - // A stray closing paren with no matching open must also be rejected. - let err = parse("SET c = c - :v)").unwrap_err(); - assert!( - matches!(&err, DynamoDbError::ValidationException(_)), - "Expected validation error for stray ')', got: {err:?}" - ); - } -} diff --git a/crates/core/src/expression/update_parser/mod.rs b/crates/core/src/expression/update_parser/mod.rs new file mode 100644 index 000000000..24764535f --- /dev/null +++ b/crates/core/src/expression/update_parser/mod.rs @@ -0,0 +1,417 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Update expression parser. +//! +//! Parses `UpdateExpression` strings into a list of `UpdateAction`s. +//! Phase 3 supports SET and REMOVE actions. +//! +//! Grammar: +//! ```text +//! update_expr → action_clause ( action_clause )* +//! action_clause → SET set_action ( ',' set_action )* +//! | REMOVE remove_action ( ',' remove_action )* +//! set_action → path '=' value_expr +//! value_expr → operand ( ('+' | '-') operand )? +//! remove_action → path +//! ``` + +use super::ast::{ArithOp, Expr, UpdateAction}; +use super::parser_common; +use super::tokenizer::Token; +use crate::error::DynamoDbError; + +/// One `SET`, `REMOVE`, `ADD`, or `DELETE` section of a parsed update +/// expression: its keyword and the range of actions it holds, in expression +/// order. A keyword may appear more than once in the parse; the semantic pass +/// reports the repetition after it has checked the section's own actions, as +/// Amazon DynamoDB does. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateSection { + pub keyword: &'static str, + pub actions: std::ops::Range, +} + +/// The grammar-level result of parsing an `UpdateExpression`: the actions in +/// expression order and the sections that group them. +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedUpdate { + pub actions: Vec, + pub sections: Vec, +} + +/// Parse an update expression token stream into a list of actions, with the +/// grammar-only checks plus the repeated-section rule. +/// +/// Callers that go on to run [`validate_update_semantics`](super::semantics::validate_update_semantics) should use +/// [`parse_update_sections`] instead; this entry point serves the storage +/// crates' tests and other callers that need only a parsed action list. +/// +/// # Errors +/// +/// Returns `ValidationException` for syntax errors. +pub fn parse_update(tokens: &[Token]) -> Result, DynamoDbError> { + let parsed = parse_update_sections(tokens, None)?; + check_sections_unique(&parsed.sections)?; + Ok(parsed.actions) +} + +/// Parse an update expression with DynamoDB-compatible syntax error messages, +/// including the repeated-section rule. +/// +/// # Errors +/// +/// Returns `ValidationException` for syntax errors. +pub fn parse_update_from( + tokens: &[Token], + source: &str, +) -> Result, DynamoDbError> { + let parsed = parse_update_sections(tokens, Some(source))?; + check_sections_unique(&parsed.sections)?; + Ok(parsed.actions) +} + +/// Grammar pass over an `UpdateExpression`. +/// +/// Only syntax is checked here: the section keywords, the shape of each +/// action, and balanced parentheses. Any identifier followed by `(` parses +/// as a function call with its name kept as written, operand counts are not +/// checked, and a section keyword may repeat. Amazon DynamoDB reports every +/// syntax error ahead of every semantic one, so the semantic rules (function +/// names, operand counts and types, undefined references, repeated sections, +/// overlapping paths) run afterwards in [`validate_update_semantics`](super::semantics::validate_update_semantics) over +/// the returned tree. +/// +/// `source`, when given, enables the service's `Syntax error; token: ..., +/// near: ...` wording for an unexpected token at section level. +/// +/// # Errors +/// +/// Returns `ValidationException` for syntax errors. +pub fn parse_update_sections( + tokens: &[Token], + source: Option<&str>, +) -> Result { + parser_common::check_redundant_parens(tokens).map_err(|msg| { + DynamoDbError::ValidationException(format!("Invalid UpdateExpression: {msg}")) + })?; + + let mut pos = 0; + let mut actions = Vec::new(); + let mut sections: Vec = Vec::new(); + + while pos < tokens.len() { + let start = actions.len(); + let keyword = match &tokens[pos] { + Token::Set => { + pos += 1; + parse_set_actions(tokens, &mut pos, &mut actions)?; + "SET" + } + Token::Remove => { + pos += 1; + parse_remove_actions(tokens, &mut pos, &mut actions)?; + "REMOVE" + } + Token::Add => { + pos += 1; + parse_add_actions(tokens, &mut pos, &mut actions)?; + "ADD" + } + Token::Delete => { + pos += 1; + parse_delete_actions(tokens, &mut pos, &mut actions)?; + "DELETE" + } + token => { + return Err(match source { + Some(source) => { + let token_text = token_display_text(token); + let near = build_near_context(source, &token_text); + validation_err(&format!( + "Invalid UpdateExpression: Syntax error; token: \"{token_text}\", near: \"{near}\"" + )) + } + None => validation_err( + "Invalid UpdateExpression: expected SET, REMOVE, ADD, or DELETE", + ), + }); + } + }; + sections.push(UpdateSection { + keyword, + actions: start..actions.len(), + }); + } + + if actions.is_empty() { + return Err(validation_err( + "Invalid UpdateExpression: expression must contain at least one action", + )); + } + + Ok(ParsedUpdate { actions, sections }) +} + +/// Each of SET, REMOVE, ADD, and DELETE may open one section. The first +/// repeated section in order of appearance is named. +pub(crate) fn check_sections_unique(sections: &[UpdateSection]) -> Result<(), DynamoDbError> { + for (i, section) in sections.iter().enumerate() { + if sections[..i].iter().any(|s| s.keyword == section.keyword) { + return Err(repeated_section_err(section.keyword)); + } + } + Ok(()) +} + +pub(crate) fn repeated_section_err(keyword: &str) -> DynamoDbError { + validation_err(&format!( + "Invalid UpdateExpression: The \"{keyword}\" section can only be used once in an update expression;" + )) +} + +fn token_display_text(token: &Token) -> String { + match token { + Token::Ident(s) => s.clone(), + Token::NameRef(s) => format!("#{s}"), + Token::Placeholder(s) => format!(":{s}"), + Token::Eq => "=".to_owned(), + Token::Ne => "<>".to_owned(), + Token::Lt => "<".to_owned(), + Token::Le => "<=".to_owned(), + Token::Gt => ">".to_owned(), + Token::Ge => ">=".to_owned(), + Token::Plus => "+".to_owned(), + Token::Minus => "-".to_owned(), + Token::Comma => ",".to_owned(), + Token::Dot => ".".to_owned(), + Token::LBracket => "[".to_owned(), + Token::RBracket => "]".to_owned(), + Token::LParen => "(".to_owned(), + Token::RParen => ")".to_owned(), + Token::And => "AND".to_owned(), + Token::Or => "OR".to_owned(), + Token::Not => "NOT".to_owned(), + Token::Between => "BETWEEN".to_owned(), + Token::In => "IN".to_owned(), + Token::Set => "SET".to_owned(), + Token::Remove => "REMOVE".to_owned(), + Token::Add => "ADD".to_owned(), + Token::Delete => "DELETE".to_owned(), + } +} + +fn build_near_context(source: &str, token_text: &str) -> String { + if source.is_empty() { + return token_text.to_owned(); + } + if let Some(start) = source.find(token_text) { + let end = std::cmp::min(start + token_text.len() + 7, source.len()); + source[start..end].trim_end().to_owned() + } else { + token_text.to_owned() + } +} + +fn parse_set_actions( + tokens: &[Token], + pos: &mut usize, + actions: &mut Vec, +) -> Result<(), DynamoDbError> { + loop { + let path = parser_common::parse_path(tokens, pos)?; + parser_common::expect_token(tokens, pos, &Token::Eq, "=", "UpdateExpression")?; + let value = parse_value_expr(tokens, pos)?; + actions.push(UpdateAction::Set { path, value }); + + if *pos < tokens.len() && tokens[*pos] == Token::Comma { + *pos += 1; + } else { + break; + } + } + Ok(()) +} + +fn parse_remove_actions( + tokens: &[Token], + pos: &mut usize, + actions: &mut Vec, +) -> Result<(), DynamoDbError> { + loop { + let path = parser_common::parse_path(tokens, pos)?; + actions.push(UpdateAction::Remove { path }); + + if *pos < tokens.len() && tokens[*pos] == Token::Comma { + *pos += 1; + } else { + break; + } + } + Ok(()) +} + +/// Parse a SET value expression: `operand` or `operand +/- operand`. +fn parse_value_expr(tokens: &[Token], pos: &mut usize) -> Result { + let left = parse_set_operand(tokens, pos)?; + + if *pos < tokens.len() { + let arith_op = match &tokens[*pos] { + Token::Plus => Some(ArithOp::Add), + Token::Minus => Some(ArithOp::Sub), + _ => None, + }; + if let Some(op) = arith_op { + *pos += 1; + let right = parse_set_operand(tokens, pos)?; + return Ok(Expr::Arithmetic { + left: Box::new(left), + op, + right: Box::new(right), + }); + } + } + + Ok(left) +} + +/// Parse a SET operand: path, placeholder, or function call. +/// +/// Any identifier followed by `(` is a function call, with the name kept as +/// written. This pass checks grammar only: whether the name exists, its case, +/// and its operand count are checked afterwards by +/// `validate_update_semantics`, over the whole parsed expression. Amazon +/// DynamoDB reports every syntax error before any semantic error (measured +/// 2026-10-14: `SET x = foo(x), b = ` reports the syntax error at the end, +/// not the unknown function name), and deferring the semantic checks gives +/// the same precedence. +fn parse_set_operand(tokens: &[Token], pos: &mut usize) -> Result { + if *pos >= tokens.len() { + return Err(validation_err("Invalid UpdateExpression: expected operand")); + } + + match &tokens[*pos] { + Token::Placeholder(name) => { + let expr = Expr::Placeholder(name.clone()); + *pos += 1; + Ok(expr) + } + Token::Ident(_) => { + if *pos + 1 < tokens.len() && tokens[*pos + 1] == Token::LParen { + parse_set_function_call(tokens, pos) + } else { + parser_common::parse_path(tokens, pos).map(Expr::Path) + } + } + Token::NameRef(_) => parser_common::parse_path(tokens, pos).map(Expr::Path), + Token::LParen => { + *pos += 1; // consume '(' + let expr = parse_value_expr(tokens, pos)?; + if *pos >= tokens.len() || tokens[*pos] != Token::RParen { + return Err(validation_err( + "Invalid UpdateExpression: expected closing ')'", + )); + } + *pos += 1; // consume ')' + Ok(expr) + } + _ => Err(validation_err( + "Invalid UpdateExpression: expected path, placeholder, or function", + )), + } +} + +fn parse_set_function_call(tokens: &[Token], pos: &mut usize) -> Result { + let name = match &tokens[*pos] { + Token::Ident(n) => n.clone(), + _ => { + return Err(validation_err( + "Invalid UpdateExpression: expected function name", + )); + } + }; + *pos += 1; + parser_common::expect_token(tokens, pos, &Token::LParen, "(", "UpdateExpression")?; + + // The grammar requires at least one operand between the parentheses. A + // call with an empty operand list, `foo()`, is a grammar error on the + // service, reported at the `)` with the text below, before the function + // name is looked at (measured 2026-10-14). + if *pos < tokens.len() && tokens[*pos] == Token::RParen { + return Err(validation_err( + "Invalid UpdateExpression: Syntax error; token: \")\", near: \"()\"", + )); + } + let mut args = vec![parse_set_operand(tokens, pos)?]; + while *pos < tokens.len() && tokens[*pos] == Token::Comma { + *pos += 1; + args.push(parse_set_operand(tokens, pos)?); + } + + parser_common::expect_token(tokens, pos, &Token::RParen, ")", "UpdateExpression")?; + Ok(Expr::Function { name, args }) +} + +/// Parse ADD actions: `ADD path value (, path value)*` +fn parse_add_actions( + tokens: &[Token], + pos: &mut usize, + actions: &mut Vec, +) -> Result<(), DynamoDbError> { + loop { + let path = parser_common::parse_path(tokens, pos)?; + let value = parse_add_delete_value(tokens, pos)?; + actions.push(UpdateAction::Add { path, value }); + if *pos < tokens.len() && tokens[*pos] == Token::Comma { + *pos += 1; + } else { + break; + } + } + Ok(()) +} + +/// Parse DELETE actions: `DELETE path value (, path value)*` +fn parse_delete_actions( + tokens: &[Token], + pos: &mut usize, + actions: &mut Vec, +) -> Result<(), DynamoDbError> { + loop { + let path = parser_common::parse_path(tokens, pos)?; + let value = parse_add_delete_value(tokens, pos)?; + actions.push(UpdateAction::Delete { path, value }); + if *pos < tokens.len() && tokens[*pos] == Token::Comma { + *pos += 1; + } else { + break; + } + } + Ok(()) +} + +/// Parse a value for ADD/DELETE. Only placeholders are valid. +fn parse_add_delete_value(tokens: &[Token], pos: &mut usize) -> Result { + if *pos >= tokens.len() { + return Err(validation_err( + "Invalid UpdateExpression: expected value for ADD/DELETE action", + )); + } + match &tokens[*pos] { + Token::Placeholder(name) => { + let expr = Expr::Placeholder(name.clone()); + *pos += 1; + Ok(expr) + } + _ => Err(validation_err( + "Invalid UpdateExpression: ADD/DELETE value must be a placeholder", + )), + } +} + +fn validation_err(msg: &str) -> DynamoDbError { + DynamoDbError::ValidationException(msg.to_owned()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/expression/update_parser/tests.rs b/crates/core/src/expression/update_parser/tests.rs new file mode 100644 index 000000000..41758716a --- /dev/null +++ b/crates/core/src/expression/update_parser/tests.rs @@ -0,0 +1,215 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The update-expression parser tests. + +use super::*; +use crate::expression::ast::{ArithOp, Expr, PathElement, UpdateAction}; +use crate::expression::tokenizer::tokenize; + +fn parse(input: &str) -> Result, DynamoDbError> { + let tokens = tokenize(input)?; + parse_update(&tokens) +} + +#[test] +fn parse_simple_set() { + let actions = parse("SET price = :p").unwrap(); + assert_eq!(actions.len(), 1); + assert!(matches!(&actions[0], UpdateAction::Set { path, .. } if path.len() == 1)); +} + +#[test] +fn parse_multiple_set_actions() { + let actions = parse("SET a = :v1, b = :v2, c = :v3").unwrap(); + assert_eq!(actions.len(), 3); + assert!( + actions + .iter() + .all(|a| matches!(a, UpdateAction::Set { .. })) + ); +} + +#[test] +fn parse_set_with_arithmetic_add() { + let actions = parse("SET price = price + :inc").unwrap(); + assert_eq!(actions.len(), 1); + if let UpdateAction::Set { value, .. } = &actions[0] { + assert!(matches!( + value, + Expr::Arithmetic { + op: ArithOp::Add, + .. + } + )); + } else { + panic!("Expected Set action"); + } +} + +#[test] +fn parse_set_with_arithmetic_sub() { + let actions = parse("SET stock = stock - :dec").unwrap(); + if let UpdateAction::Set { value, .. } = &actions[0] { + assert!(matches!( + value, + Expr::Arithmetic { + op: ArithOp::Sub, + .. + } + )); + } else { + panic!("Expected Set action"); + } +} + +#[test] +fn parse_set_with_if_not_exists() { + let actions = parse("SET price = if_not_exists(price, :default)").unwrap(); + if let UpdateAction::Set { value, .. } = &actions[0] { + if let Expr::Function { name, args } = value { + assert_eq!(name, "if_not_exists"); + assert_eq!(args.len(), 2); + } else { + panic!("Expected Function expression"); + } + } else { + panic!("Expected Set action"); + } +} + +#[test] +fn parse_set_with_list_append() { + let actions = parse("SET tags = list_append(tags, :newtags)").unwrap(); + if let UpdateAction::Set { value, .. } = &actions[0] { + assert!(matches!(value, Expr::Function { name, .. } if name == "list_append")); + } else { + panic!("Expected Set action"); + } +} + +#[test] +fn parse_simple_remove() { + let actions = parse("REMOVE oldattr").unwrap(); + assert_eq!(actions.len(), 1); + assert!(matches!(&actions[0], UpdateAction::Remove { .. })); +} + +#[test] +fn parse_multiple_remove() { + let actions = parse("REMOVE a, b, c").unwrap(); + assert_eq!(actions.len(), 3); +} + +#[test] +fn parse_set_and_remove_combined() { + let actions = parse("SET a = :v REMOVE b").unwrap(); + assert_eq!(actions.len(), 2); + assert!(matches!(&actions[0], UpdateAction::Set { .. })); + assert!(matches!(&actions[1], UpdateAction::Remove { .. })); +} + +#[test] +fn parse_add_action() { + let actions = parse("ADD counter :inc").unwrap(); + assert_eq!(actions.len(), 1); + assert!(matches!(&actions[0], UpdateAction::Add { .. })); +} + +#[test] +fn parse_delete_action() { + let actions = parse("DELETE colors :remove_colors").unwrap(); + assert_eq!(actions.len(), 1); + assert!(matches!(&actions[0], UpdateAction::Delete { .. })); +} + +#[test] +fn parse_all_four_action_types() { + let actions = parse("SET a = :v ADD b :inc DELETE c :rm REMOVE d").unwrap(); + assert_eq!(actions.len(), 4); +} + +#[test] +fn parse_nested_path_in_set() { + let actions = parse("SET address.city = :city").unwrap(); + if let UpdateAction::Set { path, .. } = &actions[0] { + assert_eq!(path.len(), 2); + assert_eq!(path[0], PathElement::Attribute("address".into())); + assert_eq!(path[1], PathElement::Attribute("city".into())); + } else { + panic!("Expected Set action"); + } +} + +#[test] +fn parse_list_index_in_set() { + let actions = parse("SET items[0] = :v").unwrap(); + if let UpdateAction::Set { path, .. } = &actions[0] { + assert_eq!(path.len(), 2); + assert_eq!(path[1], PathElement::Index(0)); + } else { + panic!("Expected Set action"); + } +} + +#[test] +fn parse_empty_expression_rejected() { + let tokens = tokenize("").unwrap(); + // Empty tokens, so parse_update must fail. + let err = parse_update(&tokens).unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("at least one action")) + ); +} + +#[test] +fn parse_invalid_action_keyword_rejected() { + let err = parse("price = :v").unwrap_err(); + assert!(matches!(err, DynamoDbError::ValidationException(msg) if msg.contains("expected SET"))); +} + +#[test] +fn parse_name_ref_in_path() { + let actions = parse("SET #n = :v").unwrap(); + if let UpdateAction::Set { path, .. } = &actions[0] { + assert_eq!(path[0], PathElement::Attribute("#n".into())); + } else { + panic!("Expected Set action"); + } +} + +#[test] +fn redundant_parens_rejected() { + let err = parse("SET c = ((c - :v))").unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(msg) if msg.contains("redundant parentheses")), + "Expected redundant parens error, got: {err:?}" + ); +} + +#[test] +fn single_parens_accepted() { + let actions = parse("SET c = (c - :v)").unwrap(); + assert!(matches!(&actions[0], UpdateAction::Set { .. })); +} + +#[test] +fn unmatched_paren_rejected() { + // Opening paren with no matching close must be a ValidationException, + // not a panic or a silently-accepted expression. + let err = parse("SET c = (c - :v").unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(msg) if msg.contains("closing")), + "Expected unmatched-paren error, got: {err:?}" + ); +} + +#[test] +fn unmatched_close_paren_rejected() { + // A stray closing paren with no matching open must also be rejected. + let err = parse("SET c = c - :v)").unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(_)), + "Expected validation error for stray ')', got: {err:?}" + ); +} diff --git a/crates/core/src/types/key_schema.rs b/crates/core/src/types/key_schema.rs index 7aa89ef3e..a80c80524 100755 --- a/crates/core/src/types/key_schema.rs +++ b/crates/core/src/types/key_schema.rs @@ -102,6 +102,13 @@ pub struct TableKeyInfo { /// validate vector-valued and search-schema attributes without an extra /// catalog round-trip. pub vector_indexes: Vec, + /// When the table was created, as seconds since the Unix epoch with the + /// catalog's sub-second precision; 0.0 when the source carries none. Read + /// from the same catalog row as the key schema, so it costs no extra + /// round-trip. The transaction handlers order their `ConsumedCapacity` + /// entries by it, oldest table first, as DynamoDB does (measured + /// 2026-10-23). + pub creation_date_time: f64, } /// Vector-index metadata needed by the write path to validate an item's @@ -294,6 +301,7 @@ mod tests { local_secondary_indexes: vec![], stream_specification: None, vector_indexes: vec![], + creation_date_time: 0.0, } } @@ -474,6 +482,7 @@ mod tests { local_secondary_indexes: vec![], stream_specification: None, vector_indexes: vec![], + creation_date_time: 0.0, }; assert_eq!(info.key_schema, info.base_key_schema); } @@ -494,6 +503,7 @@ mod tests { local_secondary_indexes: vec![], stream_specification: None, vector_indexes: vec![], + creation_date_time: 0.0, }; assert_eq!(info.key_schema, index_schema); assert_eq!(info.base_key_schema, base_schema); diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 3decb4666..c57357f59 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -1293,6 +1293,14 @@ pub fn validate_item_keys( /// Validate that a key map contains exactly the key attributes and nothing else. /// +/// A key with an attribute missing, an attribute of the wrong type, or an +/// extra attribute is refused with `The provided key element does not match +/// the schema` on every key-shaped API (GetItem, DeleteItem, UpdateItem, the +/// batch and transaction key paths; measured against Amazon DynamoDB +/// 2026-10-14). Only `PutItem`, whose key arrives inside the item, names the +/// attribute (see [`validate_item_keys`]). An empty key value keeps its own +/// message. +/// /// # Errors /// /// Returns `DynamoDbError::ValidationException` if the key has extra/missing attributes or wrong types. @@ -1300,32 +1308,54 @@ pub fn validate_key_only( key: &Item, key_schema: &[KeySchemaElement], attr_defs: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + validate_key_shape(key, key_schema, attr_defs, true) +} + +/// [`validate_key_only`] without the empty-value rule: the count, the missing +/// attribute and the type of every key attribute, and nothing else. The +/// transaction key pass uses it because the service reports an empty key value +/// only after every shape error of the same item, even one on a later key +/// attribute (measured 2026-10-27). +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` if the key has extra/missing attributes or wrong types. +pub fn validate_key_shape_only( + key: &Item, + key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + validate_key_shape(key, key_schema, attr_defs, false) +} + +fn validate_key_shape( + key: &Item, + key_schema: &[KeySchemaElement], + attr_defs: &[AttributeDefinition], + reject_empty: bool, ) -> Result<(), DynamoDbError> { // Must contain exactly the key attributes let expected_count = key_schema.len(); if key.len() != expected_count { - return Err(DynamoDbError::ValidationException( - "The provided key element does not match the schema".to_owned(), - )); + return Err(key_schema_mismatch()); } for ks in key_schema { - let value = key.get(&ks.attribute_name).ok_or_else(|| { - DynamoDbError::ValidationException(format!( - "One or more parameter values were invalid: Missing the key {} in the item", - ks.attribute_name - )) - })?; - validate_key_attribute_type(&ks.attribute_name, value, attr_defs)?; - validate_no_empty_key_value(&ks.attribute_name, value)?; + let value = key + .get(&ks.attribute_name) + .ok_or_else(key_schema_mismatch)?; + validate_key_attribute_type(&ks.attribute_name, value, attr_defs) + .map_err(remap_key_shape_errors)?; + if reject_empty { + validate_no_empty_key_value(&ks.attribute_name, value)?; + } } Ok(()) } -/// Batch-specific key validation: uses `DynamoDB`'s batch error message for type mismatches. -/// -/// Real `DynamoDB` returns "The provided key element does not match the schema" for -/// batch operations, not the single-item "Type mismatch" message. +/// Key validation for the batch and transaction key paths. Same rules and +/// message as [`validate_key_only`]; kept as a separate name for the callers. /// /// # Errors /// @@ -1335,10 +1365,12 @@ pub fn validate_batch_key_only( key_schema: &[KeySchemaElement], attr_defs: &[AttributeDefinition], ) -> Result<(), DynamoDbError> { - validate_key_only(key, key_schema, attr_defs).map_err(remap_key_type_mismatch) + validate_key_only(key, key_schema, attr_defs) } -/// Batch-specific item key validation: uses `DynamoDB`'s batch error message for type mismatches. +/// `BatchWriteItem` Put key validation: a missing or mistyped key attribute +/// inside the item is `The provided key element does not match the schema`, +/// where `PutItem` names the attribute (measured 2026-10-14). /// /// # Errors /// @@ -1348,20 +1380,24 @@ pub fn validate_batch_item_keys( key_schema: &[KeySchemaElement], attr_defs: &[AttributeDefinition], ) -> Result<(), DynamoDbError> { - validate_item_keys(item, key_schema, attr_defs).map_err(remap_key_type_mismatch) + validate_item_keys(item, key_schema, attr_defs).map_err(remap_key_shape_errors) } -/// Remap key type-mismatch errors to the batch/transaction-specific message. -/// -/// Real `DynamoDB` uses "The provided key element does not match the schema" -/// for batch and transaction operations, not the single-item "Type mismatch" -/// message. -fn remap_key_type_mismatch(err: DynamoDbError) -> DynamoDbError { +fn key_schema_mismatch() -> DynamoDbError { + DynamoDbError::ValidationException( + "The provided key element does not match the schema".to_owned(), + ) +} + +/// Remap the item-shaped key messages (`Missing the key`, `Type mismatch for +/// key`) to the key-shaped `The provided key element does not match the +/// schema`. +fn remap_key_shape_errors(err: DynamoDbError) -> DynamoDbError { match &err { - DynamoDbError::ValidationException(msg) if msg.contains("Type mismatch for key ") => { - DynamoDbError::ValidationException( - "The provided key element does not match the schema".to_owned(), - ) + DynamoDbError::ValidationException(msg) + if msg.contains("Type mismatch for key ") || msg.contains("Missing the key ") => + { + key_schema_mismatch() } _ => err, } @@ -1413,7 +1449,7 @@ fn validate_key_attribute_type( /// /// Returns `DynamoDbError::ValidationException` if `value` is an empty string /// (`S("")`) or empty binary (`B()`). -fn validate_no_empty_key_value( +pub fn validate_no_empty_key_value( attr_name: &str, value: &AttributeValue, ) -> Result<(), DynamoDbError> { @@ -1901,7 +1937,14 @@ fn validate_lsi_requires_range_key(input: &CreateTableInput) -> Result<(), Dynam Ok(()) } -fn validate_attribute_number(value: &AttributeValue) -> Result<(), DynamoDbError> { +/// Validate every number inside one attribute value (`N`, `NS`, and the +/// numbers nested in `L` and `M`) with the service's messages. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` for the first number that is +/// not a valid `DynamoDB` number. +pub fn validate_attribute_number(value: &AttributeValue) -> Result<(), DynamoDbError> { match value { AttributeValue::N(n) => { number::validate_and_normalize_number(n)?; @@ -1949,6 +1992,26 @@ where Ok(()) } +/// Validate the nesting of one `ExpressionAttributeValues` value of a +/// transaction item. The service accepts a value with as many wrappers as an +/// item may hold plus one, and refuses one more (measured 2026-10-27: a +/// `ConditionCheck` value 32 `M` wrappers deep is accepted, 33 is refused, +/// while a `Put` item attribute 32 deep is refused). A value that is stored +/// then meets the item limit, which is checked separately. +pub fn validate_expression_value_nesting_depth( + value: &AttributeValue, +) -> Result<(), DynamoDbError> { + match value { + AttributeValue::M(map) => map + .values() + .try_for_each(|v| check_attribute_value_depth(v, 0)), + AttributeValue::L(list) => list + .iter() + .try_for_each(|v| check_attribute_value_depth(v, 0)), + _ => Ok(()), + } +} + fn check_attribute_value_depth( value: &AttributeValue, current_depth: usize, diff --git a/crates/engine/src/attribute_updates.rs b/crates/engine/src/attribute_updates.rs new file mode 100644 index 000000000..e7b95babd --- /dev/null +++ b/crates/engine/src/attribute_updates.rs @@ -0,0 +1,121 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Request-time checks of the legacy `AttributeUpdates` parameter of +//! `UpdateItem`, run before the parameter is desugared into an +//! `UpdateExpression`. + +use std::collections::HashMap; + +use extenddb_core::error::DynamoDbError; +use extenddb_core::expression::attribute_type_code; +use extenddb_core::types::{AttributeValue, AttributeValueUpdate}; + +/// Refuse an `ADD` whose value is not a number or a set, and a `DELETE` with a +/// value that is not a set, with the service's text (measured 2026-10-27): +/// `1 validation error detected: One or more parameter values were invalid: +/// ADD action is not supported for the type S` and `... DELETE action with +/// value is not supported for the type N`. The check runs ahead of the table +/// lookup and of key validation. +/// +/// The service names a different entry from one call to the next when several +/// are wrong; entries are visited in attribute name order here so the answer +/// is repeatable. The service also accepts an empty list for `ADD`, which this +/// check refuses like every other type. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` for the first offending entry. +pub(crate) fn validate_action_value_types( + updates: &HashMap, +) -> Result<(), DynamoDbError> { + let mut names: Vec<&String> = updates.keys().collect(); + names.sort(); + for name in names { + let update = &updates[name]; + let Some(value) = update.value.as_ref() else { + continue; + }; + let refusal = match update.action.to_uppercase().as_str() { + "ADD" if !matches!(value, AttributeValue::N(_)) && !is_set(value) => { + "ADD action is not supported for the type" + } + "DELETE" if !is_set(value) => "DELETE action with value is not supported for the type", + _ => continue, + }; + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: One or more parameter values were invalid: {refusal} {}", + attribute_type_code(value) + ))); + } + Ok(()) +} + +fn is_set(value: &AttributeValue) -> bool { + matches!( + value, + AttributeValue::SS(_) | AttributeValue::NS(_) | AttributeValue::BS(_) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn refusal(action: &str, value: AttributeValue) -> String { + let updates = HashMap::from([( + "a".to_owned(), + AttributeValueUpdate { + value: Some(value), + action: action.to_owned(), + }, + )]); + match validate_action_value_types(&updates) { + Ok(()) => "OK".to_owned(), + Err(DynamoDbError::ValidationException(msg)) => msg, + Err(other) => panic!("unexpected {other:?}"), + } + } + + const ADD: &str = "1 validation error detected: One or more parameter values were invalid: ADD action is not supported for the type"; + const DELETE: &str = "1 validation error detected: One or more parameter values were invalid: DELETE action with value is not supported for the type"; + + // Captured from Amazon DynamoDB on 2026-10-27. + #[test] + fn add_takes_numbers_and_sets_and_delete_takes_sets() { + assert_eq!( + refusal("ADD", AttributeValue::S("x".into())), + format!("{ADD} S") + ); + assert_eq!( + refusal("ADD", AttributeValue::M(Default::default())), + format!("{ADD} M") + ); + assert_eq!( + refusal("ADD", AttributeValue::Bool(true)), + format!("{ADD} BOOL") + ); + assert_eq!( + refusal("DELETE", AttributeValue::N("1".into())), + format!("{DELETE} N") + ); + assert_eq!( + refusal("DELETE", AttributeValue::S("x".into())), + format!("{DELETE} S") + ); + assert_eq!( + refusal("DELETE", AttributeValue::L(vec![])), + format!("{DELETE} L") + ); + assert_eq!(refusal("ADD", AttributeValue::N("1".into())), "OK"); + assert_eq!( + refusal("ADD", AttributeValue::SS(["a".to_owned()].into())), + "OK" + ); + assert_eq!( + refusal("DELETE", AttributeValue::NS(["1".to_owned()].into())), + "OK" + ); + assert_eq!(refusal("PUT", AttributeValue::S("x".into())), "OK"); + } +} diff --git a/crates/engine/src/batch_get_item.rs b/crates/engine/src/batch_get_item.rs index d392c4a1c..2c9f8f354 100755 --- a/crates/engine/src/batch_get_item.rs +++ b/crates/engine/src/batch_get_item.rs @@ -3,7 +3,7 @@ //! `BatchGetItem` operation handler. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use serde_json::Value; @@ -154,7 +154,11 @@ pub async fn handle_batch_get_item( let mut total_rcu: f64 = 0.0; let mut total_pre_proj_bytes: usize = 0; let mut returned_count: u64 = 0; - let mut per_table_rcu: HashMap = HashMap::new(); + // Keyed by table name so ConsumedCapacity lists the tables in a stable + // order. The service's order is not stable: twelve calls over three + // tables returned all six orders, and one request order came back in two + // different orders (measured 2026-10-24). + let mut per_table_rcu: BTreeMap = BTreeMap::new(); for (table_name, ka) in &input.request_items { let key_info = ctx @@ -165,30 +169,36 @@ pub async fn handle_batch_get_item( let projection = table_projections.get(table_name).and_then(Option::as_ref); let mut table_items: Vec = Vec::new(); + let strongly_consistent = ka.consistent_read == Some(true); for key in &ka.keys { validate_batch_key_only(key, &key_info.key_schema, &key_info.attribute_definitions)?; validate_key_sizes(key, &key_info.key_schema, &ctx.limits)?; - if let Some(item) = ctx + let item = ctx .storage .get_item(&key_info, key) .await - .map_err(storage_err_to_dynamo)? - { - let size = item_size_bytes(&item); - let strongly_consistent = ka.consistent_read == Some(true); - let item_rcu = capacity_helpers::read_capacity_units(size, strongly_consistent); - total_rcu += item_rcu; - *per_table_rcu.entry(table_name.clone()).or_default() += item_rcu; - total_pre_proj_bytes += size; - returned_count += 1; - let item = if let Some(projection) = projection { - projection.apply(&item) - } else { - item - }; - table_items.push(item); - } + .map_err(storage_err_to_dynamo)?; + // An absent key is charged the minimum read (0.5 eventually + // consistent, 1.0 strongly consistent) and keeps its table in + // ConsumedCapacity, as GetItem does (measured 2026-10-24: one + // absent key 0.5, two 1.0, one hundred 50.0, ConsistentRead 1.0 + // each). + let size = item.as_ref().map_or(0, item_size_bytes); + let item_rcu = capacity_helpers::read_capacity_units(size, strongly_consistent); + total_rcu += item_rcu; + *per_table_rcu.entry(table_name.clone()).or_default() += item_rcu; + let Some(item) = item else { + continue; + }; + total_pre_proj_bytes += size; + returned_count += 1; + let item = if let Some(projection) = projection { + projection.apply(&item) + } else { + item + }; + table_items.push(item); } responses.insert(table_name.clone(), table_items); } diff --git a/crates/engine/src/batch_write_item.rs b/crates/engine/src/batch_write_item.rs index b0381c7a7..b95865d74 100755 --- a/crates/engine/src/batch_write_item.rs +++ b/crates/engine/src/batch_write_item.rs @@ -240,13 +240,16 @@ pub async fn handle_batch_write_item( // Per-item WCU already accumulated above (M-1: DynamoDB rounds per item, then sums). let wcu = total_wcu; + // With ReturnItemCollectionMetrics SIZE the service always returns the + // member: the populated map when a written table has a local secondary + // index, and `{}` when none does (measured 2026-10-14, as for + // TransactWriteItems). Without SIZE the member is absent. let output = BatchWriteItemOutput { unprocessed_items: HashMap::new(), consumed_capacity, - item_collection_metrics: if all_icm.is_empty() { - None - } else { - Some(all_icm) + item_collection_metrics: match input.return_item_collection_metrics { + ReturnItemCollectionMetrics::Size => Some(all_icm), + ReturnItemCollectionMetrics::None => None, }, }; let body = serialize_output(&output)?; diff --git a/crates/engine/src/delete_item.rs b/crates/engine/src/delete_item.rs index 049c368f6..f22f44f11 100755 --- a/crates/engine/src/delete_item.rs +++ b/crates/engine/src/delete_item.rs @@ -11,7 +11,7 @@ use extenddb_core::types::{DeleteItemInput, DeleteItemOutput, ReturnValues, item use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::{storage_err_to_dynamo, storage_err_to_dynamo_with_ccf}; -use crate::expression_helpers::resolve_condition; +use crate::expression_helpers::{resolve_condition, single_item_write_prefix}; use crate::serialize_output; use crate::stream_capture; use crate::{DispatchMetrics, DispatchResult}; @@ -68,13 +68,16 @@ pub async fn handle_delete_item( ], &[("ConditionExpression", has_condition)], )?; + // The single-item write APIs name no expression in the values message and + // wrap every expression-stage error in `1 validation error detected: `. extenddb_core::expression::validate_expression_param_usage( input.expression_attribute_names.as_ref(), has_condition, input.expression_attribute_values.as_ref(), has_condition, - &[extenddb_core::expression::ExpressionKind::Condition], - )?; + &[], + ) + .map_err(single_item_write_prefix)?; // ConditionalOperator requires an Expected with two or more conditions. extenddb_core::validation::validate_conditional_operator_usage( @@ -103,7 +106,8 @@ pub async fn handle_delete_item( input.expected.as_ref(), input.conditional_operator, &ctx.limits, - )?; + ) + .map_err(single_item_write_prefix)?; if input.expected.is_none() || input @@ -112,6 +116,9 @@ pub async fn handle_delete_item( .is_some_and(std::collections::HashMap::is_empty) { let exprs: Vec<&extenddb_core::expression::Expr> = condition.iter().collect(); + // The condition's references, functions, and operand types were + // checked by `resolve_condition`; the unused check follows them and + // runs ahead of the table lookup, as on the service. extenddb_core::expression::validate_unused_attributes( &maps.names, &maps.values, @@ -119,7 +126,8 @@ pub async fn handle_delete_item( &[], &std::collections::HashSet::new(), &std::collections::HashSet::new(), - )?; + ) + .map_err(single_item_write_prefix)?; } let key_info = ctx diff --git a/crates/engine/src/expression_helpers.rs b/crates/engine/src/expression_helpers.rs index 913333015..b15082c13 100755 --- a/crates/engine/src/expression_helpers.rs +++ b/crates/engine/src/expression_helpers.rs @@ -8,8 +8,9 @@ use std::collections::HashMap; use extenddb_core::error::DynamoDbError; use extenddb_core::expression::{ Expr, ExpressionKind, ExpressionMaps, KeyCondition, PathElement, Token, UpdateAction, - parse_condition_with_depth_limit, parse_key_condition, parse_projection, parse_update_from, - tokenize_for, tokenize_with_limit, validate_no_reserved_words, validate_ordering_operand_types, + parse_condition_with_depth_limit, parse_key_condition, parse_projection, parse_update_sections, + tokenize_for, tokenize_with_limit, validate_condition_semantics, validate_no_reserved_words, + validate_update_semantics, }; use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{AttributeValue, ConditionalOperator, ExpectedAttributeValue}; @@ -66,11 +67,32 @@ pub fn check_expression_size( expr: &str, kind: ExpressionKind, limits: &LimitsConfig, +) -> Result<(), DynamoDbError> { + check_expression_size_with(expr, kind, limits, kind.size_error_includes_length()) +} + +/// `check_expression_size` with the length suffix decided by the caller. +/// +/// Amazon DynamoDB appends ` expression size: N` for a `ConditionExpression` +/// inside a transaction and for a Scan `FilterExpression`, and omits it for a +/// `ConditionExpression` on PutItem, UpdateItem, or DeleteItem and for a Query +/// `FilterExpression` (measured 2026-10-14). The suffix is therefore a property +/// of the operation, not of the expression kind alone. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` when the expression is over +/// the configured byte limit. +pub fn check_expression_size_with( + expr: &str, + kind: ExpressionKind, + limits: &LimitsConfig, + include_length: bool, ) -> Result<(), DynamoDbError> { if expr.len() > limits.max_expression_length_bytes { let mut msg = format!("Invalid {kind}: Expression size has exceeded the maximum allowed size;"); - if kind.size_error_includes_length() { + if include_length { use std::fmt::Write as _; let _ = write!(msg, " expression size: {}", expr.len()); } @@ -79,33 +101,68 @@ pub fn check_expression_size( Ok(()) } -/// Tokenize, reserved-word check, and parse a required `ConditionExpression`. +/// Amazon DynamoDB wraps the expression-stage errors of the single-item write +/// APIs (PutItem, UpdateItem, DeleteItem) in `1 validation error detected: `: +/// every `Invalid ConditionExpression:` and `Invalid UpdateExpression:` message, +/// the unused-name and unused-value messages, and the names-or-values-without- +/// expression messages. The read APIs, the batch APIs, and the transactions +/// return the same messages bare. Apply this at the expression stage of those +/// three handlers only; other validation messages there ("Cannot update +/// attribute pk", key schema, item size) carry no prefix on the service. +pub fn single_item_write_prefix(err: DynamoDbError) -> DynamoDbError { + match err { + DynamoDbError::ValidationException(msg) + if !msg.starts_with("1 validation error detected") + && (msg.starts_with("Invalid ConditionExpression:") + || msg.starts_with("Invalid UpdateExpression:") + || msg.starts_with("Value provided in ExpressionAttribute") + || msg.starts_with("ExpressionAttributeNames can only be specified") + || msg.starts_with("ExpressionAttributeValues can only be specified")) => + { + DynamoDbError::ValidationException(format!("1 validation error detected: {msg}")) + } + other => other, + } +} + +/// Tokenize, reserved-word check, and parse a required `ConditionExpression`, +/// with the size-error length suffix decided by the caller (see +/// `check_expression_size_with`). /// /// Errors carry the `ConditionExpression` prefix, matching Amazon DynamoDB. /// /// # Errors /// -/// Returns `DynamoDbError::ValidationException` for syntax or reserved-word errors. -pub fn parse_condition_expr(expr: &str, limits: &LimitsConfig) -> Result { - check_expression_size(expr, ExpressionKind::Condition, limits)?; +/// Returns `DynamoDbError::ValidationException` for over-size, syntax, or +/// reserved-word errors. +pub fn parse_condition_expr_with( + expr: &str, + limits: &LimitsConfig, + include_length: bool, +) -> Result { + check_expression_size_with(expr, ExpressionKind::Condition, limits, include_length)?; tokenize_expression(expr, limits) .and_then(|tokens| parse_condition_with_depth_limit(&tokens, limits.max_expression_depth)) .map_err(|e| prefix_expression_error(e, ExpressionKind::Condition)) } -/// Tokenize, reserved-word check, and parse an `UpdateExpression`. -/// -/// Errors carry the `UpdateExpression` prefix, matching Amazon DynamoDB. +/// Size-check, tokenize, parse, and semantically validate an +/// `UpdateExpression` against the request's expression maps: every syntax +/// error first, then the service's single ordered walk (function names, +/// operand counts and types, undefined references, repeated sections), then +/// overlapping paths. See `extenddb_core::expression::validate_update_semantics`. /// /// # Errors /// -/// Returns `DynamoDbError::ValidationException` for syntax or reserved-word errors. -pub fn parse_update_expr( +/// Returns `DynamoDbError::ValidationException` with the `Invalid +/// UpdateExpression:` prefix. +pub fn parse_update_expr_checked( update_expr: &str, + maps: &ExpressionMaps, limits: &LimitsConfig, ) -> Result, DynamoDbError> { check_expression_size(update_expr, ExpressionKind::Update, limits)?; - tokenize_for( + let parsed = tokenize_for( update_expr, limits.max_expression_tokens, ExpressionKind::Update, @@ -114,42 +171,59 @@ pub fn parse_update_expr( if limits.enforce_reserved_keywords { validate_no_reserved_words(&update_tokens)?; } - parse_update_from(&update_tokens, update_expr) + parse_update_sections(&update_tokens, Some(update_expr)) }) - .map_err(|e| prefix_expression_error(e, ExpressionKind::Update)) + .map_err(|e| prefix_expression_error(e, ExpressionKind::Update))?; + validate_update_semantics(&parsed, maps)?; + Ok(parsed.actions) } -/// Parse an optional condition expression string into an AST. +/// Size-check, tokenize, parse, and semantically validate a condition-grammar +/// expression (`ConditionExpression` or `FilterExpression`) against the +/// request's expression maps. Syntax errors come first, then the service's +/// ordered walk: undefined references, function names, operand counts and +/// types, and functions in the wrong position. See +/// `extenddb_core::expression::validate_condition_semantics`. /// -/// Returns `None` if the input is `None` or empty. +/// `kind` labels the messages; `include_length` decides the size-error suffix +/// (see [`check_expression_size_with`]). /// /// # Errors /// -/// Returns `DynamoDbError::ValidationException` for syntax errors. -pub fn parse_optional_condition( - expr: Option<&str>, +/// Returns `DynamoDbError::ValidationException` with the `Invalid {kind}:` prefix. +pub fn parse_condition_checked( + expr: &str, + maps: &ExpressionMaps, limits: &LimitsConfig, -) -> Result, DynamoDbError> { - match expr { - Some(s) if !s.is_empty() => parse_condition_expr(s, limits).map(Some), - _ => Ok(None), + include_length: bool, + kind: ExpressionKind, +) -> Result { + if expr.is_empty() { + return Err(DynamoDbError::ValidationException(format!( + "Invalid {kind}: The expression can not be empty;" + ))); } + let parsed = parse_condition_expr_with(expr, limits, include_length) + .map_err(|e| prefix_expression_error(e, kind))?; + validate_condition_semantics(&parsed, maps, kind)?; + Ok(parsed) } -/// Parse an optional filter expression string into an AST. -/// -/// `FilterExpression` uses the same grammar as `ConditionExpression`. -/// Returns `None` if the input is `None` or empty. -/// -/// # Errors -/// -/// Returns `DynamoDbError::ValidationException` for syntax errors. +/// Parse an optional filter expression (grammar only, `FilterExpression` +/// prefix). The handlers run the semantic pass through +/// [`parse_condition_checked`]; this remains for the parser tests. +#[cfg(test)] pub fn parse_optional_filter( expr: Option<&str>, limits: &LimitsConfig, + include_length: bool, ) -> Result, DynamoDbError> { - parse_optional_condition(expr, limits) - .map_err(|e| prefix_expression_error(e, ExpressionKind::Filter)) + match expr { + Some(s) if !s.is_empty() => parse_condition_expr_with(s, limits, include_length) + .map(Some) + .map_err(|e| prefix_expression_error(e, ExpressionKind::Filter)), + _ => Ok(None), + } } /// Resolve a condition from either `ConditionExpression` or legacy `Expected`. @@ -201,11 +275,20 @@ pub fn resolve_condition( } let maps = build_expression_maps(names, values); - let condition = parse_optional_condition(condition_expression, limits)?; - if let Some(ref expr) = condition { - validate_ordering_operand_types(expr, &maps) - .map_err(|e| prefix_expression_error(e, ExpressionKind::Condition))?; - } + // The single-item write APIs report an over-size condition without the + // length suffix; transactions keep it (see `check_expression_size_with`). + // The whole condition, syntax and semantics, is checked here before the + // caller looks at any other expression, as the service does. + let condition = match condition_expression { + Some(s) if !s.is_empty() => Some(parse_condition_checked( + s, + &maps, + limits, + false, + ExpressionKind::Condition, + )?), + _ => None, + }; Ok((condition, maps)) } @@ -235,20 +318,29 @@ pub fn parse_projection_expr( result.map_err(|e| prefix_expression_error(e, ExpressionKind::Projection)) } -/// Size-check, tokenize, reserved-word check, and parse a `KeyConditionExpression`. +/// Size-check, tokenize, reserved-word check, and parse a +/// `KeyConditionExpression`, then reject a `#name` or `:value` the request +/// does not define. /// -/// Errors carry the `KeyConditionExpression` prefix, matching Amazon `DynamoDB`. +/// Errors carry the `KeyConditionExpression` prefix, matching Amazon +/// `DynamoDB`. The references are checked once the whole expression parses, +/// in the order they stand in the text, names and values alike (measured +/// 2026-10-27: `pk = :q AND #n = :v` names `:q`, `#b = :p AND #a > :q` names +/// `#b`). The check runs before the key schema is consulted, so an undefined +/// `#n` is reported as such rather than as a key schema mismatch (measured +/// 2026-10-14). /// /// # Errors /// -/// Returns `DynamoDbError::ValidationException` for over-size, syntax, or -/// reserved-word errors. +/// Returns `DynamoDbError::ValidationException` for over-size, syntax, +/// reserved-word, or undefined-reference errors. pub fn parse_key_condition_expr( expr: &str, + maps: &ExpressionMaps, limits: &LimitsConfig, ) -> Result { check_expression_size(expr, ExpressionKind::KeyCondition, limits)?; - tokenize_for( + let (tokens, parsed) = tokenize_for( expr, limits.max_expression_tokens, ExpressionKind::KeyCondition, @@ -257,9 +349,33 @@ pub fn parse_key_condition_expr( if limits.enforce_reserved_keywords { validate_no_reserved_words(&tokens)?; } - parse_key_condition(&tokens) + let parsed = parse_key_condition(&tokens)?; + Ok((tokens, parsed)) }) - .map_err(|e| prefix_expression_error(e, ExpressionKind::KeyCondition)) + .map_err(|e| prefix_expression_error(e, ExpressionKind::KeyCondition))?; + check_key_condition_refs(&tokens, maps)?; + Ok(parsed) +} + +/// The first `#name` or `:value` token, in text order, that `maps` does not +/// define. +fn check_key_condition_refs(tokens: &[Token], maps: &ExpressionMaps) -> Result<(), DynamoDbError> { + for token in tokens { + match token { + Token::NameRef(name) if !maps.names.contains_key(name) => { + return Err(DynamoDbError::ValidationException(format!( + "Invalid KeyConditionExpression: An expression attribute name used in the document path is not defined; attribute name: #{name}" + ))); + } + Token::Placeholder(value) if !maps.values.contains_key(value) => { + return Err(DynamoDbError::ValidationException(format!( + "Invalid KeyConditionExpression: An expression attribute value used in expression is not defined; attribute value: :{value}" + ))); + } + _ => {} + } + } + Ok(()) } /// Prefix an expression error with the expression type, matching `DynamoDB`'s format. @@ -302,7 +418,7 @@ mod tests { "((a = :v AND b = :v2))", "((NOT (a = :v)))", ] { - let err = parse_optional_condition(Some(expr), &limits).unwrap_err(); + let err = parse_condition_expr_with(expr, &limits, true).unwrap_err(); assert!( matches!(&err, DynamoDbError::ValidationException(msg) if msg == CONDITION_REDUNDANT), "expr {expr}: got {err:?}" @@ -320,7 +436,7 @@ mod tests { "(NOT (a = :v))", ] { assert!( - parse_optional_condition(Some(expr), &limits).is_ok(), + parse_condition_expr_with(expr, &limits, true).is_ok(), "expr {expr} should parse" ); } @@ -329,7 +445,7 @@ mod tests { #[test] fn filter_redundant_parens_rejected_with_filter_label() { let limits = LimitsConfig::default(); - let err = parse_optional_filter(Some("((a = :v))"), &limits).unwrap_err(); + let err = parse_optional_filter(Some("((a = :v))"), &limits, true).unwrap_err(); assert!( matches!(&err, DynamoDbError::ValidationException(msg) if msg == "Invalid FilterExpression: The expression has redundant parentheses;"), @@ -340,7 +456,7 @@ mod tests { #[test] fn filter_parser_errors_carry_filter_label() { let limits = LimitsConfig::default(); - let err = parse_optional_filter(Some("a"), &limits).unwrap_err(); + let err = parse_optional_filter(Some("a"), &limits, true).unwrap_err(); assert!( matches!(&err, DynamoDbError::ValidationException(msg) if msg.starts_with("Invalid FilterExpression:")), @@ -354,7 +470,7 @@ mod tests { let limits = LimitsConfig::default(); let path = "a".to_owned() + &"z".repeat(limits.max_expression_length_bytes); let expr = format!("{path} = :v"); - let err = parse_optional_filter(Some(&expr), &limits).unwrap_err(); + let err = parse_optional_filter(Some(&expr), &limits, true).unwrap_err(); assert!( matches!(&err, DynamoDbError::ValidationException(msg) if *msg == format!( @@ -369,7 +485,7 @@ mod tests { let limits = LimitsConfig::default(); let path = "a".to_owned() + &"z".repeat(limits.max_expression_length_bytes); let expr = format!("attribute_not_exists({path})"); - let err = parse_optional_condition(Some(&expr), &limits).unwrap_err(); + let err = parse_condition_expr_with(&expr, &limits, true).unwrap_err(); assert!( matches!(&err, DynamoDbError::ValidationException(msg) if *msg == format!( diff --git a/crates/engine/src/legacy_filter.rs b/crates/engine/src/legacy_filter.rs index 2ef07df77..97f9cab81 100644 --- a/crates/engine/src/legacy_filter.rs +++ b/crates/engine/src/legacy_filter.rs @@ -6,53 +6,115 @@ //! Converts pre-expression parameters (`KeyConditions`, `QueryFilter`, `ScanFilter`) //! into expression-based equivalents. +use std::cmp::Ordering; use std::collections::HashMap; use extenddb_core::error::DynamoDbError; use extenddb_core::expression::{ CompareOp, Expr, ExpressionMaps, KeyCondition, PathElement, SortKeyCondition, + attribute_type_code, compare_orderable, }; -use extenddb_core::types::{AttributeValue, Condition, ConditionalOperator}; +use extenddb_core::types::{ + AttributeDefinition, AttributeValue, Condition, ConditionalOperator, KeySchemaElement, KeyType, + ScalarAttributeType, +}; +use extenddb_core::validation::{validate_attribute_number, validate_no_empty_key_value}; /// Desugar legacy `KeyConditions` into a `KeyCondition` struct and expression maps. /// -/// The hash key must have `ComparisonOperator` = "EQ". -/// The optional range key supports: EQ, LE, LT, GE, GT, `BEGINS_WITH`, BETWEEN. +/// A key schema with several HASH or several RANGE attributes is handed to +/// `desugar_multipart_key_conditions`, and the engine classifies its result +/// with that schema's rules. On any other key schema this runs the checks +/// that need the key schema, in the order the service applies them (measured +/// 2026-10-24 on S/S, S/N, hash-only and GSI key schemas): +/// +/// 1. the keys are walked in schema order; an absent hash key answers +/// `Query condition missed key schema element: `, a present key whose +/// value type differs from its `AttributeDefinitions` type answers +/// `One or more parameter values were invalid: Condition parameter type +/// does not match schema type`, and a hash key with an operator other than +/// `EQ` answers `Query key condition not supported` (the type is checked +/// first: `pk LT N` on an S key names the type); +/// 2. a condition on an attribute outside the key schema answers `Query +/// condition missed key schema element: ` (the service names the +/// first absent key, and refuses three conditions on a two-key schema with +/// its own message; neither is changed here); +/// 3. an empty `S` or `B` key value, hash key first, answers `One or more +/// parameter values are not valid. The AttributeValue for a key attribute +/// cannot contain an empty string value. Key: ` (measured for every +/// range operator and both `BETWEEN` bounds; the binary wording is the +/// shared key check's and was not measured on this path). +/// +/// The request-time checks ([`validate_key_conditions`]) ran before the table +/// lookup, so a non-indexable operator never reaches this function. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the service's text. pub fn desugar_key_conditions( conditions: &HashMap, - key_schema: &[(String, bool)], // Vec of (attr_name, is_hash) + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], ) -> Result<(KeyCondition, ExpressionMaps), DynamoDbError> { - let hash_count = key_schema.iter().filter(|(_, is_hash)| *is_hash).count(); + let hash_count = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Hash) + .count(); let range_count = key_schema.len() - hash_count; if hash_count > 1 || range_count > 1 { - return desugar_multipart_key_conditions(conditions, key_schema); + let key_schema_pairs: Vec<(String, bool)> = key_schema + .iter() + .map(|ks| (ks.attribute_name.clone(), ks.key_type == KeyType::Hash)) + .collect(); + return desugar_multipart_key_conditions(conditions, &key_schema_pairs); } - // Find the hash key condition let hash_attr = key_schema .iter() - .find(|(_, is_hash)| *is_hash) - .map(|(name, _)| name.as_str()) + .find(|ks| ks.key_type == KeyType::Hash) + .map(|ks| ks.attribute_name.as_str()) .ok_or_else(|| { DynamoDbError::ValidationException("No hash key in key schema".to_owned()) })?; + let range_attr = key_schema + .iter() + .find(|ks| ks.key_type == KeyType::Range) + .map(|ks| ks.attribute_name.as_str()); + + validate_key_conditions(conditions)?; let hash_cond = conditions.get(hash_attr).ok_or_else(|| { DynamoDbError::ValidationException(format!( "Query condition missed key schema element: {hash_attr}" )) })?; - + check_key_schema_type(hash_attr, hash_cond, attribute_definitions)?; if hash_cond.comparison_operator != "EQ" { return Err(DynamoDbError::ValidationException( "Query key condition not supported".to_owned(), )); } + let range_cond = range_attr.and_then(|name| conditions.get(name).map(|cond| (name, cond))); + if let Some((name, cond)) = range_cond { + check_key_schema_type(name, cond, attribute_definitions)?; + } - if hash_cond.attribute_value_list.len() != 1 { - return Err(DynamoDbError::ValidationException( - "Query key condition must have exactly one value for EQ".to_owned(), - )); + // Check for non-key attributes in KeyConditions (invalid) + for attr_name in conditions.keys() { + if attr_name != hash_attr && range_attr != Some(attr_name.as_str()) { + return Err(DynamoDbError::ValidationException(format!( + "Query condition missed key schema element: {hash_attr}" + ))); + } + } + + for value in &hash_cond.attribute_value_list { + validate_no_empty_key_value(hash_attr, value)?; + } + if let Some((name, cond)) = range_cond { + for value in &cond.attribute_value_list { + validate_no_empty_key_value(name, value)?; + } } let mut values = HashMap::new(); @@ -65,31 +127,11 @@ pub fn desugar_key_conditions( let pk_path = vec![PathElement::Attribute(hash_attr.to_owned())]; let pk_value = Expr::Placeholder(pk_placeholder.clone()); - // Find optional range key condition - let range_attr = key_schema - .iter() - .find(|(_, is_hash)| !*is_hash) - .map(|(name, _)| name.as_str()); - - let sk_condition = if let Some(range_name) = range_attr { - if let Some(range_cond) = conditions.get(range_name) { - Some(desugar_sk_condition(range_cond, range_name, &mut values)?) - } else { - None - } - } else { - None + let sk_condition = match range_cond { + Some((name, cond)) => Some(desugar_sk_condition(cond, name, &mut values)?), + None => None, }; - // Check for non-key attributes in KeyConditions (invalid) - for attr_name in conditions.keys() { - if attr_name != hash_attr && range_attr != Some(attr_name.as_str()) { - return Err(DynamoDbError::ValidationException(format!( - "Query condition missed key schema element: {hash_attr}" - ))); - } - } - let maps = ExpressionMaps::new(HashMap::new(), values); Ok(( @@ -104,6 +146,104 @@ pub fn desugar_key_conditions( )) } +/// Refuse a key condition value whose type is not the key's +/// `AttributeDefinitions` type (measured 2026-10-24: `pk EQ N`, `EQ B`, `EQ +/// BOOL` on an S key, `sk EQ S` and `BEGINS_WITH S` on an N key, every range +/// operator, GSI keys; the message names no attribute). A key with no +/// definition is left to the storage layer, as before. +fn check_key_schema_type( + attr_name: &str, + cond: &Condition, + attribute_definitions: &[AttributeDefinition], +) -> Result<(), DynamoDbError> { + let Some(expected) = attribute_definitions + .iter() + .find(|ad| ad.attribute_name == attr_name) + .map(|ad| ad.attribute_type) + else { + return Ok(()); + }; + let mismatch = cond.attribute_value_list.iter().any(|value| { + !matches!( + (expected, value), + (ScalarAttributeType::S, AttributeValue::S(_)) + | (ScalarAttributeType::N, AttributeValue::N(_)) + | (ScalarAttributeType::B, AttributeValue::B(_)) + ) + }); + if mismatch { + return Err(DynamoDbError::ValidationException(format!( + "{INVALID_PARAMS}Condition parameter type does not match schema type" + ))); + } + Ok(()) +} + +/// Operators the service refuses on any key condition entry, whichever key +/// the entry names, with `Attempted conditional constraint is not an +/// indexable operation` (measured 2026-10-23 on the range key and 2026-10-24 +/// on the hash key: `pk NE`, `pk IN`, `pk NULL`, `pk NOT_NULL`, `pk +/// CONTAINS`). +const NON_INDEXABLE: [&str; 6] = ["NE", "NULL", "NOT_NULL", "CONTAINS", "NOT_CONTAINS", "IN"]; + +/// Request-time checks for legacy `KeyConditions`: [`validate_legacy_conditions`] +/// on every entry, then the non-indexable operator refusal on every entry. +/// Both ran before the table lookup on the service: a non-indexable operator +/// was named ahead of the Select checks that are themselves reported on a +/// table that does not exist, ahead of a missing or surplus key schema +/// element, and ahead of a schema type mismatch on another entry (measured +/// 2026-10-24). +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the service's text. +pub fn validate_key_conditions( + conditions: &HashMap, +) -> Result<(), DynamoDbError> { + validate_legacy_conditions(conditions)?; + if conditions + .values() + .any(|cond| NON_INDEXABLE.contains(&cond.comparison_operator.as_str())) + { + return Err(DynamoDbError::ValidationException(format!( + "{INVALID_PARAMS}Attempted conditional constraint is not an indexable operation" + ))); + } + Ok(()) +} + +/// Refuse a legacy `QueryFilter` entry on a key attribute of the queried +/// table or index: `QueryFilter can only contain non-primary key attributes: +/// Primary key attribute: `, the hash key named first (measured +/// 2026-10-24: table keys on a table query, index keys on a GSI query; table +/// keys pass on a GSI query and index keys pass on a table query). The +/// service reports it after the table and index lookup, the key schema and +/// empty key value checks and the Select checks, and before the key value +/// size check and the `ExclusiveStartKey` check, so the caller runs it after +/// the Select checks. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` naming the key attribute. +pub fn reject_query_filter_on_key( + filter: &HashMap, + key_schema: &[KeySchemaElement], +) -> Result<(), DynamoDbError> { + let hash_first = key_schema + .iter() + .filter(|ks| ks.key_type == KeyType::Hash) + .chain(key_schema.iter().filter(|ks| ks.key_type == KeyType::Range)); + for ks in hash_first { + if filter.contains_key(&ks.attribute_name) { + return Err(DynamoDbError::ValidationException(format!( + "QueryFilter can only contain non-primary key attributes: Primary key attribute: {}", + ks.attribute_name + ))); + } + } + Ok(()) +} + fn desugar_sk_condition( cond: &Condition, range_name: &str, @@ -112,11 +252,6 @@ fn desugar_sk_condition( let path = vec![PathElement::Attribute(range_name.to_owned())]; match cond.comparison_operator.as_str() { "EQ" | "LE" | "LT" | "GE" | "GT" => { - if cond.attribute_value_list.len() != 1 { - return Err(DynamoDbError::ValidationException( - "Invalid number of values for comparison".to_owned(), - )); - } let placeholder = "_kc_sk".to_owned(); values.insert(placeholder.clone(), cond.attribute_value_list[0].clone()); let op = match cond.comparison_operator.as_str() { @@ -134,11 +269,6 @@ fn desugar_sk_condition( }) } "BETWEEN" => { - if cond.attribute_value_list.len() != 2 { - return Err(DynamoDbError::ValidationException( - "BETWEEN requires exactly 2 values".to_owned(), - )); - } let lo_placeholder = "_kc_sk_lo".to_owned(); let hi_placeholder = "_kc_sk_hi".to_owned(); values.insert(lo_placeholder.clone(), cond.attribute_value_list[0].clone()); @@ -150,11 +280,6 @@ fn desugar_sk_condition( }) } "BEGINS_WITH" => { - if cond.attribute_value_list.len() != 1 { - return Err(DynamoDbError::ValidationException( - "BEGINS_WITH requires exactly 1 value".to_owned(), - )); - } let placeholder = "_kc_sk".to_owned(); values.insert(placeholder.clone(), cond.attribute_value_list[0].clone()); Ok(SortKeyCondition::BeginsWith { @@ -162,8 +287,9 @@ fn desugar_sk_condition( prefix: Expr::Placeholder(placeholder), }) } - other => Err(DynamoDbError::ValidationException(format!( - "Unsupported KeyCondition operator: {other}" + // validate_key_conditions refused every other operator at request time. + _ => Err(DynamoDbError::ValidationException(format!( + "{INVALID_PARAMS}Attempted conditional constraint is not an indexable operation" ))), } } @@ -371,19 +497,147 @@ pub fn desugar_filter( Ok((combined, maps)) } +const INVALID_PARAMS: &str = "One or more parameter values were invalid: "; + +/// The item-path number check, with one difference in wording. A legacy +/// condition value `{"N": ""}` answers `The parameter cannot be converted to a +/// numeric value: `, the separator kept with nothing after it, where the item +/// path answers the text without the separator. Measured on a ScanFilter `EQ` +/// condition; every other number error answers the item-path text. +fn validate_legacy_number(value: &AttributeValue) -> Result<(), DynamoDbError> { + if matches!(value, AttributeValue::N(n) if n.is_empty()) { + return Err(DynamoDbError::ValidationException( + "The parameter cannot be converted to a numeric value: ".to_owned(), + )); + } + validate_attribute_number(value) +} + +/// Validate one legacy condition (`ScanFilter`, `QueryFilter` or +/// `KeyConditions` entry) the way the service does at request time, before +/// the table is looked up and before any row is read, with its messages +/// (measured 2026-10-23 and 2026-10-24 on empty tables). In order: +/// +/// 1. every `N` in the value list, including inside `NS`, `L` and `M`, is a +/// valid number (`The parameter cannot be converted to a numeric value: +/// `, or the overflow, underflow and 38-digit messages; no prefix); +/// 2. the operator is one of the thirteen (`Invalid comparison operator`); +/// 3. the value count fits the operator: none for `NULL` and `NOT_NULL`, two +/// for `BETWEEN`, one or more for `IN`, one otherwise (`Invalid number of +/// argument(s) for the ComparisonOperator`); +/// 4. the first value's type is one the operator takes: `EQ` and `NE` take +/// every type, `BEGINS_WITH` takes `S` and `B`, `CONTAINS` and +/// `NOT_CONTAINS` take `S`, `N`, `B`, `BOOL` and `NULL`, the ordering +/// operators, `BETWEEN` and `IN` take `S`, `N` and `B` (`ComparisonOperator +/// is not valid for AttributeValue type`); +/// 5. every value has the first value's type (`AttributeValues inside +/// AttributeValueList must be of same type`); +/// 6. a `BETWEEN` lower bound is not above its upper bound (`The BETWEEN +/// condition was provided a range where the lower bound is greater than +/// the upper bound`). +/// +/// Step 1 was measured ahead of each of steps 2 to 6 (unknown operator, two +/// values, one value for NULL, mixed types, BEGINS_WITH N, inverted range). +/// The order of step 3 against step 4 within one condition was not measured; +/// each was measured against a well-formed other. +/// +/// # Errors +/// +/// Returns `DynamoDbError::ValidationException` with the service's text, +/// prefixed `One or more parameter values were invalid: ` for steps 2 to 6. +pub fn validate_legacy_condition( + operator: &str, + values: &[AttributeValue], +) -> Result<(), DynamoDbError> { + for value in values { + validate_legacy_number(value)?; + } + let refuse = |text: String| { + Err(DynamoDbError::ValidationException(format!( + "{INVALID_PARAMS}{text}" + ))) + }; + let count_ok = match operator { + "NULL" | "NOT_NULL" => values.is_empty(), + "BETWEEN" => values.len() == 2, + "IN" => !values.is_empty(), + "EQ" | "NE" | "LE" | "LT" | "GE" | "GT" | "CONTAINS" | "NOT_CONTAINS" | "BEGINS_WITH" => { + values.len() == 1 + } + _ => return refuse("Invalid comparison operator".to_owned()), + }; + if !count_ok { + return refuse(format!( + "Invalid number of argument(s) for the {operator} ComparisonOperator" + )); + } + let Some(first) = values.first() else { + return Ok(()); + }; + let scalar = matches!( + first, + AttributeValue::S(_) | AttributeValue::N(_) | AttributeValue::B(_) + ); + let type_ok = match operator { + "EQ" | "NE" => true, + "BEGINS_WITH" => matches!(first, AttributeValue::S(_) | AttributeValue::B(_)), + "CONTAINS" | "NOT_CONTAINS" => { + scalar || matches!(first, AttributeValue::Bool(_) | AttributeValue::Null) + } + _ => scalar, + }; + if !type_ok { + return refuse(format!( + "ComparisonOperator {operator} is not valid for {} AttributeValue type", + attribute_type_code(first) + )); + } + if values + .iter() + .any(|v| attribute_type_code(v) != attribute_type_code(first)) + { + return refuse("AttributeValues inside AttributeValueList must be of same type".to_owned()); + } + if operator == "BETWEEN" && compare_orderable(&values[0], &values[1]) == Some(Ordering::Greater) + { + return refuse( + "The BETWEEN condition was provided a range where the lower bound is greater than the upper bound" + .to_owned(), + ); + } + Ok(()) +} + +/// Run [`validate_legacy_condition`] over every entry of a legacy condition +/// map, in attribute name order, so `KeyConditions` can be checked before the +/// table is looked up (the service refuses a bad `QueryFilter` value before it +/// reports a missing table; a bad `KeyConditions` value was measured ahead of +/// every key schema error). +/// +/// # Errors +/// +/// The first entry's error, in attribute name order. +pub fn validate_legacy_conditions( + conditions: &HashMap, +) -> Result<(), DynamoDbError> { + let mut names: Vec<&String> = conditions.keys().collect(); + names.sort(); + for name in names { + let cond = &conditions[name]; + validate_legacy_condition(&cond.comparison_operator, &cond.attribute_value_list)?; + } + Ok(()) +} + fn desugar_one_filter_condition( path: &Expr, cond: &Condition, values: &mut HashMap, counter: &mut u32, ) -> Result { + validate_legacy_condition(&cond.comparison_operator, &cond.attribute_value_list)?; match cond.comparison_operator.as_str() { "EQ" | "NE" | "LE" | "LT" | "GE" | "GT" => { - if cond.attribute_value_list.len() != 1 { - return Err(DynamoDbError::ValidationException( - "Invalid number of attribute values for comparison".to_owned(), - )); - } let placeholder = format!("_f{counter}"); *counter += 1; values.insert(placeholder.clone(), cond.attribute_value_list[0].clone()); @@ -403,11 +657,6 @@ fn desugar_one_filter_condition( }) } "BETWEEN" => { - if cond.attribute_value_list.len() != 2 { - return Err(DynamoDbError::ValidationException( - "BETWEEN requires exactly 2 values".to_owned(), - )); - } let lo = format!("_f{counter}"); *counter += 1; let hi = format!("_f{counter}"); @@ -421,11 +670,6 @@ fn desugar_one_filter_condition( }) } "BEGINS_WITH" => { - if cond.attribute_value_list.len() != 1 { - return Err(DynamoDbError::ValidationException( - "BEGINS_WITH requires exactly 1 value".to_owned(), - )); - } let placeholder = format!("_f{counter}"); *counter += 1; values.insert(placeholder.clone(), cond.attribute_value_list[0].clone()); @@ -435,11 +679,6 @@ fn desugar_one_filter_condition( }) } "CONTAINS" => { - if cond.attribute_value_list.len() != 1 { - return Err(DynamoDbError::ValidationException( - "CONTAINS requires exactly 1 value".to_owned(), - )); - } let placeholder = format!("_f{counter}"); *counter += 1; values.insert(placeholder.clone(), cond.attribute_value_list[0].clone()); @@ -449,11 +688,6 @@ fn desugar_one_filter_condition( }) } "NOT_CONTAINS" => { - if cond.attribute_value_list.len() != 1 { - return Err(DynamoDbError::ValidationException( - "NOT_CONTAINS requires exactly 1 value".to_owned(), - )); - } let placeholder = format!("_f{counter}"); *counter += 1; values.insert(placeholder.clone(), cond.attribute_value_list[0].clone()); @@ -471,11 +705,6 @@ fn desugar_one_filter_condition( args: vec![path.clone()], }), "IN" => { - if cond.attribute_value_list.is_empty() { - return Err(DynamoDbError::ValidationException( - "IN requires at least 1 value".to_owned(), - )); - } let placeholders: Vec = cond .attribute_value_list .iter() @@ -491,8 +720,13 @@ fn desugar_one_filter_condition( list: placeholders, }) } - other => Err(DynamoDbError::ValidationException(format!( - "Unsupported filter operator: {other}" + // validate_legacy_condition refused every other operator above. + _ => Err(DynamoDbError::ValidationException(format!( + "{INVALID_PARAMS}Invalid comparison operator" ))), } } + +#[cfg(test)] +#[path = "legacy_filter_tests.rs"] +mod tests; diff --git a/crates/engine/src/legacy_filter_tests.rs b/crates/engine/src/legacy_filter_tests.rs new file mode 100644 index 000000000..42569e95d --- /dev/null +++ b/crates/engine/src/legacy_filter_tests.rs @@ -0,0 +1,592 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Tests for the legacy condition validator; every expected message was +//! measured against Amazon DynamoDB on 2026-10-23 and 2026-10-24. + +use std::collections::BTreeMap; + +use super::*; + +fn msg(op: &str, values: &[AttributeValue]) -> Option { + match validate_legacy_condition(op, values) { + Ok(()) => None, + Err(DynamoDbError::ValidationException(m)) => Some(m), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +fn refusal(text: &str) -> Option { + Some(format!("{INVALID_PARAMS}{text}")) +} + +fn type_refusal(op: &str, ty: &str) -> Option { + refusal(&format!( + "ComparisonOperator {op} is not valid for {ty} AttributeValue type" + )) +} + +fn one_of_each() -> Vec<(&'static str, AttributeValue)> { + vec![ + ("S", AttributeValue::S("a".to_owned())), + ("N", AttributeValue::N("1".to_owned())), + ("B", AttributeValue::B(vec![1])), + ("SS", AttributeValue::SS(["a".to_owned()].into())), + ("NS", AttributeValue::NS(["1".to_owned()].into())), + ("BS", AttributeValue::BS([vec![1]].into())), + ("BOOL", AttributeValue::Bool(true)), + ("NULL", AttributeValue::Null), + ("L", AttributeValue::L(Vec::new())), + ("M", AttributeValue::M(BTreeMap::new())), + ] +} + +/// The operator by type grid measured on 2026-10-23 (ScanFilter, +/// QueryFilter and KeyConditions gave the same rows). +#[test] +fn operator_by_type_grid() { + let accepted = |op: &str, ty: &str| match op { + "EQ" | "NE" => true, + "BEGINS_WITH" => matches!(ty, "S" | "B"), + "CONTAINS" | "NOT_CONTAINS" => matches!(ty, "S" | "N" | "B" | "BOOL" | "NULL"), + _ => matches!(ty, "S" | "N" | "B"), + }; + for op in [ + "EQ", + "NE", + "LE", + "LT", + "GE", + "GT", + "CONTAINS", + "NOT_CONTAINS", + "BEGINS_WITH", + "IN", + "BETWEEN", + ] { + for (ty, value) in one_of_each() { + let values = if op == "BETWEEN" { + vec![value.clone(), value] + } else { + vec![value] + }; + let expected = if accepted(op, ty) { + None + } else { + type_refusal(op, ty) + }; + assert_eq!(msg(op, &values), expected, "{op} with {ty}"); + } + } + for op in ["NULL", "NOT_NULL"] { + assert_eq!(msg(op, &[]), None); + } +} + +#[test] +fn value_counts() { + let s = AttributeValue::S("a".to_owned()); + let count = |op: &str| { + refusal(&format!( + "Invalid number of argument(s) for the {op} ComparisonOperator" + )) + }; + for op in [ + "EQ", + "NE", + "LE", + "LT", + "GE", + "GT", + "CONTAINS", + "NOT_CONTAINS", + "BEGINS_WITH", + ] { + assert_eq!(msg(op, &[]), count(op), "{op} with 0"); + assert_eq!(msg(op, &[s.clone(), s.clone()]), count(op), "{op} with 2"); + } + for op in ["NULL", "NOT_NULL"] { + assert_eq!(msg(op, std::slice::from_ref(&s)), count(op), "{op} with 1"); + } + assert_eq!(msg("IN", &[]), count("IN")); + assert_eq!(msg("IN", &[s.clone(), s.clone(), s.clone()]), None); + assert_eq!(msg("BETWEEN", std::slice::from_ref(&s)), count("BETWEEN")); + assert_eq!( + msg("BETWEEN", &[s.clone(), s.clone(), s.clone()]), + count("BETWEEN") + ); + assert_eq!( + msg("FOO", std::slice::from_ref(&s)), + refusal("Invalid comparison operator") + ); + assert_eq!( + msg("eq", std::slice::from_ref(&s)), + refusal("Invalid comparison operator") + ); +} + +#[test] +fn mixed_types_and_ranges() { + let s = AttributeValue::S("a".to_owned()); + let n = AttributeValue::N("1".to_owned()); + let l = AttributeValue::L(Vec::new()); + let b = AttributeValue::Bool(true); + let same = refusal("AttributeValues inside AttributeValueList must be of same type"); + // The first value's type is checked; the rest are held to it. + assert_eq!(msg("BETWEEN", &[s.clone(), n.clone()]), same); + assert_eq!(msg("BETWEEN", &[n.clone(), l.clone()]), same); + assert_eq!( + msg("BETWEEN", &[l.clone(), n.clone()]), + type_refusal("BETWEEN", "L") + ); + assert_eq!( + msg("BETWEEN", &[b, l.clone()]), + type_refusal("BETWEEN", "BOOL") + ); + assert_eq!(msg("IN", &[s.clone(), n.clone()]), same); + assert_eq!(msg("IN", &[l, s.clone()]), type_refusal("IN", "L")); + assert_eq!(msg("IN", &[s.clone(), AttributeValue::L(Vec::new())]), same); + let range = refusal( + "The BETWEEN condition was provided a range where the lower bound is greater than the upper bound", + ); + assert_eq!( + msg("BETWEEN", &[AttributeValue::N("2".to_owned()), n.clone()]), + range + ); + assert_eq!( + msg( + "BETWEEN", + &[ + AttributeValue::N("10".to_owned()), + AttributeValue::N("9".to_owned()) + ] + ), + range + ); + assert_eq!( + msg("BETWEEN", &[AttributeValue::S("b".to_owned()), s.clone()]), + range + ); + assert_eq!( + msg("BETWEEN", &[n.clone(), AttributeValue::N("10".to_owned())]), + None + ); + assert_eq!(msg("BETWEEN", &[n.clone(), n]), None); +} + +#[test] +fn desugar_filter_refuses_before_desugaring() { + let mut filter = HashMap::new(); + filter.insert( + "f".to_owned(), + Condition { + comparison_operator: "BEGINS_WITH".to_owned(), + attribute_value_list: vec![AttributeValue::N("1".to_owned())], + }, + ); + let err = desugar_filter(&filter, ConditionalOperator::And).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if Some(m.clone()) == type_refusal("BEGINS_WITH", "N")), + "{err:?}" + ); +} + +fn cond(op: &str, values: Vec) -> Condition { + Condition { + comparison_operator: op.to_owned(), + attribute_value_list: values, + } +} + +fn s(v: &str) -> AttributeValue { + AttributeValue::S(v.to_owned()) +} + +fn n(v: &str) -> AttributeValue { + AttributeValue::N(v.to_owned()) +} + +fn kc(entries: Vec<(&str, Condition)>) -> HashMap { + entries + .into_iter() + .map(|(name, c)| (name.to_owned(), c)) + .collect() +} + +fn schema( + keys: &[(&str, KeyType, ScalarAttributeType)], +) -> (Vec, Vec) { + ( + keys.iter() + .map(|(name, key_type, _)| KeySchemaElement { + attribute_name: (*name).to_owned(), + key_type: *key_type, + }) + .collect(), + keys.iter() + .map(|(name, _, ty)| AttributeDefinition { + attribute_name: (*name).to_owned(), + attribute_type: *ty, + }) + .collect(), + ) +} + +fn ss_schema() -> (Vec, Vec) { + schema(&[ + ("pk", KeyType::Hash, ScalarAttributeType::S), + ("sk", KeyType::Range, ScalarAttributeType::S), + ]) +} + +fn key_error( + conditions: &HashMap, + (ks, defs): &(Vec, Vec), +) -> Option { + match desugar_key_conditions(conditions, ks, defs) { + Ok(_) => None, + Err(DynamoDbError::ValidationException(m)) => Some(m), + Err(other) => panic!("unexpected error {other:?}"), + } +} + +const SCHEMA_TYPE: &str = "Condition parameter type does not match schema type"; +const NOT_INDEXABLE: &str = "Attempted conditional constraint is not an indexable operation"; + +fn empty_key(name: &str) -> Option { + Some(format!( + "One or more parameter values are not valid. The AttributeValue for a key attribute cannot contain an empty string value. Key: {name}" + )) +} + +/// A number that does not parse is reported before the operator name, the +/// count, the operator type, the same-type check and the range, inside sets, +/// lists and maps too (measured 2026-10-24). +#[test] +fn numbers_are_checked_before_the_operator_checks() { + let bad = |raw: &str| { + Some(format!( + "The parameter cannot be converted to a numeric value: {raw}" + )) + }; + assert_eq!(msg("EQ", &[n("abc")]), bad("abc")); + assert_eq!(msg("EQ", &[n("")]), bad("")); + assert_eq!(msg("EQ", &[n("12e")]), bad("12e")); + assert_eq!(msg("BEGINS_WITH", &[n("abc")]), bad("abc")); + assert_eq!(msg("FOO", &[n("abc")]), bad("abc")); + assert_eq!(msg("EQ", &[n("abc"), s("x")]), bad("abc")); + assert_eq!(msg("EQ", &[s("x"), n("abc")]), bad("abc")); + assert_eq!(msg("IN", &[s("x"), n("abc")]), bad("abc")); + assert_eq!(msg("NULL", &[n("abc")]), bad("abc")); + assert_eq!(msg("BETWEEN", &[n("2"), n("abc")]), bad("abc")); + assert_eq!(msg("BETWEEN", &[n("abc"), n("2")]), bad("abc")); + assert_eq!( + msg( + "EQ", + &[AttributeValue::NS( + ["1".to_owned(), "abc".to_owned()].into() + )] + ), + bad("abc") + ); + assert_eq!(msg("EQ", &[AttributeValue::L(vec![n("abc")])]), bad("abc")); + assert_eq!( + msg( + "EQ", + &[AttributeValue::M([("k".to_owned(), n("abc"))].into())] + ), + bad("abc") + ); + assert_eq!(msg("LT", &[AttributeValue::L(vec![n("abc")])]), bad("abc")); + assert_eq!( + msg("EQ", &[n("1e400")]), + Some( + "Number overflow. Attempting to store a number with magnitude larger than supported range" + .to_owned() + ) + ); + assert_eq!(msg("EQ", &[n("1")]), None); + assert_eq!(msg("EQ", &[n("-1.5e3")]), None); +} + +#[test] +fn key_conditions_refuse_non_indexable_operators_on_any_key_at_request_time() { + for (op, values) in [ + ("NE", vec![s("a")]), + ("NULL", Vec::new()), + ("NOT_NULL", Vec::new()), + ("CONTAINS", vec![s("a")]), + ("NOT_CONTAINS", vec![n("1")]), + ("IN", vec![s("a"), s("b")]), + ] { + let on_sk = kc(vec![ + ("pk", cond("EQ", vec![s("p")])), + ("sk", cond(op, values.clone())), + ]); + let on_pk = kc(vec![("pk", cond(op, values))]); + for conditions in [&on_sk, &on_pk] { + let err = validate_key_conditions(conditions).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if Some(m.clone()) == refusal(NOT_INDEXABLE)), + "{op}: {err:?}" + ); + } + } + // A value the operator does not take is reported first, as is a count. + let typed = kc(vec![ + ("pk", cond("EQ", vec![s("p")])), + ("sk", cond("CONTAINS", vec![AttributeValue::L(Vec::new())])), + ]); + let err = validate_key_conditions(&typed).unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if Some(m.clone()) == type_refusal("CONTAINS", "L")), + "{err:?}" + ); + // Non-indexable is reported ahead of a schema type mismatch on the other key. + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("EQ", vec![n("1")])), + ("sk", cond("NE", vec![s("a")])) + ]), + &ss_schema() + ), + refusal(NOT_INDEXABLE) + ); + assert!(validate_key_conditions(&kc(vec![("pk", cond("EQ", vec![s("p")]))])).is_ok()); +} + +#[test] +fn key_conditions_refuse_a_value_of_the_wrong_schema_type() { + let ss = ss_schema(); + let sn = schema(&[ + ("pk", KeyType::Hash, ScalarAttributeType::S), + ("sk", KeyType::Range, ScalarAttributeType::N), + ]); + let pk_s = || ("pk", cond("EQ", vec![s("p")])); + for hash in [ + n("1"), + AttributeValue::B(vec![1]), + AttributeValue::Bool(true), + ] { + assert_eq!( + key_error(&kc(vec![("pk", cond("EQ", vec![hash]))]), &ss), + refusal(SCHEMA_TYPE) + ); + } + for range in [ + cond("EQ", vec![n("1")]), + cond("LE", vec![AttributeValue::B(vec![1])]), + cond("BETWEEN", vec![n("1"), n("2")]), + cond("BEGINS_WITH", vec![AttributeValue::B(vec![1])]), + ] { + assert_eq!( + key_error(&kc(vec![pk_s(), ("sk", range)]), &ss), + refusal(SCHEMA_TYPE) + ); + } + for range in [ + cond("EQ", vec![s("abc")]), + cond("BEGINS_WITH", vec![s("1")]), + cond("BETWEEN", vec![s("a"), s("b")]), + cond("LT", vec![s("5")]), + ] { + assert_eq!( + key_error(&kc(vec![pk_s(), ("sk", range)]), &sn), + refusal(SCHEMA_TYPE) + ); + } + // Both keys wrong: one message, no attribute named. + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("EQ", vec![n("1")])), + ("sk", cond("EQ", vec![n("1")])) + ]), + &ss + ), + refusal(SCHEMA_TYPE) + ); + // The hash key's type is checked before its operator; a well-typed hash key + // with a non-EQ operator is refused before the range key's type. + assert_eq!( + key_error(&kc(vec![("pk", cond("LT", vec![n("1")]))]), &ss), + refusal(SCHEMA_TYPE) + ); + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("LT", vec![s("p")])), + ("sk", cond("EQ", vec![n("1")])) + ]), + &ss + ), + Some("Query key condition not supported".to_owned()) + ); + // Type before a condition on a non-key attribute; an absent hash key first. + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("EQ", vec![n("1")])), + ("foo", cond("EQ", vec![s("x")])) + ]), + &ss + ), + refusal(SCHEMA_TYPE) + ); + assert_eq!( + key_error(&kc(vec![("sk", cond("EQ", vec![n("1")]))]), &ss), + Some("Query condition missed key schema element: pk".to_owned()) + ); + // Well-typed conditions pass. + assert!(key_error(&kc(vec![pk_s(), ("sk", cond("EQ", vec![n("1")]))]), &sn).is_none()); + assert!( + key_error( + &kc(vec![pk_s(), ("sk", cond("BETWEEN", vec![s("a"), s("b")]))]), + &ss + ) + .is_none() + ); +} + +#[test] +fn key_conditions_refuse_an_empty_string_after_the_schema_checks() { + let ss = ss_schema(); + let pk_s = || ("pk", cond("EQ", vec![s("p")])); + assert_eq!( + key_error(&kc(vec![("pk", cond("EQ", vec![s("")]))]), &ss), + empty_key("pk") + ); + for op in ["EQ", "LE", "LT", "GE", "GT", "BEGINS_WITH"] { + assert_eq!( + key_error(&kc(vec![pk_s(), ("sk", cond(op, vec![s("")]))]), &ss), + empty_key("sk"), + "{op}" + ); + } + assert_eq!( + key_error( + &kc(vec![pk_s(), ("sk", cond("BETWEEN", vec![s(""), s("a")]))]), + &ss + ), + empty_key("sk") + ); + assert_eq!( + key_error( + &kc(vec![pk_s(), ("sk", cond("BETWEEN", vec![s(""), s("")]))]), + &ss + ), + empty_key("sk") + ); + // Hash key named first; schema type, operator and non-key checks come first. + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("EQ", vec![s("")])), + ("sk", cond("EQ", vec![s("")])) + ]), + &ss + ), + empty_key("pk") + ); + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("EQ", vec![s("")])), + ("sk", cond("EQ", vec![n("1")])) + ]), + &ss + ), + refusal(SCHEMA_TYPE) + ); + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("EQ", vec![n("1")])), + ("sk", cond("EQ", vec![s("")])) + ]), + &ss + ), + refusal(SCHEMA_TYPE) + ); + assert_eq!( + key_error(&kc(vec![("pk", cond("LT", vec![s("")]))]), &ss), + Some("Query key condition not supported".to_owned()) + ); + assert_eq!( + key_error( + &kc(vec![ + ("pk", cond("EQ", vec![s("")])), + ("foo", cond("EQ", vec![s("x")])) + ]), + &ss + ), + Some("Query condition missed key schema element: pk".to_owned()) + ); + // An inverted range is a request-time error and wins over the empty bound. + assert_eq!( + key_error( + &kc(vec![pk_s(), ("sk", cond("BETWEEN", vec![s("a"), s("")]))]), + &ss + ), + refusal( + "The BETWEEN condition was provided a range where the lower bound is greater than the upper bound" + ) + ); + // An empty B on an S key is a schema type mismatch. + assert_eq!( + key_error( + &kc(vec![( + "pk", + cond("EQ", vec![AttributeValue::B(Vec::new())]) + )]), + &ss + ), + refusal(SCHEMA_TYPE) + ); +} + +#[test] +fn query_filter_may_not_name_a_key_of_the_queried_schema() { + let (ks, _) = ss_schema(); + let on_key = |name: &str| { + Some(format!( + "QueryFilter can only contain non-primary key attributes: Primary key attribute: {name}" + )) + }; + let err = |filter: &HashMap| match reject_query_filter_on_key(filter, &ks) { + Ok(()) => None, + Err(DynamoDbError::ValidationException(m)) => Some(m), + Err(other) => panic!("unexpected error {other:?}"), + }; + assert_eq!( + err(&kc(vec![("pk", cond("EQ", vec![s("x")]))])), + on_key("pk") + ); + assert_eq!( + err(&kc(vec![("sk", cond("EQ", vec![s("x")]))])), + on_key("sk") + ); + assert_eq!( + err(&kc(vec![("pk", cond("NOT_NULL", vec![]))])), + on_key("pk") + ); + // Hash key named first whatever the map order. + assert_eq!( + err(&kc(vec![ + ("sk", cond("EQ", vec![s("x")])), + ("pk", cond("EQ", vec![s("x")])) + ])), + on_key("pk") + ); + assert_eq!(err(&kc(vec![("zz", cond("EQ", vec![s("x")]))])), None); + // Only the queried schema's keys count: a GSI schema frees the table keys. + let (gsi, _) = schema(&[ + ("gpk", KeyType::Hash, ScalarAttributeType::S), + ("gsk", KeyType::Range, ScalarAttributeType::S), + ]); + assert!(reject_query_filter_on_key(&kc(vec![("pk", cond("EQ", vec![s("x")]))]), &gsi).is_ok()); + assert!( + reject_query_filter_on_key(&kc(vec![("gsk", cond("EQ", vec![s("x")]))]), &gsi).is_err() + ); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 74083fbfa..41dc1eefe 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -7,6 +7,7 @@ //! inputs, dispatches operations by name, and translates storage errors into //! `DynamoDB`-format error responses. +mod attribute_updates; mod backup; mod batch_get_item; mod batch_write_item; @@ -25,6 +26,7 @@ mod import_export_io; mod index_helpers; mod legacy_filter; mod list_tables; +mod per_table_capacity; mod put_item; mod query; mod read_helpers; diff --git a/crates/engine/src/per_table_capacity.rs b/crates/engine/src/per_table_capacity.rs new file mode 100644 index 000000000..13033f5cf --- /dev/null +++ b/crates/engine/src/per_table_capacity.rs @@ -0,0 +1,119 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Per-table capacity totals for `TransactGetItems` and `TransactWriteItems`, +//! in the order Amazon DynamoDB lists them. + +use extenddb_core::types::TableKeyInfo; + +/// Per-table capacity totals, listed oldest table first. +/// +/// Amazon DynamoDB documents no order for the `ConsumedCapacity` entries of a +/// transaction. Measured on 2026-10-23 with three tables created on +/// 2026-10-22, 2026-10-23 11:01 and 2026-10-23 14:45 UTC, where the +/// newest table's name sorts first and the oldest table's name sorts last: +/// `TransactGetItems` and `TransactWriteItems` listed the tables oldest first +/// for the request orders [newest, middle, oldest], [middle, newest, oldest] +/// and [oldest, newest, middle]. The order follows neither the request nor +/// the names; it follows the tables' creation time. An earlier measurement +/// the same day (16 transactions over three tables created in name order) is +/// consistent with it. +/// +/// ExtendDB orders the entries the same way, by the `creation_date_time` the +/// catalog carries on [`TableKeyInfo`], oldest first; two tables with the same +/// stored creation time are listed in name order so the array is deterministic +/// on every backend. The precision of the stored value sets how close two +/// creates have to be to tie: about 0.24 microseconds on SQLite (the value +/// is an `f64` of epoch seconds), microseconds on PostgreSQL, milliseconds on +/// MongoDB and Cassandra. Before this change the +/// handlers aggregated into a `HashMap`, so the order changed from one process +/// to the next and differed between backends +/// (docs/differences-from-dynamodb.md, Capacity and Throttling). +#[derive(Default)] +pub struct PerTableCapacity(Vec); + +struct Entry { + table: String, + created: f64, + units: f64, +} + +impl PerTableCapacity { + /// Add `units` to the table `key_info` describes. + pub fn add(&mut self, key_info: &TableKeyInfo, units: f64) { + match self.0.iter_mut().find(|e| e.table == key_info.table_name) { + Some(e) => e.units += units, + None => self.0.push(Entry { + table: key_info.table_name.clone(), + created: key_info.creation_date_time, + units, + }), + } + } + + /// The totals, oldest table first; ties on creation time in name order. + pub fn ordered(&self) -> Vec<(&str, f64)> { + let mut entries: Vec<&Entry> = self.0.iter().collect(); + entries.sort_by(|a, b| { + a.created + .total_cmp(&b.created) + .then_with(|| a.table.cmp(&b.table)) + }); + entries + .into_iter() + .map(|e| (e.table.as_str(), e.units)) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn table(name: &str, created: f64) -> TableKeyInfo { + TableKeyInfo { + table_name: name.to_owned(), + creation_date_time: created, + ..Default::default() + } + } + + #[test] + fn orders_by_creation_time_not_by_request_or_name() { + // Names sort opposite to creation order, and the request names the + // newest table first: the service's order is oldest first. + let (c, b, a) = (table("c", 1.0), table("b", 2.0), table("a", 3.0)); + let mut totals = PerTableCapacity::default(); + totals.add(&a, 2.0); + totals.add(&c, 2.0); + totals.add(&b, 2.0); + totals.add(&a, 4.0); + assert_eq!( + totals.ordered(), + vec![("c", 2.0), ("b", 2.0), ("a", 6.0)], + "oldest table first, with the repeated table summed once" + ); + } + + #[test] + fn ties_on_creation_time_fall_back_to_name_order() { + let mut totals = PerTableCapacity::default(); + totals.add(&table("zeta", 5.0), 1.0); + totals.add(&table("alpha", 5.0), 1.0); + totals.add(&table("mid", 5.0), 1.0); + assert_eq!( + totals.ordered(), + vec![("alpha", 1.0), ("mid", 1.0), ("zeta", 1.0)] + ); + } + + #[test] + fn unknown_creation_time_sorts_before_known() { + // A source that carries no creation time reports 0.0 (TableKeyInfo + // doc); it lists first rather than failing the sort. + let mut totals = PerTableCapacity::default(); + totals.add(&table("known", 10.0), 1.0); + totals.add(&table("unknown", 0.0), 1.0); + assert_eq!(totals.ordered(), vec![("unknown", 1.0), ("known", 1.0)]); + } +} diff --git a/crates/engine/src/put_item.rs b/crates/engine/src/put_item.rs index ffd6f70b4..b51d54729 100755 --- a/crates/engine/src/put_item.rs +++ b/crates/engine/src/put_item.rs @@ -13,7 +13,7 @@ use extenddb_core::types::{PutItemInput, PutItemOutput, ReturnValues, item_size_ use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::{storage_err_to_dynamo, storage_err_to_dynamo_with_ccf}; -use crate::expression_helpers::resolve_condition; +use crate::expression_helpers::{resolve_condition, single_item_write_prefix}; use crate::serialize_output; use crate::stream_capture; use crate::{DispatchMetrics, DispatchResult}; @@ -109,13 +109,16 @@ pub async fn handle_put_item( ], &[("ConditionExpression", has_expression)], )?; + // The single-item write APIs name no expression in the values message and + // wrap every expression-stage error in `1 validation error detected: `. extenddb_core::expression::validate_expression_param_usage( input.expression_attribute_names.as_ref(), has_expression, input.expression_attribute_values.as_ref(), has_expression, - &[extenddb_core::expression::ExpressionKind::Condition], - )?; + &[], + ) + .map_err(single_item_write_prefix)?; // ConditionalOperator requires an Expected with two or more conditions. extenddb_core::validation::validate_conditional_operator_usage( @@ -138,7 +141,8 @@ pub async fn handle_put_item( input.expected.as_ref(), input.conditional_operator, &ctx.limits, - )?; + ) + .map_err(single_item_write_prefix)?; if input.expected.is_none() || input @@ -147,6 +151,9 @@ pub async fn handle_put_item( .is_some_and(std::collections::HashMap::is_empty) { let exprs: Vec<&extenddb_core::expression::Expr> = condition.iter().collect(); + // The condition's references, functions, and operand types were + // checked by `resolve_condition`; the unused check follows them and + // runs ahead of the table lookup, as on the service. extenddb_core::expression::validate_unused_attributes( &maps.names, &maps.values, @@ -154,7 +161,8 @@ pub async fn handle_put_item( &[], &std::collections::HashSet::new(), &std::collections::HashSet::new(), - )?; + ) + .map_err(single_item_write_prefix)?; } let key_info = ctx diff --git a/crates/engine/src/query.rs b/crates/engine/src/query.rs index 848f5fdbb..5019a95c1 100755 --- a/crates/engine/src/query.rs +++ b/crates/engine/src/query.rs @@ -3,7 +3,7 @@ //! `Query` operation handler. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use serde_json::Value; @@ -11,7 +11,7 @@ use extenddb_core::error::DynamoDbError; use extenddb_core::expression::PathElement; use extenddb_core::expression::{ExpressionKind, ExpressionMaps, Projection}; use extenddb_core::types::{ - IndexType, KeyType, ProjectionType, QueryInput, QueryOutput, Select, TableKeyInfo, + IndexType, ProjectionType, QueryInput, QueryOutput, Select, TableKeyInfo, VECTOR_INDEX_QUERY_NOT_SUPPORTED, }; use extenddb_storage::error::StorageError; @@ -19,12 +19,14 @@ use extenddb_storage::error::StorageError; use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::storage_err_to_dynamo; -use crate::expression_helpers::{build_expression_maps, parse_optional_filter}; +use crate::expression_helpers::build_expression_maps; use crate::index_helpers::{ VectorIndexReadRefusal, classify_unresolved_index_read, combined_lek_key_schema, validate_query_exclusive_start_key, }; -use crate::legacy_filter::{desugar_filter, desugar_key_conditions}; +use crate::legacy_filter::{ + desugar_filter, desugar_key_conditions, reject_query_filter_on_key, validate_key_conditions, +}; use crate::read_helpers::{PostRead, read_page}; use crate::serialize_output; use crate::{DispatchMetrics, DispatchResult}; @@ -71,6 +73,204 @@ pub async fn handle_query( )?; let input: QueryInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; + // Validate Limit >= 1 (REQ-QUERY-001) + if let Some(limit) = input.limit + && limit < 1 + { + return Err(DynamoDbError::ValidationException( + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1".to_owned(), + )); + } + + // --- Legacy vs expression mutual exclusivity checks --- + let kce_present = input.key_condition_expression.is_some(); + let has_kc = input.key_conditions.as_ref().is_some_and(|m| !m.is_empty()); + let has_fe = input.filter_expression.is_some(); + let has_qf = input.query_filter.as_ref().is_some_and(|m| !m.is_empty()); + let has_pe = input.projection_expression.is_some(); + let has_atg = input + .attributes_to_get + .as_ref() + .is_some_and(|a| !a.is_empty()); + let has_cond_op = input.conditional_operator.is_some(); + + // KeyConditions legitimately combines with FilterExpression and + // ProjectionExpression, so it joins the mixing check only when + // KeyConditionExpression is also present. + extenddb_core::validation::validate_no_expression_param_mixing( + &[ + ("AttributesToGet", has_atg), + ("QueryFilter", has_qf), + ("ConditionalOperator", has_cond_op), + ("KeyConditions", has_kc && kce_present), + ], + &[ + ("ProjectionExpression", has_pe), + ("FilterExpression", has_fe), + ("KeyConditionExpression", kce_present), + ], + )?; + + // An explicitly empty KeyConditionExpression is rejected up front, before + // the expression-parameter-usage checks (matching real DynamoDB). + reject_empty_key_condition_expression(input.key_condition_expression.as_deref())?; + + // Build expression maps from request (used for expression-based parameters) + let maps = build_expression_maps( + input.expression_attribute_names.as_ref(), + input.expression_attribute_values.as_ref(), + ); + + // Reject EAN/EAV supplied with no expression that references them. Legacy + // KeyConditions does not count as an expression. Query emits no values suffix. + let has_kce = input + .key_condition_expression + .as_ref() + .is_some_and(|s| !s.is_empty()); + let has_filter_expr = input + .filter_expression + .as_ref() + .is_some_and(|s| !s.is_empty()); + let has_proj_expr = input + .projection_expression + .as_ref() + .is_some_and(|s| !s.is_empty()); + extenddb_core::expression::validate_expression_param_usage( + input.expression_attribute_names.as_ref(), + has_kce || has_filter_expr || has_proj_expr, + input.expression_attribute_values.as_ref(), + has_kce || has_filter_expr, + &[], + )?; + + // ConditionalOperator requires a QueryFilter with two or more conditions. + extenddb_core::validation::validate_conditional_operator_usage( + has_cond_op, + input.query_filter.as_ref().map_or(0, HashMap::len), + )?; + + // Request-stage expression checks, before the table is looked up, in the + // order the service applies them (measured 2026-10-14): the + // FilterExpression (syntax, then references, functions, and operand + // types), the KeyConditionExpression (syntax, then references), the + // ProjectionExpression (syntax, then references and overlap), then unused + // names and values. A missing table is reported after all of these. + // A legacy QueryFilter skips the parser; desugar_filter checks each + // ComparisonOperator against its value count and types with the service's + // messages, still before any row is read. Legacy KeyConditions get the + // same value checks here, ahead of the table lookup, plus the + // non-indexable operator refusal; the service reported a bad QueryFilter + // value before a bad KeyConditions value, and both before any key schema + // error (measured 2026-10-23 and 2026-10-24). + let (filter, filter_maps) = + if let Some(qf) = input.query_filter.as_ref().filter(|qf| !qf.is_empty()) { + let cond_op = input.conditional_operator.unwrap_or_default(); + let (expr, fmaps) = desugar_filter(qf, cond_op)?; + (Some(expr), Some(fmaps)) + } else { + match input.filter_expression.as_deref() { + Some(expr) if !expr.is_empty() => ( + Some(crate::expression_helpers::parse_condition_checked( + expr, + &maps, + &ctx.limits, + false, + ExpressionKind::Filter, + )?), + None, + ), + _ => (None, None), + } + }; + if let Some(kc) = input.key_conditions.as_ref() { + validate_key_conditions(kc)?; + } + + let parsed_key_condition = match input.key_condition_expression.as_deref() { + Some(kce_str) => Some(crate::expression_helpers::parse_key_condition_expr( + kce_str, + &maps, + &ctx.limits, + )?), + None => None, + }; + + // Parse ProjectionExpression or desugar legacy AttributesToGet + let (effective_projection_str, extra_proj_names) = if input.projection_expression.is_some() { + (input.projection_expression.clone(), HashMap::new()) + } else if let Some(ref attrs) = input.attributes_to_get { + let mut names_map = HashMap::new(); + let placeholders: Vec = attrs + .iter() + .enumerate() + .map(|(i, attr)| { + let placeholder = format!("#_ag{i}"); + names_map.insert(placeholder.clone(), attr.clone()); + placeholder + }) + .collect(); + (Some(placeholders.join(", ")), names_map) + } else { + (None, HashMap::new()) + }; + + let projection = if let Some(ref proj_str) = effective_projection_str { + Some(crate::expression_helpers::parse_projection_expr( + proj_str, + &ctx.limits, + )?) + } else { + None + }; + + // Compile once: resolves #names and rejects overlapping paths. Overlap + // rejection is scoped to a user-supplied ProjectionExpression. + let compiled_projection = match projection { + Some(ref paths) => { + let mut names = maps.names.clone(); + for (k, v) in &extra_proj_names { + let stripped = k.strip_prefix('#').unwrap_or(k); + names.insert(stripped.to_owned(), v.clone()); + } + let proj_maps = ExpressionMaps::new(names, HashMap::new()); + Some(Projection::compile( + paths, + &proj_maps, + input.projection_expression.is_some(), + )?) + } + None => None, + }; + + // Validate unused expression attributes + { + let exprs: Vec<&extenddb_core::expression::Expr> = filter.iter().collect(); + let (mut kc_names, kc_values) = match parsed_key_condition { + Some(ref kc) => extenddb_core::expression::collect_key_condition_refs(kc), + None => (HashSet::new(), HashSet::new()), + }; + // Collect #name refs from projection paths + if let Some(ref proj) = projection { + for path in proj { + for el in path { + if let PathElement::Attribute(name) = el + && let Some(ref_name) = name.strip_prefix('#') + { + kc_names.insert(ref_name.to_owned()); + } + } + } + } + extenddb_core::expression::validate_unused_attributes( + &maps.names, + &maps.values, + &exprs, + &[], + &kc_names, + &kc_values, + )?; + } + // P118: Fetch key_info first so we can use table_id for index lookup. let key_info = ctx .table_key_info(&input.table_name) @@ -137,15 +337,6 @@ pub async fn handle_query( )?; } - // Validate Limit >= 1 (REQ-QUERY-001) - if let Some(limit) = input.limit - && limit < 1 - { - return Err(DynamoDbError::ValidationException( - "1 validation error detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1".to_owned(), - )); - } - // For index queries, build a key_info that reflects the index's key schema // so the storage layer uses the correct SK column for the index table. let query_key_info = if let Some(ref idx) = index_info { @@ -161,97 +352,31 @@ pub async fn handle_query( local_secondary_indexes: key_info.local_secondary_indexes.clone(), stream_specification: None, // Queries don't capture stream records vector_indexes: key_info.vector_indexes.clone(), + creation_date_time: key_info.creation_date_time, } } else { key_info.clone() }; - // --- Legacy vs expression mutual exclusivity checks --- - let kce_present = input.key_condition_expression.is_some(); - let has_kc = input.key_conditions.as_ref().is_some_and(|m| !m.is_empty()); - let has_fe = input.filter_expression.is_some(); - let has_qf = input.query_filter.as_ref().is_some_and(|m| !m.is_empty()); - let has_pe = input.projection_expression.is_some(); - let has_atg = input - .attributes_to_get - .as_ref() - .is_some_and(|a| !a.is_empty()); - let has_cond_op = input.conditional_operator.is_some(); - - // KeyConditions legitimately combines with FilterExpression and - // ProjectionExpression, so it joins the mixing check only when - // KeyConditionExpression is also present. - extenddb_core::validation::validate_no_expression_param_mixing( - &[ - ("AttributesToGet", has_atg), - ("QueryFilter", has_qf), - ("ConditionalOperator", has_cond_op), - ("KeyConditions", has_kc && kce_present), - ], - &[ - ("ProjectionExpression", has_pe), - ("FilterExpression", has_fe), - ("KeyConditionExpression", kce_present), - ], - )?; - - // An explicitly empty KeyConditionExpression is rejected up front, before - // the expression-parameter-usage checks (matching real DynamoDB). - reject_empty_key_condition_expression(input.key_condition_expression.as_deref())?; - - // Build expression maps from request (used for expression-based parameters) - let maps = build_expression_maps( - input.expression_attribute_names.as_ref(), - input.expression_attribute_values.as_ref(), - ); - - // Reject EAN/EAV supplied with no expression that references them. Legacy - // KeyConditions does not count as an expression. Query emits no values suffix. - let has_kce = input - .key_condition_expression - .as_ref() - .is_some_and(|s| !s.is_empty()); - let has_filter_expr = input - .filter_expression - .as_ref() - .is_some_and(|s| !s.is_empty()); - let has_proj_expr = input - .projection_expression - .as_ref() - .is_some_and(|s| !s.is_empty()); - extenddb_core::expression::validate_expression_param_usage( - input.expression_attribute_names.as_ref(), - has_kce || has_filter_expr || has_proj_expr, - input.expression_attribute_values.as_ref(), - has_kce || has_filter_expr, - &[], - )?; - - // ConditionalOperator requires a QueryFilter with two or more conditions. - extenddb_core::validation::validate_conditional_operator_usage( - has_cond_op, - input.query_filter.as_ref().map_or(0, HashMap::len), - )?; - - // Parse KeyConditionExpression or desugar legacy KeyConditions - let (mut key_condition, legacy_kc_maps) = if let Some(kce_str) = - input.key_condition_expression.as_deref() - { - let parsed = crate::expression_helpers::parse_key_condition_expr(kce_str, &ctx.limits)?; - (parsed, None) - } else if let Some(ref kc) = input.key_conditions { - let key_schema_pairs: Vec<(String, bool)> = query_key_info - .key_schema - .iter() - .map(|ks| (ks.attribute_name.clone(), ks.key_type == KeyType::Hash)) - .collect(); - let (kc_parsed, kc_maps) = desugar_key_conditions(kc, &key_schema_pairs)?; - (kc_parsed, Some(kc_maps)) - } else { - return Err(DynamoDbError::ValidationException( - "Either the KeyConditions or KeyConditionExpression parameter must be specified in the request." - .to_owned(), - )); + // The key condition: parsed above from KeyConditionExpression, or + // desugared here from legacy KeyConditions, which needs the key schema + // and the attribute types of the queried table or index. + let (mut key_condition, legacy_kc_maps) = match parsed_key_condition { + Some(parsed) => (parsed, None), + None => { + let Some(ref kc) = input.key_conditions else { + return Err(DynamoDbError::ValidationException( + "Either the KeyConditions or KeyConditionExpression parameter must be specified in the request." + .to_owned(), + )); + }; + let (kc_parsed, kc_maps) = desugar_key_conditions( + kc, + &query_key_info.key_schema, + &query_key_info.attribute_definitions, + )?; + (kc_parsed, Some(kc_maps)) + } }; // A vector index is searched only via the vector search API, never @@ -335,105 +460,6 @@ pub async fn handle_query( } } - // Parse FilterExpression or desugar legacy QueryFilter - let (filter, filter_maps) = if let Some(ref qf) = input.query_filter { - if qf.is_empty() { - ( - parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, - None, - ) - } else { - let cond_op = input.conditional_operator.unwrap_or_default(); - let (expr, fmaps) = desugar_filter(qf, cond_op)?; - (Some(expr), Some(fmaps)) - } - } else { - ( - parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, - None, - ) - }; - - // Validate #name references in filter are defined in ExpressionAttributeNames - if let Some(ref filter_expr) = filter { - let names = input.expression_attribute_names.as_ref(); - validate_name_refs_in_expr(filter_expr, names, ExpressionKind::Filter)?; - } - - // Parse ProjectionExpression or desugar legacy AttributesToGet - let (effective_projection_str, extra_proj_names) = if input.projection_expression.is_some() { - (input.projection_expression.clone(), HashMap::new()) - } else if let Some(ref attrs) = input.attributes_to_get { - let mut names_map = HashMap::new(); - let placeholders: Vec = attrs - .iter() - .enumerate() - .map(|(i, attr)| { - let placeholder = format!("#_ag{i}"); - names_map.insert(placeholder.clone(), attr.clone()); - placeholder - }) - .collect(); - (Some(placeholders.join(", ")), names_map) - } else { - (None, HashMap::new()) - }; - - let projection = if let Some(ref proj_str) = effective_projection_str { - Some(crate::expression_helpers::parse_projection_expr( - proj_str, - &ctx.limits, - )?) - } else { - None - }; - - // Compile once: resolves #names and rejects overlapping paths. Overlap - // rejection is scoped to a user-supplied ProjectionExpression. - let compiled_projection = match projection { - Some(ref paths) => { - let mut names = maps.names.clone(); - for (k, v) in &extra_proj_names { - let stripped = k.strip_prefix('#').unwrap_or(k); - names.insert(stripped.to_owned(), v.clone()); - } - let proj_maps = ExpressionMaps::new(names, HashMap::new()); - Some(Projection::compile( - paths, - &proj_maps, - input.projection_expression.is_some(), - )?) - } - None => None, - }; - - // Validate unused expression attributes - { - let exprs: Vec<&extenddb_core::expression::Expr> = filter.iter().collect(); - let (mut kc_names, kc_values) = - extenddb_core::expression::collect_key_condition_refs(&key_condition); - // Collect #name refs from projection paths - if let Some(ref proj) = projection { - for path in proj { - for el in path { - if let PathElement::Attribute(name) = el - && let Some(ref_name) = name.strip_prefix('#') - { - kc_names.insert(ref_name.to_owned()); - } - } - } - } - extenddb_core::expression::validate_unused_attributes( - &maps.names, - &maps.values, - &exprs, - &[], - &kc_names, - &kc_values, - )?; - } - // Validate Select vs ProjectionExpression and index requirements (shared // with Scan so both reject the same combinations identically). extenddb_core::validation::validate_select_projection( @@ -450,6 +476,14 @@ pub async fn handle_query( extenddb_core::validation::IS_QUERY, )?; + // A legacy QueryFilter may not name a key of the queried table or index. + // The service reports this after the key schema and Select checks above, + // and before the key value size check and the ExclusiveStartKey check + // (measured 2026-10-24). + if let Some(qf) = input.query_filter.as_ref().filter(|qf| !qf.is_empty()) { + reject_query_filter_on_key(qf, &query_key_info.key_schema)?; + } + // When Select=ALL_PROJECTED_ATTRIBUTES, capture the index info for post-read filtering. let index_proj = if matches!(input.select, Some(Select::AllProjectedAttributes)) { index_info.as_ref() @@ -478,16 +512,6 @@ pub async fn handle_query( ExpressionMaps::new(names, values) }; - // Validate begins_with operand types upfront (before any rows are read). - if let Some(ref f) = filter { - extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - } - // Validate ExclusiveStartKey matches the key schema if let Some(ref start_key) = input.exclusive_start_key { validate_query_exclusive_start_key(start_key, &key_info, index_info.as_ref())?; @@ -670,67 +694,6 @@ fn refuse_begins_with_on_number( Ok(()) } -/// Validate that all `#name` references in an expression AST are defined. -fn validate_name_refs_in_expr( - expr: &extenddb_core::expression::Expr, - names: Option<&HashMap>, - expr_type: ExpressionKind, -) -> Result<(), DynamoDbError> { - use extenddb_core::expression::Expr; - match expr { - Expr::Path(elements) => { - for el in elements { - if let PathElement::Attribute(name) = el - && let Some(ref_name) = name.strip_prefix('#') - { - let key_with_hash = format!("#{ref_name}"); - let defined = names.as_ref().is_some_and(|m| { - m.contains_key(ref_name) || m.contains_key(key_with_hash.as_str()) - }); - if !defined { - return Err(DynamoDbError::ValidationException(format!( - "Invalid {expr_type}: An expression attribute name used in the document path is not defined; attribute name: #{ref_name}" - ))); - } - } - } - Ok(()) - } - Expr::Compare { left, right, .. } => { - validate_name_refs_in_expr(left, names, expr_type)?; - validate_name_refs_in_expr(right, names, expr_type) - } - Expr::And(l, r) | Expr::Or(l, r) => { - validate_name_refs_in_expr(l, names, expr_type)?; - validate_name_refs_in_expr(r, names, expr_type) - } - Expr::Not(inner) => validate_name_refs_in_expr(inner, names, expr_type), - Expr::Between { operand, low, high } => { - validate_name_refs_in_expr(operand, names, expr_type)?; - validate_name_refs_in_expr(low, names, expr_type)?; - validate_name_refs_in_expr(high, names, expr_type) - } - Expr::In { operand, list } => { - validate_name_refs_in_expr(operand, names, expr_type)?; - for item in list { - validate_name_refs_in_expr(item, names, expr_type)?; - } - Ok(()) - } - Expr::Function { args, .. } => { - for arg in args { - validate_name_refs_in_expr(arg, names, expr_type)?; - } - Ok(()) - } - Expr::Arithmetic { left, right, .. } => { - validate_name_refs_in_expr(left, names, expr_type)?; - validate_name_refs_in_expr(right, names, expr_type) - } - Expr::Placeholder(_) => Ok(()), - } -} - /// Reject an explicitly empty `KeyConditionExpression` with DynamoDB's exact /// message. An absent expression (`None`) is allowed here; required-ness is /// enforced later. Only the empty-string case (`Some("")`) is rejected, and it diff --git a/crates/engine/src/read_helpers.rs b/crates/engine/src/read_helpers.rs index 6e3ac7bdd..19b812b67 100755 --- a/crates/engine/src/read_helpers.rs +++ b/crates/engine/src/read_helpers.rs @@ -463,7 +463,8 @@ mod tests { fn a_page_whose_items_all_fail_the_filter_ends_at_the_budget_with_a_key() { let s = store(1_000, 4_000); let filter = - parse_optional_filter(Some("payload = :nomatch"), &LimitsConfig::default()).unwrap(); + parse_optional_filter(Some("payload = :nomatch"), &LimitsConfig::default(), true) + .unwrap(); let mut values = HashMap::new(); values.insert(":nomatch".to_owned(), AttributeValue::S("never".to_owned())); let maps = crate::expression_helpers::build_expression_maps(None, Some(&values)); diff --git a/crates/engine/src/scan.rs b/crates/engine/src/scan.rs index 1c1de2274..64b0bd222 100755 --- a/crates/engine/src/scan.rs +++ b/crates/engine/src/scan.rs @@ -18,7 +18,7 @@ use extenddb_storage::error::StorageError; use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::storage_err_to_dynamo; -use crate::expression_helpers::{build_expression_maps, parse_optional_filter}; +use crate::expression_helpers::build_expression_maps; use crate::index_helpers::{ VectorIndexReadRefusal, classify_unresolved_index_read, combined_lek_key_schema, validate_scan_exclusive_start_key, @@ -68,8 +68,10 @@ pub async fn handle_scan( )?; let input: ScanInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; - // Validate Filter/Projection expressions before the existence - // check; index resolution, Segment/Limit/Select and key checks stay after. + // Limit and the Segment range checks come first, then the Filter and + // Projection expressions and the Segment pairing checks, all before the + // existence check; index resolution, the Select checks that need the index + // and the key checks stay after it. // --- Legacy vs expression mutual exclusivity checks --- let has_fe = input.filter_expression.is_some(); @@ -81,6 +83,44 @@ pub async fn handle_scan( .is_some_and(|a| !a.is_empty()); let has_cond_op = input.conditional_operator.is_some(); + // The range checks on Limit, Segment and TotalSegments are parameter-level + // checks the service runs ahead of every expression check (measured + // 2026-10-14: a Scan with an undefined filter value and Limit 0 reports the + // limit). That Segment and TotalSegments go together, and Segment is below + // TotalSegments, is checked after the expressions (see below). + if let (Some(seg), Some(total)) = (input.segment, input.total_segments) { + if total < 1 { + return Err(DynamoDbError::ValidationException( + "The parameter TotalSegments should be greater than or equal to 1".to_owned(), + )); + } + if total > 1_000_000 { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value '{total}' at 'totalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000" + ))); + } + if seg < 0 { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value '{seg}' at 'segment' failed to satisfy constraint: Member must have value greater than or equal to 0" + ))); + } + if seg > 999_999 { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Value '{seg}' at 'segment' failed to satisfy constraint: Member must have value less than or equal to 999999" + ))); + } + } + + // Validate Limit >= 1. The service's text carries neither the value nor a + // lower-case member name, for Scan as for Query (measured 2026-10-27). + if let Some(limit) = input.limit + && limit < 1 + { + return Err(DynamoDbError::ValidationException( + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1".to_owned(), + )); + } + extenddb_core::validation::validate_no_expression_param_mixing( &[ ("AttributesToGet", has_atg), @@ -120,24 +160,34 @@ pub async fn handle_scan( input.scan_filter.as_ref().map_or(0, HashMap::len), )?; - // Parse FilterExpression or desugar legacy ScanFilter - let (filter, filter_maps) = if let Some(ref sf) = input.scan_filter { - if sf.is_empty() { - ( - parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, - None, - ) - } else { + // Parse FilterExpression or desugar legacy ScanFilter. The expression + // form runs the service's ordered semantic walk (references, functions, + // operand types) here, ahead of the projection, the unused check, and the + // table lookup, so an undefined `:q` is a request error even when no row + // would be evaluated (measured 2026-10-14 on an empty table). A legacy + // ScanFilter skips the parser; desugar_filter checks each + // ComparisonOperator against its value count and types with the service's + // messages, still before any row is read. + let (filter, filter_maps) = + if let Some(sf) = input.scan_filter.as_ref().filter(|sf| !sf.is_empty()) { let cond_op = input.conditional_operator.unwrap_or_default(); let (expr, fmaps) = desugar_filter(sf, cond_op)?; (Some(expr), Some(fmaps)) - } - } else { - ( - parse_optional_filter(input.filter_expression.as_deref(), &ctx.limits)?, - None, - ) - }; + } else { + match input.filter_expression.as_deref() { + Some(expr) if !expr.is_empty() => ( + Some(crate::expression_helpers::parse_condition_checked( + expr, + &maps, + &ctx.limits, + true, + ExpressionKind::Filter, + )?), + None, + ), + _ => (None, None), + } + }; // Parse ProjectionExpression or desugar legacy AttributesToGet let (effective_projection_str, extra_proj_names) = if input.projection_expression.is_some() { @@ -227,6 +277,34 @@ pub async fn handle_scan( )?; } + // Segment and TotalSegments go together, and Segment is below + // TotalSegments. The service checks this after the expressions and the + // unused names and values, and before the table lookup (measured + // 2026-10-27: an undefined filter value is reported ahead of a Segment + // with no TotalSegments, which is reported ahead of a missing table). + // DynamoDB returns a different message per direction. + match (input.segment, input.total_segments) { + (Some(_), None) => { + return Err(DynamoDbError::ValidationException( + "The TotalSegments parameter is required but was not present in the request when Segment parameter is present" + .to_owned(), + )); + } + (None, Some(_)) => { + return Err(DynamoDbError::ValidationException( + "The Segment parameter is required but was not present in the request when parameter TotalSegments is present" + .to_owned(), + )); + } + (Some(seg), Some(total)) if seg >= total => { + return Err(DynamoDbError::ValidationException(format!( + "The Segment parameter is zero-based and must be less than parameter TotalSegments: \ + Segment: {seg} is not less than TotalSegments: {total}" + ))); + } + _ => {} + } + // P118: Fetch key_info first so we can use table_id for index lookup. let key_info = ctx .table_key_info(&input.table_name) @@ -299,60 +377,6 @@ pub async fn handle_scan( )?; } - // Validate Segment/TotalSegments — DynamoDB returns different messages per direction - match (input.segment, input.total_segments) { - (Some(_), None) => { - return Err(DynamoDbError::ValidationException( - "The TotalSegments parameter is required but was not present in the request when Segment parameter is present" - .to_owned(), - )); - } - (None, Some(_)) => { - return Err(DynamoDbError::ValidationException( - "The Segment parameter is required but was not present in the request when parameter TotalSegments is present" - .to_owned(), - )); - } - (Some(seg), Some(total)) => { - if total < 1 { - return Err(DynamoDbError::ValidationException( - "The parameter TotalSegments should be greater than or equal to 1".to_owned(), - )); - } - if total > 1_000_000 { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{total}' at 'totalSegments' failed to satisfy constraint: Member must have value less than or equal to 1000000" - ))); - } - if seg < 0 { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{seg}' at 'segment' failed to satisfy constraint: Member must have value greater than or equal to 0" - ))); - } - if seg > 999_999 { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{seg}' at 'segment' failed to satisfy constraint: Member must have value less than or equal to 999999" - ))); - } - if seg >= total { - return Err(DynamoDbError::ValidationException(format!( - "The Segment parameter is zero-based and must be less than parameter TotalSegments: \ - Segment: {seg} is not less than TotalSegments: {total}" - ))); - } - } - (None, None) => {} - } - - // Validate Limit >= 1 - if let Some(limit) = input.limit - && limit < 1 - { - return Err(DynamoDbError::ValidationException(format!( - "1 validation error detected: Value '{limit}' at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" - ))); - } - // For index scans, build a key_info that reflects the index's key schema. let scan_key_info = if let Some(ref idx) = index_info { TableKeyInfo { @@ -367,6 +391,7 @@ pub async fn handle_scan( local_secondary_indexes: key_info.local_secondary_indexes.clone(), stream_specification: None, // Scans don't capture stream records vector_indexes: key_info.vector_indexes.clone(), + creation_date_time: key_info.creation_date_time, } } else { key_info.clone() @@ -444,16 +469,6 @@ pub async fn handle_scan( } } - // Validate begins_with operand types upfront (before any rows are scanned). - if let Some(ref f) = filter { - extenddb_core::expression::validate_begins_with_operands(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - extenddb_core::expression::validate_ordering_operand_types(f, &combined_maps).map_err( - |e| crate::expression_helpers::prefix_expression_error(e, ExpressionKind::Filter), - )?; - } - // Determine which key schema to use for LastEvaluatedKey extraction. // For index scans the LEK carries the index key and the base table key, // and the same shape addresses each chunk read from storage. diff --git a/crates/engine/src/transact_get_items.rs b/crates/engine/src/transact_get_items.rs index ec6825cff..06662dc8c 100755 --- a/crates/engine/src/transact_get_items.rs +++ b/crates/engine/src/transact_get_items.rs @@ -21,6 +21,7 @@ use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::storage_err_to_dynamo; use crate::expression_helpers::build_expression_maps; +use crate::per_table_capacity::PerTableCapacity; use crate::request_limits; use crate::serialize_output; use crate::{DispatchMetrics, DispatchResult}; @@ -44,33 +45,14 @@ pub async fn handle_transact_get_items( // before authorization. request_limits::check_transact_count(input.transact_items.len(), "TransactGetItem")?; - // Resolve table key info for each item - let mut seen_keys: HashSet> = HashSet::with_capacity(input.transact_items.len()); - for tgi in &input.transact_items { - let dedup_key = - serde_json::to_vec(&(&tgi.get.table_name, &tgi.get.key)).unwrap_or_default(); - if !seen_keys.insert(dedup_key) { - return Err(DynamoDbError::ValidationException( - "Transaction request cannot include multiple operations on one item".to_owned(), - )); - } - } - - let mut key_infos = Vec::with_capacity(input.transact_items.len()); - for tgi in &input.transact_items { - let key_info = ctx - .table_key_info_uncached(&tgi.get.table_name) - .await - .map_err(storage_err_to_dynamo)?; - // Key type validation is deferred to the storage layer so that - // mismatches produce TransactionCanceledException with ValidationError - // cancellation reasons, matching real DynamoDB behavior. - key_infos.push(key_info); - } - - // Validate and compile each projection before the transactional read, so a - // malformed request fails with ValidationException instead of executing the - // transaction. DynamoDB checks overlap before the unused-name check. + // Validation runs in the order the service applies it, each phase over + // every item before the next phase starts: projection grammar and alias + // resolution (top-level ValidationException), then table lookup + // (ResourceNotFoundException), then key schema and key size + // (TransactionCanceledException with a ValidationError reason on the + // first offending item and None on every other), then duplicate keys + // (top-level ValidationException). + // Measured against DynamoDB for #108. let mut projections: Vec> = Vec::with_capacity(input.transact_items.len()); for tgi in &input.transact_items { if let Some(ref proj_str) = tgi.get.projection_expression { @@ -103,26 +85,60 @@ pub async fn handle_transact_get_items( } } - // Reject oversized primary keys as a per-item cancellation, matching real - // DynamoDB (TransactionCanceledException with a ValidationError reason for - // the offending item). Key type/emptiness is validated in the storage - // layer; this enforces the size limit the storage layer does not check. - { - let mut reasons: Vec = Vec::with_capacity(input.transact_items.len()); - let mut any_oversized = false; - for (tgi, ki) in input.transact_items.iter().zip(key_infos.iter()) { - match validation::validate_key_size_limits(&tgi.get.key, &ki.key_schema, &ctx.limits) { - Ok(()) => reasons.push(CancellationReason::none()), - Err(e) => { - any_oversized = true; - reasons.push(CancellationReason::validation_error(e.to_string())); + let mut key_infos = Vec::with_capacity(input.transact_items.len()); + for tgi in &input.transact_items { + let key_info = ctx + .table_key_info_uncached(&tgi.get.table_name) + .await + .map_err(storage_err_to_dynamo)?; + key_infos.push(key_info); + } + + // A key with the wrong type, extra or missing attributes, or over the size + // limit cancels the transaction with a ValidationError reason on the FIRST + // offending item and None on every other, even when several keys fail + // (measured 2026-10-14: items 1 and 3 mistyped report + // [ValidationError, None, None]). + let first_offender = input + .transact_items + .iter() + .zip(key_infos.iter()) + .enumerate() + .find_map(|(i, (tgi, ki))| { + validation::validate_batch_key_only( + &tgi.get.key, + &ki.key_schema, + &ki.attribute_definitions, + ) + .and_then(|()| { + validation::validate_key_size_limits(&tgi.get.key, &ki.key_schema, &ctx.limits) + }) + .err() + .map(|e| (i, e)) + }); + if let Some((index, err)) = first_offender { + let reasons: Vec = (0..input.transact_items.len()) + .map(|i| { + if i == index { + CancellationReason::validation_error(err.to_string()) + } else { + CancellationReason::none() } - } - } - if any_oversized { - return Err(storage_err_to_dynamo(StorageError::TransactionCanceled( - reasons, - ))); + }) + .collect(); + return Err(storage_err_to_dynamo(StorageError::TransactionCanceled( + reasons, + ))); + } + + let mut seen_keys: HashSet> = HashSet::with_capacity(input.transact_items.len()); + for tgi in &input.transact_items { + let dedup_key = + serde_json::to_vec(&(&tgi.get.table_name, &tgi.get.key)).unwrap_or_default(); + if !seen_keys.insert(dedup_key) { + return Err(DynamoDbError::ValidationException( + "Transaction request cannot include multiple operations on one item".to_owned(), + )); } } @@ -146,18 +162,17 @@ pub async fn handle_transact_get_items( // Capacity metering: RCU rounded per item, then summed (M-1). // Capacity metering: TransactGetItems costs 2 RCU per item (transactions // double the read cost). Missing items still cost 2 RCU. - let mut per_table_rcu: std::collections::HashMap = - std::collections::HashMap::new(); + let mut per_table_rcu = PerTableCapacity::default(); let rcu: f64 = items .iter() - .zip(input.transact_items.iter()) - .map(|(opt, tgi)| { + .zip(key_infos.iter()) + .map(|(opt, ki)| { let base_rcu = match opt { Some(item) => capacity_helpers::read_capacity_units(item_size_bytes(item), true), None => 1.0, // minimum 1 RCU for missing item }; let txn_rcu = base_rcu * 2.0; // transactions cost 2x - *per_table_rcu.entry(tgi.get.table_name.clone()).or_default() += txn_rcu; + per_table_rcu.add(ki, txn_rcu); txn_rcu }) .sum(); @@ -183,7 +198,7 @@ pub async fn handle_transact_get_items( let consumed_capacity = capacity_helpers::transact_read_capacity( input.return_consumed_capacity, - per_table_rcu.iter().map(|(t, cu)| (t.as_str(), *cu)), + per_table_rcu.ordered().into_iter(), ); let output = TransactGetItemsOutput { diff --git a/crates/engine/src/transact_write_helpers.rs b/crates/engine/src/transact_write_helpers.rs index 8295ef6db..934f16499 100755 --- a/crates/engine/src/transact_write_helpers.rs +++ b/crates/engine/src/transact_write_helpers.rs @@ -4,13 +4,23 @@ //! Helper types and functions for `TransactWriteItems`. //! //! Extracted from `transact_write_items.rs` to keep both files under the -//! 500-line limit. +//! 600-line limit. + +use std::collections::HashMap; use extenddb_core::error::DynamoDbError; -use extenddb_core::expression::{ExpressionMaps, PathElement, UpdateAction}; +use extenddb_core::expression::{ + ExpressionMaps, PathElement, UpdateAction, collect_value_placeholders, +}; +use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ - ReturnItemCollectionMetrics, ReturnValuesOnConditionCheckFailure, TableKeyInfo, - attribute_value_size, extract_key, item_size_bytes, + AttributeValue, CancellationReason, ReturnItemCollectionMetrics, + ReturnValuesOnConditionCheckFailure, TableKeyInfo, attribute_value_size, extract_key, + item_size_bytes, +}; +use extenddb_core::validation::{ + validate_attribute_values_nesting_depth, validate_expression_value_nesting_depth, + validate_key_not_empty, }; use extenddb_storage::TransactWriteOp; use hmac::{Hmac, Mac}; @@ -58,11 +68,16 @@ pub(crate) enum PreparedOp { impl PreparedOp { /// The table name for this operation. pub(crate) fn table_name(&self) -> &str { + &self.key_info().table_name + } + + /// The table this operation targets. + pub(crate) fn key_info(&self) -> &TableKeyInfo { match self { Self::Put { key_info, .. } | Self::Delete { key_info, .. } | Self::Update { key_info, .. } - | Self::ConditionCheck { key_info, .. } => &key_info.table_name, + | Self::ConditionCheck { key_info, .. } => key_info, } } @@ -206,34 +221,195 @@ impl PreparedOp { capacity_helpers::item_metrics(ricm, &key_info.key_schema, item_or_key, key_info.has_lsi) } - /// Return a `ValidationError` cancellation reason if this op's primary key - /// exceeds the size limit (hash > 2048 / range > 1024 bytes by default). + /// The per-item cancellation reason for a key that does not fit the + /// table's key schema, or `None` when the key is acceptable. /// - /// Real `DynamoDB` cancels the whole transaction with a per-item - /// `ValidationError` reason for an oversized key in ANY sub-op (Put, - /// Delete, Update, ConditionCheck). Empty-key values are handled separately - /// as a top-level `ValidationException`, so this checks size only. - pub(crate) fn oversized_key_reason( + /// Covers a key attribute of the wrong type, a `Put` item missing a key + /// attribute, a `Delete`, `Update`, or `ConditionCheck` key with extra or + /// missing attributes, and a hash or range key over its size limit. The + /// service reports all of these as a `ValidationError` cancellation reason + /// on the offending item. The reason text follows the service: a `Put` + /// item keeps the single-item wording ("Type mismatch for key ..."), a + /// `Delete`, `Update`, or `ConditionCheck` key gets "The provided key + /// element does not match the schema". An empty key value is a top-level + /// `ValidationException`; `key_pass_failure` checks it after this. + pub(crate) fn key_schema_reason( &self, limits: &extenddb_core::limits::LimitsConfig, ) -> Option { + use extenddb_core::validation::{ + validate_item_keys, validate_key_shape_only, validate_key_size_limits, + }; let (key_info, item_or_key) = match self { Self::Put { key_info, item, .. } => (key_info, item), Self::Delete { key_info, key, .. } | Self::Update { key_info, key, .. } | Self::ConditionCheck { key_info, key, .. } => (key_info, key), }; - match extenddb_core::validation::validate_key_size_limits( - item_or_key, - &key_info.key_schema, - limits, - ) { + let schema_check = match self { + Self::Put { .. } => validate_item_keys( + item_or_key, + &key_info.key_schema, + &key_info.attribute_definitions, + ), + _ => validate_key_shape_only( + item_or_key, + &key_info.key_schema, + &key_info.attribute_definitions, + ), + }; + let outcome = schema_check + .and_then(|()| validate_key_size_limits(item_or_key, &key_info.key_schema, limits)); + match outcome { Ok(()) => None, Err(e) => Some(extenddb_core::types::CancellationReason::validation_error( e.to_string(), )), } } + + /// The per-item cancellation reason for an `Update` that names a key + /// attribute in one of its actions, or `None`. The service reports it as a + /// `ValidationError` reason, not as a top-level exception (measured). + fn key_update_reason(&self) -> Option { + let Self::Update { + key_info, + actions, + maps, + .. + } = self + else { + return None; + }; + validate_no_key_updates(actions, key_info, maps) + .err() + .map(|e| match e { + DynamoDbError::ValidationException(message) => { + CancellationReason::validation_error(message) + } + other => CancellationReason::validation_error(other.to_string()), + }) + } + + /// An empty string or binary key value, as the top-level + /// `ValidationException` the service answers. + fn validate_key_value_not_empty(&self) -> Result<(), DynamoDbError> { + match self { + Self::Put { key_info, item, .. } => validate_key_not_empty(item, &key_info.key_schema), + Self::Delete { key_info, key, .. } + | Self::Update { key_info, key, .. } + | Self::ConditionCheck { key_info, key, .. } => { + validate_key_not_empty(key, &key_info.key_schema) + } + } + } +} + +/// The key pass of `TransactWriteItems`, run once every table is known. +/// +/// One item after another, the first failing item decides. Inside an item the +/// order is the service's (measured): a key that does not fit the schema, then +/// a key attribute named in an `Update`, both returned as the cancellation +/// reasons of the transaction (the failing item's reason, `None` for every +/// other item), then an empty key value, returned as a top-level +/// `ValidationException`. +pub(crate) fn key_pass_failure( + prepared: &[PreparedOp], + limits: &LimitsConfig, +) -> Result>, DynamoDbError> { + for (index, op) in prepared.iter().enumerate() { + if let Some(reason) = op + .key_schema_reason(limits) + .or_else(|| op.key_update_reason()) + { + return Ok(Some( + (0..prepared.len()) + .map(|i| { + if i == index { + reason.clone() + } else { + CancellationReason::none() + } + }) + .collect(), + )); + } + op.validate_key_value_not_empty()?; + } + Ok(None) +} + +/// The cancellation reasons for `Update` items that store a value nested as +/// deep as an item may not hold, or `None` when no item does. +/// +/// The value passes the `ExpressionAttributeValues` check +/// (`validate_value_map_nesting`) and breaks the item limit once stored. The +/// service reports `Nesting Levels have exceeded supported limits` as a +/// `ValidationError` reason on every such item, after the duplicate check and +/// alongside the other items' condition results (measured 2026-10-27). +/// ExtendDB reports the reason in the same position but gives `None` for the +/// other items, where the service gives `ConditionalCheckFailed` for an item +/// whose condition was false; the differences document carries the row. +pub(crate) fn stored_value_nesting_reasons( + prepared: &[PreparedOp], +) -> Option> { + let offends = |op: &PreparedOp| { + let PreparedOp::Update { actions, maps, .. } = op else { + return false; + }; + let mut placeholders: Vec = Vec::new(); + for action in actions { + if let UpdateAction::Set { value, .. } = action { + collect_value_placeholders(value, &mut placeholders); + } + } + validate_attribute_values_nesting_depth( + placeholders.iter().filter_map(|name| maps.values.get(name)), + ) + .is_err() + }; + let flags: Vec = prepared.iter().map(offends).collect(); + flags.iter().any(|&f| f).then(|| { + flags + .into_iter() + .map(|f| { + if f { + CancellationReason::validation_error( + "Nesting Levels have exceeded supported limits".to_owned(), + ) + } else { + CancellationReason::none() + } + }) + .collect() + }) +} + +/// Check the nesting of every `ExpressionAttributeValues` value of one item, +/// used or not, as the service does at the start of that item's validation +/// (see `validate_expression_value_nesting_depth` for the threshold). The +/// message names the offending placeholder; keys are visited in sorted order +/// so the one reported is deterministic when several values are too deep. +pub(crate) fn validate_value_map_nesting( + values: Option<&HashMap>, +) -> Result<(), DynamoDbError> { + let Some(values) = values else { + return Ok(()); + }; + let mut keys: Vec<&String> = values.keys().collect(); + keys.sort(); + for key in keys { + match validate_expression_value_nesting_depth(&values[key]) { + Ok(()) => {} + Err(DynamoDbError::ValidationException(inner)) => { + return Err(DynamoDbError::ValidationException(format!( + "ExpressionAttributeValues contains invalid value: {inner} for key {key}" + ))); + } + Err(other) => return Err(other), + } + } + Ok(()) } /// Validate `ClientRequestToken` format. diff --git a/crates/engine/src/transact_write_items.rs b/crates/engine/src/transact_write_items.rs index 069dea418..6cc914c30 100755 --- a/crates/engine/src/transact_write_items.rs +++ b/crates/engine/src/transact_write_items.rs @@ -10,12 +10,14 @@ use serde_json::Value; use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::storage_err_to_dynamo; -use crate::expression_helpers::{build_expression_maps, parse_optional_condition}; +use crate::expression_helpers::build_expression_maps; +use crate::per_table_capacity::PerTableCapacity; use crate::request_limits; use crate::serialize_output; use crate::stream_capture; use crate::transact_write_helpers::{ - PreparedOp, compute_fingerprint, validate_client_request_token, validate_no_key_updates, + PreparedOp, compute_fingerprint, key_pass_failure, stored_value_nesting_reasons, + validate_client_request_token, validate_value_map_nesting, }; use crate::{DispatchMetrics, DispatchResult}; use extenddb_core::error::DynamoDbError; @@ -24,8 +26,7 @@ use extenddb_core::types::{ CancellationReason, Item, TransactWriteItem, TransactWriteItemsInput, TransactWriteItemsOutput, }; use extenddb_core::validation::{ - validate_attribute_name_sizes, validate_attribute_values_nesting_depth, - validate_item_nesting_depth, validate_item_size, validate_key_not_empty, + validate_attribute_name_sizes, validate_item_nesting_depth, validate_item_size, }; /// Handle a `TransactWriteItems` request. @@ -75,23 +76,38 @@ pub async fn handle_transact_write_items( } } - // Resolve table key info, validate inputs, parse expressions, and check - // for duplicate targets — all in a single pass to avoid redundant - // `table_key_info` lookups. + // Validation runs in the order the service applies it, each phase over + // every item before the next phase starts: + // 1. per item, in order, with no table needed: the nesting of the item's + // ExpressionAttributeValues, expression grammar, the semantic walk, + // unused names and values, then for a Put the attribute name size, + // the item size and the item nesting (top-level ValidationException), + // 2. table lookup (ResourceNotFoundException), + // 3. the 4 MB request total (top-level ValidationException), + // 4. the key pass, per item in order: a key that does not fit the schema + // or a key attribute named in an Update (TransactionCanceledException + // with a ValidationError reason on the first offending item and None + // on every other; conditions are not evaluated when any item fails + // here), then an empty key value (top-level ValidationException), + // 5. duplicate targets (top-level ValidationException), + // 6. a value an Update stores that is as deep as an item may not hold + // (ValidationError reason on that item). + // Measured against DynamoDB; a request with two defects reports the + // earlier phase's error whichever item carries it. + let parsed: Vec = input + .transact_items + .iter() + .map(|twi| parse_write_expressions(twi, &ctx.limits)) + .collect::>()?; + let mut prepared: Vec = Vec::with_capacity(input.transact_items.len()); - let mut seen_targets: HashSet = HashSet::with_capacity(input.transact_items.len()); - for twi in &input.transact_items { - let op = prepare_write_op(twi, ctx).await?; - let target_key = op.canonical_target(); - if !seen_targets.insert(target_key) { - return Err(DynamoDbError::ValidationException( - "Transaction request cannot include multiple operations on one item".to_owned(), - )); - } - prepared.push(op); + for (twi, exprs) in input.transact_items.iter().zip(parsed) { + prepared.push(prepare_write_op(twi, exprs, ctx).await?); } - // Validate total transaction size <= 4MB + // The 4 MB request total comes after every table is looked up and before + // the key schema and duplicate checks (measured: eleven 390 KB items plus + // a mistyped key, a duplicate, or a false condition all report the size). let total_size: usize = prepared .iter() .map(super::transact_write_helpers::PreparedOp::item_size) @@ -102,31 +118,39 @@ pub async fn handle_transact_write_items( )); } - // Reject oversized primary keys as a per-item cancellation, matching real - // DynamoDB: an oversized hash/range key in any sub-op returns - // TransactionCanceledException with a ValidationError cancellation reason - // for the offending item (an EMPTY key value, by contrast, is a top-level - // ValidationException and is handled in prepare_write_op). - { - let mut reasons: Vec = - Vec::with_capacity(prepared.len()); - let mut any_oversized = false; - for op in &prepared { - match op.oversized_key_reason(&ctx.limits) { - Some(reason) => { - any_oversized = true; - reasons.push(reason); - } - None => reasons.push(extenddb_core::types::CancellationReason::none()), - } - } - if any_oversized { - return Err(storage_err_to_dynamo( - extenddb_storage::error::StorageError::TransactionCanceled(reasons), + // The key pass: one item after another, the first failing item decides. + // Inside an item a key that does not fit the schema (wrong type, missing + // or extra attribute, over the size limit) comes first, then a key + // attribute named in an Update, both as a ValidationError reason on that + // item and None on every other even when several fail (measured: items 1 + // and 3 mistyped report [ValidationError, None, None]); an empty key + // value comes last and is a top-level ValidationException. Running this + // before duplicate detection and before any condition is evaluated + // matches the service: the other items report None, not + // ConditionalCheckFailed. + if let Some(reasons) = key_pass_failure(&prepared, &ctx.limits)? { + return Err(storage_err_to_dynamo( + extenddb_storage::error::StorageError::TransactionCanceled(reasons), + )); + } + + let mut seen_targets: HashSet = HashSet::with_capacity(prepared.len()); + for op in &prepared { + if !seen_targets.insert(op.canonical_target()) { + return Err(DynamoDbError::ValidationException( + "Transaction request cannot include multiple operations on one item".to_owned(), )); } } + // A value an Update stores that is as deep as an item may not hold is a + // ValidationError reason on that item (see `stored_value_nesting_reasons`). + if let Some(reasons) = stored_value_nesting_reasons(&prepared) { + return Err(storage_err_to_dynamo( + extenddb_storage::error::StorageError::TransactionCanceled(reasons), + )); + } + // Build storage operations // Vector-valued and search-schema validation failures surface as per-item // cancellation reasons rather than a top-level ValidationException. @@ -175,18 +199,19 @@ pub async fn handle_transact_write_items( extenddb_core::types::ReturnConsumedCapacity::None => None, rcc => { let indexes = rcc == extenddb_core::types::ReturnConsumedCapacity::Indexes; - let mut per_table_rcu: HashMap = HashMap::new(); + let mut per_table_rcu = PerTableCapacity::default(); for op in &prepared { let rcu = capacity_helpers::read_capacity_units(op.write_bytes(), true) * 2.0; - *per_table_rcu.entry(op.table_name().to_owned()).or_default() += rcu; + per_table_rcu.add(op.key_info(), rcu); } Some( per_table_rcu - .iter() + .ordered() + .into_iter() .map(|(t, cu)| { extenddb_core::types::ConsumedCapacity::transact_replay_read( - t, *cu, indexes, + t, cu, indexes, ) }) .collect(), @@ -215,12 +240,12 @@ pub async fn handle_transact_write_items( // failure there leaves the item without its record. // Per-item WCU: round each item individually, then sum (M-1). - let mut per_table_wcu: HashMap = HashMap::new(); + let mut per_table_wcu = PerTableCapacity::default(); let wcu: f64 = prepared .iter() .map(|op| { let item_wcu = capacity_helpers::write_capacity_units(op.write_bytes()) * 2.0; // transactions cost 2x - *per_table_wcu.entry(op.table_name().to_owned()).or_default() += item_wcu; + per_table_wcu.add(op.key_info(), item_wcu); item_wcu }) .sum(); @@ -232,10 +257,13 @@ pub async fn handle_transact_write_items( // only for now. let consumed_capacity = capacity_helpers::transact_write_capacity( input.return_consumed_capacity, - per_table_wcu.iter().map(|(t, cu)| (t.as_str(), *cu)), + per_table_wcu.ordered().into_iter(), ); - // Collect ItemCollectionMetrics per table for write operations. + // Collect ItemCollectionMetrics per table for write operations. When SIZE + // was requested the member is present even with nothing to report (no + // table has a local secondary index): the service returns `{}` there + // (measured 2026-10-14), and omits the member only when it was not asked for. let mut all_icm: HashMap> = HashMap::new(); for p in &prepared { @@ -246,14 +274,14 @@ pub async fn handle_transact_write_items( .push(m); } } + let item_collection_metrics = match input.return_item_collection_metrics { + extenddb_core::types::ReturnItemCollectionMetrics::None => None, + extenddb_core::types::ReturnItemCollectionMetrics::Size => Some(all_icm), + }; let output = TransactWriteItemsOutput { consumed_capacity, - item_collection_metrics: if all_icm.is_empty() { - None - } else { - Some(all_icm) - }, + item_collection_metrics, }; let body = serialize_output(&output)?; Ok(DispatchResult { @@ -344,48 +372,143 @@ fn assigned_vector_attributes(actions: &[UpdateAction], maps: &ExpressionMaps) - assigned } -/// Parse and validate a single `TransactWriteItem`, returning a `PreparedOp`. +/// A `TransactWriteItem`'s expressions, parsed and checked without touching +/// storage: grammar, operand types, every `#name` and `:value` reference +/// defined, and no unused names or values. +struct ParsedExpressions { + maps: ExpressionMaps, + condition: Option, + actions: Option>, +} + +/// Phase 1 of `TransactWriteItems` validation: everything about one item that +/// needs no table. +/// +/// Runs for every item before any table is looked up, so a malformed +/// expression on item three is reported ahead of a missing table on item one. +fn parse_write_expressions( + twi: &TransactWriteItem, + limits: &extenddb_core::limits::LimitsConfig, +) -> Result { + use extenddb_core::expression::ExpressionKind; + + let (names, values, condition_expression, update_expression) = if let Some(put) = &twi.put { + ( + put.expression_attribute_names.as_ref(), + put.expression_attribute_values.as_ref(), + put.condition_expression.as_deref(), + None, + ) + } else if let Some(del) = &twi.delete { + ( + del.expression_attribute_names.as_ref(), + del.expression_attribute_values.as_ref(), + del.condition_expression.as_deref(), + None, + ) + } else if let Some(upd) = &twi.update { + ( + upd.expression_attribute_names.as_ref(), + upd.expression_attribute_values.as_ref(), + upd.condition_expression.as_deref(), + Some(upd.update_expression.as_str()), + ) + } else if let Some(cc) = &twi.condition_check { + ( + cc.expression_attribute_names.as_ref(), + cc.expression_attribute_values.as_ref(), + Some(cc.condition_expression.as_str()), + None, + ) + } else { + // Unreachable: the one-operation check above already rejected this item. + return Err(DynamoDbError::ValidationException( + "TransactWriteItem must contain exactly one operation".to_owned(), + )); + }; + + // The nesting of every ExpressionAttributeValues value, used or not, is + // the first thing the service checks about an item (measured: it beats + // the same item's syntax error, undefined reference and unused value, and + // a later item's expression errors, and yields to an earlier item's). + validate_value_map_nesting(values)?; + + // Inside a transaction the service checks an Update item's + // UpdateExpression (syntax, the ordered semantic walk, overlapping paths) + // before its ConditionExpression, the reverse of UpdateItem; then the + // unused names and values; then, for a Put, the attribute name size, the + // item's size and the item's nesting (measured: an oversize Put with an + // unused value reports the value, an oversize item one reports its size + // ahead of the next item's defects, and one item with all three defects + // reports the name first and the nesting last). + let maps = build_expression_maps(names, values); + let actions = match update_expression { + Some(expr) => Some(crate::expression_helpers::parse_update_expr_checked( + expr, &maps, limits, + )?), + None => None, + }; + let condition = match condition_expression { + // A ConditionCheck's expression is required, so an empty string is + // parsed and refused as empty (measured 2026-10-27: `Invalid + // ConditionExpression: The expression can not be empty;`). Put, + // Delete and Update treat an empty string as no condition. + Some(expr) if twi.condition_check.is_some() || !expr.is_empty() => { + Some(crate::expression_helpers::parse_condition_checked( + expr, + &maps, + limits, + true, + ExpressionKind::Condition, + )?) + } + _ => None, + }; + // Transactions accept names/values with no expression (unlike single-item + // APIs); only check for unused refs when some expression is present. + if condition.is_some() || actions.is_some() { + let exprs: Vec<&Expr> = condition.iter().collect(); + let action_refs: Vec<&UpdateAction> = actions.iter().flatten().collect(); + extenddb_core::expression::validate_unused_attributes( + &maps.names, + &maps.values, + &exprs, + &action_refs, + &HashSet::new(), + &HashSet::new(), + )?; + } + if let Some(put) = &twi.put { + validate_attribute_name_sizes(&put.item, limits)?; + validate_item_size(&put.item, limits.max_item_size_bytes)?; + validate_item_nesting_depth(&put.item)?; + } + Ok(ParsedExpressions { + maps, + condition, + actions, + }) +} + +/// Phase 2 of `TransactWriteItems` validation: look the table up and build +/// the `PreparedOp`. Nothing is validated against the table here; the key +/// pass runs once every table is known (see `key_pass_failure`). async fn prepare_write_op( twi: &TransactWriteItem, + exprs: ParsedExpressions, ctx: &OperationContext, ) -> Result { + let ParsedExpressions { + maps, + condition, + actions, + } = exprs; + if let Some(put) = &twi.put { let key_info = ctx .table_key_info_uncached(&put.table_name) .await .map_err(storage_err_to_dynamo)?; - validate_item_nesting_depth(&put.item)?; - validate_item_size(&put.item, ctx.limits.max_item_size_bytes)?; - validate_attribute_name_sizes(&put.item, &ctx.limits)?; - validate_key_not_empty(&put.item, &key_info.key_schema)?; - let maps = build_expression_maps( - put.expression_attribute_names.as_ref(), - put.expression_attribute_values.as_ref(), - ); - let condition = parse_optional_condition(put.condition_expression.as_deref(), &ctx.limits)?; - if let Some(ref expr) = condition { - extenddb_core::expression::validate_ordering_operand_types(expr, &maps).map_err( - |e| { - crate::expression_helpers::prefix_expression_error( - e, - extenddb_core::expression::ExpressionKind::Condition, - ) - }, - )?; - } - // Transactions accept names/values with no expression (unlike single-item - // APIs); only check for unused refs when a condition is present. - if condition.is_some() { - let exprs: Vec<&extenddb_core::expression::Expr> = condition.iter().collect(); - extenddb_core::expression::validate_unused_attributes( - &maps.names, - &maps.values, - &exprs, - &[], - &HashSet::new(), - &HashSet::new(), - )?; - } let stream = stream_capture::stream_view_type(&key_info).map(|vt| extenddb_storage::StreamCapture { view_type: vt, @@ -407,36 +530,6 @@ async fn prepare_write_op( .table_key_info_uncached(&del.table_name) .await .map_err(storage_err_to_dynamo)?; - // Empty or oversize key values are up-front input validation in - // DynamoDB (a top-level ValidationException), unlike a key type - // mismatch which surfaces as a per-item cancellation reason. - validate_key_not_empty(&del.key, &key_info.key_schema)?; - let maps = build_expression_maps( - del.expression_attribute_names.as_ref(), - del.expression_attribute_values.as_ref(), - ); - let condition = parse_optional_condition(del.condition_expression.as_deref(), &ctx.limits)?; - if let Some(ref expr) = condition { - extenddb_core::expression::validate_ordering_operand_types(expr, &maps).map_err( - |e| { - crate::expression_helpers::prefix_expression_error( - e, - extenddb_core::expression::ExpressionKind::Condition, - ) - }, - )?; - } - if condition.is_some() { - let exprs: Vec<&extenddb_core::expression::Expr> = condition.iter().collect(); - extenddb_core::expression::validate_unused_attributes( - &maps.names, - &maps.values, - &exprs, - &[], - &HashSet::new(), - &HashSet::new(), - )?; - } let stream = stream_capture::stream_view_type(&key_info).map(|vt| extenddb_storage::StreamCapture { view_type: vt, @@ -458,54 +551,6 @@ async fn prepare_write_op( .table_key_info_uncached(&upd.table_name) .await .map_err(storage_err_to_dynamo)?; - // Empty or oversize key values are up-front input validation in - // DynamoDB (a top-level ValidationException), unlike a key type - // mismatch which surfaces as a per-item cancellation reason. - validate_key_not_empty(&upd.key, &key_info.key_schema)?; - let maps = build_expression_maps( - upd.expression_attribute_names.as_ref(), - upd.expression_attribute_values.as_ref(), - ); - let actions = - crate::expression_helpers::parse_update_expr(&upd.update_expression, &ctx.limits)?; - validate_no_key_updates(&actions, &key_info, &maps)?; - - // Validate nesting depth of EAV values that get stored via SET actions. - { - let mut placeholders: Vec = Vec::new(); - for action in &actions { - if let extenddb_core::expression::UpdateAction::Set { value, .. } = action { - extenddb_core::expression::collect_value_placeholders(value, &mut placeholders); - } - } - let stored: Vec<&extenddb_core::types::AttributeValue> = placeholders - .iter() - .filter_map(|name| maps.values.get(name)) - .collect(); - validate_attribute_values_nesting_depth(stored)?; - } - let condition = parse_optional_condition(upd.condition_expression.as_deref(), &ctx.limits)?; - if let Some(ref expr) = condition { - extenddb_core::expression::validate_ordering_operand_types(expr, &maps).map_err( - |e| { - crate::expression_helpers::prefix_expression_error( - e, - extenddb_core::expression::ExpressionKind::Condition, - ) - }, - )?; - } - { - let exprs: Vec<&extenddb_core::expression::Expr> = condition.iter().collect(); - extenddb_core::expression::validate_unused_attributes( - &maps.names, - &maps.values, - &exprs, - &actions.iter().collect::>(), - &HashSet::new(), - &HashSet::new(), - )?; - } let stream = stream_capture::stream_view_type(&key_info).map(|vt| extenddb_storage::StreamCapture { view_type: vt, @@ -515,7 +560,7 @@ async fn prepare_write_op( return Ok(PreparedOp::Update { key_info, key: upd.key.clone(), - actions, + actions: actions.unwrap_or_default(), condition, maps, return_values_on_ccf: upd.return_values_on_condition_check_failure, @@ -528,35 +573,13 @@ async fn prepare_write_op( .table_key_info_uncached(&cc.table_name) .await .map_err(storage_err_to_dynamo)?; - // Empty or oversize key values are up-front input validation in - // DynamoDB (a top-level ValidationException), unlike a key type - // mismatch which surfaces as a per-item cancellation reason. - validate_key_not_empty(&cc.key, &key_info.key_schema)?; - let maps = build_expression_maps( - cc.expression_attribute_names.as_ref(), - cc.expression_attribute_values.as_ref(), - ); - let condition = - crate::expression_helpers::parse_condition_expr(&cc.condition_expression, &ctx.limits)?; - extenddb_core::expression::validate_ordering_operand_types(&condition, &maps).map_err( - |e| { - crate::expression_helpers::prefix_expression_error( - e, - extenddb_core::expression::ExpressionKind::Condition, - ) - }, - )?; - { - let exprs: Vec<&extenddb_core::expression::Expr> = vec![&condition]; - extenddb_core::expression::validate_unused_attributes( - &maps.names, - &maps.values, - &exprs, - &[], - &HashSet::new(), - &HashSet::new(), - )?; - } + let Some(condition) = condition else { + // `parse_write_expressions` returns a condition for every + // ConditionCheck, an empty expression included, or its error. + return Err(DynamoDbError::ValidationException( + "ConditionCheck requires a ConditionExpression".to_owned(), + )); + }; return Ok(PreparedOp::ConditionCheck { key_info, key: cc.key.clone(), diff --git a/crates/engine/src/update_item.rs b/crates/engine/src/update_item.rs index 7f74c205c..298a9948d 100755 --- a/crates/engine/src/update_item.rs +++ b/crates/engine/src/update_item.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use serde_json::Value; use extenddb_core::error::DynamoDbError; -use extenddb_core::expression::{ExpressionKind, ExpressionMaps, PathElement, UpdateAction}; +use extenddb_core::expression::{ExpressionMaps, PathElement, UpdateAction}; use extenddb_core::types::{ AttributeValue, Item, ReturnValues, TableKeyInfo, UpdateItemInput, UpdateItemOutput, item_size_bytes, @@ -20,7 +20,7 @@ use extenddb_core::types::{ use crate::OperationContext; use crate::capacity_helpers; use crate::create_table::{storage_err_to_dynamo, storage_err_to_dynamo_with_ccf}; -use crate::expression_helpers::resolve_condition; +use crate::expression_helpers::{resolve_condition, single_item_write_prefix}; use crate::serialize_output; use crate::stream_capture; use crate::{DispatchMetrics, DispatchResult}; @@ -92,6 +92,7 @@ pub async fn handle_update_item( // Desugar legacy AttributeUpdates into UpdateExpression if present. let (effective_update_expr, extra_expr_values, extra_expr_names) = if let Some(attr_updates) = &input.attribute_updates { + crate::attribute_updates::validate_action_value_types(attr_updates)?; desugar_attribute_updates(attr_updates)? } else { ( @@ -123,13 +124,16 @@ pub async fn handle_update_item( }; let has_expr = has_update_expr || has_condition; + // The single-item write APIs name no expression in the values message and + // wrap every expression-stage error in `1 validation error detected: `. extenddb_core::expression::validate_expression_param_usage( input.expression_attribute_names.as_ref(), has_expr, input.expression_attribute_values.as_ref(), has_expr, - &[ExpressionKind::Update, ExpressionKind::Condition], - )?; + &[], + ) + .map_err(single_item_write_prefix)?; // ConditionalOperator requires an Expected with two or more conditions. extenddb_core::validation::validate_conditional_operator_usage( @@ -137,6 +141,10 @@ pub async fn handle_update_item( input.expected.as_ref().map_or(0, HashMap::len), )?; + // Amazon DynamoDB checks the whole ConditionExpression (syntax, then + // references, functions, and operand types) before it looks at the + // UpdateExpression: a request with a defect in each reports the + // condition's (measured 2026-10-14). `resolve_condition` runs that pass. let (condition, maps) = resolve_condition( input.condition_expression.as_deref(), effective_expr_names.as_ref(), @@ -144,33 +152,22 @@ pub async fn handle_update_item( input.expected.as_ref(), input.conditional_operator, &ctx.limits, - )?; + ) + .map_err(single_item_write_prefix)?; // No UpdateExpression and no AttributeUpdates: no-op upsert. - // Some("") still errors via parse_update_expr. + // Some("") still errors via parse_update_expr_checked. The checked parse + // runs the service's ordered walk over the parsed actions (function + // names, operand counts and types, undefined references, repeated + // sections) and then the overlapping-path rule, all ahead of the table + // lookup. let actions = if let Some(update_expr) = effective_update_expr.as_deref() { - crate::expression_helpers::parse_update_expr(update_expr, &ctx.limits)? + crate::expression_helpers::parse_update_expr_checked(update_expr, &maps, &ctx.limits) + .map_err(single_item_write_prefix)? } else { Vec::new() }; - - // Reject an undefined #name before the existence check - // (otherwise it is only raised later, during execution). - for action in &actions { - let path = match action { - UpdateAction::Set { path, .. } - | UpdateAction::Remove { path } - | UpdateAction::Add { path, .. } - | UpdateAction::Delete { path, .. } => path, - }; - for el in path { - if let PathElement::Attribute(name) = el - && let Some(ref_name) = name.strip_prefix('#') - { - maps.resolve_name(ref_name)?; - } - } - } + let action_refs: Vec<&UpdateAction> = actions.iter().collect(); if input.expected.is_none() || input @@ -183,10 +180,11 @@ pub async fn handle_update_item( &maps.names, &maps.values, &exprs, - &actions.iter().collect::>(), + &action_refs, &std::collections::HashSet::new(), &std::collections::HashSet::new(), - )?; + ) + .map_err(single_item_write_prefix)?; } let key_info = ctx diff --git a/crates/storage-cassandra/src/backup_engine.rs b/crates/storage-cassandra/src/backup_engine.rs index 7c070934e..41b819166 100755 --- a/crates/storage-cassandra/src/backup_engine.rs +++ b/crates/storage-cassandra/src/backup_engine.rs @@ -979,6 +979,7 @@ impl CassandraEngine { local_secondary_indexes: Vec::new(), stream_specification: None, vector_indexes: Vec::new(), + creation_date_time: 0.0, }; let account_keyspace = self.account_keyspace(&job.account_id); let query = format!( diff --git a/crates/storage-cassandra/src/data/ddl.rs b/crates/storage-cassandra/src/data/ddl.rs index 9016422dd..16ab50742 100644 --- a/crates/storage-cassandra/src/data/ddl.rs +++ b/crates/storage-cassandra/src/data/ddl.rs @@ -80,7 +80,8 @@ impl CassandraEngine { let catalog_keyspace = format!("{}_catalog", self.keyspace_prefix); let query = format!( - "SELECT key_schema, attribute_definitions, table_status, table_id, stream_specification \ + "SELECT key_schema, attribute_definitions, table_status, table_id, stream_specification, \ + created_at \ FROM {catalog_keyspace}.tables WHERE account_id = ? AND table_name = ?" ); let values = cdrs_tokio::query_values!(account_id, table_name); @@ -116,6 +117,9 @@ impl CassandraEngine { .map_err(|e| StorageError::Internal(format!("Parse table_id: {e}")))?; let stream_spec_text: Option = row.get_by_name("stream_specification").ok().flatten(); + let created_at: i64 = row + .get_r_by_name("created_at") + .map_err(|e| StorageError::Internal(format!("Parse created_at: {e}")))?; if status == "CREATING" || status == "DELETING" { return Err(StorageError::TableNotFound(table_name.to_owned())); @@ -182,6 +186,7 @@ impl CassandraEngine { local_secondary_indexes, stream_specification, vector_indexes: Vec::new(), + creation_date_time: crate::cassandra_util::millis_to_seconds_f64(created_at), }) } diff --git a/crates/storage-cassandra/src/data/index.rs b/crates/storage-cassandra/src/data/index.rs index 4ebd72dac..3aa353ac6 100644 --- a/crates/storage-cassandra/src/data/index.rs +++ b/crates/storage-cassandra/src/data/index.rs @@ -147,7 +147,7 @@ pub async fn fetch_index_by_name( let row = rows .into_iter() .next() - .ok_or_else(|| StorageError::IndexNotFound(format!("Index {index_name} not found")))?; + .ok_or_else(|| StorageError::IndexNotFound(index_name.to_owned()))?; let index_id: String = get_column(&row, "index_id", "fetch_index_by_name")?; let index_type: String = get_column(&row, "index_type", "fetch_index_by_name")?; diff --git a/crates/storage-cassandra/src/data/ttl.rs b/crates/storage-cassandra/src/data/ttl.rs index e2b2b0fea..728f8d700 100644 --- a/crates/storage-cassandra/src/data/ttl.rs +++ b/crates/storage-cassandra/src/data/ttl.rs @@ -1345,6 +1345,7 @@ mod tests { local_secondary_indexes: Vec::new(), vector_indexes: Vec::new(), stream_specification: None, + creation_date_time: 0.0, } } diff --git a/crates/storage-cassandra/tests/common/mod.rs b/crates/storage-cassandra/tests/common/mod.rs index c1fd1e971..e88ff74db 100644 --- a/crates/storage-cassandra/tests/common/mod.rs +++ b/crates/storage-cassandra/tests/common/mod.rs @@ -278,6 +278,7 @@ impl TestTable { local_secondary_indexes: Vec::new(), vector_indexes: Vec::new(), stream_specification: None, + creation_date_time: 0.0, }; Self { @@ -360,6 +361,7 @@ impl TestTable { local_secondary_indexes: Vec::new(), vector_indexes: Vec::new(), stream_specification: None, + creation_date_time: 0.0, }; Self { @@ -453,6 +455,7 @@ impl TestTable { local_secondary_indexes: Vec::new(), vector_indexes: Vec::new(), stream_specification: None, + creation_date_time: 0.0, }; Self { @@ -550,6 +553,7 @@ impl TestTable { local_secondary_indexes: Vec::new(), vector_indexes: Vec::new(), stream_specification: None, + creation_date_time: 0.0, }; Self { diff --git a/crates/storage-cassandra/tests/direct/stream_ownership.rs b/crates/storage-cassandra/tests/direct/stream_ownership.rs index fa1170c59..84e6b379c 100644 --- a/crates/storage-cassandra/tests/direct/stream_ownership.rs +++ b/crates/storage-cassandra/tests/direct/stream_ownership.rs @@ -94,6 +94,7 @@ async fn stream_table_in( global_secondary_indexes: Vec::new(), local_secondary_indexes: Vec::new(), stream_specification: desc.stream_specification, + creation_date_time: 0.0, } } diff --git a/crates/storage-cassandra/tests/direct/streams.rs b/crates/storage-cassandra/tests/direct/streams.rs index 16c86540e..bf7a9ea89 100644 --- a/crates/storage-cassandra/tests/direct/streams.rs +++ b/crates/storage-cassandra/tests/direct/streams.rs @@ -106,6 +106,7 @@ pub(crate) async fn setup_stream_table( global_secondary_indexes: Vec::new(), local_secondary_indexes: Vec::new(), stream_specification: desc.stream_specification, + creation_date_time: 0.0, }; (key_info, stream_label) diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 77706c3c7..8269f330d 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -156,7 +156,12 @@ impl MongoEngine { let deletion_protection = input.deletion_protection_enabled.unwrap_or(false); let now = time::OffsetDateTime::now_utc(); - let creation_epoch = now.unix_timestamp() as f64; + // Keep the sub-second part. bson::DateTime stores milliseconds, and + // the transaction ConsumedCapacity order sorts tables by this value, + // so whole seconds make two tables created in the same second tie and + // fall back to name order, which the service does not do. + let creation_millis = now.unix_timestamp() * 1000 + i64::from(now.millisecond()); + let creation_epoch = creation_millis as f64 / 1000.0; // Build the table metadata document let key_schema_bson = @@ -243,7 +248,7 @@ impl MongoEngine { "stream_specification": stream_bson.unwrap_or(bson::Bson::Null), "table_status": table_status, "status_transition_at": status_transition_at, - "creation_date_time": bson::DateTime::from_millis((creation_epoch * 1000.0) as i64), + "creation_date_time": bson::DateTime::from_millis(creation_millis), "table_size_bytes": 0_i64, "item_count": 0_i64, "table_arn": &table_arn_val, @@ -1268,6 +1273,10 @@ impl MongoEngine { let global_secondary_indexes = grouped.gsis; let local_secondary_indexes = grouped.lsis; let has_lsi = !local_secondary_indexes.is_empty(); + let creation_date_time = table_doc + .get_datetime("creation_date_time") + .map(|dt| dt.timestamp_millis() as f64 / 1000.0) + .unwrap_or(0.0); let key_info = TableKeyInfo { table_name, @@ -1280,6 +1289,7 @@ impl MongoEngine { global_secondary_indexes, local_secondary_indexes, stream_specification, + creation_date_time, // Fields for features this backend does not implement, vector // indexes today, take their defaults. Adding one to TableKeyInfo // then does not break this build. diff --git a/crates/storage-postgres/src/data/ddl.rs b/crates/storage-postgres/src/data/ddl.rs index 16f1a3172..df33d6ece 100755 --- a/crates/storage-postgres/src/data/ddl.rs +++ b/crates/storage-postgres/src/data/ddl.rs @@ -31,13 +31,15 @@ use super::{all_sort_key_info, data_table_name, index_table_name, vector_table_n const TEXT_KEY: &str = "TEXT COLLATE \"C\""; use crate::PostgresEngine; -/// Row shape returned by the table-info query: (`key_schema`, `attr_defs`, status, `table_id`, `stream_spec`). +/// Row shape returned by the table-info query: (`key_schema`, `attr_defs`, +/// status, `table_id`, `stream_spec`, creation epoch seconds). type TableInfoRow = ( serde_json::Value, serde_json::Value, String, String, Option, + f64, ); impl PostgresEngine { @@ -373,7 +375,7 @@ impl PostgresEngine { ) -> Result { let row: Option = sqlx::query_as( "SELECT key_schema, attribute_definitions, table_status, table_id, \ - stream_specification \ + stream_specification, EXTRACT(EPOCH FROM creation_date_time)::FLOAT8 \ FROM tables \ WHERE account_id = $1 AND table_name = $2", ) @@ -383,7 +385,7 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let (ks_json, ad_json, status, table_id, stream_spec_json) = + let (ks_json, ad_json, status, table_id, stream_spec_json, creation_date_time) = row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; match status.as_str() { @@ -436,6 +438,7 @@ impl PostgresEngine { // new core field should break this site and force a decision about // whether the write path needs it, rather than silently defaulting. vector_indexes, + creation_date_time, }; // Catalog metadata that cannot describe its own sort key would make the // keyed read paths fall back to a partition-only lookup and return the diff --git a/crates/storage-sqlite/src/data/ddl.rs b/crates/storage-sqlite/src/data/ddl.rs index 739e1310e..c21405cf5 100644 --- a/crates/storage-sqlite/src/data/ddl.rs +++ b/crates/storage-sqlite/src/data/ddl.rs @@ -343,10 +343,10 @@ impl SqliteEngine { account_id: &str, table_name: &str, ) -> Result { - let row: Option<(String, String, String, String, Option)> = sqlx::query_as( + let row: Option<(String, String, String, String, Option, String)> = sqlx::query_as( "SELECT key_schema, attribute_definitions, table_status, table_id, \ - stream_specification \ - FROM tables WHERE account_id = ? AND table_name = ?", + stream_specification, creation_date_time \ + FROM tables WHERE account_id = ? AND table_name = ?", ) .bind(account_id) .bind(table_name) @@ -354,7 +354,7 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - let (ks_str, ad_str, status, table_id, stream_spec_str) = + let (ks_str, ad_str, status, table_id, stream_spec_str, created_str) = row.ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; match status.as_str() { @@ -388,6 +388,13 @@ impl SqliteEngine { // decide whether any maintenance is needed without a query, and so the // engine can validate vector attributes on writes. let vector_indexes = self.fetch_vector_index_key_info(&table_id).await?; + // CreateTable writes the column as RFC 3339 with up to nanosecond + // digits (the schema default, milliseconds, only fills a row written + // without one). The transaction handlers order tables by this value as + // epoch seconds in an f64, which resolves about 0.24 microseconds. + let creation_date_time = crate::sqlite_util::parse_timestamp(&created_str) + .map(|dt| dt.unix_timestamp_nanos() as f64 / 1e9) + .unwrap_or(0.0); let key_info = TableKeyInfo { table_name: table_name.to_owned(), @@ -404,6 +411,7 @@ impl SqliteEngine { // new core field should break this site and force a decision about // whether the write path needs it, rather than silently defaulting. vector_indexes, + creation_date_time, }; // Catalog metadata that cannot describe its own sort key would make the // keyed read paths fall back to a partition-only lookup and return the diff --git a/crates/storage-sqlite/src/data/query_scan.rs b/crates/storage-sqlite/src/data/query_scan.rs index 3eabf42ae..4ccdaa048 100644 --- a/crates/storage-sqlite/src/data/query_scan.rs +++ b/crates/storage-sqlite/src/data/query_scan.rs @@ -606,6 +606,7 @@ mod tests { local_secondary_indexes: Vec::new(), vector_indexes: Vec::new(), stream_specification: None, + creation_date_time: 0.0, } } diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 6bd1a1607..ca03aa68b 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -17,6 +17,7 @@ adaptation when switching between ExtendDB and the real service. | `TransactGetItems` overlapping an in-flight `TransactWriteItems` on the same item | May cancel the read with `TransactionCanceledException` and reason `TransactionConflict` | Never cancels for this reason on PostgreSQL: the read runs in one `REPEATABLE READ` snapshot taken at its first read, so it sees either all of the write or none of it and returns the pre-write state while the write is in flight. That is a legal serial order (the read ordered before the write) and the all-or-nothing guarantee holds, but a client that retries on `TransactionConflict` to observe the newest state will not get that signal here. SQLite behaves the same (one WAL snapshot); MongoDB with the default `snapshot` read concern likewise, and the row above covers the other read concerns. | | Transaction read concern (MongoDB backend only) | No user-configurable equivalent | `snapshot` is the default, fidelity-preserving mode. With `majority` or `local`, `TransactGetItems` is not guaranteed a single point-in-time snapshot and transaction condition reads have weaker isolation. With `local`, a condition can be evaluated against data that is later rolled back after failover. | | Contended `TransactWriteItems` | A transaction that conflicts with another in-flight transaction on the same item is canceled with a `TransactionConflict` reason ("Transaction is ongoing for the item") | **PostgreSQL** queues the second transaction on the first one's row locks and **SQLite** runs one writer at a time, so both normally commit and no `TransactionConflict` is returned. PostgreSQL locks each transaction's items in table and key order, so two `TransactWriteItems` cannot deadlock on each other. A PutItem, UpdateItem, DeleteItem or BatchWriteItem of an item that a `TransactWriteItems` has locked also waits for it, where the Amazon DynamoDB developer guide lists that request as a conflict. On PostgreSQL this includes an item that the transaction checks or deletes while it is missing: the transaction holds the key until it commits or rolls back, so a request that creates the item waits. Many transactions that check one hot missing item queue less fairly than on an existing item, because every waiter wakes at each commit and only one proceeds. Each such check or delete of a missing item also inserts and deletes its key, so it writes WAL and leaves a dead row and index entry for autovacuum, even though it changes no item. When PostgreSQL still aborts a transaction (a deadlock with another lock holder, or a serialization failure under a stricter `default_transaction_isolation`), the answer is a `TransactionCanceledException` with a `TransactionConflict` reason, not a 500. **MongoDB** retries write conflicts internally and cancels with `TransactionConflict` on every item only after sustained contention. A client that relies on `TransactionConflict` to back off does not see it under ordinary contention. | +| `CancellationReasons` beside a stored-value nesting failure in `TransactWriteItems` | When an `Update` would store a value nested deeper than an item may hold, the reason is `ValidationError` ("Nesting Levels have exceeded supported limits") on that item, and the other items carry their own condition results, `ConditionalCheckFailed` for an item whose condition was false (measured 2026-10-27) | The same `ValidationError` reason in the same position; the other items are reported as `None`, whatever their condition result | ## Authentication and Authorization (AWS IAM/STS auth surface used by DynamoDB) @@ -48,6 +49,7 @@ adaptation when switching between ExtendDB and the real service. |------|----------|------| | Table creation delay | Returns `CREATING` immediately; transitions to `ACTIVE` typically within seconds. Same behavior for on-demand and provisioned | Configurable via `control_plane_delay_seconds` runtime setting (default: 5s) | | DeletionProtectionEnabled | Enforced | Enforced (accepted and stored, DeleteTable rejects when enabled) | +| Table ARN in `TableName` | Every data-plane and table operation accepts the table's ARN (`arn:aws:dynamodb:::table/`) in place of its name | Bare table names only; an ARN is reported as a missing table or an invalid name. An ExtendDB table ARN carries ExtendDB's account and region, so a table that streams or replicates to Amazon DynamoDB would answer to two ARNs for one logical table. Keeping names as the only identifier avoids that collision. Transactions that address one item by name and by ARN are therefore two different (missing) targets here, where the service rejects them as duplicates. With authorization on, the ARN form is authorized against the table it names before the engine reports it missing (see the Authentication and Authorization table above). | ## Time to Live (TTL) @@ -96,6 +98,12 @@ adaptation when switching between ExtendDB and the real service. | Projected attributes across all indexes | At most 100 | Enforced on `CreateTable` (and so on `RestoreTableFromBackup`, which validates its target the same way) and on `UpdateTable`'s GSI `Create`: every `NonKeyAttributes` entry of every GSI and LSI is counted, so an attribute projected into two indexes counts twice. Over 100 is `ValidationException` ("Number of projected attributes in all indexes exceeds limit of 100, projected: "; the service's text was not captured) | | `Query` on a GSI with several RANGE attributes (MongoDB and Cassandra backends) | Orders results over every RANGE attribute in key-schema order and serves a key condition on each of them (equality on every one before the last, any operator on the last) | SQLite and PostgreSQL match the service. The MongoDB and Cassandra backends filter the first RANGE attribute of such an index only: a key condition on a later RANGE attribute is refused with `ValidationException` naming the attribute and the backend. MongoDB orders results by the first RANGE attribute then the base table key. Cassandra orders a page over every RANGE attribute then the base table key, and refuses a `Query` that carries an `ExclusiveStartKey` on such an index with `ValidationException` (the cursor compares the first RANGE attribute only). Every refusal the service gives for the condition's shape (one condition per key, condition count, missing HASH attribute, RANGE gap, non-key attribute, value type) is the same on every backend. | +## Query and Scan + +| Area | DynamoDB | ExtendDB | +|------|----------|------| +| Which entry of a legacy `ScanFilter`, `QueryFilter` or `KeyConditions` map is named when two or more entries are invalid | One of the invalid entries, with its own message. Measured 2026-10-23 and 2026-10-24 on 44 two- and three-entry maps: the entry named changed with the attribute names, with the order the entries were sent, and between two runs of the same request, so it is not a function of the request | The invalid entry that sorts first by attribute name, on every backend. One invalid entry is always named with the service's message; only the choice between several differs | + ## Backup and Restore Behaviour stated as measured, and every number below, was observed with release builds of this branch for each backend (PostgreSQL, SQLite, MongoDB, Cassandra) on one host with the database on loopback, except where a row says otherwise, so timings say how the backends compare, not what a networked deployment will see. Numbers marked *(earlier build)* were measured on an earlier build of this branch and not repeated on the final one. Sentences marked *(from the code)* rest on reading the code and on storage-level tests, not on a running server. @@ -149,6 +157,7 @@ The same on all four backends. | Provisioned throughput | Token bucket per table/partition | Token bucket per table/partition, matching DynamoDB's burst and refill behavior | | On-demand capacity | Automatic scaling | Fixed initial burst capacity (4000 WCU / 12000 RCU), no auto-scaling | | Throttling | Always on; throttles requests that exceed provisioned/burst capacity. No setting to disable | Configurable via `throttling_enabled` runtime setting (default: `true`) | +| Order of `ConsumedCapacity` entries in `TransactGetItems` and `TransactWriteItems` responses | Undocumented. Measured 2026-10-23 with three tables whose names sort opposite to their creation order, in three request orders: both operations listed the tables by creation time, oldest first, whatever order the request named them in (a request naming the newest table first still returned the oldest first). The order follows neither the request nor the table names | One entry per table, by the table's creation time, oldest first; two tables with the same stored creation time are listed in name order. The precision of the stored time differs per backend, so the tie does too: about 0.24 microseconds on SQLite, microseconds on PostgreSQL, milliseconds on MongoDB and Cassandra. A MongoDB catalog row written by a build before this change holds whole seconds, so two tables created in the same second still tie there. Measured on SQLite, PostgreSQL and MongoDB; Cassandra reads the same catalog column and was not served in this round. The creation time comes from the catalog row the handlers already read for the key schema, so it costs no extra round-trip. Earlier builds ordered the entries by hash-map iteration, which varied between processes and backends | ## Operations Not Implemented diff --git a/tests/test_expr_operand_type_and_size.py b/tests/test_expr_operand_type_and_size.py index 2641cb07a..580f62524 100644 --- a/tests/test_expr_operand_type_and_size.py +++ b/tests/test_expr_operand_type_and_size.py @@ -350,3 +350,193 @@ def test_query_filter_carries_filter_prefix(self, dynamodb_client, probe_table): err = _err(ei) assert err["Code"] == "ValidationException" assert "Invalid FilterExpression: Incorrect operand type" in err["Message"] + + +@pytest.fixture(scope="class") +def empty_table(dynamodb_client): + name = f"expr_legacy_{uuid.uuid4().hex[:8]}" + dynamodb_client.create_table( + TableName=name, + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + from conftest import wait_for_active + + wait_for_active(dynamodb_client, name) + yield name + dynamodb_client.delete_table(TableName=name) + + +@pytest.fixture(scope="class") +def empty_range_table(dynamodb_client): + name = f"expr_legacy_rk_{uuid.uuid4().hex[:8]}" + dynamodb_client.create_table( + TableName=name, + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + from conftest import wait_for_active + + wait_for_active(dynamodb_client, name) + yield name + dynamodb_client.delete_table(TableName=name) + + +INVALID_PARAMS = "One or more parameter values were invalid: " +LEGACY_TYPE = INVALID_PARAMS + "ComparisonOperator {} is not valid for {} AttributeValue type" +LEGACY_COUNT = INVALID_PARAMS + "Invalid number of argument(s) for the {} ComparisonOperator" +LEGACY_SAME_TYPE = INVALID_PARAMS + "AttributeValues inside AttributeValueList must be of same type" +LEGACY_RANGE = INVALID_PARAMS + "The BETWEEN condition was provided a range where the lower bound is greater than the upper bound" +LEGACY_OPERATOR = INVALID_PARAMS + "Invalid comparison operator" +LEGACY_NOT_INDEXABLE = INVALID_PARAMS + "Attempted conditional constraint is not an indexable operation" +KC_ABSENT = {"pk": {"ComparisonOperator": "EQ", "AttributeValueList": [{"S": "absent"}]}} +S_A = {"S": "a"} +N_1 = {"N": "1"} +B_1 = {"B": b"\x01"} +BOOL_T = {"BOOL": True} +L_E = {"L": []} + +# One refused type and one accepted type per operator (measured 2026-10-23; +# the full grid is in the engine's unit tests). +LEGACY_TYPE_ROWS = [ + ("LE", [{"SS": ["a"]}], "SS"), + ("LT", [BOOL_T], "BOOL"), + ("GE", [{"NULL": True}], "NULL"), + ("GT", [{"M": {}}], "M"), + ("CONTAINS", [L_E], "L"), + ("NOT_CONTAINS", [{"NS": ["1"]}], "NS"), + ("BEGINS_WITH", [N_1], "N"), + ("BEGINS_WITH", [{"BS": [b"\x01"]}], "BS"), + ("IN", [BOOL_T], "BOOL"), + ("BETWEEN", [L_E, L_E], "L"), + ("BETWEEN", [L_E, N_1], "L"), + ("BETWEEN", [BOOL_T, L_E], "BOOL"), + ("IN", [L_E, S_A], "L"), +] +LEGACY_ACCEPTED_ROWS = [ + ("EQ", [L_E]), + ("NE", [BOOL_T]), + ("LT", [B_1]), + ("GE", [N_1]), + ("CONTAINS", [BOOL_T]), + ("NOT_CONTAINS", [{"NULL": True}]), + ("BEGINS_WITH", [B_1]), + ("IN", [N_1, {"N": "2"}, {"N": "3"}]), + ("BETWEEN", [N_1, {"N": "10"}]), + ("NULL", None), + ("NOT_NULL", None), +] +LEGACY_COUNT_ROWS = [ + ("EQ", [], "EQ"), + ("EQ", [S_A, S_A], "EQ"), + ("NULL", [S_A], "NULL"), + ("NOT_NULL", [S_A], "NOT_NULL"), + ("IN", [], "IN"), + ("BETWEEN", [S_A], "BETWEEN"), + ("BETWEEN", [S_A, S_A, S_A], "BETWEEN"), + ("BEGINS_WITH", [S_A, S_A], "BEGINS_WITH"), +] +LEGACY_OTHER_ROWS = [ + ("BETWEEN", [S_A, N_1], LEGACY_SAME_TYPE), + ("BETWEEN", [N_1, L_E], LEGACY_SAME_TYPE), + ("IN", [S_A, N_1], LEGACY_SAME_TYPE), + ("IN", [S_A, L_E], LEGACY_SAME_TYPE), + ("BETWEEN", [{"N": "2"}, N_1], LEGACY_RANGE), + ("BETWEEN", [{"S": "b"}, S_A], LEGACY_RANGE), + ("FOO", [S_A], LEGACY_OPERATOR), + ("eq", [S_A], LEGACY_OPERATOR), +] + + +def _cond(operator, values): + cond = {"ComparisonOperator": operator} + if values is not None: + cond["AttributeValueList"] = values + return cond + + +def _legacy_error(call): + with pytest.raises(ClientError) as ei: + call() + assert _err(ei)["Code"] == "ValidationException" + return _err(ei)["Message"] + + +class TestLegacyFilterOperandType: + """The legacy QueryFilter, ScanFilter and KeyConditions parameters are + checked at request time, on an empty table where no row is evaluated and + before the table is looked up, with the service's messages (measured + against Amazon DynamoDB on 2026-10-23): the operator name, the value + count, the first value's type, that every value has the first value's + type, and that a BETWEEN range is not inverted. The engine's unit tests + hold the full operator by type grid; these rows cover each message on the + wire.""" + + @pytest.mark.parametrize(("operator", "values", "named_type"), LEGACY_TYPE_ROWS) + def test_scan_filter_refuses_a_type_the_operator_does_not_take(self, dynamodb_client_no_validation, empty_table, operator, values, named_type): + msg = _legacy_error(lambda: dynamodb_client_no_validation.scan(TableName=empty_table, ScanFilter={"f": _cond(operator, values)})) + assert msg == LEGACY_TYPE.format(operator, named_type) + + @pytest.mark.parametrize(("operator", "values", "named_type"), LEGACY_TYPE_ROWS) + def test_query_filter_refuses_a_type_the_operator_does_not_take(self, dynamodb_client_no_validation, empty_table, operator, values, named_type): + msg = _legacy_error(lambda: dynamodb_client_no_validation.query(TableName=empty_table, KeyConditions=KC_ABSENT, QueryFilter={"f": _cond(operator, values)})) + assert msg == LEGACY_TYPE.format(operator, named_type) + + @pytest.mark.parametrize(("operator", "values"), LEGACY_ACCEPTED_ROWS) + def test_scan_filter_accepts_the_types_the_operator_takes(self, dynamodb_client_no_validation, empty_table, operator, values): + resp = dynamodb_client_no_validation.scan(TableName=empty_table, ScanFilter={"f": _cond(operator, values)}) + assert resp["Count"] == 0 + + @pytest.mark.parametrize(("operator", "values", "named"), LEGACY_COUNT_ROWS) + def test_scan_filter_refuses_a_wrong_value_count(self, dynamodb_client_no_validation, empty_table, operator, values, named): + msg = _legacy_error(lambda: dynamodb_client_no_validation.scan(TableName=empty_table, ScanFilter={"f": _cond(operator, values)})) + assert msg == LEGACY_COUNT.format(named) + + @pytest.mark.parametrize(("operator", "values", "message"), LEGACY_OTHER_ROWS) + def test_scan_filter_mixed_types_ranges_and_operator_names(self, dynamodb_client_no_validation, empty_table, operator, values, message): + msg = _legacy_error(lambda: dynamodb_client_no_validation.scan(TableName=empty_table, ScanFilter={"f": _cond(operator, values)})) + assert msg == message + + @pytest.mark.parametrize(("operator", "values", "message"), LEGACY_OTHER_ROWS) + def test_query_filter_mixed_types_ranges_and_operator_names(self, dynamodb_client_no_validation, empty_table, operator, values, message): + msg = _legacy_error(lambda: dynamodb_client_no_validation.query(TableName=empty_table, KeyConditions=KC_ABSENT, QueryFilter={"f": _cond(operator, values)})) + assert msg == message + + def test_scan_filter_is_checked_before_the_table_lookup(self, dynamodb_client): + msg = _legacy_error(lambda: dynamodb_client.scan(TableName=f"absent_{uuid.uuid4().hex[:8]}", ScanFilter={"f": _cond("BEGINS_WITH", [N_1])})) + assert msg == LEGACY_TYPE.format("BEGINS_WITH", "N") + + def test_query_filter_is_checked_before_the_table_lookup_and_the_key_conditions(self, dynamodb_client): + msg = _legacy_error(lambda: dynamodb_client.query(TableName=f"absent_{uuid.uuid4().hex[:8]}", KeyConditions={"pk": _cond("LT", [S_A])}, QueryFilter={"f": _cond("BEGINS_WITH", [N_1])})) + assert msg == LEGACY_TYPE.format("BEGINS_WITH", "N") + + @pytest.mark.parametrize(("operator", "values", "named_type"), [("BEGINS_WITH", [N_1], "N"), ("LT", [BOOL_T], "BOOL"), ("CONTAINS", [{"SS": ["a"]}], "SS"), ("BETWEEN", [L_E, N_1], "L")]) + def test_key_conditions_get_the_same_value_checks(self, dynamodb_client_no_validation, empty_range_table, operator, values, named_type): + msg = _legacy_error(lambda: dynamodb_client_no_validation.query(TableName=empty_range_table, KeyConditions={"pk": KC_ABSENT["pk"], "sk": _cond(operator, values)})) + assert msg == LEGACY_TYPE.format(operator, named_type) + + @pytest.mark.parametrize(("operator", "values", "named"), [("EQ", [S_A, S_A], "EQ"), ("NULL", [S_A], "NULL"), ("BETWEEN", [S_A], "BETWEEN"), ("IN", [], "IN")]) + def test_key_conditions_refuse_a_wrong_value_count(self, dynamodb_client_no_validation, empty_range_table, operator, values, named): + msg = _legacy_error(lambda: dynamodb_client_no_validation.query(TableName=empty_range_table, KeyConditions={"pk": KC_ABSENT["pk"], "sk": _cond(operator, values)})) + assert msg == LEGACY_COUNT.format(named) + + @pytest.mark.parametrize(("operator", "values"), [("NE", [S_A]), ("NULL", None), ("NOT_NULL", None), ("CONTAINS", [S_A]), ("NOT_CONTAINS", [N_1]), ("IN", [S_A, {"S": "b"}])]) + def test_key_conditions_refuse_non_indexable_operators_after_the_value_checks(self, dynamodb_client_no_validation, empty_range_table, operator, values): + msg = _legacy_error(lambda: dynamodb_client_no_validation.query(TableName=empty_range_table, KeyConditions={"pk": KC_ABSENT["pk"], "sk": _cond(operator, values)})) + assert msg == LEGACY_NOT_INDEXABLE + + def test_key_conditions_inverted_between_range(self, dynamodb_client, empty_range_table): + msg = _legacy_error(lambda: dynamodb_client.query(TableName=empty_range_table, KeyConditions={"pk": KC_ABSENT["pk"], "sk": _cond("BETWEEN", [{"S": "b"}, S_A])})) + assert msg == LEGACY_RANGE + + def test_scan_filter_begins_with_string_accepted(self, dynamodb_client, empty_table): + resp = dynamodb_client.scan( + TableName=empty_table, + ScanFilter={"pk": {"ComparisonOperator": "BEGINS_WITH", "AttributeValueList": [{"S": "p"}]}}, + ) + assert resp["Count"] == 0 diff --git a/tests/test_expression_error_prefix.py b/tests/test_expression_error_prefix.py index e28fad34f..77186dcf2 100644 --- a/tests/test_expression_error_prefix.py +++ b/tests/test_expression_error_prefix.py @@ -59,7 +59,7 @@ def test_condition_prefix_put(self, dynamodb_client, hash_table): ConditionExpression="status = :v", ExpressionAttributeValues={":v": {"S": "x"}}, )) - assert msg == f"Invalid ConditionExpression: {RESERVED}" + assert msg == f"1 validation error detected: Invalid ConditionExpression: {RESERVED}" def test_condition_prefix_delete(self, dynamodb_client, hash_table): msg = _msg(lambda: dynamodb_client.delete_item( @@ -67,7 +67,7 @@ def test_condition_prefix_delete(self, dynamodb_client, hash_table): ConditionExpression="status = :v", ExpressionAttributeValues={":v": {"S": "x"}}, )) - assert msg == f"Invalid ConditionExpression: {RESERVED}" + assert msg == f"1 validation error detected: Invalid ConditionExpression: {RESERVED}" def test_update_prefix(self, dynamodb_client, hash_table): msg = _msg(lambda: dynamodb_client.update_item( @@ -75,7 +75,7 @@ def test_update_prefix(self, dynamodb_client, hash_table): UpdateExpression="SET status = :v", ExpressionAttributeValues={":v": {"S": "x"}}, )) - assert msg == f"Invalid UpdateExpression: {RESERVED}" + assert msg == f"1 validation error detected: Invalid UpdateExpression: {RESERVED}" def test_key_condition_prefix(self, dynamodb_client, hash_table): msg = _msg(lambda: dynamodb_client.query( diff --git a/tests/test_expression_param_usage.py b/tests/test_expression_param_usage.py index 653f38d04..8d5dde93f 100644 --- a/tests/test_expression_param_usage.py +++ b/tests/test_expression_param_usage.py @@ -5,8 +5,11 @@ Amazon DynamoDB rejects a request that provides ExpressionAttributeNames or ExpressionAttributeValues when there is no expression parameter that could -reference them. The names message is always suffix-free; the values message -names the null value-capable expression parameter(s) for that API. +reference them. The names message is always suffix-free. On the read APIs the +values message names the null value-capable expression parameter(s) for that +API (Scan: "FilterExpression is null"; Query: none). PutItem, UpdateItem, and +DeleteItem give both messages suffix-free and wrapped in +"1 validation error detected: " (measured 2026-10-14). Dual-target against Amazon DynamoDB and extenddb. """ @@ -19,6 +22,7 @@ from conftest import scoped_table NAMES_MSG = "ExpressionAttributeNames can only be specified when using expressions" +SINGLE_ITEM = "1 validation error detected: " VALUES_PREFIX = "ExpressionAttributeValues can only be specified when using expressions" @@ -92,7 +96,7 @@ def test_delete_item_names_no_expression(self, dynamodb_client, hash_table): Key={"pk": {"S": "k1"}}, ExpressionAttributeNames={"#a": "foo"}, ), - NAMES_MSG, + SINGLE_ITEM + NAMES_MSG, ) def test_delete_item_values_no_expression(self, dynamodb_client, hash_table): @@ -102,7 +106,7 @@ def test_delete_item_values_no_expression(self, dynamodb_client, hash_table): Key={"pk": {"S": "k1"}}, ExpressionAttributeValues={":v": {"S": "x"}}, ), - f"{VALUES_PREFIX}: ConditionExpression is null", + SINGLE_ITEM + VALUES_PREFIX, ) def test_update_item_names_no_expression(self, dynamodb_client, hash_table): @@ -112,7 +116,7 @@ def test_update_item_names_no_expression(self, dynamodb_client, hash_table): Key={"pk": {"S": "k1"}}, ExpressionAttributeNames={"#a": "foo"}, ), - NAMES_MSG, + SINGLE_ITEM + NAMES_MSG, ) def test_update_item_values_no_expression(self, dynamodb_client, hash_table): @@ -122,7 +126,7 @@ def test_update_item_values_no_expression(self, dynamodb_client, hash_table): Key={"pk": {"S": "k1"}}, ExpressionAttributeValues={":v": {"S": "x"}}, ), - f"{VALUES_PREFIX}: UpdateExpression and ConditionExpression are null", + SINGLE_ITEM + VALUES_PREFIX, ) def test_put_item_names_no_expression(self, dynamodb_client, hash_table): @@ -132,7 +136,7 @@ def test_put_item_names_no_expression(self, dynamodb_client, hash_table): Item={"pk": {"S": "k2"}}, ExpressionAttributeNames={"#a": "foo"}, ), - NAMES_MSG, + SINGLE_ITEM + NAMES_MSG, ) def test_put_item_values_no_expression(self, dynamodb_client, hash_table): @@ -142,7 +146,7 @@ def test_put_item_values_no_expression(self, dynamodb_client, hash_table): Item={"pk": {"S": "k2"}}, ExpressionAttributeValues={":v": {"S": "x"}}, ), - f"{VALUES_PREFIX}: ConditionExpression is null", + SINGLE_ITEM + VALUES_PREFIX, ) def test_query_names_no_expression(self, dynamodb_client, hash_table): diff --git a/tests/test_expression_semantics.py b/tests/test_expression_semantics.py new file mode 100644 index 000000000..e06872f70 --- /dev/null +++ b/tests/test_expression_semantics.py @@ -0,0 +1,704 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Expression semantics Amazon DynamoDB checks before it reads the table. + +After the grammar pass, the service walks the parsed expression once, in +expression order, checking each node after its operands, and only then +checks overlapping paths and unused names and values. Every expectation +below was captured from the service on 2026-10-14 (issue #108). + +UpdateExpression: + - `+` and `-` take numbers; `list_append` takes lists; `ADD` takes a number + or a set; `DELETE` takes a set. A placeholder operand of the wrong type is + refused with its type: `S`, `N`, `L` for the operators and functions, + `STRING`, `NUMBER`, `Binary`, `BOOL`, `NULL`, `LIST`, `MAP` for ADD and + DELETE. + - Function names are matched as written: `IF_NOT_EXISTS(x, :v)` and + `foo(x)` are `Invalid function name`; `size(x)` is `not allowed in an + update expression`; `list_append` is refused as an operand of a function + or of `+`. + - A repeated section is reported after that section's own actions, so + `SET a = :v SET b = :q` reports `:q` and `SET a = :v SET b = :v ADD n :s` + reports the repeated `SET`. + - Every unused name or value is listed in one set. + - UpdateItem checks the ConditionExpression before the UpdateExpression; + inside TransactWriteItems the UpdateExpression comes first. + +ConditionExpression and FilterExpression: + - the six condition functions are matched as written; `if_not_exists` and + `list_append` are `not allowed in a condition expression`; a boolean + function in operand position and `size` in condition position are `not + allowed to be used this way`; operand counts are fixed; `attribute_exists` + and `attribute_not_exists` take a document path; `begins_with` takes a + string or binary and `attribute_type` a string naming a type. + +Query and Scan: + - the FilterExpression, then the KeyConditionExpression, then the + ProjectionExpression are checked for undefined references and function + names, then unused names and values, all before the table is looked up. + +Key-shaped APIs (GetItem, DeleteItem, UpdateItem, the batch key paths, and a +BatchWriteItem Put) refuse a key with a missing, mistyped, or extra attribute +with `The provided key element does not match the schema`; PutItem names the +attribute. BatchWriteItem returns `ItemCollectionMetrics: {}` when SIZE is +requested and no written table has a local secondary index. + +Dual-target: runs against the service and against ExtendDB on every backend. +""" + +from __future__ import annotations + +import re +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +P = "1 validation error detected: " +U = "Invalid UpdateExpression: " +C = "Invalid ConditionExpression: " +F = "Invalid FilterExpression: " +K = "Invalid KeyConditionExpression: " +PR = "Invalid ProjectionExpression: " + +UNDEF_VALUE = "An expression attribute value used in expression is not defined; attribute value: {}" +UNDEF_NAME = "An expression attribute name used in the document path is not defined; attribute name: {}" +OPERAND_TYPE = "Incorrect operand type for operator or function; operator or function: {}, operand type: {}" +ADD_DELETE_TYPE = "Incorrect operand type for operator or function; operator: {}, operand type: {}, typeSet: ALLOWED_FOR_ADD_OPERAND" +INVALID_FN = "Invalid function name; function: {}" +NOT_THIS_WAY = "The function is not allowed to be used this way in an expression; function: {}" +NOT_IN_UPDATE = "The function is not allowed in an update expression; function: {}" +NOT_IN_CONDITION = "The function is not allowed in a condition expression; function: {}" +OPERAND_COUNT = "Incorrect number of operands for operator or function; operator or function: {}, number of operands: {}" +NEEDS_PATH = "Operator or function requires a document path; operator or function: {}" +SECTION_ONCE = 'The "{}" section can only be used once in an update expression;' +OVERLAP_A_AB = "Two document paths overlap with each other; must remove or rewrite one of these paths; path one: [a], path two: [a, b]" +EVAL_TYPE = "An operand in the update expression has an incorrect data type" +KEY_SCHEMA = "The provided key element does not match the schema" +EMPTY_ARGS_SYNTAX = 'Syntax error; token: ")", near: "()"' + +N = {"N": "1"} +S = {"S": "x"} +L = {"L": []} +SS = {"SS": ["a"]} +KEY = {"pk": {"S": "p"}} +WRITE_KEY = {"pk": {"S": "w"}} + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table(dynamodb_client) as name: + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": "p"}, "n": N, "s": SS, "l": L, "m": {"M": {"x": N}}}, + ) + yield name + + +@pytest.fixture(scope="module") +def range_table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + yield name + + +@pytest.fixture(scope="module") +def lsi_table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "alt", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + LocalSecondaryIndexes=[ + { + "IndexName": "byalt", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "alt", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ], + ) as name: + yield name + + +@pytest.fixture(scope="module") +def missing_table() -> str: + return f"extenddb-missing-{uuid.uuid4().hex[:12]}" + + +def _expect(func, message: str, code: str = "ValidationException"): + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == code, err + assert err["Message"] == message, err + + +def _expect_prefix(func, prefix: str, code: str = "ValidationException"): + """For syntax errors, whose inner wording ExtendDB still phrases its own way.""" + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == code, err + assert err["Message"].startswith(prefix), err + + +def _expect_unused(func, kind: str, keys: set[str]): + """The service lists every unused key in one set, in an order it does not document.""" + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + match = re.fullmatch( + rf"(?:{re.escape(P)})?Value provided in ExpressionAttribute{kind} unused in expressions: keys: \{{(.*)\}}", + err["Message"], + ) + assert match, err + assert set(match.group(1).split(", ")) == keys, err + + +def _update(client, table, expression, values=None, names=None, condition=None, key=WRITE_KEY): + kwargs = {"TableName": table, "Key": key, "UpdateExpression": expression} + if values is not None: + kwargs["ExpressionAttributeValues"] = values + if names is not None: + kwargs["ExpressionAttributeNames"] = names + if condition is not None: + kwargs["ConditionExpression"] = condition + return lambda: client.update_item(**kwargs) + + +def _transact_update(client, table, expression, values=None, condition=None): + item = {"TableName": table, "Key": WRITE_KEY, "UpdateExpression": expression} + if values is not None: + item["ExpressionAttributeValues"] = values + if condition is not None: + item["ConditionExpression"] = condition + return lambda: client.transact_write_items(TransactItems=[{"Update": item}]) + + +def _put_cond(client, table, condition, values=None, names=None): + kwargs = {"TableName": table, "Item": WRITE_KEY, "ConditionExpression": condition} + if values is not None: + kwargs["ExpressionAttributeValues"] = values + if names is not None: + kwargs["ExpressionAttributeNames"] = names + return lambda: client.put_item(**kwargs) + + +class TestUpdateOperandTypes: + @pytest.mark.parametrize( + ("operator", "value", "type_name"), + [ + ("ADD", S, "STRING"), + ("ADD", {"B": b"x"}, "Binary"), + ("ADD", {"BOOL": True}, "BOOL"), + ("ADD", {"NULL": True}, "NULL"), + ("ADD", L, "LIST"), + ("ADD", {"M": {}}, "MAP"), + ("DELETE", S, "STRING"), + ("DELETE", N, "NUMBER"), + ("DELETE", {"B": b"x"}, "Binary"), + ("DELETE", {"BOOL": True}, "BOOL"), + ("DELETE", {"NULL": True}, "NULL"), + ("DELETE", L, "LIST"), + ("DELETE", {"M": {}}, "MAP"), + ], + ) + def test_add_and_delete_refuse_the_type_at_request_time(self, dynamodb_client, table, operator, value, type_name): + _expect(_update(dynamodb_client, table, f"{operator} attr :v", {":v": value}), + P + U + ADD_DELETE_TYPE.format(operator, type_name)) + + def test_add_takes_numbers_and_sets_delete_takes_sets(self, dynamodb_client, table): + dynamodb_client.update_item(TableName=table, Key=WRITE_KEY, UpdateExpression="ADD n :n, s :ss", + ExpressionAttributeValues={":n": N, ":ss": SS}) + dynamodb_client.update_item(TableName=table, Key=WRITE_KEY, UpdateExpression="DELETE s :ss", + ExpressionAttributeValues={":ss": SS}) + + def test_set_mismatch_against_the_stored_attribute_is_an_evaluation_error(self, dynamodb_client, table): + # Request-time rules accept a set; the stored attribute decides the rest, + # and that message carries no prefix. + _expect(_update(dynamodb_client, table, "ADD n :ss", {":ss": SS}, key=KEY), EVAL_TYPE) + _expect(_update(dynamodb_client, table, "ADD s :ns", {":ns": {"NS": ["1"]}}, key=KEY), EVAL_TYPE) + + @pytest.mark.parametrize( + ("expression", "values", "symbol", "type_code"), + [ + ("SET a = :s + :n", {":s": S, ":n": N}, "+", "S"), + ("SET a = :n + :s", {":s": S, ":n": N}, "+", "S"), + ("SET a = :s + :s2", {":s": S, ":s2": {"S": "y"}}, "+", "S"), + ("SET a = n + :s", {":s": S}, "+", "S"), + ("SET a = :l - :n", {":l": L, ":n": N}, "-", "L"), + ("SET a = :b + :n", {":b": {"B": b"x"}, ":n": N}, "+", "B"), + ("SET a = :z + :n", {":z": {"NULL": True}, ":n": N}, "+", "NULL"), + ("SET a = :m + :n", {":m": {"M": {}}, ":n": N}, "+", "M"), + ("SET a = :t + :n", {":t": {"BOOL": True}, ":n": N}, "+", "BOOL"), + ("SET a = :ss + :n", {":ss": SS, ":n": N}, "+", "SS"), + ], + ) + def test_arithmetic_operands_must_be_numbers(self, dynamodb_client, table, expression, values, symbol, type_code): + _expect(_update(dynamodb_client, table, expression, values), P + U + OPERAND_TYPE.format(symbol, type_code)) + + def test_arithmetic_on_a_function_result_is_checked_at_evaluation(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "SET a = if_not_exists(a, :s) + :n", {":s": S, ":n": N}, key=KEY), + EVAL_TYPE) + + @pytest.mark.parametrize( + ("expression", "values", "type_code"), + [ + ("SET l = list_append(:v, :l)", {":v": S, ":l": L}, "S"), + ("SET l = list_append(:v, :l)", {":v": N, ":l": L}, "N"), + ("SET l = list_append(:v, :l)", {":v": {"M": {}}, ":l": L}, "M"), + ("SET l = list_append(:v, :l)", {":v": SS, ":l": L}, "SS"), + ("SET l = list_append(:v, :l)", {":v": {"BOOL": True}, ":l": L}, "BOOL"), + ("SET l = list_append(:v, :l)", {":v": {"B": b"x"}, ":l": L}, "B"), + ("SET l = list_append(:v, :l)", {":v": {"NULL": True}, ":l": L}, "NULL"), + ("SET l = list_append(:v, :l)", {":v": {"NS": ["1"]}, ":l": L}, "NS"), + ("SET l = list_append(:v, :l)", {":v": {"BS": [b"a"]}, ":l": L}, "BS"), + ("SET l = list_append(:l, :v)", {":v": S, ":l": L}, "S"), + ("SET l = list_append(l, :v)", {":v": S}, "S"), + ("SET l = list_append(:s, :n)", {":s": S, ":n": N}, "S"), + ], + ) + def test_list_append_operands_must_be_lists(self, dynamodb_client, table, expression, values, type_code): + _expect(_update(dynamodb_client, table, expression, values), P + U + OPERAND_TYPE.format("list_append", type_code)) + + def test_list_append_of_a_function_result_is_accepted(self, dynamodb_client, table): + dynamodb_client.update_item(TableName=table, Key=WRITE_KEY, + UpdateExpression="SET l = list_append(if_not_exists(l, :e), :l)", + ExpressionAttributeValues={":e": L, ":l": L}) + + +class TestUpdateFunctionNames: + @pytest.mark.parametrize( + ("expression", "values", "name"), + [ + ("SET x = IF_NOT_EXISTS(x, :v)", {":v": N}, "IF_NOT_EXISTS"), + ("SET l = List_Append(:l, :l)", {":l": L}, "List_Append"), + ("SET x = foo(x, :v)", {":v": N}, "foo"), + ("SET x = ATTRIBUTE_EXISTS(x)", None, "ATTRIBUTE_EXISTS"), + ("SET x = Size(x)", None, "Size"), + ("SET a = if_not_exists(foo(x), :v)", {":v": N}, "foo"), + ("SET a = if_not_exists(a, foo(x))", None, "foo"), + ("SET l = list_append(:l, foo(x))", {":l": L}, "foo"), + ], + ) + def test_unknown_or_miscased_names_are_refused(self, dynamodb_client, table, expression, values, name): + _expect(_update(dynamodb_client, table, expression, values), P + U + INVALID_FN.format(name)) + + @pytest.mark.parametrize("name", ["attribute_exists", "size"]) + def test_condition_functions_are_refused_by_name(self, dynamodb_client, table, name): + _expect(_update(dynamodb_client, table, f"SET x = {name}(x), y = :q"), P + U + NOT_IN_UPDATE.format(name)) + + def test_a_call_with_no_operands_is_a_syntax_error(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "SET x = foo()"), P + U + EMPTY_ARGS_SYNTAX) + + @pytest.mark.parametrize( + "expression", + ["SET l = list_append(:l, :l) + :n", "SET l = :n + list_append(:l, :l)"], + ) + def test_list_append_is_refused_as_an_arithmetic_operand(self, dynamodb_client, table, expression): + _expect(_update(dynamodb_client, table, expression, {":l": L, ":n": N}), + P + U + NOT_THIS_WAY.format("list_append")) + + def test_nested_list_append_is_reported_where_it_stands(self, dynamodb_client, table): + # The nested call is refused before its sibling operand is resolved, + # but an undefined reference inside it is found first. + _expect(_update(dynamodb_client, table, "SET l = if_not_exists(list_append(:a, :b), :q)", {":a": L, ":b": L}), + P + U + NOT_THIS_WAY.format("list_append")) + _expect(_update(dynamodb_client, table, "SET l = if_not_exists(list_append(:q, :b), :a)", {":a": L, ":b": L}), + P + U + UNDEF_VALUE.format(":q")) + + +class TestUpdateWalkOrder: + def test_references_in_expression_order(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "SET a = :q, #n = :v", {":v": N}), P + U + UNDEF_VALUE.format(":q")) + _expect(_update(dynamodb_client, table, "SET #n = :q"), P + U + UNDEF_NAME.format("#n")) + + @pytest.mark.parametrize( + ("expression", "values", "message"), + [ + ("SET a = :q SET b = :v", {":v": N}, UNDEF_VALUE.format(":q")), + ("SET a = :v SET b = :q", {":v": N}, UNDEF_VALUE.format(":q")), + ("SET #n = :v SET b = :v", {":v": N}, UNDEF_NAME.format("#n")), + ("SET a = :v REMOVE #n SET b = :v", {":v": N}, UNDEF_NAME.format("#n")), + ("SET a = :v SET b = :v ADD n :s", {":v": N, ":s": S}, SECTION_ONCE.format("SET")), + ("SET a = :v SET b = :v ADD n :q", {":v": N}, SECTION_ONCE.format("SET")), + ("SET a = :v ADD n :s SET b = :v", {":v": N, ":s": S}, ADD_DELETE_TYPE.format("ADD", "STRING")), + ("ADD n :s ADD m :n", {":s": S, ":n": N}, ADD_DELETE_TYPE.format("ADD", "STRING")), + ("DELETE s :ss DELETE t :s", {":s": S, ":ss": SS}, ADD_DELETE_TYPE.format("DELETE", "STRING")), + ("SET a = :n SET b = :s + :n", {":s": S, ":n": N}, OPERAND_TYPE.format("+", "S")), + ("SET a = :v SET x = foo(x)", {":v": N}, INVALID_FN.format("foo")), + ("SET l = list_append(list_append(:a, :a), :a) SET b = :v", {":v": N, ":a": L}, NOT_THIS_WAY.format("list_append")), + ], + ) + def test_repeated_section_is_reported_after_its_own_actions(self, dynamodb_client, table, expression, values, message): + _expect(_update(dynamodb_client, table, expression, values), P + U + message) + + @pytest.mark.parametrize( + ("expression", "values", "message"), + [ + ("SET a = :q ADD n :s", {":s": S}, UNDEF_VALUE.format(":q")), + ("ADD n :s SET a = if_not_exists(a)", {":s": S}, ADD_DELETE_TYPE.format("ADD", "STRING")), + ("SET a = if_not_exists(a) ADD n :s", {":s": S}, OPERAND_COUNT.format("if_not_exists", 1)), + ("SET a = if_not_exists(a, :q, :q2)", None, UNDEF_VALUE.format(":q")), + ("SET a = if_not_exists(a), b = :q", None, OPERAND_COUNT.format("if_not_exists", 1)), + ("SET a = if_not_exists(:a)", {":a": N}, OPERAND_COUNT.format("if_not_exists", 1)), + ("SET l = list_append(:s)", {":s": S}, OPERAND_COUNT.format("list_append", 1)), + ("SET x = foo(x), y = :q", None, INVALID_FN.format("foo")), + ("SET y = :q, x = foo(x)", None, UNDEF_VALUE.format(":q")), + ("SET x = foo(#n)", None, UNDEF_NAME.format("#n")), + ("SET x = foo(:q)", None, UNDEF_VALUE.format(":q")), + ("ADD n :q", None, UNDEF_VALUE.format(":q")), + ("ADD n :s DELETE s :s", {":s": S}, ADD_DELETE_TYPE.format("ADD", "STRING")), + ("SET a = :n, a.b = :n ADD m :s", {":s": S, ":n": N}, ADD_DELETE_TYPE.format("ADD", "STRING")), + ("SET a = :s + :n ADD n :s", {":s": S, ":n": N}, OPERAND_TYPE.format("+", "S")), + ], + ) + def test_each_node_is_checked_after_its_operands(self, dynamodb_client, table, expression, values, message): + _expect(_update(dynamodb_client, table, expression, values), P + U + message) + + def test_operand_type_and_function_name_beat_unused(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "ADD n :s", {":s": S, ":u": N}), P + U + ADD_DELETE_TYPE.format("ADD", "STRING")) + _expect(_update(dynamodb_client, table, "SET x = foo(x)", {":u": N}), P + U + INVALID_FN.format("foo")) + _expect(_update(dynamodb_client, table, "ADD n :s", {":s": S}, {"#u": "u"}), P + U + ADD_DELETE_TYPE.format("ADD", "STRING")) + + def test_a_syntax_error_anywhere_still_wins(self, dynamodb_client, table): + for expression, values in [ + ("SET a = if_not_exists(a), b = ", None), + ("SET l = list_append(list_append(:a, :a), :a), b = ", {":a": L}), + ("SET a = :s + :n, b = ", {":s": S, ":n": N}), + ("SET x = foo(x), b = ", None), + ("ADD n :s SET a = ", {":s": S}), + ]: + _expect_prefix(_update(dynamodb_client, table, expression, values), P + U) + + def test_transaction_reports_the_operand_type_bare_and_top_level(self, dynamodb_client, table): + _expect(_transact_update(dynamodb_client, table, "ADD n :s", {":s": S}), U + ADD_DELETE_TYPE.format("ADD", "STRING")) + _expect(_transact_update(dynamodb_client, table, "SET x = foo(x)"), U + INVALID_FN.format("foo")) + + def test_update_item_checks_the_condition_first(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "SET #a = :v", {":v": N}, condition="attribute_exists(#b)"), + P + C + UNDEF_NAME.format("#b")) + _expect(_update(dynamodb_client, table, "SET a = ", condition="attribute_exists(#b)"), + P + C + UNDEF_NAME.format("#b")) + _expect(_update(dynamodb_client, table, "SET a = :v", {":v": N}, condition="b = :q"), + P + C + UNDEF_VALUE.format(":q")) + _expect(_update(dynamodb_client, table, "SET a = :v, a.b = :v", {":v": N}, condition="attribute_exists(#b)"), + P + C + UNDEF_NAME.format("#b")) + _expect_prefix(_update(dynamodb_client, table, "SET #a = :v", {":v": N}, condition="attribute_exists("), P + C) + + def test_transaction_checks_the_update_first(self, dynamodb_client, table): + _expect(_transact_update(dynamodb_client, table, "SET #a = :v", {":v": N}, condition="attribute_exists(#b)"), + U + UNDEF_NAME.format("#a")) + _expect(_transact_update(dynamodb_client, table, "SET #a = :v", {":v": N}, condition="attribute_exists("), + U + UNDEF_NAME.format("#a")) + _expect(_transact_update(dynamodb_client, table, "SET a = :v, a.b = :v", {":v": N}, condition="attribute_exists(#b)"), + U + OVERLAP_A_AB) + _expect(_transact_update(dynamodb_client, table, "ADD n :s", {":s": S}, condition="attribute_exists(#b)"), + U + ADD_DELETE_TYPE.format("ADD", "STRING")) + _expect_prefix(_transact_update(dynamodb_client, table, "SET a = ", condition="attribute_exists(#b)"), U) + + +class TestUnusedKeysAreListedTogether: + def test_values(self, dynamodb_client, table): + _expect_unused(_update(dynamodb_client, table, "SET a = :u", {":u": N, ":x": N, ":y": N, ":z": N}), + "Values", {":x", ":y", ":z"}) + + def test_names(self, dynamodb_client, table): + _expect_unused(_update(dynamodb_client, table, "SET a = :u", {":u": N}, {"#x": "x", "#y": "y", "#z": "z"}), + "Names", {"#x", "#y", "#z"}) + + def test_query_names(self, dynamodb_client, table): + _expect_unused( + lambda: dynamodb_client.query(TableName=table, KeyConditionExpression="pk = :p", + ExpressionAttributeNames={"#b": "b", "#a": "a"}, + ExpressionAttributeValues={":p": S}), + "Names", {"#a", "#b"}, + ) + + +class TestConditionFunctions: + @pytest.mark.parametrize( + ("condition", "values", "message"), + [ + ("ATTRIBUTE_EXISTS(pk)", None, INVALID_FN.format("ATTRIBUTE_EXISTS")), + ("SIZE(a) > :n", {":n": N}, INVALID_FN.format("SIZE")), + ("Begins_With(pk, :v)", {":v": S}, INVALID_FN.format("Begins_With")), + ("foo(pk) AND bar(pk)", None, INVALID_FN.format("foo")), + ("foo(pk) > :n", {":n": N}, INVALID_FN.format("foo")), + ("pk = foo(pk)", None, INVALID_FN.format("foo")), + ("foo(pk, :n)", {":n": N}, INVALID_FN.format("foo")), + ("if_not_exists(a, :v) = :v", {":v": N}, NOT_IN_CONDITION.format("if_not_exists")), + ("list_append(a, :v) = :v", {":v": N}, NOT_IN_CONDITION.format("list_append")), + ("attribute_exists(pk) = :v", {":v": N}, NOT_THIS_WAY.format("attribute_exists")), + ("attribute_exists(pk) > :n", {":n": N}, NOT_THIS_WAY.format("attribute_exists")), + ("begins_with(pk, :v) = :v", {":v": S}, NOT_THIS_WAY.format("begins_with")), + ("contains(pk, :v) = :v", {":v": S}, NOT_THIS_WAY.format("contains")), + ("attribute_type(pk, :v) = :v", {":v": S}, NOT_THIS_WAY.format("attribute_type")), + ("pk IN (:v, attribute_exists(pk))", {":v": S}, NOT_THIS_WAY.format("attribute_exists")), + ("size(attribute_exists(pk)) > :n", {":n": N}, NOT_THIS_WAY.format("attribute_exists")), + ("size(pk)", None, NOT_THIS_WAY.format("size")), + ("NOT size(pk)", None, NOT_THIS_WAY.format("size")), + ("attribute_exists(size(pk))", None, NEEDS_PATH.format("attribute_exists")), + ("attribute_exists(:v)", {":v": S}, NEEDS_PATH.format("attribute_exists")), + ("attribute_not_exists(:v)", {":v": S}, NEEDS_PATH.format("attribute_not_exists")), + ("attribute_exists(pk, pk)", None, OPERAND_COUNT.format("attribute_exists", 2)), + ("attribute_exists(:v, :w)", {":v": S, ":w": S}, OPERAND_COUNT.format("attribute_exists", 2)), + ("begins_with(pk)", None, OPERAND_COUNT.format("begins_with", 1)), + ("contains(pk)", None, OPERAND_COUNT.format("contains", 1)), + ("attribute_type(pk)", None, OPERAND_COUNT.format("attribute_type", 1)), + ("size(pk, pk) > :n", {":n": N}, OPERAND_COUNT.format("size", 2)), + ("foo()", None, EMPTY_ARGS_SYNTAX), + ("attribute_exists()", None, EMPTY_ARGS_SYNTAX), + ("foo(#n)", None, UNDEF_NAME.format("#n")), + ("foo(:q)", None, UNDEF_VALUE.format(":q")), + ("attribute_exists(pk, :q)", None, UNDEF_VALUE.format(":q")), + (":q = #n", None, UNDEF_VALUE.format(":q")), + ("#n = :q", None, UNDEF_NAME.format("#n")), + ("a = :q AND #n = :v", {":v": N}, UNDEF_VALUE.format(":q")), + ("begins_with(pk, :n)", {":n": N}, OPERAND_TYPE.format("begins_with", "N")), + ("attribute_type(pk, :n)", {":n": N}, OPERAND_TYPE.format("attribute_type", "N")), + ("attribute_type(pk, :t)", {":t": {"S": "XX"}}, + "Invalid attribute type name found; type: XX, valid types: {S,SS,N,NS,B,BS,BOOL,NULL,L,M}"), + ("begins_with(pk, :n) AND #x = :v", {":n": N, ":v": S}, OPERAND_TYPE.format("begins_with", "N")), + ("#x = :v AND begins_with(pk, :n)", {":n": N, ":v": S}, UNDEF_NAME.format("#x")), + ("pk > :l AND #x = :v", {":l": L, ":v": S}, OPERAND_TYPE.format(">", "L")), + ("size(pk) BETWEEN :l AND :n", {":l": L, ":n": N}, OPERAND_TYPE.format("BETWEEN", "L")), + ], + ) + def test_put_item_refuses_with_the_prefix(self, dynamodb_client, table, condition, values, message): + _expect(_put_cond(dynamodb_client, table, condition, values), P + C + message) + + def test_function_name_beats_unused(self, dynamodb_client, table): + _expect(_put_cond(dynamodb_client, table, "foo(pk)", {":u": N}), P + C + INVALID_FN.format("foo")) + + @pytest.mark.parametrize( + ("condition", "values"), + [ + ("foo(pk) AND", None), + ("ATTRIBUTE_EXISTS(pk) AND", None), + ("begins_with(pk, :n) AND", {":n": N}), + ("attribute_exists(#x) AND", None), + ], + ) + def test_a_syntax_error_beats_the_semantic_errors(self, dynamodb_client, table, condition, values): + # The service answers 'Syntax error; token: "", near: "AND"' for each of + # these (measured 2026-10-23): the grammar pass runs over the whole text + # before the function name, the operand type or the undefined name is + # looked at. ExtendDB's syntax wording differs today (tracked under #108 + # message text); the guarantee pinned here is the order. + with pytest.raises(ClientError) as exc_info: + _put_cond(dynamodb_client, table, condition, values)() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"].startswith(P + C), err + for semantic in ("Invalid function name", "Incorrect operand type", "is not defined"): + assert semantic not in err["Message"], err + + @pytest.mark.parametrize( + ("condition", "values"), + [ + ("size(pk) = size(pk)", None), + ("begins_with(:v, pk)", {":v": S}), + ("contains(:v, :w)", {":v": S, ":w": S}), + ("attribute_type(:v, :t)", {":v": S, ":t": {"S": "S"}}), + ("attribute_type(pk, :t)", {":t": {"S": "S"}}), + ("size(pk) > :n AND attribute_exists(pk)", {":n": N}), + ], + ) + def test_accepted_shapes(self, dynamodb_client, table, condition, values): + # The request passes validation; whether the condition then holds on + # the item is a separate matter. + try: + _put_cond(dynamodb_client, table, condition, values)() + except ClientError as exc: + assert exc.response["Error"]["Code"] == "ConditionalCheckFailedException", exc.response["Error"] + + def test_transaction_and_read_apis_carry_their_own_prefixes(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + {"ConditionCheck": {"TableName": table, "Key": KEY, "ConditionExpression": "ATTRIBUTE_EXISTS(x)"}}]), + C + INVALID_FN.format("ATTRIBUTE_EXISTS"), + ) + _expect(lambda: dynamodb_client.query(TableName=table, KeyConditionExpression="pk = :p", + FilterExpression="ATTRIBUTE_EXISTS(x)", ExpressionAttributeValues={":p": S}), + F + INVALID_FN.format("ATTRIBUTE_EXISTS")) + _expect(lambda: dynamodb_client.scan(TableName=table, FilterExpression="foo(pk)"), F + INVALID_FN.format("foo")) + _expect(lambda: dynamodb_client.query(TableName=table, KeyConditionExpression="pk = :p AND Begins_With(sk, :v)", + ExpressionAttributeValues={":p": S, ":v": S}), + K + INVALID_FN.format("Begins_With")) + + +class TestReadExpressionReferences: + """Query and Scan check their expressions before the table, on an empty table too.""" + + @pytest.fixture(scope="class") + def empty(self, dynamodb_client): + with scoped_table(dynamodb_client) as name: + yield name + + def test_scan_and_query_filters(self, dynamodb_client, empty): + _expect(lambda: dynamodb_client.scan(TableName=empty, FilterExpression="x = :q"), F + UNDEF_VALUE.format(":q")) + _expect(lambda: dynamodb_client.scan(TableName=empty, FilterExpression="#n = :q", ExpressionAttributeValues={":q": S}), + F + UNDEF_NAME.format("#n")) + _expect(lambda: dynamodb_client.query(TableName=empty, KeyConditionExpression="pk = :p", FilterExpression="x = :q", + ExpressionAttributeValues={":p": S}), F + UNDEF_VALUE.format(":q")) + + def test_key_condition_references_carry_the_prefix(self, dynamodb_client, empty): + _expect(lambda: dynamodb_client.query(TableName=empty, KeyConditionExpression="pk = :p"), K + UNDEF_VALUE.format(":p")) + _expect(lambda: dynamodb_client.query(TableName=empty, KeyConditionExpression="#n = :p", ExpressionAttributeValues={":p": S}), + K + UNDEF_NAME.format("#n")) + _expect(lambda: dynamodb_client.query(TableName=empty, KeyConditionExpression="pk = :p AND #n = :q", + ExpressionAttributeValues={":p": S}), K + UNDEF_NAME.format("#n")) + + def test_filter_then_key_condition_then_projection(self, dynamodb_client, empty): + q = dynamodb_client.query + _expect(lambda: q(TableName=empty, KeyConditionExpression="pk = :p", FilterExpression="x = :q"), + F + UNDEF_VALUE.format(":q")) + _expect(lambda: q(TableName=empty, KeyConditionExpression="#n = :p", FilterExpression="#m = :p", + ExpressionAttributeValues={":p": S}), F + UNDEF_NAME.format("#m")) + _expect(lambda: q(TableName=empty, KeyConditionExpression="pk = :p", ProjectionExpression="#p"), + K + UNDEF_VALUE.format(":p")) + _expect(lambda: q(TableName=empty, KeyConditionExpression="pk = :p", FilterExpression="x = :q", ProjectionExpression="#p", + ExpressionAttributeValues={":p": S}), F + UNDEF_VALUE.format(":q")) + _expect(lambda: q(TableName=empty, KeyConditionExpression="zz = :p", ProjectionExpression="#p", + ExpressionAttributeValues={":p": S}), PR + UNDEF_NAME.format("#p")) + _expect(lambda: q(TableName=empty, KeyConditionExpression="zz = :p", FilterExpression="x = :q", + ExpressionAttributeValues={":p": S}), F + UNDEF_VALUE.format(":q")) + _expect_prefix(lambda: q(TableName=empty, KeyConditionExpression="pk = :p", FilterExpression="x = "), F) + _expect(lambda: q(TableName=empty, KeyConditionExpression="pk = ", FilterExpression="x = :q"), F + UNDEF_VALUE.format(":q")) + _expect(lambda: q(TableName=empty, KeyConditionExpression="pk = :p", ProjectionExpression="a,,b"), K + UNDEF_VALUE.format(":p")) + _expect(lambda: dynamodb_client.scan(TableName=empty, FilterExpression="x = :q", ProjectionExpression="#p"), + F + UNDEF_VALUE.format(":q")) + _expect_prefix(lambda: dynamodb_client.scan(TableName=empty, FilterExpression="x = ", ProjectionExpression="#p"), F) + + def test_unused_before_the_key_schema_match(self, dynamodb_client, empty): + _expect_unused(lambda: dynamodb_client.query(TableName=empty, KeyConditionExpression="zz = :p", + ExpressionAttributeValues={":p": S, ":u": S}), "Values", {":u"}) + + def test_everything_before_the_table_lookup(self, dynamodb_client, missing_table): + _expect(lambda: dynamodb_client.query(TableName=missing_table, KeyConditionExpression="pk = :p", FilterExpression="x = :q", + ExpressionAttributeValues={":p": S}), F + UNDEF_VALUE.format(":q")) + _expect(lambda: dynamodb_client.query(TableName=missing_table, KeyConditionExpression="pk = :p", ProjectionExpression="#p", + ExpressionAttributeValues={":p": S}), PR + UNDEF_NAME.format("#p")) + _expect(lambda: dynamodb_client.query(TableName=missing_table, KeyConditionExpression="#n = :p", + ExpressionAttributeValues={":p": S}), K + UNDEF_NAME.format("#n")) + _expect(lambda: dynamodb_client.scan(TableName=missing_table, FilterExpression="x = :q"), F + UNDEF_VALUE.format(":q")) + _expect(lambda: dynamodb_client.get_item(TableName=missing_table, Key=KEY, ProjectionExpression="#p"), + PR + UNDEF_NAME.format("#p")) + _expect_unused(lambda: dynamodb_client.scan(TableName=missing_table, FilterExpression="x = :v", + ExpressionAttributeValues={":v": N, ":u": N}), "Values", {":u"}) + _expect_unused(lambda: dynamodb_client.query(TableName=missing_table, KeyConditionExpression="pk = :p", + ExpressionAttributeValues={":p": S, ":u": N}), "Values", {":u"}) + + def test_parameter_checks_come_before_the_expressions(self, dynamodb_client, dynamodb_client_no_validation, empty): + _expect(lambda: dynamodb_client_no_validation.scan(TableName=empty, FilterExpression="#n = :q", + ExpressionAttributeValues={":q": S}, Limit=0), + P + "Value at 'Limit' failed to satisfy constraint: Member must have value greater than or equal to 1") + _expect(lambda: dynamodb_client.scan(TableName=empty, FilterExpression="x = :q", Segment=0), + F + UNDEF_VALUE.format(":q")) + _expect(lambda: dynamodb_client.scan(TableName=empty, ProjectionExpression="#p", ExpressionAttributeValues={":u": N}), + "ExpressionAttributeValues can only be specified when using expressions: FilterExpression is null") + + +class TestKeyShapeMessages: + BAD_TYPE = {"pk": {"N": "1"}} + MISSING = {"zz": {"S": "x"}} + EXTRA = {"pk": {"S": "x"}, "zz": {"S": "x"}} + + @pytest.mark.parametrize("key", [BAD_TYPE, MISSING, EXTRA], ids=["type", "missing", "extra"]) + def test_key_shaped_apis(self, dynamodb_client, table, key): + _expect(lambda: dynamodb_client.get_item(TableName=table, Key=key), KEY_SCHEMA) + _expect(lambda: dynamodb_client.delete_item(TableName=table, Key=key), KEY_SCHEMA) + _expect(lambda: dynamodb_client.update_item(TableName=table, Key=key, UpdateExpression="SET a = :v", + ExpressionAttributeValues={":v": N}), KEY_SCHEMA) + _expect(lambda: dynamodb_client.batch_get_item(RequestItems={table: {"Keys": [key]}}), KEY_SCHEMA) + _expect(lambda: dynamodb_client.batch_write_item(RequestItems={table: [{"DeleteRequest": {"Key": key}}]}), KEY_SCHEMA) + + def test_missing_range_key(self, dynamodb_client, range_table): + hash_only = {"pk": {"S": "x"}} + _expect(lambda: dynamodb_client.get_item(TableName=range_table, Key=hash_only), KEY_SCHEMA) + _expect(lambda: dynamodb_client.update_item(TableName=range_table, Key=hash_only, UpdateExpression="SET a = :v", + ExpressionAttributeValues={":v": N}), KEY_SCHEMA) + _expect(lambda: dynamodb_client.get_item(TableName=range_table, Key={"pk": {"S": "x"}, "sk": {"N": "1"}}), KEY_SCHEMA) + _expect(lambda: dynamodb_client.batch_write_item(RequestItems={range_table: [{"PutRequest": {"Item": hash_only}}]}), KEY_SCHEMA) + _expect(lambda: dynamodb_client.batch_write_item(RequestItems={range_table: [{"DeleteRequest": {"Key": hash_only}}]}), KEY_SCHEMA) + _expect(lambda: dynamodb_client.put_item(TableName=range_table, Item=hash_only), + "One or more parameter values were invalid: Missing the key sk in the item") + _expect(lambda: dynamodb_client.put_item(TableName=range_table, Item={"pk": {"S": "x"}, "sk": {"N": "1"}}), + "One or more parameter values were invalid: Type mismatch for key sk expected: S actual: N") + + def test_batch_write_put_uses_the_key_message_put_item_names_the_attribute(self, dynamodb_client, table): + bwi = dynamodb_client.batch_write_item + _expect(lambda: bwi(RequestItems={table: [{"PutRequest": {"Item": self.MISSING}}]}), KEY_SCHEMA) + _expect(lambda: bwi(RequestItems={table: [{"PutRequest": {"Item": self.BAD_TYPE}}]}), KEY_SCHEMA) + _expect(lambda: dynamodb_client.put_item(TableName=table, Item=self.MISSING), + "One or more parameter values were invalid: Missing the key pk in the item") + _expect(lambda: dynamodb_client.put_item(TableName=table, Item=self.BAD_TYPE), + "One or more parameter values were invalid: Type mismatch for key pk expected: S actual: N") + + def test_empty_and_oversize_key_values_keep_their_messages(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.get_item(TableName=table, Key={"pk": {"S": ""}}), + "One or more parameter values are not valid. The AttributeValue for a key attribute cannot contain an empty string value. Key: pk") + _expect(lambda: dynamodb_client.delete_item(TableName=table, Key={"pk": {"S": "x" * 3000}}), + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes") + + +class TestBatchWriteItemCollectionMetrics: + def test_empty_map_when_requested_without_an_lsi(self, dynamodb_client, table): + resp = dynamodb_client.batch_write_item(RequestItems={table: [{"PutRequest": {"Item": {"pk": {"S": "icm"}}}}]}, + ReturnItemCollectionMetrics="SIZE") + assert resp["ItemCollectionMetrics"] == {} + assert resp["UnprocessedItems"] == {} + + def test_absent_when_not_requested(self, dynamodb_client, table): + resp = dynamodb_client.batch_write_item(RequestItems={table: [{"PutRequest": {"Item": {"pk": {"S": "icm2"}}}}]}) + assert "ItemCollectionMetrics" not in resp + + def test_only_lsi_tables_appear(self, dynamodb_client, table, lsi_table): + resp = dynamodb_client.batch_write_item( + RequestItems={table: [{"PutRequest": {"Item": {"pk": {"S": "b"}}}}], + lsi_table: [{"PutRequest": {"Item": {"pk": {"S": "b"}, "sk": {"S": "1"}}}}]}, + ReturnItemCollectionMetrics="SIZE", + ) + assert set(resp["ItemCollectionMetrics"]) == {lsi_table} + [metric] = resp["ItemCollectionMetrics"][lsi_table] + assert metric["ItemCollectionKey"] == {"pk": {"S": "b"}} + assert metric["SizeEstimateRangeGB"] == [0.0, 1.0] + resp = dynamodb_client.batch_write_item( + RequestItems={lsi_table: [{"DeleteRequest": {"Key": {"pk": {"S": "zz"}, "sk": {"S": "1"}}}}]}, + ReturnItemCollectionMetrics="SIZE", + ) + assert set(resp["ItemCollectionMetrics"]) == {lsi_table} + + def test_single_item_apis_omit_the_member_without_an_lsi(self, dynamodb_client, table, lsi_table): + assert "ItemCollectionMetrics" not in dynamodb_client.put_item( + TableName=table, Item={"pk": {"S": "icm3"}}, ReturnItemCollectionMetrics="SIZE") + assert "ItemCollectionMetrics" not in dynamodb_client.update_item( + TableName=table, Key={"pk": {"S": "icm3"}}, UpdateExpression="SET a = :v", + ExpressionAttributeValues={":v": N}, ReturnItemCollectionMetrics="SIZE") + assert "ItemCollectionMetrics" not in dynamodb_client.delete_item( + TableName=table, Key={"pk": {"S": "icm3"}}, ReturnItemCollectionMetrics="SIZE") + resp = dynamodb_client.put_item(TableName=lsi_table, Item={"pk": {"S": "b"}, "sk": {"S": "2"}}, + ReturnItemCollectionMetrics="SIZE") + assert resp["ItemCollectionMetrics"]["ItemCollectionKey"] == {"pk": {"S": "b"}} diff --git a/tests/test_expression_size_limit.py b/tests/test_expression_size_limit.py index 09e321ef4..c298f0093 100644 --- a/tests/test_expression_size_limit.py +++ b/tests/test_expression_size_limit.py @@ -8,12 +8,16 @@ returns ``Invalid : Expression size has exceeded the maximum allowed size;``. -A captured quirk: FilterExpression and ConditionExpression append -`` expression size: `` (N = raw byte length); ProjectionExpression, -UpdateExpression, and KeyConditionExpression do not. - -Messages captured directly from Amazon DynamoDB (asomasun-admin, us-east-1). -Dual-target against Amazon DynamoDB and extenddb. +A captured quirk: the `` expression size: `` suffix (N = raw byte length) +depends on the operation, not only on the expression. A Scan FilterExpression +and a ConditionExpression inside TransactWriteItems carry it; a Query +FilterExpression, a ConditionExpression on PutItem, UpdateItem, or DeleteItem, +ProjectionExpression, UpdateExpression, and KeyConditionExpression do not. +PutItem, UpdateItem, and DeleteItem also wrap the message in +``1 validation error detected: ``. + +Messages captured directly from Amazon DynamoDB (us-east-1, re-measured +2026-10-14). Dual-target against Amazon DynamoDB and extenddb. """ from __future__ import annotations @@ -83,7 +87,7 @@ def test_filter_over_limit_has_size_suffix(self, dynamodb_client, hash_table): f"Invalid FilterExpression: {PREFIX} expression size: {len(expr)}" ) - def test_condition_over_limit_has_size_suffix(self, dynamodb_client, hash_table): + def test_put_condition_over_limit_no_suffix_with_prefix(self, dynamodb_client, hash_table): path = _name_of_len(MAX + 1) expr = f"attribute_not_exists({path})" with pytest.raises(ClientError) as exc_info: @@ -94,6 +98,27 @@ def test_condition_over_limit_has_size_suffix(self, dynamodb_client, hash_table) ) err = exc_info.value.response["Error"] assert err["Code"] == "ValidationException" + assert err["Message"] == ( + f"1 validation error detected: Invalid ConditionExpression: {PREFIX}" + ) + + def test_transact_condition_over_limit_has_size_suffix(self, dynamodb_client, hash_table): + path = _name_of_len(MAX + 1) + expr = f"attribute_not_exists({path})" + with pytest.raises(ClientError) as exc_info: + dynamodb_client.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "TableName": hash_table, + "Key": {"pk": {"S": "k1"}}, + "ConditionExpression": expr, + } + } + ] + ) + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException" assert err["Message"] == ( f"Invalid ConditionExpression: {PREFIX} expression size: {len(expr)}" ) @@ -123,7 +148,9 @@ def test_update_over_limit_no_suffix(self, dynamodb_client, hash_table): ) err = exc_info.value.response["Error"] assert err["Code"] == "ValidationException" - assert err["Message"] == f"Invalid UpdateExpression: {PREFIX}" + assert err["Message"] == ( + f"1 validation error detected: Invalid UpdateExpression: {PREFIX}" + ) def test_size_measured_on_raw_text_not_substituted(self, dynamodb_client, hash_table): # 686 short #aN placeholders => raw ~4005 bytes (under limit) even though diff --git a/tests/test_legacy_condition_parity.py b/tests/test_legacy_condition_parity.py new file mode 100644 index 000000000..ba1769048 --- /dev/null +++ b/tests/test_legacy_condition_parity.py @@ -0,0 +1,348 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +"""Legacy QueryFilter, ScanFilter and KeyConditions: the checks that need the +key schema, the number check on legacy values, and BatchGetItem capacity for +absent keys. Every expected response was measured against Amazon DynamoDB in +us-east-1 on 2026-10-24 (issue #477), and the cases marked 2026-10-31 on that +date. +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import wait_for_active, wait_for_gsi_items + +INVALID_PARAMS = "One or more parameter values were invalid: " +SCHEMA_TYPE = INVALID_PARAMS + "Condition parameter type does not match schema type" +NOT_INDEXABLE = INVALID_PARAMS + "Attempted conditional constraint is not an indexable operation" +LEGACY_TYPE = INVALID_PARAMS + "ComparisonOperator {} is not valid for {} AttributeValue type" +LEGACY_COUNT = INVALID_PARAMS + "Invalid number of argument(s) for the {} ComparisonOperator" +LEGACY_RANGE = INVALID_PARAMS + "The BETWEEN condition was provided a range where the lower bound is greater than the upper bound" +NOT_SUPPORTED = "Query key condition not supported" +EMPTY_KEY = "One or more parameter values are not valid. The AttributeValue for a key attribute cannot contain an empty string value. Key: {}" +FILTER_ON_KEY = "QueryFilter can only contain non-primary key attributes: Primary key attribute: {}" +NOT_A_NUMBER = "The parameter cannot be converted to a numeric value: {}" + + +def _cond(operator, values=None): + cond = {"ComparisonOperator": operator} + if values is not None: + cond["AttributeValueList"] = values + return cond + + +def _s(v): + return {"S": v} + + +def _n(v): + return {"N": v} + + +PK_X = {"pk": _cond("EQ", [_s("x")])} + + +def _validation_error(call): + with pytest.raises(ClientError) as ei: + call() + err = ei.value.response["Error"] + assert err["Code"] == "ValidationException", err + return err["Message"] + + +@pytest.fixture(scope="module") +def ss_table(dynamodb_client): + """pk S, sk S, with a GSI on gpk S / gsk S and two items under pk p1.""" + name = f"legacy_parity_{uuid.uuid4().hex[:8]}" + dynamodb_client.create_table( + TableName=name, + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + {"AttributeName": "gpk", "AttributeType": "S"}, + {"AttributeName": "gsk", "AttributeType": "S"}, + ], + GlobalSecondaryIndexes=[ + { + "IndexName": "gsi1", + "KeySchema": [{"AttributeName": "gpk", "KeyType": "HASH"}, {"AttributeName": "gsk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}, + } + ], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + dynamodb_client.put_item(TableName=name, Item={"pk": _s("p1"), "sk": _s("a"), "gpk": _s("g1"), "gsk": _s("a")}) + dynamodb_client.put_item(TableName=name, Item={"pk": _s("p1"), "sk": _s("b"), "gpk": _s("g1"), "gsk": _s("b")}) + wait_for_gsi_items(lambda: dynamodb_client.query(TableName=name, IndexName="gsi1", KeyConditions={"gpk": _cond("EQ", [_s("g1")])})["Items"], 2) + yield name + dynamodb_client.delete_table(TableName=name) + + +@pytest.fixture(scope="module") +def sn_table(dynamodb_client): + """pk S, sk N, two items under pk p1.""" + name = f"legacy_parity_n_{uuid.uuid4().hex[:8]}" + dynamodb_client.create_table( + TableName=name, + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "N"}], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(dynamodb_client, name) + dynamodb_client.put_item(TableName=name, Item={"pk": _s("p1"), "sk": _n("1")}) + dynamodb_client.put_item(TableName=name, Item={"pk": _s("p1"), "sk": _n("2")}) + yield name + dynamodb_client.delete_table(TableName=name) + + +class TestLegacyNumberValues: + """A legacy N value that is not a number is refused with the number + message, ahead of the operator checks on the same entry and ahead of the + table lookup; sets, lists and maps are searched too.""" + + @pytest.mark.parametrize( + ("operator", "values"), + [ + ("EQ", [_n("abc")]), + ("BEGINS_WITH", [_n("abc")]), + ("FOO", [_n("abc")]), + ("EQ", [_n("abc"), _s("x")]), + ("EQ", [_s("x"), _n("abc")]), + ("IN", [_s("x"), _n("abc")]), + ("NULL", [_n("abc")]), + ("BETWEEN", [_n("2"), _n("abc")]), + ("BETWEEN", [_n("abc"), _n("2")]), + ("EQ", [{"NS": ["1", "abc"]}]), + ("EQ", [{"L": [_n("abc")]}]), + ("EQ", [{"M": {"k": _n("abc")}}]), + ("LT", [{"L": [_n("abc")]}]), + ], + ) + def test_scan_filter_number_is_checked_first(self, dynamodb_client_no_validation, ss_table, operator, values): + msg = _validation_error(lambda: dynamodb_client_no_validation.scan(TableName=ss_table, ScanFilter={"a": _cond(operator, values)})) + assert msg == NOT_A_NUMBER.format("abc") + + @pytest.mark.parametrize(("raw", "message"), [("", NOT_A_NUMBER.format("")), ("12e", NOT_A_NUMBER.format("12e")), (" 1", NOT_A_NUMBER.format(" 1")), ("1e400", "Number overflow. Attempting to store a number with magnitude larger than supported range"), ("1e-400", "Number underflow. Attempting to store a number with magnitude smaller than supported range"), ("1" * 39, "Attempting to store more than 38 significant digits in a Number")]) + def test_scan_filter_number_messages(self, dynamodb_client_no_validation, ss_table, raw, message): + msg = _validation_error(lambda: dynamodb_client_no_validation.scan(TableName=ss_table, ScanFilter={"a": _cond("EQ", [_n(raw)])})) + assert msg == message + + def test_query_filter_and_key_conditions_numbers(self, dynamodb_client_no_validation, ss_table, sn_table): + client = dynamodb_client_no_validation + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=PK_X, QueryFilter={"a": _cond("EQ", [_n("abc")])})) == NOT_A_NUMBER.format("abc") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_n("abc")])})) == NOT_A_NUMBER.format("abc") + assert _validation_error(lambda: client.query(TableName=sn_table, KeyConditions={"pk": _cond("EQ", [_s("x")]), "sk": _cond("EQ", [_n("abc")])})) == NOT_A_NUMBER.format("abc") + # A QueryFilter error is reported before a KeyConditions number. + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_n("abc")])}, QueryFilter={"a": _cond("BEGINS_WITH", [_n("1")])})) == LEGACY_TYPE.format("BEGINS_WITH", "N") + + def test_number_is_checked_before_the_table_lookup_and_select(self, dynamodb_client_no_validation): + client = dynamodb_client_no_validation + absent = f"absent_{uuid.uuid4().hex[:8]}" + assert _validation_error(lambda: client.scan(TableName=absent, ScanFilter={"a": _cond("EQ", [_n("abc")])})) == NOT_A_NUMBER.format("abc") + assert _validation_error(lambda: client.scan(TableName=absent, ScanFilter={"a": _cond("EQ", [_n("abc")])}, IndexName="nope")) == NOT_A_NUMBER.format("abc") + + @pytest.mark.parametrize(("segments", "absent_table"), [({"Segment": 0}, False), ({"Segment": 2, "TotalSegments": 2}, False), ({"Segment": 0}, True)]) + def test_number_is_checked_before_the_segment_checks(self, dynamodb_client_no_validation, ss_table, segments, absent_table): + # Measured 2026-10-31: the number message ahead of the Segment and + # TotalSegments pairing checks, on a table that does not exist too. + table = f"absent_{uuid.uuid4().hex[:8]}" if absent_table else ss_table + msg = _validation_error(lambda: dynamodb_client_no_validation.scan(TableName=table, ScanFilter={"x": _cond("EQ", [_n("abc")])}, **segments)) + assert msg == NOT_A_NUMBER.format("abc") + + def test_segment_check_answers_when_the_number_is_valid(self, dynamodb_client_no_validation, ss_table): + # Measured 2026-10-31. + msg = _validation_error(lambda: dynamodb_client_no_validation.scan(TableName=ss_table, ScanFilter={"x": _cond("EQ", [_n("5")])}, Segment=0)) + assert msg == "The TotalSegments parameter is required but was not present in the request when Segment parameter is present" + + def test_both_forms_conflict_is_reported_before_the_number(self, dynamodb_client_no_validation, ss_table): + msg = _validation_error(lambda: dynamodb_client_no_validation.scan(TableName=ss_table, ScanFilter={"a": _cond("EQ", [_n("abc")])}, FilterExpression="b = :v", ExpressionAttributeValues={":v": _s("x")})) + assert msg.startswith("Can not use both expression and non-expression parameters") + + def test_valid_numbers_pass(self, dynamodb_client, ss_table): + assert dynamodb_client.scan(TableName=ss_table, ScanFilter={"a": _cond("EQ", [_n("-1.5e3")])})["Count"] == 0 + + +class TestKeyConditionsSchemaChecks: + """KeyConditions values are held to the key's AttributeDefinitions type, + non-indexable operators are refused on either key at request time, empty + key strings are refused after the schema checks, and the checks run in + the measured order.""" + + @pytest.mark.parametrize("value", [_n("1"), {"B": b"\x01"}, {"BOOL": True}, {"B": b""}]) + def test_hash_key_value_of_the_wrong_type(self, dynamodb_client_no_validation, ss_table, value): + msg = _validation_error(lambda: dynamodb_client_no_validation.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [value])})) + assert msg == SCHEMA_TYPE + + @pytest.mark.parametrize("cond", [_cond("EQ", [_n("1")]), _cond("LE", [{"B": b"\x01"}]), _cond("BETWEEN", [_n("1"), _n("2")]), _cond("BEGINS_WITH", [{"B": b"\x01"}]), _cond("BEGINS_WITH", [{"B": b""}])]) + def test_range_key_value_of_the_wrong_type_on_a_string_key(self, dynamodb_client_no_validation, ss_table, cond): + msg = _validation_error(lambda: dynamodb_client_no_validation.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": cond})) + assert msg == SCHEMA_TYPE + + @pytest.mark.parametrize("cond", [_cond("EQ", [_s("abc")]), _cond("BEGINS_WITH", [_s("1")]), _cond("BETWEEN", [_s("a"), _s("b")]), _cond("LT", [_s("5")])]) + def test_range_key_value_of_the_wrong_type_on_a_number_key(self, dynamodb_client, sn_table, cond): + msg = _validation_error(lambda: dynamodb_client.query(TableName=sn_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": cond})) + assert msg == SCHEMA_TYPE + + def test_well_typed_conditions_read_rows(self, dynamodb_client, sn_table, ss_table): + assert dynamodb_client.query(TableName=sn_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond("EQ", [_n("1")])})["Count"] == 1 + assert dynamodb_client.query(TableName=sn_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond("BETWEEN", [_n("1"), _n("2")])})["Count"] == 2 + assert dynamodb_client.query(TableName=ss_table, IndexName="gsi1", KeyConditions={"gpk": _cond("EQ", [_s("g1")]), "gsk": _cond("EQ", [_s("a")])})["Count"] == 1 + + def test_index_keys_are_held_to_the_index_schema(self, dynamodb_client, ss_table): + assert _validation_error(lambda: dynamodb_client.query(TableName=ss_table, IndexName="gsi1", KeyConditions={"gpk": _cond("EQ", [_n("1")])})) == SCHEMA_TYPE + assert _validation_error(lambda: dynamodb_client.query(TableName=ss_table, IndexName="gsi1", KeyConditions={"gpk": _cond("EQ", [_s("g1")]), "gsk": _cond("EQ", [_n("1")])})) == SCHEMA_TYPE + assert _validation_error(lambda: dynamodb_client.query(TableName=ss_table, IndexName="gsi1", KeyConditions={"gpk": _cond("EQ", [_s("")])})) == EMPTY_KEY.format("gpk") + assert _validation_error(lambda: dynamodb_client.query(TableName=ss_table, IndexName="gsi1", KeyConditions={"gpk": _cond("EQ", [_s("g1")]), "gsk": _cond("BEGINS_WITH", [_s("")])})) == EMPTY_KEY.format("gsk") + + @pytest.mark.parametrize(("operator", "values"), [("NE", [_s("x")]), ("IN", [_s("x")]), ("NULL", None), ("NOT_NULL", None), ("CONTAINS", [_s("x")])]) + def test_non_indexable_operator_on_the_hash_key(self, dynamodb_client_no_validation, ss_table, operator, values): + msg = _validation_error(lambda: dynamodb_client_no_validation.query(TableName=ss_table, KeyConditions={"pk": _cond(operator, values)})) + assert msg == NOT_INDEXABLE + + def test_non_indexable_is_reported_before_the_schema_checks(self, dynamodb_client, ss_table): + client = dynamodb_client + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_n("1")]), "sk": _cond("NE", [_s("x")])})) == NOT_INDEXABLE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("x")]), "sk": _cond("NE", [_n("1")])})) == NOT_INDEXABLE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"sk": _cond("NE", [_s("x")])})) == NOT_INDEXABLE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("")]), "sk": _cond("NE", [_s("x")])})) == NOT_INDEXABLE + + def test_hash_key_order_type_then_operator(self, dynamodb_client, ss_table): + client = dynamodb_client + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("LT", [_n("1")])})) == SCHEMA_TYPE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("LT", [_s("x")]), "sk": _cond("EQ", [_n("1")])})) == NOT_SUPPORTED + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("LT", [_s("")])})) == NOT_SUPPORTED + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("x"), _s("y")]), "sk": _cond("EQ", [_n("1")])})) == LEGACY_COUNT.format("EQ") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"sk": _cond("EQ", [_n("1")])})) == "Query condition missed key schema element: pk" + + def test_schema_type_is_reported_after_the_lookups_and_before_the_start_key(self, dynamodb_client, ss_table): + client = dynamodb_client + bad = {"pk": _cond("EQ", [_n("1")])} + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=bad, IndexName="nope")) == "The table does not have the specified index: nope" + assert _validation_error(lambda: client.query(TableName=ss_table, IndexName="gsi1", ConsistentRead=True, KeyConditions={"gpk": _cond("EQ", [_n("1")])})) == "Consistent reads are not supported on global secondary indexes" + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=bad, ExclusiveStartKey={"zz": _s("x")})) == SCHEMA_TYPE + with pytest.raises(ClientError) as ei: + client.query(TableName=f"absent_{uuid.uuid4().hex[:8]}", KeyConditions=bad) + assert ei.value.response["Error"]["Code"] == "ResourceNotFoundException" + + @pytest.mark.parametrize("operator", ["EQ", "LE", "LT", "GE", "GT", "BEGINS_WITH"]) + def test_empty_string_on_the_range_key(self, dynamodb_client, ss_table, operator): + msg = _validation_error(lambda: dynamodb_client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond(operator, [_s("")])})) + assert msg == EMPTY_KEY.format("sk") + + def test_empty_string_on_the_hash_key_and_between_bounds(self, dynamodb_client, ss_table): + client = dynamodb_client + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("")])})) == EMPTY_KEY.format("pk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("")]), "sk": _cond("EQ", [_s("")])})) == EMPTY_KEY.format("pk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond("BETWEEN", [_s(""), _s("a")])})) == EMPTY_KEY.format("sk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond("BETWEEN", [_s(""), _s("")])})) == EMPTY_KEY.format("sk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond("BETWEEN", [_s("a"), _s("")])})) == LEGACY_RANGE + + def test_empty_string_is_reported_after_the_schema_checks(self, dynamodb_client, ss_table): + client = dynamodb_client + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("")]), "sk": _cond("EQ", [_n("1")])})) == SCHEMA_TYPE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_n("1")]), "sk": _cond("EQ", [_s("")])})) == SCHEMA_TYPE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("")])}, QueryFilter={"a": _cond("BEGINS_WITH", [_n("1")])})) == LEGACY_TYPE.format("BEGINS_WITH", "N") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("")])}, ExclusiveStartKey={"zz": _s("x")})) == EMPTY_KEY.format("pk") + # A QueryFilter on a key is the last check. + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("")])}, QueryFilter={"pk": _cond("EQ", [_s("x")])})) == EMPTY_KEY.format("pk") + # An empty filter value on a non-key attribute passes. + assert client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")])}, QueryFilter={"a": _cond("EQ", [_s("")])})["Count"] == 0 + + +class TestQueryFilterOnKeyAttribute: + """A legacy QueryFilter entry on a key of the queried table or index is + refused, after every other Query error.""" + + def test_table_keys_are_refused_hash_named_first(self, dynamodb_client, ss_table): + client = dynamodb_client + kc = {"pk": _cond("EQ", [_s("p1")])} + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter={"pk": _cond("EQ", [_s("p1")])})) == FILTER_ON_KEY.format("pk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter={"sk": _cond("EQ", [_s("a")])})) == FILTER_ON_KEY.format("sk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter={"sk": _cond("EQ", [_s("a")]), "pk": _cond("EQ", [_s("p1")])})) == FILTER_ON_KEY.format("pk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter={"pk": _cond("NOT_NULL")})) == FILTER_ON_KEY.format("pk") + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter={"pk": _cond("EQ", [_s("p1")]), "zz": _cond("EQ", [_s("x")])}, ConditionalOperator="OR")) == FILTER_ON_KEY.format("pk") + + def test_index_query_refuses_index_keys_and_passes_table_keys(self, dynamodb_client, ss_table): + client = dynamodb_client + kc = {"gpk": _cond("EQ", [_s("g1")])} + assert _validation_error(lambda: client.query(TableName=ss_table, IndexName="gsi1", KeyConditions=kc, QueryFilter={"gpk": _cond("EQ", [_s("g1")])})) == FILTER_ON_KEY.format("gpk") + assert _validation_error(lambda: client.query(TableName=ss_table, IndexName="gsi1", KeyConditions=kc, QueryFilter={"gsk": _cond("EQ", [_s("a")])})) == FILTER_ON_KEY.format("gsk") + assert client.query(TableName=ss_table, IndexName="gsi1", KeyConditions=kc, QueryFilter={"pk": _cond("EQ", [_s("x")])})["Count"] == 0 + assert client.query(TableName=ss_table, IndexName="gsi1", KeyConditions=kc, QueryFilter={"sk": _cond("EQ", [_s("a")])})["Count"] == 1 + assert client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")])}, QueryFilter={"gpk": _cond("EQ", [_s("g1")])})["Count"] == 2 + + def test_reported_after_every_other_query_error(self, dynamodb_client, ss_table): + client = dynamodb_client + qf = {"pk": _cond("EQ", [_s("x")])} + kc = {"pk": _cond("EQ", [_s("p1")])} + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter=qf, IndexName="nope")) == "The table does not have the specified index: nope" + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_n("1")])}, QueryFilter=qf)) == SCHEMA_TYPE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"sk": _cond("EQ", [_s("a")])}, QueryFilter=qf)) == "Query condition missed key schema element: pk" + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond("NE", [_s("a")])}, QueryFilter=qf)) == NOT_INDEXABLE + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("LT", [_s("p1")])}, QueryFilter=qf)) == NOT_SUPPORTED + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions={"pk": _cond("EQ", [_s("p1")]), "sk": _cond("EQ", [_s("")])}, QueryFilter=qf)) == EMPTY_KEY.format("sk") + assert _validation_error(lambda: client.query(TableName=ss_table, QueryFilter=qf)) == "Either the KeyConditions or KeyConditionExpression parameter must be specified in the request." + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter={"pk": _cond("EQ", [_s("x")]), "zz": _cond("BEGINS_WITH", [_n("1")])})) == LEGACY_TYPE.format("BEGINS_WITH", "N") + assert _validation_error(lambda: client.query(TableName=ss_table, IndexName="gsi1", ConsistentRead=True, KeyConditions={"gpk": _cond("EQ", [_s("g1")])}, QueryFilter=qf)) == "Consistent reads are not supported on global secondary indexes" + # The Select check comes first (measured 2026-10-31). + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter=qf, Select="SPECIFIC_ATTRIBUTES")) == "1 validation error detected: Must specify the AttributesToGet or ProjectionExpression when choosing to get SPECIFIC_ATTRIBUTES" + assert _validation_error(lambda: client.query(TableName=ss_table, KeyConditions=kc, QueryFilter=qf, ExclusiveStartKey={"zz": _s("x")})) == FILTER_ON_KEY.format("pk") + with pytest.raises(ClientError) as ei: + client.query(TableName=f"absent_{uuid.uuid4().hex[:8]}", KeyConditions=kc, QueryFilter=qf) + assert ei.value.response["Error"]["Code"] == "ResourceNotFoundException" + + def test_scan_filter_on_a_key_passes(self, dynamodb_client, ss_table): + assert dynamodb_client.scan(TableName=ss_table, ScanFilter={"pk": _cond("EQ", [_s("p1")])})["Count"] == 2 + + +class TestBatchGetAbsentKeyCapacity: + """BatchGetItem charges each absent key the minimum read and lists every + requested table in ConsumedCapacity. The service lists the tables in no + stable order, so the order is not asserted.""" + + def test_absent_keys_are_charged(self, dynamodb_client, sn_table): + client = dynamodb_client + absent = {"pk": _s("nope"), "sk": _n("1")} + absent2 = {"pk": _s("nope"), "sk": _n("2")} + present = {"pk": _s("p1"), "sk": _n("1")} + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent]}}, ReturnConsumedCapacity="TOTAL") + assert resp["Responses"] == {sn_table: []} + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 0.5}] + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent, absent2]}}, ReturnConsumedCapacity="TOTAL") + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 1.0}] + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [present, absent]}}, ReturnConsumedCapacity="TOTAL") + assert len(resp["Responses"][sn_table]) == 1 + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 1.0}] + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent], "ConsistentRead": True}}, ReturnConsumedCapacity="TOTAL") + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 1.0}] + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent, absent2], "ConsistentRead": True}}, ReturnConsumedCapacity="TOTAL") + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 2.0}] + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent], "ProjectionExpression": "pk"}}, ReturnConsumedCapacity="TOTAL") + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 0.5}] + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [{"pk": _s("nope"), "sk": _n(str(i))} for i in range(100)]}}, ReturnConsumedCapacity="TOTAL") + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 50.0}] + + def test_indexes_shape_and_none(self, dynamodb_client, sn_table): + client = dynamodb_client + absent = {"pk": _s("nope"), "sk": _n("1")} + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent]}}, ReturnConsumedCapacity="INDEXES") + assert resp["ConsumedCapacity"] == [{"TableName": sn_table, "CapacityUnits": 0.5, "Table": {"CapacityUnits": 0.5}}] + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent]}}, ReturnConsumedCapacity="NONE") + assert "ConsumedCapacity" not in resp + resp = client.batch_get_item(RequestItems={sn_table: {"Keys": [absent]}}) + assert "ConsumedCapacity" not in resp + + def test_two_tables_all_absent_each_listed_once(self, dynamodb_client, ss_table, sn_table): + client = dynamodb_client + items = {sn_table: {"Keys": [{"pk": _s("nope"), "sk": _n("1")}]}, ss_table: {"Keys": [{"pk": _s("nope"), "sk": _s("x")}]}} + for order in (list(items), list(reversed(list(items)))): + resp = client.batch_get_item(RequestItems={t: items[t] for t in order}, ReturnConsumedCapacity="TOTAL") + assert resp["Responses"] == {ss_table: [], sn_table: []} + listed = sorted(resp["ConsumedCapacity"], key=lambda entry: entry["TableName"]) + assert listed == [{"TableName": t, "CapacityUnits": 0.5} for t in sorted(items)] diff --git a/tests/test_request_stage_checks.py b/tests/test_request_stage_checks.py new file mode 100644 index 000000000..cd1f86abe --- /dev/null +++ b/tests/test_request_stage_checks.py @@ -0,0 +1,282 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Request-stage checks of Query, Scan and UpdateItem, with the service's text. + +Each row below was measured against Amazon DynamoDB on 2026-10-27 (two +samples, identical bytes). These checks run before the table is read, so the +tables here are only needed for the cases that name one. + +Dual-target: runs against the service and against ExtendDB on every backend. +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +S = {"S": "s"} +N = {"N": "1"} +KEY = {"pk": {"S": "p"}, "sk": {"S": "s"}} +UNDEF_NAME = "An expression attribute name used in the document path is not defined; attribute name: {}" +UNDEF_VALUE = "An expression attribute value used in expression is not defined; attribute value: {}" +KEY_CONDITION = "Invalid KeyConditionExpression: " +LIMIT = ( + "1 validation error detected: Value at 'Limit' failed to satisfy constraint: " + "Member must have value greater than or equal to 1" +) + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """Composite-key table (pk S, sk S) with one item.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "p"}, "sk": {"S": "s"}, "n": {"N": "1"}}) + yield name + + +@pytest.fixture(scope="module") +def missing_table() -> str: + return f"extenddb-missing-{uuid.uuid4().hex[:12]}" + + +def _refused(func, message: str): + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"] == message, err + + +class TestKeyConditionReferenceOrder: + """An undefined `#name` or `:value` is named in the order the text has them.""" + + @pytest.mark.parametrize( + ("expression", "values", "undefined"), + [ + ("#b = :p AND #a > :q", {":p": S, ":q": S}, UNDEF_NAME.format("#b")), + ("#a = :p AND #b > :q", {":p": S, ":q": S}, UNDEF_NAME.format("#a")), + ("pk = :q AND #n = :v", {":v": S}, UNDEF_VALUE.format(":q")), + ("pk = :z AND sk > :y", None, UNDEF_VALUE.format(":z")), + ("pk = :y AND sk > :z", None, UNDEF_VALUE.format(":y")), + ], + ids=["names-b-a", "names-a-b", "value-before-name", "values-z-y", "values-y-z"], + ) + def test_the_first_undefined_reference_in_the_text_is_named( + self, dynamodb_client, table, expression, values, undefined + ): + extra = {} if values is None else {"ExpressionAttributeValues": values} + _refused( + lambda: dynamodb_client.query(TableName=table, KeyConditionExpression=expression, **extra), + KEY_CONDITION + undefined, + ) + + +class TestKeyConditionFunctionNames: + """A function in a key condition: the text depends on the name as written.""" + + NOT_AN_OPERATOR = KEY_CONDITION + "Invalid operator used in KeyConditionExpression" + NESTED = KEY_CONDITION + "KeyConditionExpressions cannot contain nested operations" + + @pytest.mark.parametrize( + ("expression", "values", "message"), + [ + ("pk = :p AND contains(sk, :v)", {":p": S, ":v": S}, NOT_AN_OPERATOR), + ("pk = :p AND attribute_exists(sk)", {":p": S}, NOT_AN_OPERATOR), + ("pk = :p AND attribute_not_exists(sk)", {":p": S}, NOT_AN_OPERATOR), + ("pk = :p AND attribute_type(sk, :t)", {":p": S, ":t": {"S": "S"}}, NOT_AN_OPERATOR), + ("contains(pk, :v) AND sk = :s", {":v": S, ":s": S}, NOT_AN_OPERATOR), + ("pk = :p AND size(sk) > :n", {":p": S, ":n": N}, NESTED), + ("pk = :p AND foo(sk, :v)", {":p": S, ":v": S}, KEY_CONDITION + "Invalid function name; function: foo"), + ( + "pk = :p AND Begins_With(sk, :v)", + {":p": S, ":v": S}, + KEY_CONDITION + "Invalid function name; function: Begins_With", + ), + ], + ids=["contains", "attribute_exists", "attribute_not_exists", "attribute_type", "contains-on-pk", + "size", "unknown", "miscased"], + ) + def test_the_refusal_follows_the_name(self, dynamodb_client, table, expression, values, message): + _refused( + lambda: dynamodb_client.query( + TableName=table, KeyConditionExpression=expression, ExpressionAttributeValues=values + ), + message, + ) + + def test_begins_with_is_accepted(self, dynamodb_client, table): + response = dynamodb_client.query( + TableName=table, + KeyConditionExpression="pk = :p AND begins_with(sk, :v)", + ExpressionAttributeValues={":p": {"S": "p"}, ":v": {"S": "s"}}, + ) + assert response["Count"] == 1 + + +class TestLimitBelowOne: + """Query and Scan refuse a Limit under 1 with one text, ahead of the table and of every expression.""" + + @pytest.mark.parametrize("limit", [0, -1]) + def test_the_text_is_the_same_on_an_existing_and_a_missing_table( + self, dynamodb_client_no_validation, table, missing_table, limit + ): + client = dynamodb_client_no_validation + for name in (table, missing_table): + _refused( + lambda: client.query( + TableName=name, KeyConditionExpression="pk = :p", ExpressionAttributeValues={":p": S}, Limit=limit + ), + LIMIT, + ) + _refused(lambda: client.scan(TableName=name, Limit=limit), LIMIT) + + def test_it_comes_before_every_expression_check(self, dynamodb_client_no_validation, table): + client = dynamodb_client_no_validation + _refused(lambda: client.query(TableName=table, KeyConditionExpression="pk = :q", Limit=0), LIMIT) + _refused( + lambda: client.query( + TableName=table, + KeyConditionExpression="pk = :p", + FilterExpression="n = ", + ExpressionAttributeValues={":p": S}, + Limit=0, + ), + LIMIT, + ) + _refused(lambda: client.scan(TableName=table, FilterExpression="n = ", Limit=0), LIMIT) + + +class TestLegacyAttributeUpdates: + """The value type of an ADD or DELETE is refused with the legacy text, ahead of the table and the key.""" + + ADD = "1 validation error detected: One or more parameter values were invalid: ADD action is not supported for the type {}" + DELETE = ( + "1 validation error detected: One or more parameter values were invalid: " + "DELETE action with value is not supported for the type {}" + ) + + @pytest.mark.parametrize( + ("action", "value", "message"), + [ + ("ADD", {"S": "x"}, ADD.format("S")), + ("ADD", {"M": {}}, ADD.format("M")), + ("ADD", {"BOOL": True}, ADD.format("BOOL")), + ("DELETE", {"N": "1"}, DELETE.format("N")), + ("DELETE", {"S": "x"}, DELETE.format("S")), + ("DELETE", {"L": []}, DELETE.format("L")), + ], + ids=["add-S", "add-M", "add-BOOL", "delete-N", "delete-S", "delete-L"], + ) + def test_the_value_type_is_refused(self, dynamodb_client, table, action, value, message): + _refused( + lambda: dynamodb_client.update_item( + TableName=table, Key=KEY, AttributeUpdates={"a": {"Action": action, "Value": value}} + ), + message, + ) + + def test_the_check_comes_before_the_table_and_the_key(self, dynamodb_client, table, missing_table): + refusal = self.ADD.format("S") + update = {"a": {"Action": "ADD", "Value": {"S": "x"}}} + for name, key in ( + (missing_table, KEY), + (table, {"pk": {"N": "1"}, "sk": {"S": "s"}}), + (table, {"pk": {"S": "p"}}), + ): + _refused( + lambda: dynamodb_client.update_item(TableName=name, Key=key, AttributeUpdates=update), refusal + ) + + +class TestEmptyExpressionStrings: + """An empty expression string is refused as empty.""" + + def test_a_condition_check_with_an_empty_condition(self, dynamodb_client_no_validation, table): + _refused( + lambda: dynamodb_client_no_validation.transact_write_items( + TransactItems=[{"ConditionCheck": {"TableName": table, "Key": KEY, "ConditionExpression": ""}}] + ), + "Invalid ConditionExpression: The expression can not be empty;", + ) + + def test_an_update_with_an_empty_update_expression(self, dynamodb_client_no_validation, table): + client = dynamodb_client_no_validation + _refused( + lambda: client.transact_write_items( + TransactItems=[{"Update": {"TableName": table, "Key": KEY, "UpdateExpression": ""}}] + ), + "Invalid UpdateExpression: The expression can not be empty;", + ) + _refused( + lambda: client.update_item(TableName=table, Key=KEY, UpdateExpression=""), + "1 validation error detected: Invalid UpdateExpression: The expression can not be empty;", + ) + + +class TestScanSegmentChecks: + """Segment and TotalSegments go together; the service checks that after the + expressions and the unused names, and before the table lookup.""" + + TOTAL_REQUIRED = ( + "The TotalSegments parameter is required but was not present in the request " + "when Segment parameter is present" + ) + SEGMENT_REQUIRED = ( + "The Segment parameter is required but was not present in the request " + "when parameter TotalSegments is present" + ) + ZERO_BASED = "The Segment parameter is zero-based and must be less than parameter TotalSegments" + UNDEFINED_VALUE = "Invalid FilterExpression: " + UNDEF_VALUE.format(":q") + + def test_the_messages(self, dynamodb_client, table): + _refused(lambda: dynamodb_client.scan(TableName=table, Segment=0), self.TOTAL_REQUIRED) + _refused(lambda: dynamodb_client.scan(TableName=table, TotalSegments=2), self.SEGMENT_REQUIRED) + # The service ends its text there; ExtendDB adds the two values. + with pytest.raises(ClientError) as exc_info: + dynamodb_client.scan(TableName=table, Segment=5, TotalSegments=3) + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"].startswith(self.ZERO_BASED), err + + def test_they_come_after_the_expressions(self, dynamodb_client, table): + for pair in ({"Segment": 0}, {"TotalSegments": 2}, {"Segment": 5, "TotalSegments": 3}): + _refused( + lambda: dynamodb_client.scan(TableName=table, FilterExpression="x = :q", **pair), + self.UNDEFINED_VALUE, + ) + _refused( + lambda: dynamodb_client.scan( + TableName=table, + FilterExpression="x = :v", + ExpressionAttributeValues={":v": N, ":u": N}, + Segment=0, + ), + "Value provided in ExpressionAttributeValues unused in expressions: keys: {:u}", + ) + _refused( + lambda: dynamodb_client.scan(TableName=table, ProjectionExpression="#p", Segment=0), + "Invalid ProjectionExpression: " + UNDEF_NAME.format("#p"), + ) + + def test_they_come_before_the_table_lookup(self, dynamodb_client, missing_table): + _refused(lambda: dynamodb_client.scan(TableName=missing_table, Segment=0), self.TOTAL_REQUIRED) + + def test_limit_still_comes_first(self, dynamodb_client_no_validation, table): + client = dynamodb_client_no_validation + _refused(lambda: client.scan(TableName=table, Segment=0, Limit=0), LIMIT) + _refused(lambda: client.scan(TableName=table, Segment=5, TotalSegments=3, Limit=0), LIMIT) diff --git a/tests/test_transact_capacity_order.py b/tests/test_transact_capacity_order.py new file mode 100644 index 000000000..6da1705bf --- /dev/null +++ b/tests/test_transact_capacity_order.py @@ -0,0 +1,113 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Order of the ConsumedCapacity entries of a multi-table transaction. + +Amazon DynamoDB documents no order for the ``ConsumedCapacity`` array of +``TransactGetItems`` and ``TransactWriteItems``. Measured on 2026-10-23 with +three tables whose names sort opposite to their creation order: +both operations listed the tables oldest first, for every request order tried +([newest, middle, oldest], [middle, newest, oldest], [oldest, newest, middle]). +The order follows neither the request nor the names. + +The fixture below creates its three tables so that the name order is the +reverse of the creation order, waiting for ACTIVE between them, so a server +that ordered by name, or by first appearance in the request, fails these +tests. Dual-target: runs against the service and against ExtendDB on every +backend. +""" + +from __future__ import annotations + +import uuid + +import pytest + +from conftest import wait_for_active, wait_for_deleted + + +@pytest.fixture(scope="module") +def tables_oldest_first(dynamodb_client) -> list[str]: + """Three tables, returned oldest first; their names sort newest first.""" + suffix = uuid.uuid4().hex[:12] + names = [f"extenddb-test-c-{suffix}", f"extenddb-test-b-{suffix}", f"extenddb-test-a-{suffix}"] + created: list[str] = [] + try: + for name in names: + dynamodb_client.create_table( + TableName=name, + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + BillingMode="PAY_PER_REQUEST", + ) + created.append(name) + # Each table is ACTIVE before the next is created, so the creation + # times are distinct and in list order. + wait_for_active(dynamodb_client, name) + yield created + finally: + for name in created: + try: + dynamodb_client.delete_table(TableName=name) + except dynamodb_client.exceptions.ResourceNotFoundException: + continue + wait_for_deleted(dynamodb_client, name) + + +def _request_orders(oldest_first: list[str]) -> list[list[str]]: + oldest, middle, newest = oldest_first + return [ + [newest, middle, oldest], + [middle, newest, oldest], + [oldest, newest, middle], + ] + + +def _table_names(resp: dict) -> list[str]: + return [c["TableName"] for c in resp["ConsumedCapacity"]] + + +class TestTransactionConsumedCapacityOrder: + """Measured against Amazon DynamoDB on 2026-10-23.""" + + def test_fixture_names_sort_opposite_to_creation(self, tables_oldest_first): + assert sorted(tables_oldest_first) == list(reversed(tables_oldest_first)) + + @pytest.mark.parametrize("granularity", ["TOTAL", "INDEXES"]) + def test_transact_get_items_lists_tables_oldest_first( + self, dynamodb_client, tables_oldest_first, granularity + ): + for order in _request_orders(tables_oldest_first): + resp = dynamodb_client.transact_get_items( + ReturnConsumedCapacity=granularity, + TransactItems=[{"Get": {"TableName": t, "Key": {"pk": {"S": "k"}}}} for t in order], + ) + assert _table_names(resp) == tables_oldest_first, order + + @pytest.mark.parametrize("granularity", ["TOTAL", "INDEXES"]) + def test_transact_write_items_lists_tables_oldest_first( + self, dynamodb_client, tables_oldest_first, granularity + ): + for n, order in enumerate(_request_orders(tables_oldest_first)): + resp = dynamodb_client.transact_write_items( + ReturnConsumedCapacity=granularity, + TransactItems=[ + {"Put": {"TableName": t, "Item": {"pk": {"S": f"{granularity}-{n}"}}}} for t in order + ], + ) + assert _table_names(resp) == tables_oldest_first, order + + def test_a_table_named_twice_has_one_entry_in_the_same_order( + self, dynamodb_client, tables_oldest_first + ): + oldest, middle, newest = tables_oldest_first + resp = dynamodb_client.transact_get_items( + ReturnConsumedCapacity="TOTAL", + TransactItems=[ + {"Get": {"TableName": newest, "Key": {"pk": {"S": "k1"}}}}, + {"Get": {"TableName": oldest, "Key": {"pk": {"S": "k1"}}}}, + {"Get": {"TableName": newest, "Key": {"pk": {"S": "k2"}}}}, + {"Get": {"TableName": middle, "Key": {"pk": {"S": "k1"}}}}, + ], + ) + assert _table_names(resp) == tables_oldest_first diff --git a/tests/test_transact_validation_order.py b/tests/test_transact_validation_order.py new file mode 100644 index 000000000..115574208 --- /dev/null +++ b/tests/test_transact_validation_order.py @@ -0,0 +1,664 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Validation order inside TransactGetItems and TransactWriteItems. + +When a transaction carries two defects, Amazon DynamoDB reports the earlier +of these phases whichever item carries it: + + 1. per item, in order: expression grammar and the semantic walk (a + `#name` or `:value` that is not defined counts here, not as a + cancellation reason), unused names and values, then the item's 400 KB + size, all as a top-level ValidationException; + 2. a table that does not exist, as ResourceNotFoundException; + 3. the 4 MB request total, as a top-level ValidationException; + 4. a key that does not fit the table's key schema, or a key attribute + named in an Update, as TransactionCanceledException with a + ValidationError reason on the first offending item and None on every + other item; conditions are not evaluated and runtime update errors are + not reported when any item fails here; + 5. two operations on one item, as a top-level ValidationException. + +Captured from Amazon DynamoDB (issue #108); the per-item checks of phases 1 +and 4 are in test_transact_write_phase_order.py. Dual-target (all backends). +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +DUPLICATE = "Transaction request cannot include multiple operations on one item" +RESERVED_PROJECTION = ( + "Invalid ProjectionExpression: Attribute name is a reserved keyword; " + "reserved keyword: COMMENT" +) +UNUSED_NAME = "Value provided in ExpressionAttributeNames unused in expressions: keys: {#unused}" +MISSING_PROJECTION_NAME = ( + "Invalid ProjectionExpression: An expression attribute name used in the " + "document path is not defined; attribute name: #missing" +) +MISSING_CONDITION_NAME = ( + "Invalid ConditionExpression: An expression attribute name used in the " + "document path is not defined; attribute name: #missing" +) +MISSING_CONDITION_VALUE = ( + "Invalid ConditionExpression: An expression attribute value used in " + "expression is not defined; attribute value: :missing" +) +# The service's full text is 'Invalid UpdateExpression: Syntax error; token: +# "", near: "+"'. ExtendDB's parser wording differs today (tracked under +# #108 message text); this file asserts the class and prefix, which is what +# the ordering guarantees. +UPDATE_SYNTAX_PREFIX = "Invalid UpdateExpression:" +# A key that does not fit the schema: a Delete, Update, ConditionCheck, or Get +# key gets the first text; a Put item keeps the single-item wording. +KEY_SCHEMA_REASONS = { + "The provided key element does not match the schema", + "One or more parameter values were invalid: Type mismatch for key pk expected: S actual: N", +} + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + """Composite-key table (pk S, sk S) with one item.""" + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": "p"}, "sk": {"S": "s"}, "note": {"S": "old"}}, + ) + yield name + + +@pytest.fixture(scope="module") +def missing_table() -> str: + return f"extenddb-missing-{uuid.uuid4().hex[:12]}" + + +GOOD_KEY = {"pk": {"S": "p"}, "sk": {"S": "s"}} +OTHER_KEY = {"pk": {"S": "other"}, "sk": {"S": "s"}} +BAD_TYPE_KEY = {"pk": {"N": "1"}, "sk": {"S": "s"}} + + +def _get(table_name: str, key: dict, **extra) -> dict: + return {"Get": {"TableName": table_name, "Key": key, **extra}} + + +def _expect(func, code: str, message: str | None = None): + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == code, err + if message is not None: + assert err["Message"] == message, err + return exc_info.value.response + + +def _expect_prefix(func, prefix: str): + """For syntax errors, whose inner wording ExtendDB still phrases its own way.""" + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"].startswith(prefix), err + + +def _expect_cancelled(func, reasons: list[str | None]): + """Assert a cancellation whose per-item codes are exactly `reasons`. + + An item that did not fail carries the literal code "None" and no message. + """ + response = _expect(func, "TransactionCanceledException") + codes = [r.get("Code") for r in response["CancellationReasons"]] + assert codes == ["None" if r is None else r for r in reasons], response["CancellationReasons"] + for reason in response["CancellationReasons"]: + if reason["Code"] == "ValidationError": + assert reason["Message"] in KEY_SCHEMA_REASONS, reason + expected_text = ", ".join("None" if r is None else r for r in reasons) + assert response["Error"]["Message"] == ( + "Transaction cancelled, please refer cancellation reasons for specific " + f"reasons [{expected_text}]" + ) + + +class TestTransactGetItemsOrder: + def test_projection_grammar_beats_duplicate_keys(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_get_items( + TransactItems=[ + _get(table, GOOD_KEY, ProjectionExpression="COMMENT"), + _get(table, GOOD_KEY), + _get(table, GOOD_KEY), + ] + ), + "ValidationException", + RESERVED_PROJECTION, + ) + + def test_later_projection_grammar_beats_earlier_duplicate_keys(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_get_items( + TransactItems=[ + _get(table, GOOD_KEY), + _get(table, GOOD_KEY), + _get(table, OTHER_KEY, ProjectionExpression="COMMENT"), + ] + ), + "ValidationException", + RESERVED_PROJECTION, + ) + + def test_unused_projection_name_beats_duplicate_keys(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_get_items( + TransactItems=[ + _get( + table, + GOOD_KEY, + ProjectionExpression="#ok", + ExpressionAttributeNames={"#ok": "ok", "#unused": "COMMENT"}, + ), + _get(table, GOOD_KEY), + ] + ), + "ValidationException", + UNUSED_NAME, + ) + + def test_undefined_projection_name_beats_duplicate_keys(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_get_items( + TransactItems=[ + _get(table, GOOD_KEY, ProjectionExpression="#missing"), + _get(table, GOOD_KEY), + ] + ), + "ValidationException", + MISSING_PROJECTION_NAME, + ) + + def test_projection_grammar_beats_later_missing_table(self, dynamodb_client, table, missing_table): + _expect( + lambda: dynamodb_client.transact_get_items( + TransactItems=[ + _get(table, GOOD_KEY, ProjectionExpression="COMMENT"), + _get(missing_table, GOOD_KEY), + ] + ), + "ValidationException", + RESERVED_PROJECTION, + ) + + def test_missing_table_beats_duplicate_keys_either_order(self, dynamodb_client, table, missing_table): + for items in ( + [_get(missing_table, GOOD_KEY), _get(table, GOOD_KEY), _get(table, GOOD_KEY)], + [_get(table, GOOD_KEY), _get(table, GOOD_KEY), _get(missing_table, GOOD_KEY)], + ): + _expect( + lambda items=items: dynamodb_client.transact_get_items(TransactItems=items), + "ResourceNotFoundException", + "Requested resource not found", + ) + + def test_key_schema_beats_duplicate_keys_either_order(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_get_items( + TransactItems=[ + _get(table, GOOD_KEY), + _get(table, GOOD_KEY), + _get(table, BAD_TYPE_KEY), + ] + ), + [None, None, "ValidationError"], + ) + _expect_cancelled( + lambda: dynamodb_client.transact_get_items( + TransactItems=[ + _get(table, BAD_TYPE_KEY), + _get(table, GOOD_KEY), + _get(table, GOOD_KEY), + ] + ), + ["ValidationError", None, None], + ) + + def test_duplicate_keys_alone_still_refused(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_get_items( + TransactItems=[_get(table, GOOD_KEY), _get(table, GOOD_KEY)] + ), + "ValidationException", + DUPLICATE, + ) + + +class TestTransactWriteItemsOrder: + def test_undefined_condition_value_is_top_level(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "#c = :v", + "ExpressionAttributeNames": {"#c": "COMMENT"}, + } + } + ] + ), + "ValidationException", + "Invalid ConditionExpression: An expression attribute value used in " + "expression is not defined; attribute value: :v", + ) + + def test_undefined_value_beats_later_false_condition(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "note = :missing", + } + }, + { + "ConditionCheck": { + "TableName": table, + "Key": {"pk": {"S": "q"}, "sk": {"S": "s"}}, + "ConditionExpression": "note = :v", + "ExpressionAttributeValues": {":v": {"S": "different"}}, + } + }, + ] + ), + "ValidationException", + MISSING_CONDITION_VALUE, + ) + + def test_undefined_value_beats_key_schema_mismatch(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "note = :missing", + } + }, + {"Delete": {"TableName": table, "Key": BAD_TYPE_KEY}}, + ] + ), + "ValidationException", + MISSING_CONDITION_VALUE, + ) + + def test_undefined_name_beats_put_key_schema_mismatch(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "Delete": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "#missing = :v", + "ExpressionAttributeValues": {":v": {"S": "old"}}, + } + }, + { + "Put": { + "TableName": table, + "Item": {"pk": {"N": "1"}, "sk": {"S": "s"}, "note": {"S": "x"}}, + } + }, + ] + ), + "ValidationException", + MISSING_CONDITION_NAME, + ) + + def test_unused_name_beats_earlier_missing_table(self, dynamodb_client, table, missing_table): + _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "TableName": missing_table, + "Key": GOOD_KEY, + "ConditionExpression": "attribute_exists(pk)", + } + }, + { + "Put": { + "TableName": table, + "Item": {"pk": {"S": "p"}, "sk": {"S": "s2"}, "note": {"S": "new"}}, + "ConditionExpression": "attribute_not_exists(pk)", + "ExpressionAttributeNames": {"#unused": "COMMENT"}, + } + }, + ] + ), + "ValidationException", + UNUSED_NAME, + ) + + def test_update_grammar_beats_missing_table_either_order(self, dynamodb_client, table, missing_table): + bad_update = { + "Update": { + "TableName": table, + "Key": GOOD_KEY, + "UpdateExpression": "SET note = :v +", + "ExpressionAttributeValues": {":v": {"N": "1"}}, + } + } + missing = { + "ConditionCheck": { + "TableName": missing_table, + "Key": GOOD_KEY, + "ConditionExpression": "attribute_exists(pk)", + } + } + for items in ([bad_update, missing], [missing, bad_update]): + response = _expect( + lambda items=items: dynamodb_client.transact_write_items(TransactItems=items), + "ValidationException", + ) + assert response["Error"]["Message"].startswith(UPDATE_SYNTAX_PREFIX), response["Error"] + + def test_key_schema_mismatch_suppresses_condition_evaluation(self, dynamodb_client, table): + false_condition = { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "note = :v", + "ExpressionAttributeValues": {":v": {"S": "different"}}, + } + } + bad_delete = {"Delete": {"TableName": table, "Key": BAD_TYPE_KEY}} + _expect_cancelled( + lambda: dynamodb_client.transact_write_items(TransactItems=[false_condition, bad_delete]), + [None, "ValidationError"], + ) + _expect_cancelled( + lambda: dynamodb_client.transact_write_items(TransactItems=[bad_delete, false_condition]), + ["ValidationError", None], + ) + + def test_key_schema_mismatch_suppresses_runtime_update_error(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "Update": { + "TableName": table, + "Key": GOOD_KEY, + "UpdateExpression": "SET n = n + note", + } + }, + {"Delete": {"TableName": table, "Key": BAD_TYPE_KEY}}, + ] + ), + [None, "ValidationError"], + ) + + def test_key_schema_mismatch_beats_duplicate_items(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "Put": { + "TableName": table, + "Item": {"pk": {"S": "p"}, "sk": {"S": "s"}, "note": {"S": "new"}}, + } + }, + { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "attribute_exists(pk)", + } + }, + {"Delete": {"TableName": table, "Key": BAD_TYPE_KEY}}, + ] + ), + [None, None, "ValidationError"], + ) + + def test_duplicate_items_alone_still_refused(self, dynamodb_client, table): + _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + {"Put": {"TableName": table, "Item": {"pk": {"S": "p"}, "sk": {"S": "s"}}}}, + { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "attribute_exists(pk)", + } + }, + ] + ), + "ValidationException", + DUPLICATE, + ) + + def test_key_mismatch_reason_text_follows_the_operation(self, dynamodb_client, table): + """A Put keeps the single-item wording; a Delete gets the key-element wording.""" + response = _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + {"Put": {"TableName": table, "Item": {"pk": {"N": "1"}, "sk": {"S": "s"}}}}, + ] + ), + "TransactionCanceledException", + ) + assert response["CancellationReasons"][0]["Message"] == ( + "One or more parameter values were invalid: Type mismatch for key pk expected: S actual: N" + ) + response = _expect( + lambda: dynamodb_client.transact_write_items( + TransactItems=[{"Delete": {"TableName": table, "Key": BAD_TYPE_KEY}}] + ), + "TransactionCanceledException", + ) + assert response["CancellationReasons"][0]["Message"] == ( + "The provided key element does not match the schema" + ) + + def test_false_condition_alone_still_cancels(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_write_items( + TransactItems=[ + { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "note = :v", + "ExpressionAttributeValues": {":v": {"S": "different"}}, + } + } + ] + ), + ["ConditionalCheckFailed"], + ) + + +class TestTransactWriteItemsResponseShape: + """Measured against Amazon DynamoDB on 2026-10-14.""" + + def test_item_collection_metrics_is_an_empty_map_when_requested_without_lsi(self, dynamodb_client, table): + resp = dynamodb_client.transact_write_items( + ReturnConsumedCapacity="TOTAL", + ReturnItemCollectionMetrics="SIZE", + TransactItems=[{"Put": {"TableName": table, "Item": {"pk": {"S": "icm"}, "sk": {"S": "s"}}}}], + ) + assert resp["ItemCollectionMetrics"] == {} + assert [c["TableName"] for c in resp["ConsumedCapacity"]] == [table] + + def test_item_collection_metrics_absent_when_not_requested(self, dynamodb_client, table): + resp = dynamodb_client.transact_write_items( + TransactItems=[{"Put": {"TableName": table, "Item": {"pk": {"S": "icm2"}, "sk": {"S": "s"}}}}], + ) + assert "ItemCollectionMetrics" not in resp + assert "ConsumedCapacity" not in resp + + +BIG_ITEM = {"pk": {"S": "big"}, "sk": {"S": "s"}, "blob": {"S": "x" * 410_000}} +ITEM_SIZE = "Item size has exceeded the maximum allowed size" +UNDEF_UPDATE_VALUE = ( + "Invalid UpdateExpression: An expression attribute value used in " + "expression is not defined; attribute value: :q" +) + + +def _put(table_name: str, item: dict, **extra) -> dict: + return {"Put": {"TableName": table_name, "Item": item, **extra}} + + +def _update(table_name: str, key: dict, expression: str = "SET a = :v", values: dict | None = None) -> dict: + return {"Update": {"TableName": table_name, "Key": key, "UpdateExpression": expression, + "ExpressionAttributeValues": values if values is not None else {":v": {"N": "1"}}}} + + +class TestKeySchemaFirstOffenderOnly: + """Measured 2026-10-14: only the first item whose key fails carries ValidationError.""" + + def test_write_items_first_and_third(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + _update(table, BAD_TYPE_KEY), _update(table, GOOD_KEY), _update(table, {"pk": {"N": "3"}, "sk": {"S": "s"}})]), + ["ValidationError", None, None], + ) + + def test_write_items_all_three(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + _update(table, BAD_TYPE_KEY), _update(table, {"pk": {"N": "2"}, "sk": {"S": "s"}}), + _update(table, {"pk": {"N": "3"}, "sk": {"S": "s"}})]), + ["ValidationError", None, None], + ) + + def test_write_items_middle_only(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + _update(table, GOOD_KEY), _update(table, BAD_TYPE_KEY), _update(table, OTHER_KEY)]), + [None, "ValidationError", None], + ) + + def test_false_condition_between_two_offenders_is_not_evaluated(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + _update(table, BAD_TYPE_KEY), + {"ConditionCheck": {"TableName": table, "Key": GOOD_KEY, "ConditionExpression": "note = :z", + "ExpressionAttributeValues": {":z": {"S": "never"}}}}, + _update(table, {"pk": {"N": "3"}, "sk": {"S": "s"}})]), + ["ValidationError", None, None], + ) + + def test_get_items(self, dynamodb_client, table): + _expect_cancelled( + lambda: dynamodb_client.transact_get_items(TransactItems=[ + _get(table, BAD_TYPE_KEY), _get(table, GOOD_KEY), _get(table, {"pk": {"N": "3"}, "sk": {"S": "s"}})]), + ["ValidationError", None, None], + ) + _expect_cancelled( + lambda: dynamodb_client.transact_get_items(TransactItems=[ + _get(table, BAD_TYPE_KEY), _get(table, {"pk": {"N": "2"}, "sk": {"S": "s"}}), + _get(table, {"pk": {"N": "3"}, "sk": {"S": "s"}})]), + ["ValidationError", None, None], + ) + + +class TestItemSizePhase: + """The 400 KB item size follows the item's own expressions and precedes the + next item's, the table lookup, the key schema, and duplicates (measured + 2026-10-14).""" + + def test_size_follows_the_same_items_expressions(self, dynamodb_client, table): + _expect_prefix( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, BIG_ITEM, ConditionExpression="attribute_exists(")]), + "Invalid ConditionExpression:", + ) + _expect( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, BIG_ITEM, ConditionExpression="a = :q")]), + "ValidationException", MISSING_CONDITION_VALUE.replace(":missing", ":q"), + ) + _expect( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, BIG_ITEM, ConditionExpression="a = :v", + ExpressionAttributeValues={":v": {"N": "1"}, ":u": {"N": "1"}})]), + "ValidationException", "Value provided in ExpressionAttributeValues unused in expressions: keys: {:u}", + ) + + def test_size_precedes_the_next_items_expressions(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, BIG_ITEM), _update(table, GOOD_KEY, "SET a = :q")]), "ValidationException", ITEM_SIZE) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _update(table, GOOD_KEY, "SET a = :q"), _put(table, BIG_ITEM)]), "ValidationException", UNDEF_UPDATE_VALUE) + _expect_prefix(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _update(table, GOOD_KEY, "SET a = "), _put(table, BIG_ITEM)]), UPDATE_SYNTAX_PREFIX) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, BIG_ITEM), _update(table, GOOD_KEY, "SET a = ")]), "ValidationException", ITEM_SIZE) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, BIG_ITEM), + _put(table, GOOD_KEY, ConditionExpression="a = :v", + ExpressionAttributeValues={":v": {"N": "1"}, ":u": {"N": "1"}})]), + "ValidationException", ITEM_SIZE) + + def test_size_precedes_table_lookup_key_schema_and_duplicates(self, dynamodb_client, table, missing_table): + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(missing_table, GOOD_KEY), _put(table, BIG_ITEM)]), "ValidationException", ITEM_SIZE) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _update(table, BAD_TYPE_KEY), _put(table, BIG_ITEM)]), "ValidationException", ITEM_SIZE) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, GOOD_KEY), _put(table, GOOD_KEY), _put(table, BIG_ITEM)]), "ValidationException", ITEM_SIZE) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, {"pk": {"N": "1"}, "sk": {"S": "s"}, "blob": {"S": "x" * 410_000}})]), + "ValidationException", ITEM_SIZE) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=[ + _put(table, {"zz": {"S": "1"}, "blob": {"S": "x" * 410_000}})]), "ValidationException", ITEM_SIZE) + + def test_request_total_follows_table_lookup_and_precedes_key_schema_and_duplicates( + self, dynamodb_client, table, missing_table + ): + # The service's message carries its own serialized payload size + # ("Transaction payload size cannot exceed 4MB. Payload Size: N"); + # ExtendDB states the limit without a size. Both name the 4 MB limit. + mids = [_put(table, {"pk": {"S": f"m{i}"}, "sk": {"S": "s"}, "blob": {"S": "y" * 390_000}}) for i in range(11)] + + def expect_total(items): + with pytest.raises(ClientError) as exc_info: + dynamodb_client.transact_write_items(TransactItems=items) + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert "4MB" in err["Message"] or "4 MB" in err["Message"], err + + expect_total(mids) + expect_total(mids + [_update(table, BAD_TYPE_KEY)]) + expect_total(mids + [_put(table, GOOD_KEY), _put(table, GOOD_KEY)]) + expect_total(mids + [{"ConditionCheck": {"TableName": table, "Key": GOOD_KEY, "ConditionExpression": "note = :z", + "ExpressionAttributeValues": {":z": {"S": "never"}}}}]) + _expect(lambda: dynamodb_client.transact_write_items(TransactItems=mids + [_put(missing_table, GOOD_KEY)]), + "ResourceNotFoundException") + _expect_prefix(lambda: dynamodb_client.transact_write_items(TransactItems=mids + [_update(table, GOOD_KEY, "SET a = ")]), + UPDATE_SYNTAX_PREFIX) + _expect_prefix(lambda: dynamodb_client.transact_write_items(TransactItems=[_update(table, GOOD_KEY, "SET a = ")] + mids), + UPDATE_SYNTAX_PREFIX) diff --git a/tests/test_transact_value_nesting.py b/tests/test_transact_value_nesting.py new file mode 100644 index 000000000..078b8c5a8 --- /dev/null +++ b/tests/test_transact_value_nesting.py @@ -0,0 +1,183 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Nesting limits on the values of TransactWriteItems. + +Measured against Amazon DynamoDB on 2026-10-27 (two samples, identical +bytes). A value is counted in `M` or `L` wrappers around a leaf. + + * an ExpressionAttributeValues value, used or not, may hold 32 wrappers and + is refused at 33, as a top-level ValidationException worded + "ExpressionAttributeValues contains invalid value: ... for key :v"; + * a Put item attribute may hold 31 wrappers and is refused at 32, as a + top-level ValidationException; + * a value an Update stores with SET is an attribute of the item, so 32 + wrappers pass the first rule and break the item limit: a ValidationError + cancellation reason "Nesting Levels have exceeded supported limits", + reported after the lookup, the 4 MB total, the key pass and the duplicate + check. + +Dual-target: runs against the service and against ExtendDB on every backend. +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +ITEM_NESTING = ( + "Nesting Levels have exceeded supported limits: " + "Attributes in the item have nested levels beyond supported limit" +) +STORED_NESTING = "Nesting Levels have exceeded supported limits" +GOOD_KEY = {"pk": {"S": "p"}, "sk": {"S": "s"}} +OTHER_KEY = {"pk": {"S": "other"}, "sk": {"S": "s"}} +BAD_KEY = {"pk": {"N": "1"}, "sk": {"S": "s"}} +KEY_MISMATCH = "The provided key element does not match the schema" +KEY_UPDATE = ( + "One or more parameter values were invalid: Cannot update attribute pk. " + "This attribute is part of the key" +) +EMPTY_KEY = ( + "One or more parameter values are not valid. The AttributeValue for a key " + "attribute cannot contain an empty string value. Key: pk" +) +DUPLICATE = "Transaction request cannot include multiple operations on one item" + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + dynamodb_client.put_item(TableName=name, Item=GOOD_KEY) + yield name + + +@pytest.fixture(scope="module") +def missing_table() -> str: + return f"extenddb-missing-{uuid.uuid4().hex[:12]}" + + +def _deep(wrappers: int) -> dict: + value: dict = {"S": "leaf"} + for _ in range(wrappers): + value = {"M": {"a": value}} + return value + + +def _set_value(table_name, wrappers, key=GOOD_KEY): + return { + "Update": { + "TableName": table_name, + "Key": key, + "UpdateExpression": "SET d = :v", + "ExpressionAttributeValues": {":v": _deep(wrappers)}, + } + } + + +def _fail(client, items, code): + with pytest.raises(ClientError) as exc_info: + client.transact_write_items(TransactItems=items) + err = exc_info.value.response["Error"] + assert err["Code"] == code, err + return exc_info.value.response + + +def _top(client, items, message): + assert _fail(client, items, "ValidationException")["Error"]["Message"] == message + + +def _reasons(client, items): + """(Code, Message) of every cancellation reason; an item that did not fail has no message.""" + got = _fail(client, items, "TransactionCanceledException")["CancellationReasons"] + return [(r["Code"], None if r["Code"] == "None" else r.get("Message")) for r in got] + + +class TestExpressionAttributeValues: + def test_a_condition_check_value_32_deep_is_accepted_and_33_is_refused(self, dynamodb_client, table): + def check(wrappers): + return { + "ConditionCheck": { + "TableName": table, + "Key": GOOD_KEY, + "ConditionExpression": "attribute_exists(pk) OR :d = :d", + "ExpressionAttributeValues": {":d": _deep(wrappers)}, + } + } + + dynamodb_client.transact_write_items(TransactItems=[check(32)]) + _top( + dynamodb_client, + [check(33)], + f"ExpressionAttributeValues contains invalid value: {ITEM_NESTING} for key :d", + ) + + def test_a_value_33_deep_is_refused_before_the_lookup_in_either_order( + self, dynamodb_client, table, missing_table + ): + message = f"ExpressionAttributeValues contains invalid value: {ITEM_NESTING} for key :v" + absent = {"Put": {"TableName": missing_table, "Item": GOOD_KEY}} + _top(dynamodb_client, [_set_value(table, 33), absent], message) + _top(dynamodb_client, [absent, _set_value(table, 33)], message) + + +class TestStoredValues: + def test_a_put_attribute_31_deep_is_stored_and_32_is_refused(self, dynamodb_client, table): + def put(wrappers): + item = {"pk": {"S": f"put{wrappers}"}, "sk": {"S": "s"}, "d": _deep(wrappers)} + return {"Put": {"TableName": table, "Item": item}} + + dynamodb_client.transact_write_items(TransactItems=[put(31)]) + _top(dynamodb_client, [put(32)], ITEM_NESTING) + + def test_an_update_stores_31_deep_and_cancels_at_32(self, dynamodb_client, table): + dynamodb_client.transact_write_items(TransactItems=[_set_value(table, 31)]) + assert _reasons(dynamodb_client, [_set_value(table, 32)]) == [("ValidationError", STORED_NESTING)] + + def test_every_offending_update_is_flagged(self, dynamodb_client, table): + items = [_set_value(table, 32), _set_value(table, 32, OTHER_KEY)] + assert _reasons(dynamodb_client, items) == [("ValidationError", STORED_NESTING)] * 2 + + def test_the_reason_comes_after_every_other_phase(self, dynamodb_client, table, missing_table): + stored = _set_value(table, 32) + twin = {"Put": {"TableName": table, "Item": GOOD_KEY}} + mismatch = {"Delete": {"TableName": table, "Key": BAD_KEY}} + key_update = { + "Update": { + "TableName": table, "Key": OTHER_KEY, "UpdateExpression": "SET pk = :v", + "ExpressionAttributeValues": {":v": {"S": "z"}}, + } + } + empty_key = { + "ConditionCheck": { + "TableName": table, "Key": {"pk": {"S": ""}, "sk": {"S": "a"}}, + "ConditionExpression": "attribute_exists(pk)", + } + } + absent = {"Put": {"TableName": missing_table, "Item": GOOD_KEY}} + for items in ([stored, absent], [absent, stored]): + _fail(dynamodb_client, items, "ResourceNotFoundException") + fail = ("ValidationError", KEY_MISMATCH) + assert _reasons(dynamodb_client, [stored, mismatch]) == [("None", None), fail] + assert _reasons(dynamodb_client, [mismatch, stored]) == [fail, ("None", None)] + fail = ("ValidationError", KEY_UPDATE) + assert _reasons(dynamodb_client, [stored, key_update]) == [("None", None), fail] + assert _reasons(dynamodb_client, [key_update, stored]) == [fail, ("None", None)] + for items in ([stored, empty_key], [empty_key, stored]): + _top(dynamodb_client, items, EMPTY_KEY) + for items in ([stored, twin], [twin, stored]): + _top(dynamodb_client, items, DUPLICATE) diff --git a/tests/test_transact_write_phase_order.py b/tests/test_transact_write_phase_order.py new file mode 100644 index 000000000..fcc7686f5 --- /dev/null +++ b/tests/test_transact_write_phase_order.py @@ -0,0 +1,282 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Where four per-item checks sit in TransactWriteItems validation. + +Amazon DynamoDB reports them in two places (measured 2026-10-27): + + * before any table is looked up, one item after another, in the pass that + also checks the expressions: the nesting of every ExpressionAttributeValues + value of the item, and for a Put the attribute name size, then the item + size, then the item nesting; + * after every table is looked up and after the 4 MB total, in the key pass, + one item after another: a key that does not fit the schema, then a key + attribute named in an Update (both a ValidationError cancellation reason + on that item and None on the others), then an empty key value (a + top-level ValidationException). + +The attribute name size is worded differently by the two targets, so only its +class and the words "attribute name" are compared. + +Dual-target: runs against the service and against ExtendDB on every backend. +""" + +from __future__ import annotations + +import uuid + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +NESTING = ( + "Nesting Levels have exceeded supported limits: " + "Attributes in the item have nested levels beyond supported limit" +) +VALUE_NESTING = f"ExpressionAttributeValues contains invalid value: {NESTING} for key :v" +EMPTY_KEY = ( + "One or more parameter values are not valid. The AttributeValue for a key " + "attribute cannot contain an empty string value. Key: pk" +) +KEY_UPDATE = ( + "One or more parameter values were invalid: Cannot update attribute pk. " + "This attribute is part of the key" +) +KEY_UPDATE_SK = KEY_UPDATE.replace("attribute pk.", "attribute sk.") +KEY_MISMATCH = "The provided key element does not match the schema" +PUT_MISMATCH_SK = "One or more parameter values were invalid: Type mismatch for key sk expected: S actual: N" +ITEM_SIZE = "Item size has exceeded the maximum allowed size" +UNDEFINED_VALUE = ( + "Invalid ConditionExpression: An expression attribute value used in " + "expression is not defined; attribute value: :q" +) +NOT_FOUND = "Requested resource not found" + +GOOD_KEY = {"pk": {"S": "p"}, "sk": {"S": "s"}} +BAD_KEY = {"pk": {"N": "1"}, "sk": {"S": "s"}} +EMPTY_PK = {"pk": {"S": ""}, "sk": {"S": "a"}} + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table( + dynamodb_client, + attribute_definitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "S"}, + ], + key_schema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + ) as name: + dynamodb_client.put_item(TableName=name, Item={**GOOD_KEY, "note": {"S": "old"}}) + yield name + + +@pytest.fixture(scope="module") +def missing_table() -> str: + return f"extenddb-missing-{uuid.uuid4().hex[:12]}" + + +def _deep(levels: int) -> dict: + root: dict = {"L": []} + cur = root + for _ in range(levels - 1): + nxt: dict = {"L": []} + cur["L"].append(nxt) + cur = nxt + return root + + +def _put(table_name, item, **extra): + return {"Put": {"TableName": table_name, "Item": item, **extra}} + + +def _delete(table_name, key): + return {"Delete": {"TableName": table_name, "Key": key}} + + +def _update(table_name, key, expression, values=None): + body = {"TableName": table_name, "Key": key, "UpdateExpression": expression} + if values is not None: + body["ExpressionAttributeValues"] = values + return {"Update": body} + + +def _check(table_name, key, expression): + return {"ConditionCheck": {"TableName": table_name, "Key": key, "ConditionExpression": expression}} + + +def _fail(client, items, code): + with pytest.raises(ClientError) as exc_info: + client.transact_write_items(TransactItems=items) + err = exc_info.value.response["Error"] + assert err["Code"] == code, err + return exc_info.value.response + + +def _top(client, items, message): + assert _fail(client, items, "ValidationException")["Error"]["Message"] == message + + +def _cancelled(client, items, reasons): + """`reasons` holds None for an item that did not fail and a message otherwise.""" + response = _fail(client, items, "TransactionCanceledException") + got = response["CancellationReasons"] + assert [r["Code"] for r in got] == ["None" if r is None else "ValidationError" for r in reasons], got + assert [r["Message"] for r in got if r["Code"] == "ValidationError"] == [r for r in reasons if r], got + + +def _both_orders(client, defect, other, check): + check(client, [defect, other]) + check(client, [other, defect]) + + +def _nested_put(table): + return _put(table, {"pk": {"S": "nd"}, "sk": {"S": "a"}, "d": _deep(40)}) + + +def _long_name_put(table): + return _put(table, {"pk": {"S": "an"}, "sk": {"S": "a"}, "a" * 65536: {"S": "v"}}) + + +def _deep_value_update(table): + return _update(table, GOOD_KEY, "SET d = :v", {":v": _deep(40)}) + + +def _attribute_name_error(client, items): + message = _fail(client, items, "ValidationException")["Error"]["Message"] + assert "attribute name" in message.lower(), message + + +class TestBeforeTheLookup: + def test_item_nesting_beats_a_missing_table_in_either_order(self, dynamodb_client, table, missing_table): + absent = _put(missing_table, {"pk": {"S": "x"}, "sk": {"S": "a"}}) + _both_orders(dynamodb_client, _nested_put(table), absent, lambda c, i: _top(c, i, NESTING)) + + def test_value_nesting_beats_a_missing_table_in_either_order(self, dynamodb_client, table, missing_table): + absent = _put(missing_table, {"pk": {"S": "x"}, "sk": {"S": "a"}}) + check = lambda c, i: _top(c, i, VALUE_NESTING) # noqa: E731 + _both_orders(dynamodb_client, _deep_value_update(table), absent, check) + used_in_a_condition = { + "ConditionCheck": { + "TableName": table, "Key": GOOD_KEY, "ConditionExpression": "note = :v", + "ExpressionAttributeValues": {":v": _deep(40)}, + } + } + _both_orders(dynamodb_client, used_in_a_condition, absent, check) + + def test_attribute_name_size_beats_a_missing_table_in_either_order(self, dynamodb_client, table, missing_table): + absent = _put(missing_table, {"pk": {"S": "x"}, "sk": {"S": "a"}}) + _both_orders(dynamodb_client, _long_name_put(table), absent, _attribute_name_error) + + def test_the_checks_follow_item_order_against_an_expression_error(self, dynamodb_client, table): + undefined = _check(table, GOOD_KEY, "note = :q") + for defect, check in ( + (_nested_put(table), lambda c, i: _top(c, i, NESTING)), + (_deep_value_update(table), lambda c, i: _top(c, i, VALUE_NESTING)), + (_long_name_put(table), _attribute_name_error), + ): + check(dynamodb_client, [defect, undefined]) + _top(dynamodb_client, [undefined, defect], UNDEFINED_VALUE) + + def test_a_put_reports_name_size_then_item_size_then_nesting(self, dynamodb_client, table): + pad = {"S": "x" * 410_000} + name_and_size = {"pk": {"S": "w"}, "sk": {"S": "a"}, "a" * 65536: {"S": "v"}, "pad": pad} + nesting_and_size = {"pk": {"S": "w"}, "sk": {"S": "a"}, "d": _deep(40), "pad": pad} + nesting_and_name = {"pk": {"S": "w"}, "sk": {"S": "a"}, "d": _deep(40), "a" * 65536: {"S": "v"}} + _attribute_name_error(dynamodb_client, [_put(table, name_and_size)]) + _top(dynamodb_client, [_put(table, nesting_and_size)], ITEM_SIZE) + _attribute_name_error(dynamodb_client, [_put(table, nesting_and_name)]) + + def test_inside_one_item_the_expressions_come_first_and_the_value_nesting_before_them( + self, dynamodb_client, table + ): + deep_item = {"pk": {"S": "w"}, "sk": {"S": "a"}, "d": _deep(40)} + _top( + dynamodb_client, + [_put(table, deep_item, ConditionExpression="note = :q")], + UNDEFINED_VALUE, + ) + _top(dynamodb_client, [_update(table, GOOD_KEY, "SET d = ", {":v": _deep(40)})], VALUE_NESTING) + _top( + dynamodb_client, + [_update(table, GOOD_KEY, "SET d = :v", {":v": _deep(40), ":u": {"N": "1"}})], + VALUE_NESTING, + ) + + +class TestKeyPass: + @pytest.mark.parametrize("kind", ["check", "put", "delete", "update"]) + def test_an_empty_key_value_waits_for_every_lookup(self, dynamodb_client, table, missing_table, kind): + defect = { + "check": _check(table, EMPTY_PK, "attribute_exists(pk)"), + "put": _put(table, {"pk": {"S": ""}, "sk": {"S": "a"}}), + "delete": _delete(table, EMPTY_PK), + "update": _update(table, EMPTY_PK, "SET x = :v", {":v": {"S": "1"}}), + }[kind] + absent = _put(missing_table, {"pk": {"S": "x"}, "sk": {"S": "a"}}) + _both_orders(dynamodb_client, defect, absent, lambda c, i: _expect_not_found(c, i)) + _top(dynamodb_client, [defect], EMPTY_KEY) + + def test_a_key_attribute_update_waits_for_every_lookup(self, dynamodb_client, table, missing_table): + absent = _put(missing_table, {"pk": {"S": "x"}, "sk": {"S": "a"}}) + key_update = _update(table, GOOD_KEY, "SET pk = :v", {":v": {"S": "z"}}) + _both_orders(dynamodb_client, key_update, absent, lambda c, i: _expect_not_found(c, i)) + + def test_a_key_attribute_update_is_a_cancellation_reason(self, dynamodb_client, table): + _cancelled(dynamodb_client, [_update(table, GOOD_KEY, "SET pk = :v", {":v": {"S": "z"}})], [KEY_UPDATE]) + _cancelled(dynamodb_client, [_update(table, GOOD_KEY, "REMOVE sk")], [KEY_UPDATE_SK]) + + def test_the_request_total_beats_the_key_pass(self, dynamodb_client, table): + big = [ + _put(table, {"pk": {"S": f"big{i}"}, "sk": {"S": "a"}, "pad": {"S": "x" * 389_000}}) + for i in range(11) + ] + for items in ( + [_check(table, EMPTY_PK, "attribute_exists(pk)")] + big, + big + [_update(table, GOOD_KEY, "SET pk = :v", {":v": {"S": "z"}})], + ): + message = _fail(dynamodb_client, items, "ValidationException")["Error"]["Message"] + assert "4MB" in message or "4 MB" in message, message + + def test_items_are_examined_in_order(self, dynamodb_client, table): + empty = _check(table, EMPTY_PK, "attribute_exists(pk)") + key_update = _update(table, GOOD_KEY, "SET pk = :v", {":v": {"S": "z"}}) + mismatch = _delete(table, BAD_KEY) + _cancelled(dynamodb_client, [key_update, empty], [KEY_UPDATE, None]) + _top(dynamodb_client, [empty, key_update], EMPTY_KEY) + _top(dynamodb_client, [empty, mismatch], EMPTY_KEY) + _cancelled(dynamodb_client, [mismatch, empty], [KEY_MISMATCH, None]) + _cancelled(dynamodb_client, [key_update, mismatch], [KEY_UPDATE, None]) + _cancelled(dynamodb_client, [mismatch, key_update], [KEY_MISMATCH, None]) + + def test_inside_one_item_a_mismatch_beats_a_key_update_beats_an_empty_value(self, dynamodb_client, table): + _cancelled(dynamodb_client, [_delete(table, {"pk": {"S": ""}, "sk": {"N": "1"}})], [KEY_MISMATCH]) + _cancelled( + dynamodb_client, + [_update(table, {"pk": {"N": "1"}, "sk": {"S": "a"}}, "SET pk = :v", {":v": {"S": "z"}})], + [KEY_MISMATCH], + ) + _cancelled( + dynamodb_client, [_update(table, EMPTY_PK, "SET pk = :v", {":v": {"S": "z"}})], [KEY_UPDATE] + ) + _cancelled( + dynamodb_client, [_put(table, {"pk": {"S": ""}, "sk": {"N": "1"}})], [PUT_MISMATCH_SK] + ) + + def test_the_key_pass_beats_duplicate_items(self, dynamodb_client, table): + twin = _put(table, {"pk": {"S": "dup"}, "sk": {"S": "a"}}) + key_update = _update(table, GOOD_KEY, "SET pk = :v", {":v": {"S": "z"}}) + empty = _check(table, EMPTY_PK, "attribute_exists(pk)") + _cancelled(dynamodb_client, [key_update, twin, twin], [KEY_UPDATE, None, None]) + _cancelled(dynamodb_client, [twin, twin, key_update], [None, None, KEY_UPDATE]) + _top(dynamodb_client, [empty, twin, twin], EMPTY_KEY) + _top(dynamodb_client, [twin, twin, empty], EMPTY_KEY) + + +def _expect_not_found(client, items): + response = _fail(client, items, "ResourceNotFoundException") + assert response["Error"]["Message"] == NOT_FOUND diff --git a/tests/test_update_expression_rules.py b/tests/test_update_expression_rules.py new file mode 100644 index 000000000..ac1874ebf --- /dev/null +++ b/tests/test_update_expression_rules.py @@ -0,0 +1,303 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""UpdateExpression request-level rules, and the single-item write prefix. + +Amazon DynamoDB rejects these before it reads the table: + + - a repeated section (`SET ... SET ...`, likewise REMOVE, ADD, DELETE), + reported after the whole expression has parsed, so a syntax error + anywhere wins over it, and after that section's own actions have been + checked (test_expression_semantics.py pins that order); + - `if_not_exists` and `list_append` with other than two operands; + - `if_not_exists` whose first operand is not a document path; + - `list_append` as an operand of another function + (`list_append(if_not_exists(l, :e), :v)` is allowed; the reverse is not); + - two actions on overlapping paths, across sections, after `#name` + resolution, reported as `path one: [...], path two: [...]`. + +Order on the service when a request carries two of these: syntax error, +then one walk over the actions in expression order (undefined `#name` or +`:value`, function rules, and a repeated section at the end of its section), +then overlap, then unused name or value. + +PutItem, UpdateItem, and DeleteItem wrap every expression-stage error in +`1 validation error detected: `; TransactWriteItems and the read APIs do not. +Every expected message was captured from Amazon DynamoDB on 2026-10-14. +Dual-target. +""" + +from __future__ import annotations + +import pytest +from botocore.exceptions import ClientError + +from conftest import scoped_table + +P = "1 validation error detected: " +U = "Invalid UpdateExpression: " +SECTION_ONCE = U + 'The "{}" section can only be used once in an update expression;' +OPERANDS = U + "Incorrect number of operands for operator or function; operator or function: {}, number of operands: {}" +NEEDS_PATH = U + "Operator or function requires a document path; operator or function: if_not_exists" +NOT_THIS_WAY = U + "The function is not allowed to be used this way in an expression; function: list_append" +OVERLAP = U + "Two document paths overlap with each other; must remove or rewrite one of these paths; path one: {}, path two: {}" + + +@pytest.fixture(scope="module") +def table(dynamodb_client): + with scoped_table(dynamodb_client) as name: + dynamodb_client.put_item( + TableName=name, + Item={"pk": {"S": "p"}, "fieldx": {"N": "1"}, "mapx": {"M": {"num": {"N": "1"}}}, + "lst": {"L": [{"M": {"ok": {"N": "1"}}}, {"M": {"x": {"N": "1"}}}]}}, + ) + yield name + + +KEY = {"pk": {"S": "p"}} +V = {":v": {"N": "1"}} + + +def _expect(func, message: str): + with pytest.raises(ClientError) as exc_info: + func() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"] == message, err + + +def _update(client, table, expression, names=None, values=None): + kwargs = {"TableName": table, "Key": KEY, "UpdateExpression": expression} + if names: + kwargs["ExpressionAttributeNames"] = names + if values: + kwargs["ExpressionAttributeValues"] = values + return lambda: client.update_item(**kwargs) + + +def _transact_update(client, table, expression, names=None, values=None): + item = {"TableName": table, "Key": KEY, "UpdateExpression": expression} + if names: + item["ExpressionAttributeNames"] = names + if values: + item["ExpressionAttributeValues"] = values + return lambda: client.transact_write_items(TransactItems=[{"Update": item}]) + + +class TestRepeatedSections: + @pytest.mark.parametrize( + ("expression", "section"), + [ + ("SET fieldx = :v SET otherx = :v", "SET"), + ("REMOVE a REMOVE b", "REMOVE"), + ("ADD a :v ADD b :v", "ADD"), + ("DELETE a :s DELETE b :s", "DELETE"), + ], + ) + def test_section_used_twice(self, dynamodb_client, table, expression, section): + values = {":s": {"SS": ["x"]}} if "DELETE" in expression else V + _expect(_update(dynamodb_client, table, expression, values=values), P + SECTION_ONCE.format(section)) + + def test_section_check_follows_the_grammar_pass(self, dynamodb_client, table): + # The service answers 'Syntax error; token: "", near: "+"'; ExtendDB's + # parser wording differs today (tracked under #108 message text). The + # guarantee pinned here is the order: the syntax error wins over the + # repeated-section error. + with pytest.raises(ClientError) as exc_info: + _update(dynamodb_client, table, "SET a = :v SET b = :v +", values=V)() + err = exc_info.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"].startswith(P + U), err + assert "section can only be used once" not in err["Message"], err + + def test_repeated_section_beats_overlap(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "SET a = :v SET a = :v", values=V), P + SECTION_ONCE.format("SET")) + + def test_transaction_reports_it_without_the_prefix(self, dynamodb_client, table): + _expect( + _transact_update(dynamodb_client, table, "SET fieldx = :v SET otherx = :v", values=V), + SECTION_ONCE.format("SET"), + ) + + +class TestFunctionOperands: + def test_if_not_exists_one_operand(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "SET a = if_not_exists(a)"), P + OPERANDS.format("if_not_exists", 1)) + + def test_if_not_exists_three_operands(self, dynamodb_client, table): + _expect( + _update(dynamodb_client, table, "SET a = if_not_exists(a, :v, :v)", values=V), + P + OPERANDS.format("if_not_exists", 3), + ) + + def test_list_append_one_operand(self, dynamodb_client, table): + _expect( + _update(dynamodb_client, table, "SET l = list_append(:l)", values={":l": {"L": []}}), + P + OPERANDS.format("list_append", 1), + ) + + def test_if_not_exists_first_operand_must_be_a_path(self, dynamodb_client, table): + _expect(_update(dynamodb_client, table, "SET a = if_not_exists(:v, :v)", values=V), P + NEEDS_PATH) + + def test_nested_if_not_exists_is_not_a_path(self, dynamodb_client, table): + _expect( + _update( + dynamodb_client, table, "SET #r = if_not_exists(if_not_exists(n, :zero), :one)", + names={"#r": "result"}, values={":zero": {"N": "0"}, ":one": {"N": "1"}}, + ), + P + NEEDS_PATH, + ) + + def test_list_append_inside_list_append(self, dynamodb_client, table): + _expect( + _update( + dynamodb_client, table, "SET #r = list_append(list_append(l, :tail), :tail2)", + names={"#r": "result"}, values={":tail": {"L": [{"S": "b"}]}, ":tail2": {"L": [{"S": "c"}]}}, + ), + P + NOT_THIS_WAY, + ) + + def test_list_append_inside_if_not_exists(self, dynamodb_client, table): + _expect( + _update( + dynamodb_client, table, "SET l = if_not_exists(l, list_append(:e, :v))", + values={":e": {"L": []}, ":v": {"L": [{"S": "x"}]}}, + ), + P + NOT_THIS_WAY, + ) + + def test_if_not_exists_inside_list_append_is_allowed(self, dynamodb_client, table): + dynamodb_client.update_item( + TableName=table, Key=KEY, + UpdateExpression="SET l2 = list_append(if_not_exists(l2, :e), :v)", + ExpressionAttributeValues={":e": {"L": []}, ":v": {"L": [{"S": "x"}]}}, + ) + item = dynamodb_client.get_item(TableName=table, Key=KEY)["Item"] + assert item["l2"] == {"L": [{"S": "x"}]} + + +class TestOverlappingPaths: + @pytest.mark.parametrize( + ("expression", "names", "values", "one", "two"), + [ + ("SET fieldx = :v, fieldx = :w", None, {":v": {"N": "1"}, ":w": {"N": "2"}}, "[fieldx]", "[fieldx]"), + ("SET mapx = :m, mapx.num = :v", None, {":m": {"M": {}}, ":v": {"N": "2"}}, "[mapx]", "[mapx, num]"), + ("SET #a = :v, fieldx = :v", {"#a": "fieldx"}, V, "[fieldx]", "[fieldx]"), + ("SET a = :v REMOVE a", None, V, "[a]", "[a]"), + ("SET a.b = :v REMOVE a", None, V, "[a, b]", "[a]"), + ("REMOVE a, a.b", None, None, "[a]", "[a, b]"), + ("SET a = :v ADD a :v", None, V, "[a]", "[a]"), + ("SET #l[1] = :v, #l[1].x = :v", {"#l": "lst"}, V, "[lst, [1]]", "[lst, [1], x]"), + ], + ) + def test_overlap(self, dynamodb_client, table, expression, names, values, one, two): + _expect(_update(dynamodb_client, table, expression, names=names, values=values), P + OVERLAP.format(one, two)) + + def test_siblings_do_not_overlap(self, dynamodb_client, table): + dynamodb_client.update_item( + TableName=table, Key=KEY, + UpdateExpression="SET mapx.a = :v, mapx.b = :v", + ExpressionAttributeValues=V, + ) + + def test_undefined_value_beats_overlap(self, dynamodb_client, table): + _expect( + _update(dynamodb_client, table, "SET a = :v, a = :missing", values=V), + P + U + "An expression attribute value used in expression is not defined; attribute value: :missing", + ) + + def test_undefined_name_beats_overlap(self, dynamodb_client, table): + _expect( + _update(dynamodb_client, table, "SET a = :v, a = :v, #m = :v", values=V), + P + U + "An expression attribute name used in the document path is not defined; attribute name: #m", + ) + + def test_overlap_beats_unused_name(self, dynamodb_client, table): + _expect( + _update(dynamodb_client, table, "SET a = :v, a = :v", names={"#u": "x"}, values=V), + P + OVERLAP.format("[a]", "[a]"), + ) + + def test_transaction_overlap_without_the_prefix(self, dynamodb_client, table): + _expect( + _transact_update(dynamodb_client, table, "SET fieldx = :s, fieldx = :o", + values={":s": {"S": "new"}, ":o": {"S": "other"}}), + OVERLAP.format("[fieldx]", "[fieldx]"), + ) + _expect( + _transact_update(dynamodb_client, table, "SET lst[0] = :v, lst[0].ok = :v", values=V), + OVERLAP.format("[lst, [0]]", "[lst, [0], ok]"), + ) + + +class TestSingleItemWritePrefix: + """PutItem, UpdateItem, DeleteItem prefix expression-stage errors; the read APIs do not.""" + + RESERVED_C = "Invalid ConditionExpression: Attribute name is a reserved keyword; reserved keyword: count" + UNUSED_N = "Value provided in ExpressionAttributeNames unused in expressions: keys: {#u}" + UNUSED_V = "Value provided in ExpressionAttributeValues unused in expressions: keys: {:u}" + NAMES_NO_EXPR = "ExpressionAttributeNames can only be specified when using expressions" + VALUES_NO_EXPR = "ExpressionAttributeValues can only be specified when using expressions" + UNDEF_V = "Invalid ConditionExpression: An expression attribute value used in expression is not defined; attribute value: :missing" + UNDEF_N = "Invalid ConditionExpression: An expression attribute name used in the document path is not defined; attribute name: #m" + SIZE_C = "Invalid ConditionExpression: Expression size has exceeded the maximum allowed size;" + SIZE_U = "Invalid UpdateExpression: Expression size has exceeded the maximum allowed size;" + SIZE_F = "Invalid FilterExpression: Expression size has exceeded the maximum allowed size;" + + def test_put_reserved_word(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.put_item(TableName=table, Item=KEY, ConditionExpression="count = :v", + ExpressionAttributeValues=V), P + self.RESERVED_C) + + def test_delete_reserved_word(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.delete_item(TableName=table, Key=KEY, ConditionExpression="count = :v", + ExpressionAttributeValues=V), P + self.RESERVED_C) + + def test_unused_names_and_values(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.update_item(TableName=table, Key=KEY, UpdateExpression="SET a = :v", + ExpressionAttributeNames={"#u": "x"}, ExpressionAttributeValues=V), P + self.UNUSED_N) + _expect(lambda: dynamodb_client.put_item(TableName=table, Item=KEY, ConditionExpression="attribute_exists(pk)", + ExpressionAttributeNames={"#u": "x"}), P + self.UNUSED_N) + _expect(lambda: dynamodb_client.delete_item(TableName=table, Key=KEY, ConditionExpression="attribute_exists(pk)", + ExpressionAttributeValues={":u": {"N": "1"}}), P + self.UNUSED_V) + + def test_names_or_values_without_expression(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.put_item(TableName=table, Item=KEY, ExpressionAttributeNames={"#u": "x"}), + P + self.NAMES_NO_EXPR) + _expect(lambda: dynamodb_client.delete_item(TableName=table, Key=KEY, ExpressionAttributeValues=V), + P + self.VALUES_NO_EXPR) + _expect(lambda: dynamodb_client.update_item(TableName=table, Key=KEY, ExpressionAttributeValues=V), + P + self.VALUES_NO_EXPR) + + def test_undefined_references(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.put_item(TableName=table, Item=KEY, ConditionExpression="a = :missing"), + P + self.UNDEF_V) + _expect(lambda: dynamodb_client.delete_item(TableName=table, Key=KEY, ConditionExpression="#m = :v", + ExpressionAttributeValues=V), P + self.UNDEF_N) + + def test_over_size_condition_has_no_length_suffix(self, dynamodb_client, table): + big = "attribute_exists(" + "a" * 4097 + ")" + _expect(lambda: dynamodb_client.put_item(TableName=table, Item=KEY, ConditionExpression=big), P + self.SIZE_C) + _expect(lambda: dynamodb_client.update_item(TableName=table, Key=KEY, UpdateExpression="SET a = :v", + ConditionExpression=big, ExpressionAttributeValues=V), P + self.SIZE_C) + _expect(lambda: dynamodb_client.update_item(TableName=table, Key=KEY, UpdateExpression="SET " + "a" * 4097 + " = :v", + ExpressionAttributeValues=V), P + self.SIZE_U) + + def test_transaction_condition_keeps_the_length_suffix(self, dynamodb_client, table): + big = "attribute_exists(" + "a" * 4097 + ")" + _expect( + lambda: dynamodb_client.transact_write_items(TransactItems=[ + {"ConditionCheck": {"TableName": table, "Key": KEY, "ConditionExpression": big}}]), + self.SIZE_C + f" expression size: {len(big)}", + ) + + def test_query_filter_has_no_suffix_scan_filter_has_one(self, dynamodb_client, table): + big = "attribute_exists(" + "a" * 4097 + ")" + _expect(lambda: dynamodb_client.query(TableName=table, KeyConditionExpression="pk = :v", FilterExpression=big, + ExpressionAttributeValues={":v": {"S": "p"}}), self.SIZE_F) + _expect(lambda: dynamodb_client.scan(TableName=table, FilterExpression=big), self.SIZE_F + f" expression size: {len(big)}") + + def test_read_apis_carry_no_prefix(self, dynamodb_client, table): + _expect(lambda: dynamodb_client.get_item(TableName=table, Key=KEY, ProjectionExpression="#a", + ExpressionAttributeNames={"#a": "a", "#u": "x"}), self.UNUSED_N) + _expect(lambda: dynamodb_client.query(TableName=table, KeyConditionExpression="pk = :v", + ExpressionAttributeValues={":v": {"S": "p"}, ":u": {"N": "1"}}), self.UNUSED_V)