diff --git a/Makefile b/Makefile index 37a77a8..7431f29 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \ $(TEST_DIR)/test_classifier.cpp \ $(TEST_DIR)/test_expression.cpp \ $(TEST_DIR)/test_set.cpp \ + $(TEST_DIR)/test_user_variable.cpp \ $(TEST_DIR)/test_select.cpp \ $(TEST_DIR)/test_emitter.cpp \ $(TEST_DIR)/test_stmt_cache.cpp \ diff --git a/include/sql_parser/ast.h b/include/sql_parser/ast.h index f51bfa4..4b81509 100644 --- a/include/sql_parser/ast.h +++ b/include/sql_parser/ast.h @@ -3,6 +3,7 @@ #include "sql_parser/common.h" #include "sql_parser/arena.h" +#include "sql_parser/token.h" #include #include @@ -12,17 +13,25 @@ struct AstNode { AstNode* first_child; AstNode* next_sibling; const char* value_ptr; + const char* source_ptr; uint32_t value_len; + uint32_t source_len; NodeType type; uint16_t flags; StringRef value() const { return StringRef{value_ptr, value_len}; } + StringRef source() const { return StringRef{source_ptr, source_len}; } void set_value(StringRef ref) { value_ptr = ref.ptr; value_len = ref.len; } + void set_source(StringRef ref) { + source_ptr = ref.ptr; + source_len = ref.len; + } + void add_child(AstNode* child) { if (!child) return; if (!first_child) { @@ -34,7 +43,7 @@ struct AstNode { last->next_sibling = child; } }; -static_assert(sizeof(AstNode) == 32, "AstNode must be 32 bytes"); +static_assert(sizeof(AstNode) == 48, "AstNode layout changed unexpectedly"); static_assert(std::is_trivially_copyable_v); inline AstNode* make_node(Arena& arena, NodeType type, StringRef value = {}, @@ -48,6 +57,13 @@ inline AstNode* make_node(Arena& arena, NodeType type, StringRef value = {}, return node; } +inline AstNode* make_node_from_token(Arena& arena, NodeType type, + const Token& token, uint16_t flags = 0) { + AstNode* node = make_node(arena, type, token.text, flags); + if (node) node->set_source(token.source); + return node; +} + } // namespace sql_parser #endif // SQL_PARSER_AST_H diff --git a/include/sql_parser/common.h b/include/sql_parser/common.h index a994da9..7cc651d 100644 --- a/include/sql_parser/common.h +++ b/include/sql_parser/common.h @@ -231,6 +231,12 @@ enum class NodeType : uint16_t { NODE_SET_ROLE, // SET [LOCAL] ROLE |NONE|DEFAULT NODE_SET_SESSION_AUTHORIZATION, // SET SESSION AUTHORIZATION |DEFAULT NODE_SET_CONSTRAINTS, // SET CONSTRAINTS {ALL|[,...]} {DEFERRED|IMMEDIATE} + + // MySQL lossless user-variable/literal nodes. Keep appended so existing + // enum values remain stable for consumers that index by NodeType. + NODE_USER_VARIABLE, + NODE_LITERAL_HEX, + NODE_LITERAL_BIT, }; } // namespace sql_parser diff --git a/include/sql_parser/digest.h b/include/sql_parser/digest.h index 2acd52a..3508c34 100644 --- a/include/sql_parser/digest.h +++ b/include/sql_parser/digest.h @@ -63,15 +63,17 @@ class Digest { // Helper: check if a token type is a keyword (not an identifier, literal, or operator) static bool is_keyword_token(TokenType type) { - // Keywords start at TK_SELECT and go through TK_EXCEPT - return static_cast(type) >= static_cast(TokenType::TK_SELECT); + return static_cast(type) >= static_cast(TokenType::TK_SELECT) && + static_cast(type) <= static_cast(TokenType::TK_RECURSIVE); } // Helper: check if a token type is a literal value that should become ? static bool is_literal_token(TokenType type) { return type == TokenType::TK_INTEGER || type == TokenType::TK_FLOAT || - type == TokenType::TK_STRING; + type == TokenType::TK_STRING || + type == TokenType::TK_HEX_LITERAL || + type == TokenType::TK_BIT_LITERAL; } // Helper: uppercase a character @@ -104,7 +106,12 @@ class Digest { // Emit a single token to the string builder, uppercasing keywords, replacing literals with ? void emit_token(StringBuilder& sb, const Token& t, TokenType prev) { - bool space = (prev != TokenType::TK_EOF) && needs_space_before(prev, t.type); + bool quoted_user_after_account = t.type == TokenType::TK_USER_VARIABLE && + t.source.len >= 2 && + (t.source.ptr[1] == '\'' || t.source.ptr[1] == '"' || t.source.ptr[1] == '`') && + (prev == TokenType::TK_STRING || prev == TokenType::TK_QUESTION); + bool space = (prev != TokenType::TK_EOF) && + !quoted_user_after_account && needs_space_before(prev, t.type); if (space) sb.append_char(' '); if (is_literal_token(t.type)) { @@ -115,6 +122,13 @@ class Digest { sb.append(t.text.ptr, t.text.len); } else if (t.type == TokenType::TK_QUESTION) { sb.append_char('?'); + } else if (t.type == TokenType::TK_USER_VARIABLE) { + if (t.source.len >= 2 && + (t.source.ptr[1] == '\'' || t.source.ptr[1] == '"' || t.source.ptr[1] == '`')) { + sb.append("@?", 2); + } else { + sb.append(t.source); + } } else if (t.type == TokenType::TK_COMMA) { sb.append(",", 1); } else { diff --git a/include/sql_parser/emitter.h b/include/sql_parser/emitter.h index 8263fff..39f6596 100644 --- a/include/sql_parser/emitter.h +++ b/include/sql_parser/emitter.h @@ -123,6 +123,8 @@ class Emitter { case NodeType::NODE_ARRAY_SUBSCRIPT: emit_array_subscript(node); break; case NodeType::NODE_FIELD_ACCESS: emit_field_access(node); break; case NodeType::NODE_SUBQUERY: emit_subquery(node); break; + case NodeType::NODE_EXPRESSION: emit_parenthesized_expression(node); break; + case NodeType::NODE_USER_VARIABLE: emit_user_variable(node); break; // ---- Leaf nodes (emit value directly) ---- case NodeType::NODE_PLACEHOLDER: @@ -131,6 +133,8 @@ class Emitter { // ---- Leaf nodes (emit value directly) ---- case NodeType::NODE_LITERAL_INT: case NodeType::NODE_LITERAL_FLOAT: + case NodeType::NODE_LITERAL_HEX: + case NodeType::NODE_LITERAL_BIT: if (mode_ == EmitMode::DIGEST) { sb_.append_char('?'); break; } emit_value(node); break; case NodeType::NODE_LITERAL_NULL: @@ -152,6 +156,30 @@ class Emitter { sb_.append(node->value_ptr, node->value_len); } + void emit_user_variable(const AstNode* node) { + if (mode_ == EmitMode::DIGEST) { + StringRef source = node->source(); + if (source.len >= 2 && + (source.ptr[1] == '\'' || source.ptr[1] == '"' || source.ptr[1] == '`')) { + sb_.append("@?", 2); + return; + } + } + StringRef source = node->source(); + if (!source.empty()) { + sb_.append(source.ptr, source.len); + return; + } + sb_.append_char('@'); + emit_value(node); + } + + void emit_parenthesized_expression(const AstNode* node) { + sb_.append_char('('); + if (node->first_child) emit_node(node->first_child); + sb_.append_char(')'); + } + void emit_string_literal(const AstNode* node) { sb_.append_char('\''); sb_.append(node->value_ptr, node->value_len); diff --git a/include/sql_parser/expression_parser.h b/include/sql_parser/expression_parser.h index f6e412c..51b15c8 100644 --- a/include/sql_parser/expression_parser.h +++ b/include/sql_parser/expression_parser.h @@ -6,6 +6,7 @@ #include "sql_parser/tokenizer.h" #include "sql_parser/ast.h" #include "sql_parser/arena.h" +#include "sql_parser/user_variable.h" namespace sql_parser { @@ -99,19 +100,27 @@ class ExpressionParser { switch (t.type) { case TokenType::TK_INTEGER: { tok_.skip(); - return make_node(arena_, NodeType::NODE_LITERAL_INT, t.text); + return make_node_from_token(arena_, NodeType::NODE_LITERAL_INT, t); } case TokenType::TK_FLOAT: { tok_.skip(); - return make_node(arena_, NodeType::NODE_LITERAL_FLOAT, t.text); + return make_node_from_token(arena_, NodeType::NODE_LITERAL_FLOAT, t); + } + case TokenType::TK_HEX_LITERAL: { + tok_.skip(); + return make_node_from_token(arena_, NodeType::NODE_LITERAL_HEX, t); + } + case TokenType::TK_BIT_LITERAL: { + tok_.skip(); + return make_node_from_token(arena_, NodeType::NODE_LITERAL_BIT, t); } case TokenType::TK_STRING: { tok_.skip(); - return make_node(arena_, NodeType::NODE_LITERAL_STRING, t.text); + return make_node_from_token(arena_, NodeType::NODE_LITERAL_STRING, t); } case TokenType::TK_NULL: { tok_.skip(); - return make_node(arena_, NodeType::NODE_LITERAL_NULL, t.text); + return make_node_from_token(arena_, NodeType::NODE_LITERAL_NULL, t); } case TokenType::TK_TRUE: case TokenType::TK_FALSE: { @@ -150,6 +159,10 @@ class ExpressionParser { static_cast((name.text.ptr + name.text.len) - t.text.ptr)}; return make_node(arena_, NodeType::NODE_COLUMN_REF, full); } + case TokenType::TK_USER_VARIABLE: { + tok_.skip(); + return make_mysql_user_variable_node(arena_, t); + } case TokenType::TK_DOUBLE_AT: { // System variable: @@name or @@scope.name tok_.skip(); @@ -174,19 +187,26 @@ class ExpressionParser { AstNode* operand = parse(Precedence::UNARY); if (!operand) return nullptr; AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text); + set_span_through_node_(node, t.source, operand); node->add_child(operand); return node; } case TokenType::TK_PLUS: { // Unary plus tok_.skip(); - return parse(Precedence::UNARY); + AstNode* operand = parse(Precedence::UNARY); + if (!operand) return nullptr; + AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text); + set_span_through_node_(node, t.source, operand); + node->add_child(operand); + return node; } case TokenType::TK_NOT: { tok_.skip(); AstNode* operand = parse(Precedence::NOT); if (!operand) return nullptr; AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text); + set_span_through_node_(node, t.source, operand); node->add_child(operand); return node; } @@ -266,7 +286,12 @@ class ExpressionParser { return parse_postfix(tuple); } if (tok_.peek().type == TokenType::TK_RPAREN) { - tok_.skip(); + Token close = tok_.next_token(); + AstNode* wrapper = make_node(arena_, NodeType::NODE_EXPRESSION); + wrapper->set_source(StringRef{t.source.ptr, + static_cast(close.source.ptr + close.source.len - t.source.ptr)}); + wrapper->add_child(expr); + return parse_postfix(wrapper); } // Check for postfix: (expr).field or (expr)[index] return parse_postfix(expr); @@ -287,11 +312,39 @@ class ExpressionParser { } } + static void set_span_through_node_(AstNode* node, StringRef start, + const AstNode* end_node) { + if (!node || !start.ptr || !end_node) return; + StringRef end = end_node->source(); + if (end.empty()) end = end_node->value(); + if (!end.ptr || end.ptr < start.ptr) return; + node->set_source(StringRef{start.ptr, + static_cast(end.ptr + end.len - start.ptr)}); + } + AstNode* parse_identifier_or_function(const Token& name_token) { // Check for function call: name( if (tok_.peek().type == TokenType::TK_LPAREN) { tok_.skip(); // consume ( AstNode* func = make_node(arena_, NodeType::NODE_FUNCTION_CALL, name_token.text); + // CAST uses `CAST(expr AS type)` rather than a comma-separated + // argument list. Model it as a function call so consumers can + // reject or handle the expression without leaving valid input + // unconsumed. + if (name_token.text.equals_ci("CAST", 4)) { + AstNode* arg = parse(); + if (!arg || tok_.peek().type != TokenType::TK_AS) return func; + func->add_child(arg); + tok_.skip(); + Token type = tok_.next_token(); + if (type.type == TokenType::TK_EOF || + type.type == TokenType::TK_RPAREN) { + return func; + } + func->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, type.text)); + if (tok_.peek().type == TokenType::TK_RPAREN) tok_.skip(); + return func; + } // Parse argument list if (tok_.peek().type != TokenType::TK_RPAREN) { while (true) { diff --git a/include/sql_parser/parse_result.h b/include/sql_parser/parse_result.h index 41853a7..66ed202 100644 --- a/include/sql_parser/parse_result.h +++ b/include/sql_parser/parse_result.h @@ -39,6 +39,8 @@ struct ParseResult { AstNode* ast = nullptr; ErrorInfo error; StringRef remaining; + bool full_input = false; + bool has_user_variables = false; StringRef table_name; StringRef schema_name; diff --git a/include/sql_parser/parser.h b/include/sql_parser/parser.h index f150353..c8d7f7b 100644 --- a/include/sql_parser/parser.h +++ b/include/sql_parser/parser.h @@ -7,6 +7,7 @@ #include "sql_parser/ast.h" #include "sql_parser/parse_result.h" #include "sql_parser/stmt_cache.h" +#include "sql_parser/user_variable.h" namespace sql_parser { diff --git a/include/sql_parser/set_parser.h b/include/sql_parser/set_parser.h index c734c52..a0df255 100644 --- a/include/sql_parser/set_parser.h +++ b/include/sql_parser/set_parser.h @@ -352,9 +352,14 @@ class SetParser { } } else { while (tok_.peek().type == TokenType::TK_COMMA) { - tok_.skip(); + Token comma = tok_.next_token(); AstNode* next_assign = parse_comma_item(); - if (next_assign) root->add_child(next_assign); + if (next_assign) { + root->add_child(next_assign); + } else { + tok_.flag_error_at(comma.source); + break; + } } } @@ -551,7 +556,17 @@ class SetParser { } Token var = tok_.peek(); - if (var.type == TokenType::TK_AT) { + bool user_variable_target = false; + if (var.type == TokenType::TK_USER_VARIABLE) { + user_variable_target = true; + tok_.skip(); + AstNode* variable = make_mysql_user_variable_node(arena_, var); + if (!variable) { + tok_.flag_error_at(var.source); + return nullptr; + } + target->add_child(variable); + } else if (var.type == TokenType::TK_AT) { // User variable @name. The name may be backtick/double-quoted; // in that case the source bytes between `@` and the name include // the opening delimiter (and the closing delimiter sits one past @@ -624,13 +639,20 @@ class SetParser { // Expect = or := (MySQL) or TO (PostgreSQL) Token eq = tok_.peek(); + bool has_assignment_operator = false; if (eq.type == TokenType::TK_EQUAL || eq.type == TokenType::TK_COLON_EQUAL) { tok_.skip(); + has_assignment_operator = true; } else if constexpr (D == Dialect::PostgreSQL) { if (eq.type == TokenType::TK_TO) { tok_.skip(); + has_assignment_operator = true; } } + if (user_variable_target && !has_assignment_operator) { + tok_.flag_error_at(eq.source); + return nullptr; + } // Parse RHS expression. If the parser couldn't produce one -- // typically because the input is truncated (`SET x =`), starts diff --git a/include/sql_parser/token.h b/include/sql_parser/token.h index 2643a97..7f1f75f 100644 --- a/include/sql_parser/token.h +++ b/include/sql_parser/token.h @@ -127,11 +127,18 @@ enum class TokenType : uint16_t { // CTE tokens TK_WITH, TK_RECURSIVE, + + // MySQL lossless user-variable/literal tokens. Keep appended so existing + // enum values remain stable for consumers that index by TokenType. + TK_USER_VARIABLE, + TK_HEX_LITERAL, + TK_BIT_LITERAL, }; struct Token { TokenType type = TokenType::TK_EOF; StringRef text; + StringRef source; uint32_t offset = 0; }; diff --git a/include/sql_parser/tokenizer.h b/include/sql_parser/tokenizer.h index e49328a..c1edeeb 100644 --- a/include/sql_parser/tokenizer.h +++ b/include/sql_parser/tokenizer.h @@ -16,6 +16,11 @@ class Tokenizer { end_ = input + len; has_peeked_ = false; has_error_ = false; + has_fatal_error_ = false; + has_user_variables_ = false; + paren_depth_ = 0; + first_open_paren_ = nullptr; + error_source_ = {}; } // True iff the tokenizer has emitted at least one TK_ERROR token since @@ -23,12 +28,23 @@ class Tokenizer { // from "the input was syntactically invalid" and surface the latter // as ParseResult::ERROR rather than PARTIAL. bool has_error() const { return has_error_; } + bool has_fatal_error() const { return has_fatal_error_; } + bool has_user_variables() const { return has_user_variables_; } + StringRef error_source() const { return error_source_; } // Hook for parser-level (non-tokenizer) errors -- when the parser // detects clearly invalid input (e.g. `SET = X`, `SET x = ;`, // `SET x = ,foo`) it can flag the error so the eventual ParseResult // is ERROR rather than PARTIAL with a null AST. void flag_error() { has_error_ = true; } + void flag_error_at(StringRef source) { + has_error_ = true; + if (error_source_.empty()) error_source_ = source; + } + void flag_fatal_error_at(StringRef source) { + has_fatal_error_ = true; + flag_error_at(source); + } Token next_token() { if (has_peeked_) { @@ -67,6 +83,11 @@ class Tokenizer { Token peeked_; bool has_peeked_ = false; bool has_error_ = false; + bool has_fatal_error_ = false; + bool has_user_variables_ = false; + uint32_t paren_depth_ = 0; + const char* first_open_paren_ = nullptr; + StringRef error_source_; uint32_t offset() const { return static_cast(cursor_ - start_); @@ -93,8 +114,14 @@ class Tokenizer { continue; } - // -- line comment (MySQL requires space after --, PgSQL doesn't but we handle both) - if (c == '-' && peek_char(1) == '-') { + // PostgreSQL accepts any `--` line comment. MySQL requires the + // second dash to be followed by whitespace or a control byte. + const bool dash_comment = c == '-' && peek_char(1) == '-' && + (D == Dialect::PostgreSQL || + (cursor_ + 2 < end_ && + (static_cast(peek_char(2)) <= 0x20 || + static_cast(peek_char(2)) == 0x7f))); + if (dash_comment) { cursor_ += 2; while (cursor_ < end_ && *cursor_ != '\n') ++cursor_; continue; @@ -111,6 +138,7 @@ class Tokenizer { // /* block comment */ if (c == '/' && peek_char(1) == '*') { + const char* comment_start = cursor_; cursor_ += 2; if constexpr (D == Dialect::PostgreSQL) { // PostgreSQL supports nested block comments @@ -126,15 +154,38 @@ class Tokenizer { ++cursor_; } } + if (depth != 0) { + flag_fatal_error_at(StringRef{comment_start, + static_cast(end_ - comment_start)}); + } } else { // MySQL: no nesting + const bool executable = cursor_ < end_ && + (*cursor_ == '!' || + (cursor_ + 1 < end_ && cursor_[0] == 'M' && cursor_[1] == '!')); + bool has_user_variable_marker = false; + bool closed = false; while (cursor_ < end_) { if (*cursor_ == '*' && peek_char(1) == '/') { cursor_ += 2; + closed = true; break; } + if (*cursor_ == '@') has_user_variable_marker = true; ++cursor_; } + if (!closed) { + flag_fatal_error_at(StringRef{comment_start, + static_cast(end_ - comment_start)}); + } + // MySQL and MariaDB executable comments run as SQL. Until + // their contents are parsed exactly, preserve any possible + // user variable use and force conservative classification. + if (executable && has_user_variable_marker) { + has_user_variables_ = true; + flag_fatal_error_at(StringRef{comment_start, + static_cast(cursor_ - comment_start)}); + } } continue; } @@ -143,10 +194,29 @@ class Tokenizer { } } - Token make_token(TokenType type, const char* start, uint32_t len) { - if (type == TokenType::TK_ERROR) has_error_ = true; - return Token{type, StringRef{start, len}, - static_cast(start - start_)}; + Token make_token(TokenType type, const char* text_start, uint32_t text_len, + const char* source_start = nullptr, uint32_t source_len = 0) { + if (!source_start) { + source_start = text_start; + source_len = text_len; + } + if (type == TokenType::TK_ERROR) { + has_error_ = true; + if (error_source_.empty()) error_source_ = StringRef{source_start, source_len}; + } + if (type == TokenType::TK_LPAREN) { + if (paren_depth_ == 0) first_open_paren_ = source_start; + ++paren_depth_; + } else if (type == TokenType::TK_RPAREN && paren_depth_ > 0) { + if (--paren_depth_ == 0) first_open_paren_ = nullptr; + } else if (type == TokenType::TK_EOF && paren_depth_ > 0) { + flag_fatal_error_at(StringRef{first_open_paren_, + static_cast(end_ - first_open_paren_)}); + } + if (type == TokenType::TK_USER_VARIABLE) has_user_variables_ = true; + return Token{type, StringRef{text_start, text_len}, + StringRef{source_start, source_len}, + static_cast(source_start - start_)}; } Token scan_identifier_or_keyword() { @@ -184,23 +254,78 @@ class Tokenizer { Token scan_number() { const char* start = cursor_; bool has_dot = false; - while (cursor_ < end_) { - char c = *cursor_; - if (c >= '0' && c <= '9') { - ++cursor_; - } else if (c == '.' && !has_dot) { - has_dot = true; - ++cursor_; - } else { - break; + while (cursor_ < end_ && *cursor_ >= '0' && *cursor_ <= '9') ++cursor_; + if (cursor_ < end_ && *cursor_ == '.') { + has_dot = true; + ++cursor_; + while (cursor_ < end_ && *cursor_ >= '0' && *cursor_ <= '9') ++cursor_; + } + bool has_exponent = false; + if (cursor_ < end_ && (*cursor_ == 'e' || *cursor_ == 'E')) { + has_exponent = true; + ++cursor_; + if (cursor_ < end_ && (*cursor_ == '+' || *cursor_ == '-')) ++cursor_; + const char* exponent_digits = cursor_; + while (cursor_ < end_ && *cursor_ >= '0' && *cursor_ <= '9') ++cursor_; + if (cursor_ == exponent_digits) { + return make_token(TokenType::TK_ERROR, start, + static_cast(cursor_ - start)); } } uint32_t len = static_cast(cursor_ - start); - return make_token(has_dot ? TokenType::TK_FLOAT : TokenType::TK_INTEGER, + return make_token(has_dot || has_exponent ? TokenType::TK_FLOAT : TokenType::TK_INTEGER, + start, len); + } + + static bool is_hex_digit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); + } + + static bool is_token_word_char(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || c == '_' || c == '$'; + } + + Token scan_prefixed_base_literal(bool hex) { + const char* start = cursor_; + cursor_ += 2; + const char* digits = cursor_; + while (cursor_ < end_ && (hex ? is_hex_digit(*cursor_) : (*cursor_ == '0' || *cursor_ == '1'))) { + ++cursor_; + } + bool invalid = cursor_ == digits; + if (cursor_ < end_ && is_token_word_char(*cursor_)) { + invalid = true; + while (cursor_ < end_ && is_token_word_char(*cursor_)) ++cursor_; + } + uint32_t len = static_cast(cursor_ - start); + return make_token(invalid ? TokenType::TK_ERROR : + (hex ? TokenType::TK_HEX_LITERAL : TokenType::TK_BIT_LITERAL), + start, len); + } + + Token scan_quoted_base_literal(bool hex) { + const char* start = cursor_; + cursor_ += 2; // prefix and opening quote + bool invalid = false; + while (cursor_ < end_ && *cursor_ != '\'') { + if (hex ? !is_hex_digit(*cursor_) : (*cursor_ != '0' && *cursor_ != '1')) invalid = true; + ++cursor_; + } + if (cursor_ >= end_) { + return make_token(TokenType::TK_ERROR, start, + static_cast(cursor_ - start)); + } + ++cursor_; + uint32_t len = static_cast(cursor_ - start); + return make_token(invalid ? TokenType::TK_ERROR : + (hex ? TokenType::TK_HEX_LITERAL : TokenType::TK_BIT_LITERAL), start, len); } Token scan_single_quoted_string() { + const char* source_start = cursor_; ++cursor_; // skip opening quote const char* content_start = cursor_; while (cursor_ < end_) { @@ -220,8 +345,91 @@ class Tokenizer { } } uint32_t len = static_cast(cursor_ - content_start); - if (cursor_ < end_) ++cursor_; // skip closing quote - return make_token(TokenType::TK_STRING, content_start, len); + if (cursor_ >= end_) { + return make_token(TokenType::TK_ERROR, source_start, + static_cast(cursor_ - source_start)); + } + ++cursor_; // skip closing quote + return make_token(TokenType::TK_STRING, content_start, len, source_start, + static_cast(cursor_ - source_start)); + } + + Token scan_double_quoted_string() { + const char* source_start = cursor_; + ++cursor_; + const char* content_start = cursor_; + while (cursor_ < end_) { + if (*cursor_ == '"') { + if (cursor_ + 1 < end_ && cursor_[1] == '"') { + cursor_ += 2; + continue; + } + break; + } + if (*cursor_ == '\\') { + ++cursor_; + if (cursor_ < end_) ++cursor_; + } else { + ++cursor_; + } + } + uint32_t len = static_cast(cursor_ - content_start); + if (cursor_ >= end_) { + return make_token(TokenType::TK_ERROR, source_start, + static_cast(cursor_ - source_start)); + } + ++cursor_; + return make_token(TokenType::TK_STRING, content_start, len, source_start, + static_cast(cursor_ - source_start)); + } + + Token scan_mysql_user_variable() { + has_user_variables_ = true; + const char* source_start = cursor_; + ++cursor_; // @ + if (cursor_ >= end_) return make_token(TokenType::TK_AT, source_start, 1); + + char delimiter = *cursor_; + if (delimiter == '\'' || delimiter == '"' || delimiter == '`') { + ++cursor_; + const char* content_start = cursor_; + while (cursor_ < end_) { + if (*cursor_ == delimiter) { + if (cursor_ + 1 < end_ && cursor_[1] == delimiter) { + cursor_ += 2; + continue; + } + uint32_t text_len = static_cast(cursor_ - content_start); + ++cursor_; + return make_token(TokenType::TK_USER_VARIABLE, content_start, text_len, + source_start, + static_cast(cursor_ - source_start)); + } + if (*cursor_ == '\\') { + ++cursor_; + if (cursor_ < end_) ++cursor_; + } else { + ++cursor_; + } + } + return make_token(TokenType::TK_ERROR, source_start, + static_cast(cursor_ - source_start)); + } + + const char* name_start = cursor_; + while (cursor_ < end_) { + char c = *cursor_; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '$') { + ++cursor_; + } else { + break; + } + } + if (cursor_ == name_start) return make_token(TokenType::TK_AT, source_start, 1); + return make_token(TokenType::TK_USER_VARIABLE, name_start, + static_cast(cursor_ - name_start), source_start, + static_cast(cursor_ - source_start)); } // MySQL: backtick-quoted identifier @@ -240,7 +448,8 @@ class Tokenizer { } uint32_t len = static_cast(cursor_ - content_start); ++cursor_; // skip closing backtick - return make_token(TokenType::TK_IDENTIFIER, content_start, len); + return make_token(TokenType::TK_IDENTIFIER, content_start, len, open_pos, + static_cast(cursor_ - open_pos)); } // PostgreSQL: double-quoted identifier @@ -258,7 +467,8 @@ class Tokenizer { } uint32_t len = static_cast(cursor_ - content_start); ++cursor_; // skip closing quote - return make_token(TokenType::TK_IDENTIFIER, content_start, len); + return make_token(TokenType::TK_IDENTIFIER, content_start, len, open_pos, + static_cast(cursor_ - open_pos)); } // PostgreSQL: $$...$$ dollar-quoted string @@ -288,6 +498,21 @@ class Tokenizer { char c = *cursor_; + if constexpr (D == Dialect::MySQL) { + if ((c == 'x' || c == 'X') && peek_char(1) == '\'') { + return scan_quoted_base_literal(true); + } + if ((c == 'b' || c == 'B') && peek_char(1) == '\'') { + return scan_quoted_base_literal(false); + } + if (c == '0' && (peek_char(1) == 'x' || peek_char(1) == 'X')) { + return scan_prefixed_base_literal(true); + } + if (c == '0' && (peek_char(1) == 'b' || peek_char(1) == 'B')) { + return scan_prefixed_base_literal(false); + } + } + // Identifiers and keywords if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { return scan_identifier_or_keyword(); @@ -310,16 +535,7 @@ class Tokenizer { // MySQL: double-quoted strings; PostgreSQL: double-quoted identifiers if (c == '"') { if constexpr (D == Dialect::MySQL) { - // In MySQL, double quotes are strings (unless ANSI_QUOTES mode) - ++cursor_; - const char* content_start = cursor_; - while (cursor_ < end_ && *cursor_ != '"') { - if (*cursor_ == '\\') { ++cursor_; if (cursor_ < end_) ++cursor_; } - else ++cursor_; - } - uint32_t len = static_cast(cursor_ - content_start); - if (cursor_ < end_) ++cursor_; - return make_token(TokenType::TK_STRING, content_start, len); + return scan_double_quoted_string(); } else { return scan_double_quoted_identifier(); } @@ -337,9 +553,13 @@ class Tokenizer { cursor_ += 2; return make_token(TokenType::TK_DOUBLE_AT, s, 2); } - const char* s = cursor_; - ++cursor_; - return make_token(TokenType::TK_AT, s, 1); + if constexpr (D == Dialect::MySQL) { + return scan_mysql_user_variable(); + } else { + const char* s = cursor_; + ++cursor_; + return make_token(TokenType::TK_AT, s, 1); + } } // $ — PostgreSQL: $N placeholder or $$string$$ diff --git a/include/sql_parser/user_variable.h b/include/sql_parser/user_variable.h new file mode 100644 index 0000000..96c4abb --- /dev/null +++ b/include/sql_parser/user_variable.h @@ -0,0 +1,62 @@ +#ifndef SQL_PARSER_USER_VARIABLE_H +#define SQL_PARSER_USER_VARIABLE_H + +#include "sql_parser/ast.h" +#include "sql_parser/parse_result.h" +#include "sql_parser/token.h" + +#include + +namespace sql_parser { + +enum class UserVariableUsage : uint8_t { + NO_USER_VARIABLE, + READ_ONLY, + UNSAFE_OR_UNKNOWN +}; + +// Decode a TK_USER_VARIABLE into a stable identity while retaining its exact +// replayable spelling. Delimiter doubling is unescaped; backslashes are kept +// verbatim because their meaning depends on the session SQL mode. +inline AstNode* make_mysql_user_variable_node(Arena& arena, const Token& token) { + if (token.type != TokenType::TK_USER_VARIABLE || !token.source.ptr || + token.source.len < 2 || token.source.ptr[0] != '@') { + return nullptr; + } + + const char* source = token.source.ptr; + uint32_t source_len = token.source.len; + StringRef decoded; + char delimiter = source[1]; + bool quoted = delimiter == '\'' || delimiter == '"' || delimiter == '`'; + + if (!quoted) { + decoded = StringRef{source + 1, source_len - 1}; + if (decoded.len > 64) return nullptr; + } else { + if (source_len < 3 || source[source_len - 1] != delimiter) return nullptr; + uint32_t inner_len = source_len - 3; + char* buffer = static_cast(arena.allocate(inner_len)); + if (!buffer && inner_len != 0) return nullptr; + uint32_t written = 0; + for (uint32_t i = 2; i + 1 < source_len; ++i) { + char c = source[i]; + if (c == delimiter && i + 2 < source_len && source[i + 1] == delimiter) { + ++i; + } + if (written == 64) return nullptr; + buffer[written++] = c; + } + decoded = StringRef{buffer, written}; + } + + AstNode* node = make_node(arena, NodeType::NODE_USER_VARIABLE, decoded); + if (node) node->set_source(token.source); + return node; +} + +UserVariableUsage classify_mysql_user_variable_usage(const ParseResult& result); + +} // namespace sql_parser + +#endif // SQL_PARSER_USER_VARIABLE_H diff --git a/src/sql_parser/parser.cpp b/src/sql_parser/parser.cpp index ae9c383..cd7de68 100644 --- a/src/sql_parser/parser.cpp +++ b/src/sql_parser/parser.cpp @@ -23,8 +23,17 @@ void Parser::reset() { template ParseResult Parser::parse(const char* sql, size_t len) { arena_.reset(); + bool has_user_variables = false; + if constexpr (D == Dialect::MySQL) { + Tokenizer detector; + detector.reset(sql, len); + while (detector.next_token().type != TokenType::TK_EOF) {} + has_user_variables = detector.has_user_variables(); + } tokenizer_.reset(sql, len); - return classify_and_dispatch(); + ParseResult result = classify_and_dispatch(); + result.has_user_variables = has_user_variables || tokenizer_.has_user_variables(); + return result; } template @@ -927,21 +936,39 @@ Token Parser::read_table_name(StringRef& schema_out) { template void Parser::scan_to_end(ParseResult& result) { - while (true) { - Token t = tokenizer_.next_token(); - if (t.type == TokenType::TK_EOF) break; - if (t.type == TokenType::TK_SEMICOLON) { - Token next = tokenizer_.peek(); - if (next.type != TokenType::TK_EOF) { - const char* remaining_start = next.text.ptr; - const char* input_end = tokenizer_.input_end(); - result.remaining = StringRef{ - remaining_start, - static_cast(input_end - remaining_start) - }; + Token first = tokenizer_.next_token(); + if (first.type == TokenType::TK_EOF) { + StringRef error_source = tokenizer_.error_source(); + if (!error_source.empty()) { + if (tokenizer_.has_fatal_error()) result.status = ParseResult::ERROR; + result.remaining = StringRef{error_source.ptr, + static_cast(tokenizer_.input_end() - error_source.ptr)}; + } else { + result.full_input = true; + } + return; + } + + if (first.type == TokenType::TK_SEMICOLON) { + Token next = tokenizer_.next_token(); + if (next.type == TokenType::TK_EOF) { + StringRef error_source = tokenizer_.error_source(); + if (tokenizer_.has_fatal_error() && !error_source.empty()) { + result.status = ParseResult::ERROR; + result.remaining = StringRef{error_source.ptr, + static_cast(tokenizer_.input_end() - error_source.ptr)}; + return; } - break; + result.full_input = true; + return; } + first = next; + } + + const char* remaining_start = first.source.ptr ? first.source.ptr : first.text.ptr; + if (remaining_start) { + result.remaining = StringRef{remaining_start, + static_cast(tokenizer_.input_end() - remaining_start)}; } } @@ -1294,6 +1321,90 @@ ParseResult Parser::parse_with() { return r; } +namespace { + +bool is_forbidden_user_variable_context(NodeType type) { + switch (type) { + case NodeType::NODE_FUNCTION_CALL: + case NodeType::NODE_CALL_STMT: + case NodeType::NODE_DO_STMT: + case NodeType::NODE_PLACEHOLDER: + case NodeType::NODE_SUBQUERY: + return true; + default: + return false; + } +} + +bool is_allowed_read_ancestor(NodeType type) { + switch (type) { + case NodeType::NODE_SELECT_STMT: + case NodeType::NODE_SELECT_ITEM_LIST: + case NodeType::NODE_SELECT_ITEM: + case NodeType::NODE_WHERE_CLAUSE: + case NodeType::NODE_GROUP_BY_CLAUSE: + case NodeType::NODE_HAVING_CLAUSE: + case NodeType::NODE_ORDER_BY_CLAUSE: + case NodeType::NODE_ORDER_BY_ITEM: + case NodeType::NODE_LIMIT_CLAUSE: + case NodeType::NODE_EXPRESSION: + case NodeType::NODE_BINARY_OP: + case NodeType::NODE_UNARY_OP: + case NodeType::NODE_IS_NULL: + case NodeType::NODE_IS_NOT_NULL: + case NodeType::NODE_BETWEEN: + case NodeType::NODE_IN_LIST: + return true; + default: + return false; + } +} + +struct UsageWalk { + bool found = false; + bool unsafe = false; +}; + +void walk_user_variable_usage(const AstNode* node, bool path_is_read_safe, + bool write_context, UsageWalk& walk) { + if (!node || walk.unsafe) return; + if (is_forbidden_user_variable_context(node->type)) { + walk.unsafe = true; + return; + } + + bool child_write_context = write_context || + node->type == NodeType::NODE_VAR_TARGET || + node->type == NodeType::NODE_INTO_CLAUSE; + + if (node->type == NodeType::NODE_USER_VARIABLE) { + walk.found = true; + if (write_context || !path_is_read_safe) walk.unsafe = true; + return; + } + + bool child_path_is_read_safe = path_is_read_safe && + is_allowed_read_ancestor(node->type); + for (const AstNode* child = node->first_child; child; child = child->next_sibling) { + walk_user_variable_usage(child, child_path_is_read_safe, + child_write_context, walk); + } +} + +} // namespace + +UserVariableUsage classify_mysql_user_variable_usage(const ParseResult& result) { + if (!result.has_user_variables) return UserVariableUsage::NO_USER_VARIABLE; + if (result.status != ParseResult::OK || !result.full_input || !result.ast) { + return UserVariableUsage::UNSAFE_OR_UNKNOWN; + } + + UsageWalk walk; + walk_user_variable_usage(result.ast, true, false, walk); + if (walk.unsafe || !walk.found) return UserVariableUsage::UNSAFE_OR_UNKNOWN; + return UserVariableUsage::READ_ONLY; +} + // ---- Explicit template instantiations ---- template class Parser; diff --git a/tests/test_digest.cpp b/tests/test_digest.cpp index 7ce878e..f22972c 100644 --- a/tests/test_digest.cpp +++ b/tests/test_digest.cpp @@ -154,6 +154,20 @@ TEST_F(MySQLDigestTest, SetVariableDigest) { EXPECT_EQ(d1.hash, d2.hash); } +TEST_F(MySQLDigestTest, QuotedUserVariablesNormalizeSafelyAndConsistently) { + const char* plain_sql = "SELECT @plain"; + EXPECT_EQ(normalized(plain_sql), "SELECT @plain"); + EXPECT_EQ(normalized_token(plain_sql), "SELECT @plain"); + + const char* select_sql = "SELECT @'a-b'"; + EXPECT_EQ(normalized(select_sql), "SELECT @?"); + EXPECT_EQ(normalized_token(select_sql), "SELECT @?"); + + const char* set_sql = "SET @`a``b` = 1"; + EXPECT_EQ(normalized(set_sql), "SET @? = ?"); + EXPECT_EQ(normalized_token(set_sql), "SET @? = ?"); +} + // ========== NULL and boolean literals ========== TEST_F(MySQLDigestTest, NullPreserved) { diff --git a/tests/test_emitter.cpp b/tests/test_emitter.cpp index f0a8774..770c68b 100644 --- a/tests/test_emitter.cpp +++ b/tests/test_emitter.cpp @@ -92,9 +92,14 @@ TEST_F(MySQLEmitterTest, SetDottedUserVariable) { EXPECT_EQ(out, "SET @user.var = 7"); } +TEST_F(MySQLEmitterTest, QuotedUserVariableRetainsExactReplaySpelling) { + EXPECT_EQ(round_trip("SELECT @'a-b'"), "SELECT @'a-b'"); + EXPECT_EQ(round_trip("SET @`a``b` = 1"), "SET @`a``b` = 1"); +} + TEST_F(MySQLEmitterTest, SetScopedCommaItemAfterUserVariable) { std::string out = round_trip("SET @'mix' := 1, LOCAL wait_timeout := 20"); - EXPECT_EQ(out, "SET @mix = 1, LOCAL wait_timeout = 20"); + EXPECT_EQ(out, "SET @'mix' = 1, LOCAL wait_timeout = 20"); } TEST_F(MySQLEmitterTest, SetTransaction) { diff --git a/tests/test_expression.cpp b/tests/test_expression.cpp index 03adc27..ef98bbb 100644 --- a/tests/test_expression.cpp +++ b/tests/test_expression.cpp @@ -99,14 +99,102 @@ TEST_F(ExpressionTest, DefaultKeyword) { TEST_F(ExpressionTest, UserVariable) { AstNode* node = parse_expr("@my_var"); ASSERT_NE(node, nullptr); - EXPECT_EQ(node->type, NodeType::NODE_COLUMN_REF); + EXPECT_EQ(node->type, NodeType::NODE_USER_VARIABLE); + EXPECT_EQ(std::string(node->value().ptr, node->value().len), "my_var"); + EXPECT_EQ(std::string(node->source().ptr, node->source().len), "@my_var"); +} + +TEST_F(ExpressionTest, LosslessLiteralSourcesAndTypes) { + struct LiteralCase { + const char* sql; + NodeType type; + }; + const LiteralCase cases[] = { + {"1", NodeType::NODE_LITERAL_INT}, + {"1.25", NodeType::NODE_LITERAL_FLOAT}, + {".25", NodeType::NODE_LITERAL_FLOAT}, + {"1.", NodeType::NODE_LITERAL_FLOAT}, + {"1e3", NodeType::NODE_LITERAL_FLOAT}, + {"1.2E-3", NodeType::NODE_LITERAL_FLOAT}, + {"0xCAFE", NodeType::NODE_LITERAL_HEX}, + {"X'CAFE'", NodeType::NODE_LITERAL_HEX}, + {"0b101", NodeType::NODE_LITERAL_BIT}, + {"B'101'", NodeType::NODE_LITERAL_BIT}, + {"'a''b'", NodeType::NODE_LITERAL_STRING}, + {"\"a\\\"b\"", NodeType::NODE_LITERAL_STRING}, + {"NULL", NodeType::NODE_LITERAL_NULL}, + }; + + for (const auto& tc : cases) { + SCOPED_TRACE(tc.sql); + arena.reset(); + AstNode* node = parse_expr(tc.sql); + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->type, tc.type); + EXPECT_EQ(std::string(node->source().ptr, node->source().len), tc.sql); + } +} + +TEST_F(ExpressionTest, UserVariableNamesDecodeWithoutLosingSource) { + struct VariableCase { + const char* sql; + const char* decoded; + }; + const VariableCase cases[] = { + {"@plain", "plain"}, + {"@with.dot", "with.dot"}, + {"@with$dollar", "with$dollar"}, + {"@'quoted-name'", "quoted-name"}, + {"@\"quoted-name\"", "quoted-name"}, + {"@`quoted-name`", "quoted-name"}, + {"@'a''b'", "a'b"}, + {"@\"a\"\"b\"", "a\"b"}, + {"@`a``b`", "a`b"}, + {"@'back\\\\slash'", "back\\\\slash"}, + }; + + for (const auto& tc : cases) { + SCOPED_TRACE(tc.sql); + arena.reset(); + AstNode* node = parse_expr(tc.sql); + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->type, NodeType::NODE_USER_VARIABLE); + EXPECT_EQ(std::string(node->value().ptr, node->value().len), tc.decoded); + EXPECT_EQ(std::string(node->source().ptr, node->source().len), tc.sql); + } +} + +TEST_F(ExpressionTest, UserVariableNamesEnforceDecodedLengthLimit) { + std::string accepted = "@" + std::string(64, 'a'); + AstNode* node = parse_expr(accepted.c_str()); + ASSERT_NE(node, nullptr); + EXPECT_EQ(node->value().len, 64u); + + arena.reset(); + std::string rejected = "@" + std::string(65, 'a'); + EXPECT_EQ(parse_expr(rejected.c_str()), nullptr); +} + +TEST_F(ExpressionTest, UnarySignsRetainWholeSourceSpan) { + AstNode* negative = parse_expr("-1.2E-3"); + ASSERT_NE(negative, nullptr); + EXPECT_EQ(negative->type, NodeType::NODE_UNARY_OP); + EXPECT_EQ(std::string(negative->source().ptr, negative->source().len), "-1.2E-3"); + + arena.reset(); + AstNode* positive = parse_expr("+0xCAFE"); + ASSERT_NE(positive, nullptr); + EXPECT_EQ(positive->type, NodeType::NODE_UNARY_OP); + EXPECT_EQ(std::string(positive->source().ptr, positive->source().len), "+0xCAFE"); } TEST_F(ExpressionTest, ParenthesizedExpression) { AstNode* node = parse_expr("(42)"); ASSERT_NE(node, nullptr); - EXPECT_EQ(node->type, NodeType::NODE_LITERAL_INT); - EXPECT_EQ(std::string(node->value_ptr, node->value_len), "42"); + EXPECT_EQ(node->type, NodeType::NODE_EXPRESSION); + EXPECT_EQ(std::string(node->source().ptr, node->source().len), "(42)"); + ASSERT_NE(node->first_child, nullptr); + EXPECT_EQ(node->first_child->type, NodeType::NODE_LITERAL_INT); } // ===== Task 2: Binary Operators, IS NULL, BETWEEN, IN, Functions ===== @@ -229,8 +317,11 @@ TEST_F(ExpressionTest, NestedParens) { EXPECT_EQ(std::string(node->value_ptr, node->value_len), "*"); // Left child should be 1+2 ASSERT_NE(node->first_child, nullptr); - EXPECT_EQ(node->first_child->type, NodeType::NODE_BINARY_OP); - EXPECT_EQ(std::string(node->first_child->value_ptr, node->first_child->value_len), "+"); + EXPECT_EQ(node->first_child->type, NodeType::NODE_EXPRESSION); + ASSERT_NE(node->first_child->first_child, nullptr); + EXPECT_EQ(node->first_child->first_child->type, NodeType::NODE_BINARY_OP); + EXPECT_EQ(std::string(node->first_child->first_child->value_ptr, + node->first_child->first_child->value_len), "+"); } TEST_F(ExpressionTest, LikeOperator) { diff --git a/tests/test_set.cpp b/tests/test_set.cpp index f4a2541..02b0f2d 100644 --- a/tests/test_set.cpp +++ b/tests/test_set.cpp @@ -5,6 +5,65 @@ using namespace sql_parser; +namespace { +std::string ref_string(StringRef ref) { + return ref.ptr ? std::string(ref.ptr, ref.len) : std::string(); +} +} + +TEST(MySQLSetUserVariable, TargetHasDecodedValueAndExactSource) { + Parser parser; + const char* sql = "SET @`a``b` = -1.25"; + ParseResult r = parser.parse(sql, strlen(sql)); + ASSERT_EQ(r.status, ParseResult::OK); + ASSERT_NE(r.ast, nullptr); + AstNode* assignment = r.ast->first_child; + ASSERT_NE(assignment, nullptr); + AstNode* target = assignment->first_child; + ASSERT_NE(target, nullptr); + AstNode* variable = target->first_child; + ASSERT_NE(variable, nullptr); + EXPECT_EQ(variable->type, NodeType::NODE_USER_VARIABLE); + EXPECT_EQ(ref_string(variable->value()), "a`b"); + EXPECT_EQ(ref_string(variable->source()), "@`a``b`"); + AstNode* rhs = target->next_sibling; + ASSERT_NE(rhs, nullptr); + EXPECT_EQ(rhs->type, NodeType::NODE_UNARY_OP); + EXPECT_EQ(ref_string(rhs->source()), "-1.25"); +} + +TEST(MySQLSetCompleteness, OnlyEofOrOneTrailingSemicolonIsFullInput) { + Parser parser; + struct Case { const char* sql; bool full; const char* remaining; }; + const Case cases[] = { + {"SET @x=1", true, ""}, + {"SET @x=1;", true, ""}, + {"SET @x=1; ", true, ""}, + {"SET @x=1 trailing", false, "trailing"}, + {"SET @x=1,", false, ","}, + {"SET @x=1; SELECT 1", false, "SELECT 1"}, + {"SET @x=1;;", false, ";"}, + }; + + for (const auto& tc : cases) { + SCOPED_TRACE(tc.sql); + ParseResult r = parser.parse(tc.sql, strlen(tc.sql)); + EXPECT_EQ(r.full_input, tc.full); + EXPECT_EQ(ref_string(r.remaining), tc.remaining); + } +} + +TEST(MySQLSetCompleteness, MissingClosingParenthesisIsAnError) { + Parser parser; + const char* cases[] = {"SET @x=(1;", "SET @x=(1,2", "SET @x=(1"}; + for (const char* sql : cases) { + SCOPED_TRACE(sql); + ParseResult r = parser.parse(sql, strlen(sql)); + EXPECT_EQ(r.status, ParseResult::ERROR); + EXPECT_FALSE(r.full_input); + } +} + // ============================================================================ // Data-driven test infrastructure // ============================================================================ @@ -453,7 +512,9 @@ TEST_F(MySQLSetTest, SetDottedUserVariable) { AstNode* target = r.ast->first_child->first_child; ASSERT_NE(target, nullptr); ASSERT_NE(target->first_child, nullptr); - EXPECT_EQ(value(target->first_child), "@user.var"); + EXPECT_EQ(target->first_child->type, NodeType::NODE_USER_VARIABLE); + EXPECT_EQ(value(target->first_child), "user.var"); + EXPECT_EQ(ref_string(target->first_child->source()), "@user.var"); } TEST_F(MySQLSetTest, SetScopedCommaItemAfterUserVariable) { @@ -692,7 +753,7 @@ TEST_F(MySQLSetTest, SetUserVariableRHS) { ASSERT_NE(target, nullptr); AstNode* rhs = target->next_sibling; ASSERT_NE(rhs, nullptr); - EXPECT_EQ(rhs->type, NodeType::NODE_COLUMN_REF); + EXPECT_EQ(rhs->type, NodeType::NODE_USER_VARIABLE); } // ============================================================================ @@ -718,7 +779,8 @@ TEST_F(PgSQLSetTest, SetVarEqualValue) { } TEST_F(PgSQLSetTest, SetLocalVar) { - auto r = parser.parse("SET LOCAL timezone = 'UTC'", 25); + const char* sql = "SET LOCAL timezone = 'UTC'"; + auto r = parser.parse(sql, strlen(sql)); EXPECT_EQ(r.status, ParseResult::OK); ASSERT_NE(r.ast, nullptr); } @@ -918,7 +980,7 @@ TEST(MySQLSetBulk, LenientAcceptsUnusualSyntax) { // ============================================================================ // Helper: walk to first VAR_ASSIGNMENT's VAR_TARGET, then to its single -// IDENTIFIER child (single-target case). Returns nullptr if shape differs. +// variable child (single-target case). Returns nullptr if shape differs. static const AstNode* first_target_identifier(const AstNode* set_stmt) { if (!set_stmt || set_stmt->type != NodeType::NODE_SET_STMT) return nullptr; const AstNode* va = set_stmt->first_child; @@ -926,7 +988,8 @@ static const AstNode* first_target_identifier(const AstNode* set_stmt) { const AstNode* vt = va->first_child; if (!vt || vt->type != NodeType::NODE_VAR_TARGET) return nullptr; const AstNode* id = vt->first_child; - if (!id || id->type != NodeType::NODE_IDENTIFIER) return nullptr; + if (!id || (id->type != NodeType::NODE_IDENTIFIER && + id->type != NodeType::NODE_USER_VARIABLE)) return nullptr; return id; } @@ -962,7 +1025,9 @@ TEST(MySQLSetP6, AtBacktickedNameKeepsFullText) { EXPECT_EQ(r.status, ParseResult::OK); const AstNode* id = first_target_identifier(r.ast); ASSERT_NE(id, nullptr); - EXPECT_EQ(std::string(id->value_ptr, id->value_len), "@my_var"); + EXPECT_EQ(id->type, NodeType::NODE_USER_VARIABLE); + EXPECT_EQ(ref_string(id->value()), "my_var"); + EXPECT_EQ(ref_string(id->source()), "@`my_var`"); } TEST(MySQLSetP6, DoubleAtScopedBacktickedNameKeepsFullText) { diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp index 0ea194e..ea094a7 100644 --- a/tests/test_tokenizer.cpp +++ b/tests/test_tokenizer.cpp @@ -102,10 +102,8 @@ TEST_F(MySQLTokenizerTest, AtVariables) { tok.reset(sql, strlen(sql)); Token t = tok.next_token(); - EXPECT_EQ(t.type, TokenType::TK_AT); - - t = tok.next_token(); - EXPECT_EQ(t.type, TokenType::TK_IDENTIFIER); + EXPECT_EQ(t.type, TokenType::TK_USER_VARIABLE); + EXPECT_EQ(std::string(t.source.ptr, t.source.len), "@myvar"); t = tok.next_token(); EXPECT_EQ(t.type, TokenType::TK_DOUBLE_AT); @@ -114,6 +112,120 @@ TEST_F(MySQLTokenizerTest, AtVariables) { EXPECT_EQ(t.type, TokenType::TK_IDENTIFIER); } +TEST_F(MySQLTokenizerTest, LosslessLiteralTokens) { + struct LiteralCase { + const char* sql; + TokenType type; + const char* text; + }; + const LiteralCase cases[] = { + {"1", TokenType::TK_INTEGER, "1"}, + {"1.25", TokenType::TK_FLOAT, "1.25"}, + {".25", TokenType::TK_FLOAT, ".25"}, + {"1.", TokenType::TK_FLOAT, "1."}, + {"1e3", TokenType::TK_FLOAT, "1e3"}, + {"1.2E-3", TokenType::TK_FLOAT, "1.2E-3"}, + {"0xCAFE", TokenType::TK_HEX_LITERAL, "0xCAFE"}, + {"X'CAFE'", TokenType::TK_HEX_LITERAL, "X'CAFE'"}, + {"0b101", TokenType::TK_BIT_LITERAL, "0b101"}, + {"B'101'", TokenType::TK_BIT_LITERAL, "B'101'"}, + {"'a''b'", TokenType::TK_STRING, "a''b"}, + {"\"a\\\"b\"", TokenType::TK_STRING, "a\\\"b"}, + {"NULL", TokenType::TK_NULL, "NULL"}, + }; + + for (const auto& tc : cases) { + SCOPED_TRACE(tc.sql); + tok.reset(tc.sql, strlen(tc.sql)); + Token t = tok.next_token(); + EXPECT_EQ(t.type, tc.type); + EXPECT_EQ(std::string(t.text.ptr, t.text.len), tc.text); + EXPECT_EQ(std::string(t.source.ptr, t.source.len), tc.sql); + EXPECT_EQ(tok.next_token().type, TokenType::TK_EOF); + } +} + +TEST_F(MySQLTokenizerTest, UserVariableIsOneLosslessToken) { + const char* cases[] = { + "@plain", "@with.dot", "@with$dollar", "@'quoted-name'", + "@\"quoted-name\"", "@`quoted-name`", "@'a''b'", "@`a``b`", + "@'back\\\\slash'", + }; + + for (const char* sql : cases) { + SCOPED_TRACE(sql); + tok.reset(sql, strlen(sql)); + Token t = tok.next_token(); + EXPECT_EQ(t.type, TokenType::TK_USER_VARIABLE); + EXPECT_EQ(std::string(t.source.ptr, t.source.len), sql); + EXPECT_EQ(tok.next_token().type, TokenType::TK_EOF); + } + + tok.reset("@@session.sql_mode", strlen("@@session.sql_mode")); + EXPECT_EQ(tok.next_token().type, TokenType::TK_DOUBLE_AT); +} + +TEST_F(MySQLTokenizerTest, MalformedLosslessTokensAreErrors) { + const char* cases[] = { + "0x", "0xGG", "0xCAFG", "0b", "0b2", "0b102", + "X'", "X'GG'", "X'CAFE", "B'", "B'2'", "B'101", + "1e", "1e+", "1.2E-", "'unterminated", "\"unterminated", + "@'unterminated", "@\"unterminated", "@`unterminated", + }; + + for (const char* sql : cases) { + SCOPED_TRACE(sql); + tok.reset(sql, strlen(sql)); + EXPECT_EQ(tok.next_token().type, TokenType::TK_ERROR); + EXPECT_TRUE(tok.has_error()); + } +} + +TEST_F(MySQLTokenizerTest, DoubleDashRequiresFollowingWhitespaceOrControl) { + const char* sql = "SELECT 1--@x"; + tok.reset(sql, strlen(sql)); + EXPECT_EQ(tok.next_token().type, TokenType::TK_SELECT); + EXPECT_EQ(tok.next_token().type, TokenType::TK_INTEGER); + EXPECT_EQ(tok.next_token().type, TokenType::TK_MINUS); + EXPECT_EQ(tok.next_token().type, TokenType::TK_MINUS); + EXPECT_EQ(tok.next_token().type, TokenType::TK_USER_VARIABLE); + EXPECT_TRUE(tok.has_user_variables()); + + sql = "SELECT 1-- @x\n"; + tok.reset(sql, strlen(sql)); + EXPECT_EQ(tok.next_token().type, TokenType::TK_SELECT); + EXPECT_EQ(tok.next_token().type, TokenType::TK_INTEGER); + EXPECT_EQ(tok.next_token().type, TokenType::TK_EOF); + EXPECT_FALSE(tok.has_user_variables()); + + const char control_sql[] = "SELECT 1--\x7f@x\n"; + tok.reset(control_sql, sizeof(control_sql) - 1); + EXPECT_EQ(tok.next_token().type, TokenType::TK_SELECT); + EXPECT_EQ(tok.next_token().type, TokenType::TK_INTEGER); + EXPECT_EQ(tok.next_token().type, TokenType::TK_EOF); + EXPECT_FALSE(tok.has_user_variables()); +} + +TEST_F(MySQLTokenizerTest, UnterminatedBlockCommentIsAnError) { + const char* sql = "SELECT @x /* unterminated"; + tok.reset(sql, strlen(sql)); + while (tok.next_token().type != TokenType::TK_EOF) {} + EXPECT_TRUE(tok.has_error()); +} + +TEST_F(MySQLTokenizerTest, MariaDBExecutableCommentsExposeUserVariables) { + const char* cases[] = { + "/*M!100100 SET @x=1 */", + "/*M! SET @x=1 */", + }; + for (const char* sql : cases) { + SCOPED_TRACE(sql); + tok.reset(sql, strlen(sql)); + while (tok.next_token().type != TokenType::TK_EOF) {} + EXPECT_TRUE(tok.has_user_variables()); + } +} + TEST_F(MySQLTokenizerTest, Placeholder) { const char* sql = "?"; tok.reset(sql, strlen(sql)); diff --git a/tests/test_user_variable.cpp b/tests/test_user_variable.cpp new file mode 100644 index 0000000..848fecb --- /dev/null +++ b/tests/test_user_variable.cpp @@ -0,0 +1,82 @@ +#include +#include "sql_parser/parser.h" +#include "sql_parser/user_variable.h" + +#include + +using namespace sql_parser; + +namespace { +UserVariableUsage classify(const char* sql) { + Parser parser; + ParseResult result = parser.parse(sql, std::strlen(sql)); + return classify_mysql_user_variable_usage(result); +} +} + +TEST(MySQLUserVariableUsage, IgnoresStringsAndComments) { + EXPECT_EQ(classify("SELECT 1"), UserVariableUsage::NO_USER_VARIABLE); + EXPECT_EQ(classify("SELECT '@x'"), UserVariableUsage::NO_USER_VARIABLE); + EXPECT_EQ(classify("SELECT 1 /* @x */"), UserVariableUsage::NO_USER_VARIABLE); +} + +TEST(MySQLUserVariableUsage, AllowsDirectReadsAndPredicates) { + EXPECT_EQ(classify("SELECT @x"), UserVariableUsage::READ_ONLY); + EXPECT_EQ(classify("SELECT -@x + 1 WHERE @y = 2 AND @z IS NOT NULL"), + UserVariableUsage::READ_ONLY); +} + +TEST(MySQLUserVariableUsage, RejectsWritesAndIntoTargets) { + EXPECT_EQ(classify("SET @x=1"), UserVariableUsage::UNSAFE_OR_UNKNOWN); + EXPECT_EQ(classify("SELECT @x:=1"), UserVariableUsage::UNSAFE_OR_UNKNOWN); + EXPECT_EQ(classify("SELECT id INTO @x FROM test.uv_source"), + UserVariableUsage::UNSAFE_OR_UNKNOWN); +} + +TEST(MySQLUserVariableUsage, RejectsSensitiveOrUnknownContexts) { + const char* cases[] = { + "SELECT COALESCE(@x, 1)", + "CALL p(@x)", + "DO @x", + "SELECT ? + @x", + "SELECT (@x IN (SELECT id FROM t))", + "SELECT @x FROM (SELECT 1) AS s", + }; + for (const char* sql : cases) { + SCOPED_TRACE(sql); + EXPECT_EQ(classify(sql), UserVariableUsage::UNSAFE_OR_UNKNOWN); + } +} + +TEST(MySQLUserVariableUsage, RejectsMalformedAndIncompleteParses) { + const char* cases[] = { + "SELECT @", "SELECT @x trailing extra", "SELECT @x; SELECT 1", + "SET @x=", "SELECT @'unterminated", + }; + for (const char* sql : cases) { + SCOPED_TRACE(sql); + EXPECT_EQ(classify(sql), UserVariableUsage::UNSAFE_OR_UNKNOWN); + } +} + +TEST(MySQLUserVariableUsage, HandlesMySQLCommentBoundariesConservatively) { + EXPECT_EQ(classify("SELECT 1--@x"), UserVariableUsage::READ_ONLY); + EXPECT_EQ(classify("/*!40101 SET @x=1 */"), + UserVariableUsage::UNSAFE_OR_UNKNOWN); + EXPECT_EQ(classify("/*M!100100 SET @x=1 */"), + UserVariableUsage::UNSAFE_OR_UNKNOWN); + EXPECT_EQ(classify("/*M! SET @x=1 */"), + UserVariableUsage::UNSAFE_OR_UNKNOWN); + EXPECT_EQ(classify("SELECT @x /* unterminated"), + UserVariableUsage::UNSAFE_OR_UNKNOWN); + EXPECT_EQ(classify("SET @x=1 /* unterminated"), + UserVariableUsage::UNSAFE_OR_UNKNOWN); +} + +TEST(MySQLUserVariableUsage, UnterminatedBlockCommentIsNotFullInput) { + Parser parser; + const char* sql = "SET @x=1 /* unterminated"; + ParseResult result = parser.parse(sql, std::strlen(sql)); + EXPECT_EQ(result.status, ParseResult::ERROR); + EXPECT_FALSE(result.full_input); +}