feat: expose lossless MySQL user variables

Makefile

@@ -42,6 +42,7 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
             $(TEST_DIR)/test_classifier.cpp \
             $(TEST_DIR)/test_expression.cpp \
             $(TEST_DIR)/test_set.cpp \
+            $(TEST_DIR)/test_user_variable.cpp \
             $(TEST_DIR)/test_select.cpp \
             $(TEST_DIR)/test_emitter.cpp \
             $(TEST_DIR)/test_stmt_cache.cpp \

include/sql_parser/ast.h

@@ -3,6 +3,7 @@
 
 #include "sql_parser/common.h"
 #include "sql_parser/arena.h"
+#include "sql_parser/token.h"
 #include <cstdint>
 #include <type_traits>
 
@@ -12,17 +13,25 @@ struct AstNode {
     AstNode* first_child;
     AstNode* next_sibling;
     const char* value_ptr;
+    const char* source_ptr;
     uint32_t value_len;
+    uint32_t source_len;
     NodeType type;
     uint16_t flags;
 
     StringRef value() const { return StringRef{value_ptr, value_len}; }
+    StringRef source() const { return StringRef{source_ptr, source_len}; }
 
     void set_value(StringRef ref) {
         value_ptr = ref.ptr;
         value_len = ref.len;
     }
 
+    void set_source(StringRef ref) {
+        source_ptr = ref.ptr;
+        source_len = ref.len;
+    }
+
     void add_child(AstNode* child) {
         if (!child) return;
         if (!first_child) {
@@ -34,7 +43,7 @@ struct AstNode {
         last->next_sibling = child;
     }
 };
-static_assert(sizeof(AstNode) == 32, "AstNode must be 32 bytes");
+static_assert(sizeof(AstNode) == 48, "AstNode layout changed unexpectedly");
 static_assert(std::is_trivially_copyable_v<AstNode>);
 
 inline AstNode* make_node(Arena& arena, NodeType type, StringRef value = {},
@@ -48,6 +57,13 @@ inline AstNode* make_node(Arena& arena, NodeType type, StringRef value = {},
     return node;
 }
 
+inline AstNode* make_node_from_token(Arena& arena, NodeType type,
+                                     const Token& token, uint16_t flags = 0) {
+    AstNode* node = make_node(arena, type, token.text, flags);
+    if (node) node->set_source(token.source);
+    return node;
+}
+
 } // namespace sql_parser
 
 #endif // SQL_PARSER_AST_H

include/sql_parser/common.h

@@ -231,6 +231,12 @@ enum class NodeType : uint16_t {
     NODE_SET_ROLE,                  // SET [LOCAL] ROLE <name>|NONE|DEFAULT
     NODE_SET_SESSION_AUTHORIZATION, // SET SESSION AUTHORIZATION <name>|DEFAULT
     NODE_SET_CONSTRAINTS,           // SET CONSTRAINTS {ALL|<name>[,...]} {DEFERRED|IMMEDIATE}
+
+    // MySQL lossless user-variable/literal nodes. Keep appended so existing
+    // enum values remain stable for consumers that index by NodeType.
+    NODE_USER_VARIABLE,
+    NODE_LITERAL_HEX,
+    NODE_LITERAL_BIT,
 };
 
 } // namespace sql_parser

include/sql_parser/digest.h

@@ -63,15 +63,17 @@ class Digest {
 
     // Helper: check if a token type is a keyword (not an identifier, literal, or operator)
     static bool is_keyword_token(TokenType type) {
-        // Keywords start at TK_SELECT and go through TK_EXCEPT
-        return static_cast<uint16_t>(type) >= static_cast<uint16_t>(TokenType::TK_SELECT);
+        return static_cast<uint16_t>(type) >= static_cast<uint16_t>(TokenType::TK_SELECT) &&
+               static_cast<uint16_t>(type) <= static_cast<uint16_t>(TokenType::TK_RECURSIVE);
     }
 
     // Helper: check if a token type is a literal value that should become ?
     static bool is_literal_token(TokenType type) {
         return type == TokenType::TK_INTEGER ||
                type == TokenType::TK_FLOAT ||
-               type == TokenType::TK_STRING;
+               type == TokenType::TK_STRING ||
+               type == TokenType::TK_HEX_LITERAL ||
+               type == TokenType::TK_BIT_LITERAL;
     }
 
     // Helper: uppercase a character
@@ -104,7 +106,12 @@ class Digest {
 
     // Emit a single token to the string builder, uppercasing keywords, replacing literals with ?
     void emit_token(StringBuilder& sb, const Token& t, TokenType prev) {
-        bool space = (prev != TokenType::TK_EOF) && needs_space_before(prev, t.type);
+        bool quoted_user_after_account = t.type == TokenType::TK_USER_VARIABLE &&
+            t.source.len >= 2 &&
+            (t.source.ptr[1] == '\'' || t.source.ptr[1] == '"' || t.source.ptr[1] == '`') &&
+            (prev == TokenType::TK_STRING || prev == TokenType::TK_QUESTION);
+        bool space = (prev != TokenType::TK_EOF) &&
+                     !quoted_user_after_account && needs_space_before(prev, t.type);
         if (space) sb.append_char(' ');
 
         if (is_literal_token(t.type)) {
@@ -115,6 +122,13 @@ class Digest {
             sb.append(t.text.ptr, t.text.len);
         } else if (t.type == TokenType::TK_QUESTION) {
             sb.append_char('?');
+        } else if (t.type == TokenType::TK_USER_VARIABLE) {
+            if (t.source.len >= 2 &&
+                (t.source.ptr[1] == '\'' || t.source.ptr[1] == '"' || t.source.ptr[1] == '`')) {
+                sb.append("@?", 2);
+            } else {
+                sb.append(t.source);
+            }
         } else if (t.type == TokenType::TK_COMMA) {
             sb.append(",", 1);
         } else {

include/sql_parser/emitter.h

@@ -123,6 +123,8 @@ class Emitter {
             case NodeType::NODE_ARRAY_SUBSCRIPT: emit_array_subscript(node); break;
             case NodeType::NODE_FIELD_ACCESS:    emit_field_access(node); break;
             case NodeType::NODE_SUBQUERY:        emit_subquery(node); break;
+            case NodeType::NODE_EXPRESSION:      emit_parenthesized_expression(node); break;
+            case NodeType::NODE_USER_VARIABLE:   emit_user_variable(node); break;
 
             // ---- Leaf nodes (emit value directly) ----
             case NodeType::NODE_PLACEHOLDER:
@@ -131,6 +133,8 @@ class Emitter {
             // ---- Leaf nodes (emit value directly) ----
             case NodeType::NODE_LITERAL_INT:
             case NodeType::NODE_LITERAL_FLOAT:
+            case NodeType::NODE_LITERAL_HEX:
+            case NodeType::NODE_LITERAL_BIT:
                 if (mode_ == EmitMode::DIGEST) { sb_.append_char('?'); break; }
                 emit_value(node); break;
             case NodeType::NODE_LITERAL_NULL:
@@ -152,6 +156,30 @@ class Emitter {
         sb_.append(node->value_ptr, node->value_len);
     }
 
+    void emit_user_variable(const AstNode* node) {
+        if (mode_ == EmitMode::DIGEST) {
+            StringRef source = node->source();
+            if (source.len >= 2 &&
+                (source.ptr[1] == '\'' || source.ptr[1] == '"' || source.ptr[1] == '`')) {
+                sb_.append("@?", 2);
+                return;
+            }
+        }
+        StringRef source = node->source();
+        if (!source.empty()) {
+            sb_.append(source.ptr, source.len);
+            return;
+        }
+        sb_.append_char('@');
+        emit_value(node);
+    }
+
+    void emit_parenthesized_expression(const AstNode* node) {
+        sb_.append_char('(');
+        if (node->first_child) emit_node(node->first_child);
+        sb_.append_char(')');
+    }
+
     void emit_string_literal(const AstNode* node) {
         sb_.append_char('\'');
         sb_.append(node->value_ptr, node->value_len);

include/sql_parser/expression_parser.h

@@ -6,6 +6,7 @@
 #include "sql_parser/tokenizer.h"
 #include "sql_parser/ast.h"
 #include "sql_parser/arena.h"
+#include "sql_parser/user_variable.h"
 
 namespace sql_parser {
 
@@ -99,19 +100,27 @@ class ExpressionParser {
         switch (t.type) {
             case TokenType::TK_INTEGER: {
                 tok_.skip();
-                return make_node(arena_, NodeType::NODE_LITERAL_INT, t.text);
+                return make_node_from_token(arena_, NodeType::NODE_LITERAL_INT, t);
             }
             case TokenType::TK_FLOAT: {
                 tok_.skip();
-                return make_node(arena_, NodeType::NODE_LITERAL_FLOAT, t.text);
+                return make_node_from_token(arena_, NodeType::NODE_LITERAL_FLOAT, t);
+            }
+            case TokenType::TK_HEX_LITERAL: {
+                tok_.skip();
+                return make_node_from_token(arena_, NodeType::NODE_LITERAL_HEX, t);
+            }
+            case TokenType::TK_BIT_LITERAL: {
+                tok_.skip();
+                return make_node_from_token(arena_, NodeType::NODE_LITERAL_BIT, t);
             }
             case TokenType::TK_STRING: {
                 tok_.skip();
-                return make_node(arena_, NodeType::NODE_LITERAL_STRING, t.text);
+                return make_node_from_token(arena_, NodeType::NODE_LITERAL_STRING, t);
             }
             case TokenType::TK_NULL: {
                 tok_.skip();
-                return make_node(arena_, NodeType::NODE_LITERAL_NULL, t.text);
+                return make_node_from_token(arena_, NodeType::NODE_LITERAL_NULL, t);
             }
             case TokenType::TK_TRUE:
             case TokenType::TK_FALSE: {
@@ -150,6 +159,10 @@ class ExpressionParser {
                     static_cast<uint32_t>((name.text.ptr + name.text.len) - t.text.ptr)};
                 return make_node(arena_, NodeType::NODE_COLUMN_REF, full);
             }
+            case TokenType::TK_USER_VARIABLE: {
+                tok_.skip();
+                return make_mysql_user_variable_node(arena_, t);
+            }
             case TokenType::TK_DOUBLE_AT: {
                 // System variable: @@name or @@scope.name
                 tok_.skip();
@@ -174,19 +187,26 @@ class ExpressionParser {
                 AstNode* operand = parse(Precedence::UNARY);
                 if (!operand) return nullptr;
                 AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text);
+                set_span_through_node_(node, t.source, operand);
                 node->add_child(operand);
                 return node;
             }
             case TokenType::TK_PLUS: {
                 // Unary plus
                 tok_.skip();
-                return parse(Precedence::UNARY);
+                AstNode* operand = parse(Precedence::UNARY);
+                if (!operand) return nullptr;
+                AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text);
+                set_span_through_node_(node, t.source, operand);
+                node->add_child(operand);
+                return node;
             }
             case TokenType::TK_NOT: {
                 tok_.skip();
                 AstNode* operand = parse(Precedence::NOT);
                 if (!operand) return nullptr;
                 AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text);
+                set_span_through_node_(node, t.source, operand);
                 node->add_child(operand);
                 return node;
             }
@@ -266,7 +286,12 @@ class ExpressionParser {
                     return parse_postfix(tuple);
                 }
                 if (tok_.peek().type == TokenType::TK_RPAREN) {
-                    tok_.skip();
+                    Token close = tok_.next_token();
+                    AstNode* wrapper = make_node(arena_, NodeType::NODE_EXPRESSION);
+                    wrapper->set_source(StringRef{t.source.ptr,
+                        static_cast<uint32_t>(close.source.ptr + close.source.len - t.source.ptr)});
+                    wrapper->add_child(expr);
+                    return parse_postfix(wrapper);
                 }
                 // Check for postfix: (expr).field or (expr)[index]
                 return parse_postfix(expr);
@@ -287,11 +312,39 @@ class ExpressionParser {
         }
     }
 
+    static void set_span_through_node_(AstNode* node, StringRef start,
+                                       const AstNode* end_node) {
+        if (!node || !start.ptr || !end_node) return;
+        StringRef end = end_node->source();
+        if (end.empty()) end = end_node->value();
+        if (!end.ptr || end.ptr < start.ptr) return;
+        node->set_source(StringRef{start.ptr,
+            static_cast<uint32_t>(end.ptr + end.len - start.ptr)});
+    }
+
     AstNode* parse_identifier_or_function(const Token& name_token) {
         // Check for function call: name(
         if (tok_.peek().type == TokenType::TK_LPAREN) {
             tok_.skip();  // consume (
             AstNode* func = make_node(arena_, NodeType::NODE_FUNCTION_CALL, name_token.text);
+            // CAST uses `CAST(expr AS type)` rather than a comma-separated
+            // argument list. Model it as a function call so consumers can
+            // reject or handle the expression without leaving valid input
+            // unconsumed.
+            if (name_token.text.equals_ci("CAST", 4)) {
+                AstNode* arg = parse();
+                if (!arg || tok_.peek().type != TokenType::TK_AS) return func;
+                func->add_child(arg);
+                tok_.skip();
+                Token type = tok_.next_token();
+                if (type.type == TokenType::TK_EOF ||
+                    type.type == TokenType::TK_RPAREN) {
+                    return func;
+                }
+                func->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, type.text));
+                if (tok_.peek().type == TokenType::TK_RPAREN) tok_.skip();
+                return func;
+            }
             // Parse argument list
             if (tok_.peek().type != TokenType::TK_RPAREN) {
                 while (true) {

include/sql_parser/parse_result.h

@@ -39,6 +39,8 @@ struct ParseResult {
     AstNode* ast = nullptr;
     ErrorInfo error;
     StringRef remaining;
+    bool full_input = false;
+    bool has_user_variables = false;
 
     StringRef table_name;
     StringRef schema_name;

include/sql_parser/parser.h

@@ -7,6 +7,7 @@
 #include "sql_parser/ast.h"
 #include "sql_parser/parse_result.h"
 #include "sql_parser/stmt_cache.h"
+#include "sql_parser/user_variable.h"
 
 namespace sql_parser {
 

include/sql_parser/set_parser.h

@@ -352,9 +352,14 @@ class SetParser {
             }
         } else {
             while (tok_.peek().type == TokenType::TK_COMMA) {
-                tok_.skip();
+                Token comma = tok_.next_token();
                 AstNode* next_assign = parse_comma_item();
-                if (next_assign) root->add_child(next_assign);
+                if (next_assign) {
+                    root->add_child(next_assign);
+                } else {
+                    tok_.flag_error_at(comma.source);
+                    break;
+                }
             }
         }
 
@@ -551,7 +556,17 @@ class SetParser {
         }
 
         Token var = tok_.peek();
-        if (var.type == TokenType::TK_AT) {
+        bool user_variable_target = false;
+        if (var.type == TokenType::TK_USER_VARIABLE) {
+            user_variable_target = true;
+            tok_.skip();
+            AstNode* variable = make_mysql_user_variable_node(arena_, var);
+            if (!variable) {
+                tok_.flag_error_at(var.source);
+                return nullptr;
+            }
+            target->add_child(variable);
+        } else if (var.type == TokenType::TK_AT) {
             // User variable @name. The name may be backtick/double-quoted;
             // in that case the source bytes between `@` and the name include
             // the opening delimiter (and the closing delimiter sits one past
@@ -624,13 +639,20 @@ class SetParser {
 
         // Expect = or := (MySQL) or TO (PostgreSQL)
         Token eq = tok_.peek();
+        bool has_assignment_operator = false;
         if (eq.type == TokenType::TK_EQUAL || eq.type == TokenType::TK_COLON_EQUAL) {
             tok_.skip();
+            has_assignment_operator = true;
         } else if constexpr (D == Dialect::PostgreSQL) {
             if (eq.type == TokenType::TK_TO) {
                 tok_.skip();
+                has_assignment_operator = true;
             }
         }
+        if (user_variable_target && !has_assignment_operator) {
+            tok_.flag_error_at(eq.source);
+            return nullptr;
+        }
 
         // Parse RHS expression. If the parser couldn't produce one --
         // typically because the input is truncated (`SET x =`), starts