diff --git a/clickhouse/types/type_parser.cpp b/clickhouse/types/type_parser.cpp index c1df828c..d305a398 100644 --- a/clickhouse/types/type_parser.cpp +++ b/clickhouse/types/type_parser.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #if defined _win_ @@ -76,6 +77,20 @@ static const std::unordered_map kTypeCode = { { "JSON", Type::JSON }, }; +static constexpr auto kUnescapeMap = [](){ + std::array m{}; + // std::fill or m.fill are not yet constant expressions in C++17 + for (auto& x : m) x = (char)-1; + m['0'] = '\0'; + m['\\'] = '\\'; + m['b'] = '\b'; + m['f'] = '\f'; + m['n'] = '\n'; + m['r'] = '\r'; + m['t'] = '\t'; + return m; +}(); + template inline int CompateStringsCaseInsensitive(const L& left, const R& right) { int64_t size_diff = left.size() - right.size(); @@ -247,19 +262,26 @@ TypeParser::Token TypeParser::NextToken() { return Token{Token::Comma, StringView(cur_++, 1)}; case '\'': { - const auto end_quote_length = 1; - const StringView end_quote{cur_, end_quote_length}; - // Fast forward to the closing quote. - const auto start = cur_++; - for (; cur_ < end_ - end_quote_length; ++cur_) { - // TODO (nemkov): handle escaping ? - if (end_quote == StringView{cur_, end_quote_length}) { - cur_ += end_quote_length; - - return Token{Token::QuotedString, StringView{start, cur_}}; + scratch_.clear(); + scratch_ += *(cur_++); // quotes must be included + for (; end_ - cur_ > 1; ++cur_) { + if (*cur_ == '\\' && end_ - cur_ > 1 && kUnescapeMap[(uint8_t)*(cur_ + 1)] != (char)-1) { + scratch_ += kUnescapeMap[(uint8_t)*(cur_ + 1)]; + ++cur_; + } + else if (*cur_ == '\\' && end_ - cur_ > 1 && *(cur_ + 1) == '\'') { + scratch_ += '\''; + ++cur_; + } + else if ('\'' == *cur_) { + scratch_ += *(cur_++); + return Token{Token::QuotedString, StringView(scratch_)}; + } + else { + scratch_ += *cur_; } } - return Token{Token::QuotedString, StringView(cur_++, 1)}; + return Token{Token::QuotedString, StringView(scratch_)}; } case '"': case '`': diff --git a/clickhouse/types/type_parser.h b/clickhouse/types/type_parser.h index fb58ec46..cdef18ee 100644 --- a/clickhouse/types/type_parser.h +++ b/clickhouse/types/type_parser.h @@ -77,6 +77,7 @@ class TypeParser { bool Parse(TypeAst* type); private: + // WARNING: The StringView in this token is only valid until the next NextToken() call. Token NextToken(); private: @@ -85,10 +86,11 @@ class TypeParser { TypeAst* type_; std::stack open_elements_; - // Backing storage for unescaped QuotedIdentifier token values. When a - // quoted identifier contains escape sequences the unescaped content is - // written here and the returned StringView points into this string. - // Valid only until the next NextToken() call. + + // Backing storage for identifiers with processed escape sequences. When an + // identifier contains escape sequences, the cleaned content is written here and + // the returned StringView points into this scratch. + // WARNING: Valid only until the next NextToken() call. std::string scratch_; }; diff --git a/clickhouse/types/types.cpp b/clickhouse/types/types.cpp index 4a1276b1..9b52255f 100644 --- a/clickhouse/types/types.cpp +++ b/clickhouse/types/types.cpp @@ -5,9 +5,25 @@ #include #include +#include namespace clickhouse { +static constexpr auto kEscapeMap = []{ + std::array m{}; + // std::fill or m.fill are not yet constant expressions in C++17 + for (auto& x : m) x = (char)-1; + m['\''] = '\''; + m['\0'] = '0'; + m['\\'] = '\\'; + m['\b'] = 'b'; + m['\f'] = 'f'; + m['\n'] = 'n'; + m['\r'] = 'r'; + m['\t'] = 't'; + return m; +}(); + Type::Type(const Code code) : code_(code) , type_unique_id_(0) @@ -304,6 +320,22 @@ DecimalType::DecimalType(size_t precision, size_t scale) // TODO: assert(precision <= 38 && precision > 0); } +static std::string EscapeStringLiteral(std::string_view in) +{ + std::string ret{}; + ret.reserve(in.size()); + for (char c : in) { + if (kEscapeMap[(uint8_t)c] != (char)-1) { + ret.push_back('\\'); + ret.push_back(kEscapeMap[(uint8_t)c]); + } + else { + ret.push_back(c); + } + } + return ret; +} + std::string DecimalType::GetName() const { switch (GetCode()) { case Decimal: @@ -340,7 +372,7 @@ std::string EnumType::GetName() const { for (auto ei = value_to_name_.begin(); ei != value_to_name_.end();) { result += "'"; - result += ei->second; + result += EscapeStringLiteral(ei->second); result += "' = "; result += std::to_string(ei->first); @@ -413,7 +445,7 @@ std::string DateTimeType::GetName() const { std::string datetime_representation = "DateTime"; const auto & timezone = Timezone(); if (!timezone.empty()) - datetime_representation += "('" + timezone + "')"; + datetime_representation += "('" + EscapeStringLiteral(timezone) + "')"; return datetime_representation; } @@ -436,7 +468,7 @@ std::string DateTime64Type::GetName() const { const auto & timezone = Timezone(); if (!timezone.empty()) { - datetime64_representation += ", '" + timezone + "'"; + datetime64_representation += ", '" + EscapeStringLiteral(timezone) + "'"; } datetime64_representation += ")"; diff --git a/ut/CreateColumnByType_ut.cpp b/ut/CreateColumnByType_ut.cpp index e312c116..279a19cc 100644 --- a/ut/CreateColumnByType_ut.cpp +++ b/ut/CreateColumnByType_ut.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,64 @@ TEST(CreateColumnByType, DateTime) { ASSERT_EQ(CreateColumnByType("DateTime('UTC')")->As()->Timezone(), "UTC"); ASSERT_EQ(CreateColumnByType("DateTime64(3, 'UTC')")->As()->Timezone(), "UTC"); + ASSERT_EQ(CreateColumnByType("DateTime('Etc/Can\\'t')")->As()->Timezone(), "Etc/Can't"); + ASSERT_EQ(CreateColumnByType("DateTime64(3, 'A\\\\B')")->As()->Timezone(), "A\\B"); +} + +TEST(CreateColumnByType, EnumEscapedNames) { + const std::vector enum_items = {{"can't", 1}, {"a\\b", 2}, {"a,b=(c)", 3}, {"", 4}}; + auto col = CreateColumnByType("Enum8('can\\'t' = 1, 'a\\\\b' = 2, 'a,b=(c)' = 3, '' = 4)"); + ASSERT_NE(nullptr, col); + ASSERT_TRUE(col->Type()->IsEqual(Type::CreateEnum8(enum_items))); +} + +// Round-trip: build a Type from raw values, render it via GetName() (escape), +// re-parse the rendered name via CreateColumnByType (unescape) and verify the +// raw values survive the render->parse cycle. Uses only escape symbols that are +// currently implemented. + +TEST(CreateColumnByType, RoundTrip_Enum) { + const std::vector enum_items = {{"can't", 1}, {"a\\b", 2}, {"tab\there", 3}, {"line\nbreak", 4}}; + auto type = Type::CreateEnum8(enum_items); + + auto col = CreateColumnByType(type->GetName()); + ASSERT_NE(nullptr, col); + EXPECT_TRUE(col->Type()->IsEqual(type)); + + const auto* enum_type = col->Type()->As(); + ASSERT_NE(nullptr, enum_type); + EXPECT_EQ(enum_type->GetEnumName(1), "can't"); + EXPECT_EQ(enum_type->GetEnumValue("a\\b"), 2); + EXPECT_EQ(enum_type->GetEnumName(3), "tab\there"); + EXPECT_EQ(enum_type->GetEnumValue("line\nbreak"), 4); +} + +TEST(CreateColumnByType, RoundTrip_DateTime) { + auto type = Type::CreateDateTime("Etc/Can't"); + + auto col = CreateColumnByType(type->GetName()); + ASSERT_NE(nullptr, col); + EXPECT_EQ(col->As()->Timezone(), "Etc/Can't"); +} + +TEST(CreateColumnByType, RoundTrip_DateTime64) { + auto type = Type::CreateDateTime64(3, "A\\B"); + + auto col = CreateColumnByType(type->GetName()); + ASSERT_NE(nullptr, col); + EXPECT_EQ(col->As()->Timezone(), "A\\B"); +} + +TEST(CreateColumnByType, RoundTrip_Tuple) { + const std::vector item_names = {"a`b", "c.d"}; + auto type = Type::CreateTuple({Type::CreateSimple(), Type::CreateString()}, item_names); + + auto col = CreateColumnByType(type->GetName()); + ASSERT_NE(nullptr, col); + + const auto* tuple_type = col->Type()->As(); + ASSERT_NE(nullptr, tuple_type); + EXPECT_EQ(tuple_type->GetItemNames(), item_names); } TEST(CreateColumnByType, AggregateFunction) { diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 8ff874a0..dff1400e 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -835,6 +835,60 @@ TEST_P(ClientCase, Enum) { EXPECT_EQ(sizeof(TEST_DATA)/sizeof(TEST_DATA[0]), row); } +// Live-server round-trip of escape sequences in an Enum type: hand-write a +// CREATE TABLE whose Enum labels contain escape sequences, insert matching +// values via the streaming BeginInsert/SendInsertBlock/EndInsert API (so the +// inserted block's columns are derived from the server-reported schema), select +// them back and verify the client decodes the labels unchanged. +// NOTE: Includes backspace ('\b') and other escape sequences to verify the client unescapes Enum labels correctly. +// NOTE: DateTime/DateTime64 are intentionally not covered here because the +// server validates timezone names, so escaped/unknown timezones can't be +// stored (those cases stay covered by the client-side unit tests). +TEST_P(ClientCase, EscapeRoundtrip_Enum) { + const std::vector enum_items = { + {"q'q", 1}, {"bs\\bs", 2}, {"tab\ttab", 3}, {"nl\nnl", 4}, {"cr\rcr", 5}, {"bsp\bbsp", 6}, + }; + + client_->Execute("DROP TEMPORARY TABLE IF EXISTS test_clickhouse_cpp_escape_enum;"); + client_->Execute( + "CREATE TEMPORARY TABLE IF NOT EXISTS test_clickhouse_cpp_escape_enum " + "(id UInt64, e Enum8('q\\'q' = 1, 'bs\\\\bs' = 2, 'tab\\ttab' = 3, 'nl\\nnl' = 4, 'cr\\rcr' = 5, 'bsp\\bbsp' = 6))"); + + { + auto block = client_->BeginInsert("INSERT INTO test_clickhouse_cpp_escape_enum VALUES"); + ASSERT_EQ(size_t(2), block.GetColumnCount()); + auto id = block[0]->As(); + auto e = block[1]->As(); + for (const auto& [name, value] : enum_items) { + id->Append(static_cast(value)); + e->Append(name); + } + block.RefreshRowCount(); + client_->SendInsertBlock(block); + client_->EndInsert(); + } + + size_t row = 0; + client_->Select("SELECT id, e FROM test_clickhouse_cpp_escape_enum ORDER BY id", + [&](const Block& block) { + if (block.GetRowCount() == 0) { + return; + } + const auto* enum_type = block[1]->Type()->As(); + ASSERT_NE(nullptr, enum_type); + for (size_t i = 0; i < block.GetRowCount(); ++i, ++row) { + ASSERT_LT(row, enum_items.size()); + const auto& [name, value] = enum_items[row]; + EXPECT_EQ(static_cast(value), (*block[0]->As())[i]); + EXPECT_EQ(value, block[1]->As()->At(i)); + EXPECT_EQ(name, block[1]->As()->NameAt(i)); + EXPECT_EQ(name, enum_type->GetEnumName(value)); + EXPECT_EQ(value, enum_type->GetEnumValue(name)); + } + }); + EXPECT_EQ(enum_items.size(), row); +} + TEST_P(ClientCase, Decimal) { client_->Execute( "CREATE TEMPORARY TABLE IF NOT EXISTS " diff --git a/ut/type_parser_ut.cpp b/ut/type_parser_ut.cpp index 561b1c9d..76bedd7a 100644 --- a/ut/type_parser_ut.cpp +++ b/ut/type_parser_ut.cpp @@ -96,6 +96,70 @@ TEST(TypeParserCase, ParseEnum) { } } +TEST(TypeParserCase, ParseEnum_AllEscapedBytes) { + // {fragment as seen by the parser, expected decoded raw byte} + const std::vector> cases = { + {"\\0", '\0'}, // NUL + {"\\b", '\b'}, // Backspace + {"\\t", '\t'}, // Tab + {"\\n", '\n'}, // Line feed + {"\\f", '\f'}, // Form feed + {"\\r", '\r'}, // Carriage return + {"\\\\", '\\'}, // Backslash + {"\\'", '\''}, // Single quote + }; + + // Each escaped byte, embedded in the middle of a label. + for (const auto& [fragment, raw] : cases) { + TypeAst ast; + ASSERT_TRUE(TypeParser("Enum8('x" + fragment + "y' = 1)").Parse(&ast)) + << "failed to parse fragment: " << fragment; + ASSERT_EQ(ast.meta, TypeAst::Enum); + ASSERT_EQ(ast.elements.size(), 2u); + EXPECT_EQ(ast.elements[0].value_string, std::string("x") + raw + "y") + << "wrong decoding for fragment: " << fragment; + } + + // Escaped byte as the only/first/last character of the label. + for (const auto& [fragment, raw] : cases) { + TypeAst only; + ASSERT_TRUE(TypeParser("Enum8('" + fragment + "' = 1)").Parse(&only)); + EXPECT_EQ(only.elements[0].value_string, std::string(1, raw)); + + TypeAst first; + ASSERT_TRUE(TypeParser("Enum8('" + fragment + "z' = 1)").Parse(&first)); + EXPECT_EQ(first.elements[0].value_string, std::string(1, raw) + "z"); + + TypeAst last; + ASSERT_TRUE(TypeParser("Enum8('z" + fragment + "' = 1)").Parse(&last)); + EXPECT_EQ(last.elements[0].value_string, std::string("z") + raw); + } + + // All escaped bytes combined in a single label. + { + std::string escaped; + std::string expected; + for (const auto& [fragment, raw] : cases) { + escaped += fragment; + expected += raw; + } + TypeAst ast; + ASSERT_TRUE(TypeParser("Enum8('" + escaped + "' = 1)").Parse(&ast)); + EXPECT_EQ(ast.elements[0].value_string, expected); + } + + // Bytes that are NOT escaped must pass through unchanged. + { + TypeAst ast; + ASSERT_TRUE(TypeParser(std::string("Enum8('x\x07y' = 1)")).Parse(&ast)); // BEL + EXPECT_EQ(ast.elements[0].value_string, std::string("x\x07y")); + + TypeAst emoji; + ASSERT_TRUE(TypeParser("Enum8('😀' = 1)").Parse(&emoji)); + EXPECT_EQ(emoji.elements[0].value_string, "😀"); + } +} + TEST(TypeParserCase, ParseTuple) { TypeAst ast; TypeParser( @@ -278,6 +342,79 @@ TEST(TypeParserCase, ParseDateTime_MINSK_TIMEZONE) { ASSERT_EQ(ast.elements[0].meta, TypeAst::Terminal); } +TEST(TypeParserCase, ParseTimezoneEscapedStrings) { + const std::vector escaped_timezones = {"can\\'t", "a\\tb", "a\\\\b", "a\\nb", "a\\rb", "😀", "a,b=(c)"}; + const std::vector timezones = {"can't", "a\tb", "a\\b", "a\nb", "a\rb", "😀", "a,b=(c)"}; + + for (size_t i = 0; i < timezones.size(); ++i) { + TypeAst date_time; + ASSERT_TRUE(TypeParser("DateTime('" + escaped_timezones[i] + "')").Parse(&date_time)); + ASSERT_EQ(date_time.code, Type::DateTime); + ASSERT_EQ(date_time.elements.size(), 1u); + EXPECT_EQ(date_time.elements[0].value_string, timezones[i]); + + TypeAst date_time64; + ASSERT_TRUE(TypeParser("DateTime64(3, '" + escaped_timezones[i] + "')").Parse(&date_time64)); + ASSERT_EQ(date_time64.code, Type::DateTime64); + ASSERT_EQ(date_time64.elements.size(), 2u); + EXPECT_EQ(date_time64.elements[1].value_string, timezones[i]); + } +} + +TEST(TypeParserCase, ParseTimezone_AllEscapedBytes) { + // {fragment as seen by the parser, expected decoded raw byte} + const std::vector> cases = { + {"\\0", '\0'}, // NUL + {"\\b", '\b'}, // Backspace + {"\\t", '\t'}, // Tab + {"\\n", '\n'}, // Line feed + {"\\f", '\f'}, // Form feed + {"\\r", '\r'}, // Carriage return + {"\\\\", '\\'}, // Backslash + {"\\'", '\''}, // Single quote + }; + + for (const auto& [fragment, raw] : cases) { + const std::string expected = std::string("A") + raw + "B"; + + TypeAst date_time; + ASSERT_TRUE(TypeParser("DateTime('A" + fragment + "B')").Parse(&date_time)) + << "failed to parse fragment: " << fragment; + ASSERT_EQ(date_time.code, Type::DateTime); + ASSERT_EQ(date_time.elements.size(), 1u); + EXPECT_EQ(date_time.elements[0].value_string, expected) + << "wrong decoding for fragment: " << fragment; + + TypeAst date_time64; + ASSERT_TRUE(TypeParser("DateTime64(3, 'A" + fragment + "B')").Parse(&date_time64)) + << "failed to parse fragment: " << fragment; + ASSERT_EQ(date_time64.code, Type::DateTime64); + ASSERT_EQ(date_time64.elements.size(), 2u); + EXPECT_EQ(date_time64.elements[1].value_string, expected) + << "wrong decoding for fragment: " << fragment; + } + + // All escaped bytes combined in a single timezone string. + { + std::string escaped; + std::string expected; + for (const auto& [fragment, raw] : cases) { + escaped += fragment; + expected += raw; + } + + TypeAst date_time; + ASSERT_TRUE(TypeParser("DateTime('" + escaped + "')").Parse(&date_time)); + ASSERT_EQ(date_time.elements.size(), 1u); + EXPECT_EQ(date_time.elements[0].value_string, expected); + + TypeAst date_time64; + ASSERT_TRUE(TypeParser("DateTime64(3, '" + escaped + "')").Parse(&date_time64)); + ASSERT_EQ(date_time64.elements.size(), 2u); + EXPECT_EQ(date_time64.elements[1].value_string, expected); + } +} + TEST(TypeParserCase, EqualityIncludesValueString) { TypeAst utc; TypeAst minsk; diff --git a/ut/types_ut.cpp b/ut/types_ut.cpp index 2a18241a..51ee3aff 100644 --- a/ut/types_ut.cpp +++ b/ut/types_ut.cpp @@ -138,10 +138,45 @@ TEST(TypesCase, EnumTypesEmpty) { ASSERT_EQ("Enum16()", Type::CreateEnum16({})->GetName()); } +TEST(TypesCase, EnumTypeNameEscapesLabels) { + // One special byte per label; {raw label, expected escaped rendering}. + const std::vector> cases = { + {"a'b", "a\\'b"}, // single quote + {"a\\b", "a\\\\b"}, // backslash + {"a\tb", "a\\tb"}, // tab + {"a\nb", "a\\nb"}, // line feed + {"a\rb", "a\\rb"}, // carriage return + {"a\bb", "a\\bb"}, // backspace + {"a\fb", "a\\fb"}, // form feed + {std::string("a\0b", 3), "a\\0b"}, // NUL + }; + for (const auto& [label, escaped] : cases) { + EXPECT_EQ(Type::CreateEnum8({{label, 1}})->GetName(), "Enum8('" + escaped + "' = 1)"); + } +} + TEST(TypesCase, DecimalTypes) { // TODO: implement this test. } +TEST(TypesCase, DateTimeTypeNameEscapesTimezone) { + // One special byte per timezone; {raw timezone, expected escaped rendering}. + const std::vector> cases = { + {"a'b", "a\\'b"}, // single quote + {"a\\b", "a\\\\b"}, // backslash + {"a\tb", "a\\tb"}, // tab + {"a\nb", "a\\nb"}, // line feed + {"a\rb", "a\\rb"}, // carriage return + {"a\bb", "a\\bb"}, // backspace + {"a\fb", "a\\fb"}, // form feed + {std::string("a\0b", 3), "a\\0b"}, // NUL + }; + for (const auto& [tz, escaped] : cases) { + EXPECT_EQ(Type::CreateDateTime(tz)->GetName(), "DateTime('" + escaped + "')"); + EXPECT_EQ(Type::CreateDateTime64(3, tz)->GetName(), "DateTime64(3, '" + escaped + "')"); + } +} + TEST(TypesCase, IsEqual) { const std::string type_names[] = { #if !CH_MAP_BOOL_TO_UINT8