From a353c4e004148517094cfb04f4062bdc370b2a7d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 8 Jul 2026 01:23:26 +0000 Subject: [PATCH] ReDoS-safe @pattern: linear-time regex engine for constraint validation Generated @pattern validation evaluated patterns with std::regex, a backtracking engine, so a catastrophic pattern like ^([0-9]+)+$ plus a request-sized non-matching input could hang the dispatch thread. - smithy::Regex (smithy/core/regex.h): a Thompson-NFA engine simulated breadth-first (Pike VM), O(pattern x input) for every pattern/input combination. Covers the ECMA-262 subset Smithy patterns use over UTF-8 bytes with std::regex_search partial-match semantics: literals, '.', character classes, class escapes, \xHH/\uHHHH, ^ $ \b \B, greedy and lazy quantifiers including {n,m}, groups, and alternation. Compiled program size, repeat counts, and group nesting are capped; Outcome-based errors per ADR-0003. Zero new dependencies. - ValidationGenerator emits smithy::Regex instead of std::regex, failing closed if a pattern were ever uncompilable at runtime, and rejects backreferences and lookaround (inherently backtracking constructs) at generation time with an error naming the pattern and the fix. - Tests: engine unit suite including linearity checks on classic ReDoS bombs and a differential test against std::regex on random inputs; a regex fuzz harness (smoke-tested in every CI job); jsonrpc2 authored conformance suite grows @pattern coverage with JsonRpc2PatternMismatch and JsonRpc2PatternReDoSInput (the catastrophic pattern answered promptly); codegen unit test pins the generation-time rejection. - Regenerated fixture output; stale ReDoS notes in the exclusion list and server-guide replaced with the new contract; runtime.md lists the engine. Closes #38 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01SyQAo21Pv6GYhHrkbQj8xQ --- .../codegen/ValidationGenerator.java | 69 +- .../codegen/protocol-test-exclusions.txt | 3 - .../codegen/CppCodegenPluginTest.java | 41 ++ docs/runtime.md | 2 +- docs/server-guide.md | 10 +- .../roundtrip/rest/generated/src/server.cc | 14 +- examples/weather/generated/src/server.cc | 14 +- fuzz/BUILD.bazel | 1 + fuzz/regex_fuzz.cc | 24 + .../smithy/protocoltests/jsonrpc2/types.h | 2 + .../jsonrpc2/generated/src/serde.cc | 24 + .../jsonrpc2/generated/src/server.cc | 15 + .../generated/tests/server_malformed_tests.cc | 28 + protocol-tests/jsonrpc2/model/jsonrpc2.smithy | 54 ++ runtime/BUILD.bazel | 3 + runtime/include/smithy/core/regex.h | 78 ++ runtime/src/core/regex.cc | 664 ++++++++++++++++++ runtime/tests/core/regex_test.cc | 180 +++++ 18 files changed, 1201 insertions(+), 25 deletions(-) create mode 100644 fuzz/regex_fuzz.cc create mode 100644 runtime/include/smithy/core/regex.h create mode 100644 runtime/src/core/regex.cc create mode 100644 runtime/tests/core/regex_test.cc diff --git a/codegen/smithy-cpp-codegen/src/main/java/io/smithycpp/codegen/ValidationGenerator.java b/codegen/smithy-cpp-codegen/src/main/java/io/smithycpp/codegen/ValidationGenerator.java index 5c9c9c57..a9d75f47 100644 --- a/codegen/smithy-cpp-codegen/src/main/java/io/smithycpp/codegen/ValidationGenerator.java +++ b/codegen/smithy-cpp-codegen/src/main/java/io/smithycpp/codegen/ValidationGenerator.java @@ -461,15 +461,20 @@ private void writeRangeCheck(CppWriter w, RangeTrait range, String valueExpr, St private void writePatternCheck( CppWriter w, PatternTrait pattern, String valueExpr, String pathVar) { + rejectUnsupportedPattern(pattern); String variable = "kPattern" + patternCounter++; - w.addInclude(""); - // Raw string literal keeps the regex byte-exact; the failure message needs - // C++ escaping instead. + w.addInclude("\"smithy/core/regex.h\""); + // smithy::Regex is a linear-time engine, so no pattern/input combination + // can backtrack catastrophically (ReDoS). The raw string literal keeps + // the regex byte-exact; the failure message needs C++ escaping instead. + // Compile failure (impossible for generator-accepted patterns) fails + // closed: every value is rejected rather than skipping the check. w.write( - "static const std::regex $L{R\"__smithy($L)__smithy\", std::regex::ECMAScript};", + "static const smithy::Outcome $L =" + + " smithy::Regex::Compile(R\"__smithy($L)__smithy\");", variable, pattern.getValue()); - w.openBlock("if (!std::regex_search($L, $L)) {", valueExpr, variable); + w.openBlock("if (!$L.ok() || !$L->Search($L)) {", variable, variable, valueExpr); w.write( "AddValidationFailure(failures, $L, \"Value at '\" + $L + \"' failed to satisfy " + "constraint: Member must satisfy regular expression pattern: \" + " @@ -480,6 +485,60 @@ private void writePatternCheck( w.closeBlock("}"); } + /** + * Rejects @pattern regexes the linear-time runtime engine cannot support — backreferences and + * lookaround are inherently backtracking constructs. Everything else ECMA-262 offers (classes, + * anchors, quantifiers, groups, alternation) compiles to the NFA. Failing here keeps the contract + * visible at generation time instead of surfacing as an always-failing validator. + */ + private static void rejectUnsupportedPattern(PatternTrait pattern) { + String value = pattern.getValue(); + boolean inClass = false; + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c == '\\') { + char next = i + 1 < value.length() ? value.charAt(i + 1) : '\0'; + if (!inClass && next >= '1' && next <= '9') { + throw new software.amazon.smithy.codegen.core.CodegenException( + "@pattern " + + value + + " (at " + + pattern.getSourceLocation() + + ") uses a backreference (\\" + + next + + "), which the linear-time ReDoS-safe engine cannot support; rewrite the" + + " pattern to repeat the group instead of referencing its capture"); + } + i++; // Skip the escaped character. + continue; + } + if (inClass) { + inClass = c != ']'; + continue; + } + if (c == '[') { + inClass = true; + continue; + } + if (c == '(' && i + 1 < value.length() && value.charAt(i + 1) == '?') { + char kind = i + 2 < value.length() ? value.charAt(i + 2) : '\0'; + boolean lookbehind = + kind == '<' + && i + 3 < value.length() + && (value.charAt(i + 3) == '=' || value.charAt(i + 3) == '!'); + if (kind == '=' || kind == '!' || lookbehind) { + throw new software.amazon.smithy.codegen.core.CodegenException( + "@pattern " + + value + + " (at " + + pattern.getSourceLocation() + + ") uses lookaround, which the linear-time ReDoS-safe engine cannot support;" + + " rewrite the pattern to match the text directly"); + } + } + } + } + private void writeUniqueItemsCheck(CppWriter w, String valueExpr, String pathVar) { w.openBlock("{"); w.write("bool unique = true;"); diff --git a/codegen/smithy-cpp-codegen/src/main/resources/io/smithycpp/codegen/protocol-test-exclusions.txt b/codegen/smithy-cpp-codegen/src/main/resources/io/smithycpp/codegen/protocol-test-exclusions.txt index 7d3f2506..3153aa92 100644 --- a/codegen/smithy-cpp-codegen/src/main/resources/io/smithycpp/codegen/protocol-test-exclusions.txt +++ b/codegen/smithy-cpp-codegen/src/main/resources/io/smithycpp/codegen/protocol-test-exclusions.txt @@ -26,9 +26,6 @@ smithy.protocoltests.rpcv2Cbor#RpcV2Protocol server-request RpcV2CborSupportsNaN # Absent query list params deserialize as unset members, not engaged empty lists. -# std::regex is a backtracking engine, so the deliberately catastrophic -# pattern ^([0-9]+)+$ hangs; ReDoS-safe @pattern matching needs a linear-time -# regex engine (tracked in PLAN). # Nested @required absences (inside collections/structures) surface as strict # deserialization errors without JSON-pointer paths; reporting them as fieldList diff --git a/codegen/smithy-cpp-codegen/src/test/java/io/smithycpp/codegen/CppCodegenPluginTest.java b/codegen/smithy-cpp-codegen/src/test/java/io/smithycpp/codegen/CppCodegenPluginTest.java index 89497a5f..9116e2e6 100644 --- a/codegen/smithy-cpp-codegen/src/test/java/io/smithycpp/codegen/CppCodegenPluginTest.java +++ b/codegen/smithy-cpp-codegen/src/test/java/io/smithycpp/codegen/CppCodegenPluginTest.java @@ -177,6 +177,47 @@ void generatesRecursiveShapes() { assertTrue(header.contains("struct TreeNode;"), header); } + @Test + void rejectsBacktrackingOnlyPatterns() { + // Backreferences and lookaround need a backtracking engine; the + // linear-time ReDoS-safe matcher refuses them at generation time. + Model model = + Model.assembler() + .discoverModels(CppCodegenPluginTest.class.getClassLoader()) + .addUnparsedModel( + "backref.smithy", + """ + $version: "2.0" + namespace test.redos + use smithy.cpp.protocols#jsonRpc2 + + @jsonRpc2 + service Svc { version: "1", operations: [Op] } + operation Op { + input := { + @pattern("^(a+)\\\\1$") + doubled: String + } + } + """) + .assemble() + .unwrap(); + PluginContext context = + PluginContext.builder() + .fileManifest(new MockManifest()) + .model(model) + .settings( + Node.objectNodeBuilder() + .withMember("service", "test.redos#Svc") + .withMember("namespace", "test::redos") + .build()) + .build(); + CodegenException error = + assertThrows(CodegenException.class, () -> new CppCodegenPlugin().execute(context)); + assertTrue(error.getMessage().contains("backreference")); + assertTrue(error.getMessage().contains("^(a+)\\1$")); + } + @Test void rejectsRecursionThroughUnionMembers() { Model model = diff --git a/docs/runtime.md b/docs/runtime.md index ee5fd818..5eac77c7 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -7,7 +7,7 @@ crates (PLAN §3.2a). | Bazel target | Namespace | Contents | |---|---|---| -| `//runtime:core` | `smithy` | `Outcome` + `Error` (ADR-0003), `Blob`, `Timestamp` (epoch-seconds / RFC 3339 date-time / IMF-fixdate http-date), `Document` (dynamic value + serde pivot), base64 | +| `//runtime:core` | `smithy` | `Outcome` + `Error` (ADR-0003), `Blob`, `Timestamp` (epoch-seconds / RFC 3339 date-time / IMF-fixdate http-date), `Document` (dynamic value + serde pivot), base64, `Regex` (linear-time NFA engine behind generated `@pattern` validation — ReDoS-safe) | | `//runtime:json` | `smithy::json` | `Document` ⇄ JSON text via nlohmann (blobs as base64, timestamps per stored format) | | `//runtime:cbor` | `smithy::cbor` | `Document` ⇄ deterministic CBOR (RFC 8949; tag-1 timestamps; tolerant decoder) — ADR-0005 | | `//runtime:http` | `smithy::http` | `Headers` (case-insensitive), URI percent-encoding per the Smithy HTTP binding rules, `HttpRequest`/`HttpResponse`, `HttpClient`/`HttpServerTransport` interfaces, `Loopback` in-memory transport, built-in `SocketHttpClient`/`SocketHttpServer` (test/reference only — ADR-0006), W3C `TraceContext` parse/format/generate | diff --git a/docs/server-guide.md b/docs/server-guide.md index 1ce07a92..e7d5f008 100644 --- a/docs/server-guide.md +++ b/docs/server-guide.md @@ -81,6 +81,13 @@ responds with the standard 400 `ValidationException` wire shape (`message` summa the exact message formats the official validation conformance suite pins. `@internal` enum members stay accepted on the wire but are omitted from the advertised value set. +`@pattern` evaluates on a linear-time NFA engine (`smithy/core/regex.h`), so no pattern/input +combination can backtrack catastrophically — the classic ReDoS pattern `^([0-9]+)+$` validates +request-sized inputs in microseconds instead of hanging the dispatch thread. The engine covers +the ECMA-262 subset Smithy patterns use; backreferences and lookaround (inherently +backtracking constructs) are rejected at generation time with an error naming the shape and +the fix. + ## Conformance Generated servers pass the official server-mode `httpRequestTests`/`httpResponseTests` suites @@ -101,5 +108,4 @@ serialization error) — see [production-guide.md](production-guide.md). Nested `@required` absences as `fieldList` entries and a server-strict serde variant (clients must skip null dense-map values and accept UTC-offset timestamps in responses; servers share -that serde today), ReDoS-safe `@pattern` matching (`std::regex` backtracks, so the suite's -deliberately catastrophic pattern is excluded), and `@streaming` payloads (Phase 8). +that serde today), and `@streaming` payloads (Phase 8). diff --git a/examples/roundtrip/rest/generated/src/server.cc b/examples/roundtrip/rest/generated/src/server.cc index 916865f2..310408f6 100644 --- a/examples/roundtrip/rest/generated/src/server.cc +++ b/examples/roundtrip/rest/generated/src/server.cc @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -20,6 +19,7 @@ #include "smithy/core/blob.h" #include "smithy/core/document.h" #include "smithy/core/document_serde.h" +#include "smithy/core/regex.h" #include "smithy/core/text.h" #include "smithy/http/headers.h" #include "smithy/json/json.h" @@ -136,8 +136,8 @@ void ValidateDescribeSinkInput(const DescribeSinkInput& value, const std::string AddValidationFailure(failures, member_path, "Value with length " + std::to_string(member_length) + " at '" + member_path + "' failed to satisfy constraint: Member must have length between 1 and 32, inclusive"); } } - static const std::regex kPattern0{R"__smithy(^[A-Za-z0-9]+$)__smithy", std::regex::ECMAScript}; - if (!std::regex_search(value.sinkId, kPattern0)) { + static const smithy::Outcome kPattern0 = smithy::Regex::Compile(R"__smithy(^[A-Za-z0-9]+$)__smithy"); + if (!kPattern0.ok() || !kPattern0->Search(value.sinkId)) { AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^[A-Za-z0-9]+$")); } } @@ -175,8 +175,8 @@ void ValidatePutSinkInput(const PutSinkInput& value, const std::string& path, st AddValidationFailure(failures, member_path, "Value with length " + std::to_string(member_length) + " at '" + member_path + "' failed to satisfy constraint: Member must have length between 1 and 32, inclusive"); } } - static const std::regex kPattern1{R"__smithy(^[A-Za-z0-9]+$)__smithy", std::regex::ECMAScript}; - if (!std::regex_search(value.sinkId, kPattern1)) { + static const smithy::Outcome kPattern1 = smithy::Regex::Compile(R"__smithy(^[A-Za-z0-9]+$)__smithy"); + if (!kPattern1.ok() || !kPattern1->Search(value.sinkId)) { AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^[A-Za-z0-9]+$")); } } @@ -207,8 +207,8 @@ void ValidateUploadAttachmentInput(const UploadAttachmentInput& value, const std AddValidationFailure(failures, member_path, "Value with length " + std::to_string(member_length) + " at '" + member_path + "' failed to satisfy constraint: Member must have length between 1 and 32, inclusive"); } } - static const std::regex kPattern2{R"__smithy(^[A-Za-z0-9]+$)__smithy", std::regex::ECMAScript}; - if (!std::regex_search(value.sinkId, kPattern2)) { + static const smithy::Outcome kPattern2 = smithy::Regex::Compile(R"__smithy(^[A-Za-z0-9]+$)__smithy"); + if (!kPattern2.ok() || !kPattern2->Search(value.sinkId)) { AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^[A-Za-z0-9]+$")); } } diff --git a/examples/weather/generated/src/server.cc b/examples/weather/generated/src/server.cc index fbae868e..380759f2 100644 --- a/examples/weather/generated/src/server.cc +++ b/examples/weather/generated/src/server.cc @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -20,6 +19,7 @@ #include "smithy/core/blob.h" #include "smithy/core/document.h" #include "smithy/core/document_serde.h" +#include "smithy/core/regex.h" #include "smithy/http/headers.h" #include "smithy/json/json.h" #include "smithy/server/router.h" @@ -110,8 +110,8 @@ void AddValidationFailure(std::vector* failur void ValidateDeleteCityInput(const DeleteCityInput& value, const std::string& path, std::vector* failures) { { const std::string member_path = path + "/cityId"; - static const std::regex kPattern0{R"__smithy(^[A-Za-z0-9 ]+$)__smithy", std::regex::ECMAScript}; - if (!std::regex_search(value.cityId, kPattern0)) { + static const smithy::Outcome kPattern0 = smithy::Regex::Compile(R"__smithy(^[A-Za-z0-9 ]+$)__smithy"); + if (!kPattern0.ok() || !kPattern0->Search(value.cityId)) { AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^[A-Za-z0-9 ]+$")); } } @@ -120,8 +120,8 @@ void ValidateDeleteCityInput(const DeleteCityInput& value, const std::string& pa void ValidateGetForecastInput(const GetForecastInput& value, const std::string& path, std::vector* failures) { { const std::string member_path = path + "/cityId"; - static const std::regex kPattern1{R"__smithy(^[A-Za-z0-9 ]+$)__smithy", std::regex::ECMAScript}; - if (!std::regex_search(value.cityId, kPattern1)) { + static const smithy::Outcome kPattern1 = smithy::Regex::Compile(R"__smithy(^[A-Za-z0-9 ]+$)__smithy"); + if (!kPattern1.ok() || !kPattern1->Search(value.cityId)) { AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^[A-Za-z0-9 ]+$")); } } @@ -130,8 +130,8 @@ void ValidateGetForecastInput(const GetForecastInput& value, const std::string& void ValidateGetCityInput(const GetCityInput& value, const std::string& path, std::vector* failures) { { const std::string member_path = path + "/cityId"; - static const std::regex kPattern2{R"__smithy(^[A-Za-z0-9 ]+$)__smithy", std::regex::ECMAScript}; - if (!std::regex_search(value.cityId, kPattern2)) { + static const smithy::Outcome kPattern2 = smithy::Regex::Compile(R"__smithy(^[A-Za-z0-9 ]+$)__smithy"); + if (!kPattern2.ok() || !kPattern2->Search(value.cityId)) { AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^[A-Za-z0-9 ]+$")); } } diff --git a/fuzz/BUILD.bazel b/fuzz/BUILD.bazel index f3bc5f19..b3774896 100644 --- a/fuzz/BUILD.bazel +++ b/fuzz/BUILD.bazel @@ -11,6 +11,7 @@ FUZZ_TARGETS = { "//runtime:cbor", "//runtime:core", ], + "regex": ["//runtime:core"], "uri": ["//runtime:http"], "server_dispatch": [ "//examples/weather/generated:server", diff --git a/fuzz/regex_fuzz.cc b/fuzz/regex_fuzz.cc new file mode 100644 index 00000000..4e949045 --- /dev/null +++ b/fuzz/regex_fuzz.cc @@ -0,0 +1,24 @@ +// Fuzz target: the linear-time @pattern engine. Compile must never crash on +// arbitrary pattern bytes, and anything it accepts must Search arbitrary +// input in bounded time (the harness splits the record into pattern and +// text at the first NUL). ReDoS-resistance is the engine's contract, so a +// hang here is a finding, not flakiness. +#include +#include +#include + +#include "smithy/core/regex.h" + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) { + const std::string_view record(reinterpret_cast(data), size); + const std::size_t split = record.find('\0'); + const std::string_view pattern = + split == std::string_view::npos ? record : record.substr(0, split); + const std::string_view text = split == std::string_view::npos ? "" : record.substr(split + 1); + auto re = smithy::Regex::Compile(pattern); + if (re.ok()) { + (void)re->Search(text); + (void)re->Search(pattern); + } + return 0; +} diff --git a/protocol-tests/jsonrpc2/generated/include/smithy/protocoltests/jsonrpc2/types.h b/protocol-tests/jsonrpc2/generated/include/smithy/protocoltests/jsonrpc2/types.h index b49b0c65..a7a93ac3 100644 --- a/protocol-tests/jsonrpc2/generated/include/smithy/protocoltests/jsonrpc2/types.h +++ b/protocol-tests/jsonrpc2/generated/include/smithy/protocoltests/jsonrpc2/types.h @@ -73,6 +73,8 @@ struct NoArgsOutput { struct PutConstrainedInput { std::string name{}; std::optional limit{}; + std::optional slug{}; + std::optional evilDigits{}; friend bool operator==(const PutConstrainedInput&, const PutConstrainedInput&) = default; }; diff --git a/protocol-tests/jsonrpc2/generated/src/serde.cc b/protocol-tests/jsonrpc2/generated/src/serde.cc index a91627e3..06f39c86 100644 --- a/protocol-tests/jsonrpc2/generated/src/serde.cc +++ b/protocol-tests/jsonrpc2/generated/src/serde.cc @@ -372,6 +372,12 @@ smithy::Document SerializePutConstrainedInput(const PutConstrainedInput& value) if (value.limit.has_value()) { map.emplace("limit", smithy::Document(static_cast((*value.limit)))); } + if (value.slug.has_value()) { + map.emplace("slug", smithy::Document((*value.slug))); + } + if (value.evilDigits.has_value()) { + map.emplace("evilDigits", smithy::Document((*value.evilDigits))); + } return smithy::Document(std::move(map)); } @@ -396,6 +402,24 @@ smithy::Outcome DeserializePutConstrainedInput(const smithy out.limit = std::move(parsed_member); } } + { + const smithy::Document* member = doc.Find("slug"); + if (member != nullptr && !member->is_null()) { + std::string parsed_member{}; + if (!member->is_string()) return smithy::Error::Serialization("PutConstrainedInput.slug: unexpected type on the wire"); + parsed_member = member->as_string(); + out.slug = std::move(parsed_member); + } + } + { + const smithy::Document* member = doc.Find("evilDigits"); + if (member != nullptr && !member->is_null()) { + std::string parsed_member{}; + if (!member->is_string()) return smithy::Error::Serialization("PutConstrainedInput.evilDigits: unexpected type on the wire"); + parsed_member = member->as_string(); + out.evilDigits = std::move(parsed_member); + } + } return out; } diff --git a/protocol-tests/jsonrpc2/generated/src/server.cc b/protocol-tests/jsonrpc2/generated/src/server.cc index d2b0d343..468a032c 100644 --- a/protocol-tests/jsonrpc2/generated/src/server.cc +++ b/protocol-tests/jsonrpc2/generated/src/server.cc @@ -8,6 +8,7 @@ #include #include "smithy/core/document.h" +#include "smithy/core/regex.h" #include "smithy/core/text.h" #include "smithy/http/headers.h" #include "smithy/json/json.h" @@ -96,6 +97,20 @@ void ValidatePutConstrainedInput(const PutConstrainedInput& value, const std::st AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must be between 1 and 100, inclusive"); } } + if (value.slug.has_value()) { + const std::string member_path = path + "/slug"; + static const smithy::Outcome kPattern0 = smithy::Regex::Compile(R"__smithy(^[a-z0-9-]+$)__smithy"); + if (!kPattern0.ok() || !kPattern0->Search((*value.slug))) { + AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^[a-z0-9-]+$")); + } + } + if (value.evilDigits.has_value()) { + const std::string member_path = path + "/evilDigits"; + static const smithy::Outcome kPattern1 = smithy::Regex::Compile(R"__smithy(^([0-9]+)+$)__smithy"); + if (!kPattern1.ok() || !kPattern1->Search((*value.evilDigits))) { + AddValidationFailure(failures, member_path, "Value at '" + member_path + "' failed to satisfy constraint: Member must satisfy regular expression pattern: " + std::string("^([0-9]+)+$")); + } + } } smithy::http::HttpResponse ValidationErrorResponse(const std::vector& failures, const smithy::Document& id) { diff --git a/protocol-tests/jsonrpc2/generated/tests/server_malformed_tests.cc b/protocol-tests/jsonrpc2/generated/tests/server_malformed_tests.cc index a82559cb..eb18c29b 100644 --- a/protocol-tests/jsonrpc2/generated/tests/server_malformed_tests.cc +++ b/protocol-tests/jsonrpc2/generated/tests/server_malformed_tests.cc @@ -173,4 +173,32 @@ TEST(JsonRpc2ProtocolServerMalformedTest, JsonRpc2ValidationFailure) { EXPECT_TRUE(smithy::testing::JsonBodyEquals("{\"jsonrpc\":\"2.0\",\"error\":{\"code\":400,\"message\":\"1 validation error detected. Value with length 0 at '/name' failed to satisfy constraint: Member must have length between 1 and 8, inclusive\",\"data\":{\"__type\":\"smithy.framework#ValidationException\",\"fieldList\":[{\"message\":\"Value with length 0 at '/name' failed to satisfy constraint: Member must have length between 1 and 8, inclusive\",\"path\":\"/name\"}]}},\"id\":9}", response.body)) << response.body; } +// @pattern violations report the suite-exact message with the pattern text. +TEST(JsonRpc2ProtocolServerMalformedTest, JsonRpc2PatternMismatch) { + JsonRpc2ProtocolServer server(std::make_shared()); + smithy::http::HttpRequest request; + request.method = "POST"; + request.target = "/"; + request.headers.Set("content-type", "application/json"); + request.body = "{\"jsonrpc\":\"2.0\",\"method\":\"PutConstrained\",\"params\":{\"name\":\"ok\",\"slug\":\"Not Valid!\"},\"id\":10}"; + const smithy::http::HttpResponse response = server.Handler()(request); + EXPECT_EQ(response.status, 200) << response.body; + EXPECT_EQ(response.headers.Get("content-type").value_or(""), "application/json"); + EXPECT_TRUE(smithy::testing::JsonBodyEquals("{\"jsonrpc\":\"2.0\",\"error\":{\"code\":400,\"message\":\"1 validation error detected. Value at '/slug' failed to satisfy constraint: Member must satisfy regular expression pattern: ^[a-z0-9-]+$\",\"data\":{\"__type\":\"smithy.framework#ValidationException\",\"fieldList\":[{\"message\":\"Value at '/slug' failed to satisfy constraint: Member must satisfy regular expression pattern: ^[a-z0-9-]+$\",\"path\":\"/slug\"}]}},\"id\":10}", response.body)) << response.body; +} + +// When the pattern is susceptible to catastrophic backtracking, the server answers promptly instead of hanging while evaluating it (linear-time engine). +TEST(JsonRpc2ProtocolServerMalformedTest, JsonRpc2PatternReDoSInput) { + JsonRpc2ProtocolServer server(std::make_shared()); + smithy::http::HttpRequest request; + request.method = "POST"; + request.target = "/"; + request.headers.Set("content-type", "application/json"); + request.body = "{\"jsonrpc\":\"2.0\",\"method\":\"PutConstrained\",\"params\":{\"name\":\"ok\",\"evilDigits\":\"00000000000000000000000000000000000000000000000000!\"},\"id\":11}"; + const smithy::http::HttpResponse response = server.Handler()(request); + EXPECT_EQ(response.status, 200) << response.body; + EXPECT_EQ(response.headers.Get("content-type").value_or(""), "application/json"); + EXPECT_TRUE(smithy::testing::JsonBodyEquals("{\"jsonrpc\":\"2.0\",\"error\":{\"code\":400,\"message\":\"1 validation error detected. Value at '/evilDigits' failed to satisfy constraint: Member must satisfy regular expression pattern: ^([0-9]+)+$\",\"data\":{\"__type\":\"smithy.framework#ValidationException\",\"fieldList\":[{\"message\":\"Value at '/evilDigits' failed to satisfy constraint: Member must satisfy regular expression pattern: ^([0-9]+)+$\",\"path\":\"/evilDigits\"}]}},\"id\":11}", response.body)) << response.body; +} + } // namespace smithy::protocoltests::jsonrpc2 diff --git a/protocol-tests/jsonrpc2/model/jsonrpc2.smithy b/protocol-tests/jsonrpc2/model/jsonrpc2.smithy index f5516f9e..c44e6ad1 100644 --- a/protocol-tests/jsonrpc2/model/jsonrpc2.smithy +++ b/protocol-tests/jsonrpc2/model/jsonrpc2.smithy @@ -321,6 +321,14 @@ operation PutConstrained { @range(min: 1, max: 100) limit: Integer + + @pattern("^[a-z0-9-]+$") + slug: String + + // Catastrophic under a backtracking engine (ReDoS); the generated + // validator must evaluate it in linear time. + @pattern("^([0-9]+)+$") + evilDigits: String } output := { @@ -353,6 +361,52 @@ apply PutConstrained @httpMalformedRequestTests([ } } } + { + id: "JsonRpc2PatternMismatch" + documentation: "@pattern violations report the suite-exact message with the pattern text." + protocol: jsonRpc2 + request: { + method: "POST" + uri: "/" + body: """ + {"jsonrpc":"2.0","method":"PutConstrained","params":{"name":"ok","slug":"Not Valid!"},"id":10}""" + headers: { "content-type": "application/json" } + } + response: { + code: 200 + headers: { "content-type": "application/json" } + body: { + mediaType: "application/json" + assertion: { + contents: """ + {"jsonrpc":"2.0","error":{"code":400,"message":"1 validation error detected. Value at '/slug' failed to satisfy constraint: Member must satisfy regular expression pattern: ^[a-z0-9-]+$","data":{"__type":"smithy.framework#ValidationException","fieldList":[{"message":"Value at '/slug' failed to satisfy constraint: Member must satisfy regular expression pattern: ^[a-z0-9-]+$","path":"/slug"}]}},"id":10}""" + } + } + } + } + { + id: "JsonRpc2PatternReDoSInput" + documentation: "When the pattern is susceptible to catastrophic backtracking, the server answers promptly instead of hanging while evaluating it (linear-time engine)." + protocol: jsonRpc2 + request: { + method: "POST" + uri: "/" + body: """ + {"jsonrpc":"2.0","method":"PutConstrained","params":{"name":"ok","evilDigits":"00000000000000000000000000000000000000000000000000!"},"id":11}""" + headers: { "content-type": "application/json" } + } + response: { + code: 200 + headers: { "content-type": "application/json" } + body: { + mediaType: "application/json" + assertion: { + contents: """ + {"jsonrpc":"2.0","error":{"code":400,"message":"1 validation error detected. Value at '/evilDigits' failed to satisfy constraint: Member must satisfy regular expression pattern: ^([0-9]+)+$","data":{"__type":"smithy.framework#ValidationException","fieldList":[{"message":"Value at '/evilDigits' failed to satisfy constraint: Member must satisfy regular expression pattern: ^([0-9]+)+$","path":"/evilDigits"}]}},"id":11}""" + } + } + } + } ]) @error("client") diff --git a/runtime/BUILD.bazel b/runtime/BUILD.bazel index b26f4e9c..bb0d30b4 100644 --- a/runtime/BUILD.bazel +++ b/runtime/BUILD.bazel @@ -15,6 +15,7 @@ cc_library( srcs = [ "src/core/base64.cc", "src/core/document_serde.cc", + "src/core/regex.cc", "src/core/text.cc", "src/core/timestamp.cc", "src/core/uuid.cc", @@ -28,6 +29,7 @@ cc_library( "include/smithy/core/document_serde.h", "include/smithy/core/error.h", "include/smithy/core/outcome.h", + "include/smithy/core/regex.h", "include/smithy/core/text.h", "include/smithy/core/timestamp.h", "include/smithy/core/uuid.h", @@ -427,6 +429,7 @@ cc_test( "tests/core/boxed_test.cc", "tests/core/core_test.cc", "tests/core/document_serde_test.cc", + "tests/core/regex_test.cc", "tests/core/timestamp_differential_test.cc", "tests/core/timestamp_test.cc", "tests/core/version_test.cc", diff --git a/runtime/include/smithy/core/regex.h b/runtime/include/smithy/core/regex.h new file mode 100644 index 00000000..77f98bd6 --- /dev/null +++ b/runtime/include/smithy/core/regex.h @@ -0,0 +1,78 @@ +#ifndef SMITHY_CORE_REGEX_H_ +#define SMITHY_CORE_REGEX_H_ + +#include +#include +#include +#include + +#include "smithy/core/outcome.h" + +namespace smithy { + +// Linear-time regular expressions for generated @pattern validation. The +// pattern compiles to a Thompson NFA that Search() simulates breadth-first +// (a Pike VM), so matching costs O(program size x input bytes) for every +// pattern/input combination — a deliberately catastrophic pattern like +// ^([0-9]+)+$ evaluates in linear time instead of hanging the dispatch +// thread the way a backtracking engine does (ReDoS). +// +// Supported syntax is the ECMA-262 subset Smithy @pattern uses, evaluated +// over UTF-8 bytes exactly like the std::regex engine this replaces: +// literals, '.', character classes (ranges, negation, class escapes), +// \d \D \w \W \s \S, \xHH and \uHHHH escapes, ^ $ \b \B assertions, +// greedy and lazy quantifiers (* + ? {n} {n,} {n,m}), groups (capturing, +// non-capturing, named), and alternation. Constructs a linear-time engine +// cannot support — backreferences and lookaround — are Compile() errors; +// the code generator rejects such patterns before any C++ exists. +class Regex { + public: + // Compiles the pattern; kSerialization error on invalid syntax or on an + // unsupported construct. + static Outcome Compile(std::string_view pattern); + + // True when text contains a match, std::regex_search semantics: partial + // match anywhere in text unless the pattern anchors itself with ^/$. + bool Search(std::string_view text) const; + + // Implementation detail, public only so the compiler/parser in regex.cc + // can name it; not part of the supported API. + struct Inst { + enum class Op : std::uint8_t { + kByte, // match one input byte equal to `byte` + kClass, // match one input byte in classes_[arg] + kSplit, // continue at both pc+1 and arg + kJmp, // continue at arg + kAssert, // zero-width check `assert_kind`, continue at pc+1 + kMatch, // a match exists + }; + enum class Assert : std::uint8_t { + kInputStart, // ^ + kInputEnd, // $ + kWordBoundary, // \b + kNotWordBoundary, // \B + }; + Op op; + Assert assert_kind = Assert::kInputStart; + std::uint8_t byte = 0; + std::uint32_t arg = 0; + }; + + private: + Regex() = default; + + // Adds pc (following epsilon transitions and pos-applicable assertions) to + // the thread list; returns true when a kMatch instruction is reached. + bool AddThread(std::vector* list, std::vector* seen_stamp, + std::uint32_t stamp, std::uint32_t pc, std::string_view text, + std::size_t pos) const; + + std::vector program_; + std::vector> classes_; + + friend class RegexCompiler; +}; + +} // namespace smithy + +#endif // SMITHY_CORE_REGEX_H_ diff --git a/runtime/src/core/regex.cc b/runtime/src/core/regex.cc new file mode 100644 index 00000000..7715bb2d --- /dev/null +++ b/runtime/src/core/regex.cc @@ -0,0 +1,664 @@ +#include "smithy/core/regex.h" + +#include +#include +#include + +#include "smithy/core/error.h" + +namespace smithy { +namespace { + +// Compiled programs are bounded so counted repetition ({n,m} expands by +// duplication) cannot balloon memory; patterns come from the model, so the +// caps only need to be generous, not tight. +constexpr std::size_t kMaxProgramSize = 1 << 16; +constexpr int kMaxRepeatCount = 1024; +constexpr int kMaxGroupDepth = 128; + +bool IsWordByte(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_'; +} + +std::bitset<256> DigitClass() { + std::bitset<256> s; + for (int c = '0'; c <= '9'; ++c) s.set(c); + return s; +} + +std::bitset<256> WordClass() { + std::bitset<256> s; + for (int c = 0; c < 256; ++c) { + if (IsWordByte(static_cast(c))) s.set(c); + } + return s; +} + +std::bitset<256> SpaceClass() { + std::bitset<256> s; + // ECMA WhiteSpace + LineTerminator, restricted to single bytes. + for (unsigned char c : {' ', '\t', '\n', '\r', '\f', '\v'}) s.set(c); + return s; +} + +std::bitset<256> DotClass() { + // '.' over bytes: everything except the single-byte line terminators, + // matching the byte-oriented std::regex behavior this engine replaces. + std::bitset<256> s; + s.set(); + s.reset('\n'); + s.reset('\r'); + return s; +} + +// Parse tree. +struct Node; +using NodePtr = std::unique_ptr; +struct Node { + enum class Kind { + kEmpty, // matches the empty string + kByte, // one literal byte + kClass, // one byte from a set + kConcat, // children in sequence + kAlt, // any one child + kRepeat, // child repeated [min, max] times (max < 0 = unbounded) + kAssert, // zero-width assertion + }; + explicit Node(Kind k) : kind(k) {} + Kind kind; + std::uint8_t byte = 0; + std::bitset<256> cls; + std::vector children; + int min = 0; + int max = -1; + Regex::Inst::Assert assert_kind = Regex::Inst::Assert::kInputStart; +}; + +NodePtr MakeByte(std::uint8_t b) { + auto n = std::make_unique(Node::Kind::kByte); + n->byte = b; + return n; +} + +NodePtr MakeClass(std::bitset<256> cls) { + auto n = std::make_unique(Node::Kind::kClass); + n->cls = cls; + return n; +} + +NodePtr MakeAssert(Regex::Inst::Assert kind) { + auto n = std::make_unique(Node::Kind::kAssert); + n->assert_kind = kind; + return n; +} + +// Recursive-descent parser over the pattern bytes. +class Parser { + public: + explicit Parser(std::string_view pattern) : pattern_(pattern) {} + + Outcome Parse() { + auto node = ParseAlternation(0); + if (!node) return std::move(node).error(); + if (!AtEnd()) return Fail("unbalanced ')'"); + return std::move(node); + } + + private: + bool AtEnd() const { return pos_ >= pattern_.size(); } + unsigned char Peek() const { return static_cast(pattern_[pos_]); } + unsigned char Take() { return static_cast(pattern_[pos_++]); } + bool Eat(char c) { + if (AtEnd() || pattern_[pos_] != c) return false; + ++pos_; + return true; + } + + static Error Fail(const std::string& why) { return Error::Serialization("Regex: " + why); } + + Outcome ParseAlternation(int depth) { + auto first = ParseConcat(depth); + if (!first) return first; + if (AtEnd() || Peek() != '|') return first; + auto alt = std::make_unique(Node::Kind::kAlt); + alt->children.push_back(std::move(*first)); + while (Eat('|')) { + auto branch = ParseConcat(depth); + if (!branch) return branch; + alt->children.push_back(std::move(*branch)); + } + return NodePtr(std::move(alt)); + } + + Outcome ParseConcat(int depth) { + auto concat = std::make_unique(Node::Kind::kConcat); + while (!AtEnd() && Peek() != '|' && Peek() != ')') { + auto piece = ParseRepeat(depth); + if (!piece) return piece; + concat->children.push_back(std::move(*piece)); + } + if (concat->children.empty()) return NodePtr(std::make_unique(Node::Kind::kEmpty)); + if (concat->children.size() == 1) return std::move(concat->children.front()); + return NodePtr(std::move(concat)); + } + + Outcome ParseRepeat(int depth) { + auto atom = ParseAtom(depth); + if (!atom) return atom; + int min = 0; + int max = -1; + unsigned char q = AtEnd() ? 0 : Peek(); + if (q == '*') { + ++pos_; + } else if (q == '+') { + ++pos_; + min = 1; + } else if (q == '?') { + ++pos_; + max = 1; + } else if (q == '{') { + std::size_t saved = pos_; + if (!ParseBoundedQuantifier(&min, &max)) { + // Not a well-formed {n,m} quantifier: '{' is a literal (the lenient + // Annex-B behavior real engines implement). + pos_ = saved; + return atom; + } + } else { + return atom; + } + if ((*atom)->kind == Node::Kind::kAssert) { + return Fail("quantifier applied to an assertion"); + } + Eat('?'); // Lazy quantifiers exist-match identically; accept and ignore. + if (!AtEnd() && (Peek() == '*' || Peek() == '+')) { + return Fail("double quantifier"); + } + auto repeat = std::make_unique(Node::Kind::kRepeat); + repeat->min = min; + repeat->max = max; + repeat->children.push_back(std::move(*atom)); + return NodePtr(std::move(repeat)); + } + + // Parses {n}, {n,}, {n,m} starting at '{'; false when not a quantifier. + bool ParseBoundedQuantifier(int* min, int* max) { + ++pos_; // '{' + int n = 0; + if (!ParseInt(&n)) return false; + if (Eat('}')) { + *min = n; + *max = n; + return true; + } + if (!Eat(',')) return false; + if (Eat('}')) { + *min = n; + *max = -1; + return true; + } + int m = 0; + if (!ParseInt(&m) || !Eat('}') || m < n) return false; + *min = n; + *max = m; + return true; + } + + bool ParseInt(int* out) { + if (AtEnd() || Peek() < '0' || Peek() > '9') return false; + long value = 0; + while (!AtEnd() && Peek() >= '0' && Peek() <= '9') { + value = value * 10 + (Take() - '0'); + if (value > kMaxRepeatCount) return false; + } + *out = static_cast(value); + return true; + } + + Outcome ParseAtom(int depth) { + unsigned char c = Take(); + switch (c) { + case '^': + return MakeAssert(Regex::Inst::Assert::kInputStart); + case '$': + return MakeAssert(Regex::Inst::Assert::kInputEnd); + case '.': + return MakeClass(DotClass()); + case '(': + return ParseGroup(depth); + case '[': + return ParseClass(); + case '\\': + return ParseEscape(/*in_class=*/false); + case '*': + case '+': + case '?': + return Fail("quantifier with nothing to repeat"); + default: + return MakeByte(c); + } + } + + Outcome ParseGroup(int depth) { + if (depth >= kMaxGroupDepth) return Fail("groups nested too deeply"); + if (Eat('?')) { + if (Eat(':')) { + // Non-capturing group. + } else if (!AtEnd() && (Peek() == '=' || Peek() == '!')) { + return Fail("lookahead is not supported by the linear-time engine"); + } else if (Eat('<')) { + if (!AtEnd() && (Peek() == '=' || Peek() == '!')) { + return Fail("lookbehind is not supported by the linear-time engine"); + } + // Named capturing group: skip the name, match as a plain group. + while (!AtEnd() && Peek() != '>') ++pos_; + if (!Eat('>')) return Fail("unterminated group name"); + } else { + return Fail("unsupported group syntax '(?'"); + } + } + auto body = ParseAlternation(depth + 1); + if (!body) return body; + if (!Eat(')')) return Fail("missing ')'"); + return body; + } + + Outcome ParseClass() { + std::bitset<256> cls; + bool negate = Eat('^'); + bool first = true; + while (true) { + if (AtEnd()) return Fail("unterminated character class"); + if (Peek() == ']' && !first) { + ++pos_; + break; + } + first = false; + // One class member: a literal byte or an escape (which may itself be a + // whole class, e.g. [\d-] — a class escape cannot start a range). + std::bitset<256> lo_class; + int lo = -1; + unsigned char c = Take(); + if (c == '\\') { + auto escaped = ParseClassEscape(&lo_class, &lo); + if (!escaped.ok()) return std::move(escaped).error(); + } else { + lo = c; + } + if (lo < 0) { + cls |= lo_class; + continue; + } + if (!AtEnd() && Peek() == '-' && pos_ + 1 < pattern_.size() && pattern_[pos_ + 1] != ']') { + ++pos_; // '-' + int hi = -1; + std::bitset<256> hi_class; + unsigned char h = Take(); + if (h == '\\') { + auto escaped = ParseClassEscape(&hi_class, &hi); + if (!escaped.ok()) return std::move(escaped).error(); + } else { + hi = h; + } + if (hi < 0) return Fail("class escape cannot end a range"); + if (hi < lo) return Fail("character range is out of order"); + for (int b = lo; b <= hi; ++b) cls.set(static_cast(b)); + } else { + cls.set(static_cast(lo)); + } + } + if (negate) cls.flip(); + return MakeClass(cls); + } + + // After a backslash inside a class: either a single byte (*byte >= 0) or a + // class escape (*byte stays -1 and *cls is filled in). + Outcome ParseClassEscape(std::bitset<256>* cls, int* byte) { + if (AtEnd()) return Fail("dangling escape"); + unsigned char c = Take(); + switch (c) { + case 'd': + *cls = DigitClass(); + return Unit{}; + case 'D': + *cls = ~DigitClass(); + return Unit{}; + case 'w': + *cls = WordClass(); + return Unit{}; + case 'W': + *cls = ~WordClass(); + return Unit{}; + case 's': + *cls = SpaceClass(); + return Unit{}; + case 'S': + *cls = ~SpaceClass(); + return Unit{}; + case 'b': + *byte = 0x08; + return Unit{}; // \b inside a class = backspace + default: { + int b = SimpleEscapeByte(c); + if (b < 0) return Fail(std::string("unsupported escape '\\") + static_cast(c) + "'"); + *byte = b; + return Unit{}; + } + } + } + + Outcome ParseEscape(bool /*in_class*/) { + if (AtEnd()) return Fail("dangling escape"); + unsigned char c = Take(); + switch (c) { + case 'd': + return MakeClass(DigitClass()); + case 'D': + return MakeClass(~DigitClass()); + case 'w': + return MakeClass(WordClass()); + case 'W': + return MakeClass(~WordClass()); + case 's': + return MakeClass(SpaceClass()); + case 'S': + return MakeClass(~SpaceClass()); + case 'b': + return MakeAssert(Regex::Inst::Assert::kWordBoundary); + case 'B': + return MakeAssert(Regex::Inst::Assert::kNotWordBoundary); + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return Fail("backreferences are not supported by the linear-time engine"); + case 'u': { + long code = ParseHex(4); + if (code < 0) return Fail("malformed \\uHHHH escape"); + return Utf8Literal(code); + } + default: { + int b = SimpleEscapeByte(c); + if (b < 0) return Fail(std::string("unsupported escape '\\") + static_cast(c) + "'"); + return MakeByte(static_cast(b)); + } + } + } + + // Escapes shared by class and non-class contexts; -1 when unknown. + int SimpleEscapeByte(unsigned char c) { + switch (c) { + case 'n': + return '\n'; + case 'r': + return '\r'; + case 't': + return '\t'; + case 'f': + return '\f'; + case 'v': + return '\v'; + case '0': + return '\0'; + case 'x': { + long b = ParseHex(2); + return b < 0 ? -1 : static_cast(b); + } + default: + // ECMA identity escapes: any non-alphanumeric escapes to itself. + if (IsWordByte(c)) return -1; + return c; + } + } + + long ParseHex(int digits) { + long value = 0; + for (int i = 0; i < digits; ++i) { + if (AtEnd()) return -1; + unsigned char c = Take(); + int d = -1; + if (c >= '0' && c <= '9') + d = c - '0'; + else if (c >= 'a' && c <= 'f') + d = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') + d = c - 'A' + 10; + if (d < 0) return -1; + value = value * 16 + d; + } + return value; + } + + // A \uHHHH escape matches the code point's UTF-8 byte sequence. + static Outcome Utf8Literal(long code) { + if (code >= 0xD800 && code <= 0xDFFF) { + return Fail("surrogate \\u escapes are not supported"); + } + if (code < 0x80) return MakeByte(static_cast(code)); + auto concat = std::make_unique(Node::Kind::kConcat); + if (code < 0x800) { + concat->children.push_back(MakeByte(static_cast(0xC0 | (code >> 6)))); + } else { + concat->children.push_back(MakeByte(static_cast(0xE0 | (code >> 12)))); + concat->children.push_back(MakeByte(static_cast(0x80 | ((code >> 6) & 0x3F)))); + } + concat->children.push_back(MakeByte(static_cast(0x80 | (code & 0x3F)))); + return NodePtr(std::move(concat)); + } + + std::string_view pattern_; + std::size_t pos_ = 0; +}; + +} // namespace + +// Emits the parse tree as a Thompson NFA program. +class RegexCompiler { + public: + static Outcome Compile(std::string_view pattern) { + Parser parser(pattern); + auto tree = parser.Parse(); + if (!tree) return std::move(tree).error(); + Regex re; + RegexCompiler compiler(&re); + if (auto emitted = compiler.Emit(**tree); !emitted.ok()) { + return std::move(emitted).error(); + } + if (auto added = compiler.Add({Regex::Inst::Op::kMatch}); !added.ok()) { + return std::move(added).error(); + } + return re; + } + + private: + using Inst = Regex::Inst; + + explicit RegexCompiler(Regex* re) : re_(re) {} + + Outcome Add(Inst inst) { + if (re_->program_.size() >= kMaxProgramSize) { + return Error::Serialization("Regex: pattern compiles to too large a program"); + } + re_->program_.push_back(inst); + return Unit{}; + } + + std::uint32_t Here() const { return static_cast(re_->program_.size()); } + + std::uint32_t AddClass(const std::bitset<256>& cls) { + for (std::size_t i = 0; i < re_->classes_.size(); ++i) { + if (re_->classes_[i] == cls) return static_cast(i); + } + re_->classes_.push_back(cls); + return static_cast(re_->classes_.size() - 1); + } + + Outcome Emit(const Node& node) { + switch (node.kind) { + case Node::Kind::kEmpty: + return Unit{}; + case Node::Kind::kByte: { + Inst inst{Inst::Op::kByte}; + inst.byte = node.byte; + return Add(inst); + } + case Node::Kind::kClass: { + Inst inst{Inst::Op::kClass}; + inst.arg = AddClass(node.cls); + return Add(inst); + } + case Node::Kind::kAssert: { + Inst inst{Inst::Op::kAssert}; + inst.assert_kind = node.assert_kind; + return Add(inst); + } + case Node::Kind::kConcat: + for (const NodePtr& child : node.children) { + if (auto emitted = Emit(*child); !emitted.ok()) return emitted; + } + return Unit{}; + case Node::Kind::kAlt: { + // split; A; jmp end; split; B; jmp end; ...; C + std::vector jumps_to_end; + for (std::size_t i = 0; i < node.children.size(); ++i) { + const bool last = i + 1 == node.children.size(); + std::uint32_t split_pc = 0; + if (!last) { + split_pc = Here(); + if (auto added = Add({Inst::Op::kSplit}); !added.ok()) return added; + } + if (auto emitted = Emit(*node.children[i]); !emitted.ok()) return emitted; + if (!last) { + jumps_to_end.push_back(Here()); + if (auto added = Add({Inst::Op::kJmp}); !added.ok()) return added; + re_->program_[split_pc].arg = Here(); + } + } + for (std::uint32_t pc : jumps_to_end) re_->program_[pc].arg = Here(); + return Unit{}; + } + case Node::Kind::kRepeat: + return EmitRepeat(node); + } + return Unit{}; // Unreachable; keeps -Werror switch analysis happy. + } + + Outcome EmitRepeat(const Node& node) { + const Node& body = *node.children.front(); + for (int i = 0; i < node.min; ++i) { + if (auto emitted = Emit(body); !emitted.ok()) return emitted; + } + if (node.max < 0) { + // body{min,} — one looping optional copy: L: split(end); body; jmp L + std::uint32_t loop = Here(); + if (auto added = Add({Inst::Op::kSplit}); !added.ok()) return added; + if (auto emitted = Emit(body); !emitted.ok()) return emitted; + Inst jmp{Inst::Op::kJmp}; + jmp.arg = loop; + if (auto added = Add(jmp); !added.ok()) return added; + re_->program_[loop].arg = Here(); + return Unit{}; + } + // body{min,max} — (max - min) nested optional copies: each split jumps + // past everything that remains. + std::vector splits; + for (int i = node.min; i < node.max; ++i) { + splits.push_back(Here()); + if (auto added = Add({Inst::Op::kSplit}); !added.ok()) return added; + if (auto emitted = Emit(body); !emitted.ok()) return emitted; + } + for (std::uint32_t pc : splits) re_->program_[pc].arg = Here(); + return Unit{}; + } + + Regex* re_; +}; + +Outcome Regex::Compile(std::string_view pattern) { return RegexCompiler::Compile(pattern); } + +bool Regex::AddThread(std::vector* list, std::vector* seen_stamp, + std::uint32_t stamp, std::uint32_t pc, std::string_view text, + std::size_t pos) const { + // Iterative epsilon closure; the explicit stack keeps deeply split + // programs from overflowing the call stack. + std::vector work{pc}; + while (!work.empty()) { + std::uint32_t at = work.back(); + work.pop_back(); + if ((*seen_stamp)[at] == stamp) continue; + (*seen_stamp)[at] = stamp; + const Inst& inst = program_[at]; + switch (inst.op) { + case Inst::Op::kMatch: + return true; + case Inst::Op::kJmp: + work.push_back(inst.arg); + break; + case Inst::Op::kSplit: + work.push_back(at + 1); + work.push_back(inst.arg); + break; + case Inst::Op::kAssert: { + bool before = pos > 0 && IsWordByte(static_cast(text[pos - 1])); + bool after = pos < text.size() && IsWordByte(static_cast(text[pos])); + bool holds = false; + switch (inst.assert_kind) { + case Inst::Assert::kInputStart: + holds = pos == 0; + break; + case Inst::Assert::kInputEnd: + holds = pos == text.size(); + break; + case Inst::Assert::kWordBoundary: + holds = before != after; + break; + case Inst::Assert::kNotWordBoundary: + holds = before == after; + break; + } + if (holds) work.push_back(at + 1); + break; + } + case Inst::Op::kByte: + case Inst::Op::kClass: + list->push_back(at); + break; + } + } + return false; +} + +bool Regex::Search(std::string_view text) const { + if (program_.empty()) return false; + std::vector current; + std::vector next; + // Stamps deduplicate threads per input position without clearing a + // visited set on every byte. 0 means "never seen"; stamps start at 1. + std::vector seen_stamp(program_.size(), 0); + for (std::size_t pos = 0; pos <= text.size(); ++pos) { + // Unanchored search: a fresh attempt starts at every position. + if (AddThread(¤t, &seen_stamp, static_cast(pos) + 1, 0, text, pos)) { + return true; + } + if (pos == text.size()) break; + const auto c = static_cast(text[pos]); + next.clear(); + for (std::uint32_t pc : current) { + const Inst& inst = program_[pc]; + bool matches = inst.op == Inst::Op::kByte ? inst.byte == c : classes_[inst.arg].test(c); + if (matches && AddThread(&next, &seen_stamp, static_cast(pos) + 2, pc + 1, + text, pos + 1)) { + return true; + } + } + current.swap(next); + } + return false; +} + +} // namespace smithy diff --git a/runtime/tests/core/regex_test.cc b/runtime/tests/core/regex_test.cc new file mode 100644 index 00000000..bb4c2879 --- /dev/null +++ b/runtime/tests/core/regex_test.cc @@ -0,0 +1,180 @@ +#include "smithy/core/regex.h" + +#include + +#include +#include +#include +#include + +namespace { + +bool Search(const std::string& pattern, const std::string& text) { + auto re = smithy::Regex::Compile(pattern); + EXPECT_TRUE(re.ok()) << pattern << ": " << (re.ok() ? "" : re.error().message()); + return re.ok() && re->Search(text); +} + +bool Compiles(const std::string& pattern) { return smithy::Regex::Compile(pattern).ok(); } + +TEST(RegexTest, LiteralsUsePartialMatchSemantics) { + EXPECT_TRUE(Search("bc", "abcd")); // regex_search, not regex_match + EXPECT_FALSE(Search("bc", "b c")); + EXPECT_TRUE(Search("", "anything")); // empty pattern matches everywhere + EXPECT_TRUE(Search("a", "a")); + EXPECT_FALSE(Search("a", "")); +} + +TEST(RegexTest, AnchorsPinTheMatch) { + EXPECT_TRUE(Search("^ab", "abc")); + EXPECT_FALSE(Search("^bc", "abc")); + EXPECT_TRUE(Search("bc$", "abc")); + EXPECT_FALSE(Search("ab$", "abc")); + EXPECT_TRUE(Search("^abc$", "abc")); + EXPECT_FALSE(Search("^abc$", "abcd")); + EXPECT_TRUE(Search("^$", "")); + EXPECT_FALSE(Search("^$", "x")); +} + +TEST(RegexTest, TheFixturePatterns) { + // Every @pattern in the checked-in fixture models. + EXPECT_TRUE(Search("^[A-Za-z0-9 ]+$", "Seattle 98101")); + EXPECT_FALSE(Search("^[A-Za-z0-9 ]+$", "nope!")); + EXPECT_FALSE(Search("^[A-Za-z0-9 ]+$", "")); +} + +TEST(RegexTest, ClassesRangesAndNegation) { + EXPECT_TRUE(Search("[a-c]", "b")); + EXPECT_FALSE(Search("[a-c]", "d")); + EXPECT_TRUE(Search("[^a-c]", "d")); + EXPECT_FALSE(Search("^[^a-c]+$", "abc")); + EXPECT_TRUE(Search("[]a]", "]")); // ']' first in a class is a literal + EXPECT_TRUE(Search("[a-]", "-")); // trailing '-' is a literal + EXPECT_TRUE(Search("[-a]", "-")); + EXPECT_TRUE(Search("[\\d]", "7")); + EXPECT_FALSE(Search("^[\\d-]+$", "7x")); +} + +TEST(RegexTest, ClassEscapes) { + EXPECT_TRUE(Search("^\\d+$", "0123456789")); + EXPECT_FALSE(Search("^\\d+$", "12a")); + EXPECT_TRUE(Search("^\\w+$", "az_AZ09")); + EXPECT_FALSE(Search("^\\w+$", "a b")); + EXPECT_TRUE(Search("^\\s$", "\t")); + EXPECT_TRUE(Search("^\\S+$", "abc")); + EXPECT_FALSE(Search("^\\D$", "5")); + EXPECT_TRUE(Search("^\\W$", "!")); +} + +TEST(RegexTest, QuantifiersIncludingCounted) { + EXPECT_TRUE(Search("^a*$", "")); + EXPECT_TRUE(Search("^a+$", "aaa")); + EXPECT_FALSE(Search("^a+$", "")); + EXPECT_TRUE(Search("^ab?c$", "ac")); + EXPECT_TRUE(Search("^a{3}$", "aaa")); + EXPECT_FALSE(Search("^a{3}$", "aa")); + EXPECT_TRUE(Search("^a{2,}$", "aaaa")); + EXPECT_FALSE(Search("^a{2,}$", "a")); + EXPECT_TRUE(Search("^a{1,3}$", "aa")); + EXPECT_FALSE(Search("^a{1,3}$", "aaaa")); + EXPECT_TRUE(Search("^a*?b$", "aab")); // lazy quantifiers accepted + EXPECT_TRUE(Search("a{,3}", "a{,3}")); // not a quantifier: literal '{' +} + +TEST(RegexTest, GroupsAndAlternation) { + EXPECT_TRUE(Search("^(ab)+$", "ababab")); + EXPECT_FALSE(Search("^(ab)+$", "aba")); + EXPECT_TRUE(Search("^(a|bc)d$", "bcd")); + EXPECT_TRUE(Search("^(?:xy){2}$", "xyxy")); + EXPECT_TRUE(Search("cat|dog", "hotdog")); + EXPECT_FALSE(Search("^(cat|dog)$", "cow")); + EXPECT_TRUE(Search("^(?ab)$", "ab")); // named group = plain group +} + +TEST(RegexTest, DotAndEscapedLiterals) { + EXPECT_TRUE(Search("^a.c$", "abc")); + EXPECT_FALSE(Search("^a.c$", "a\nc")); + EXPECT_TRUE(Search("^a\\.c$", "a.c")); + EXPECT_FALSE(Search("^a\\.c$", "abc")); + EXPECT_TRUE(Search("^\\x41$", "A")); + EXPECT_TRUE(Search("^\\u0041$", "A")); + EXPECT_TRUE(Search("^\\u00e9$", "\xc3\xa9")); // é as UTF-8 bytes + EXPECT_TRUE(Search("\\$\\^\\(\\)\\[\\]", "$^()[]")); +} + +TEST(RegexTest, WordBoundaries) { + EXPECT_TRUE(Search("\\bcat\\b", "a cat sat")); + EXPECT_FALSE(Search("\\bcat\\b", "concatenate")); + EXPECT_TRUE(Search("\\Bcat\\B", "concatenate")); + EXPECT_FALSE(Search("\\Bcat\\B", "a cat sat")); +} + +TEST(RegexTest, UnsupportedConstructsFailAtCompileTime) { + EXPECT_FALSE(Compiles("(a)\\1")); // backreference + EXPECT_FALSE(Compiles("a(?=b)")); // lookahead + EXPECT_FALSE(Compiles("a(?!b)")); // negative lookahead + EXPECT_FALSE(Compiles("(?<=a)b")); // lookbehind + EXPECT_FALSE(Compiles("(?Search(evil)); + const auto elapsed = std::chrono::steady_clock::now() - start; + EXPECT_LT(elapsed, std::chrono::seconds(5)); + std::string good(100000, '7'); + EXPECT_TRUE(re->Search(good)); +} + +TEST(RegexTest, MoreNestedQuantifierBombs) { + std::string as(50000, 'a'); + EXPECT_FALSE(Search("^(a+)+$", as + "b")); + EXPECT_FALSE(Search("^(a|a)+$", as + "b")); + EXPECT_FALSE(Search("^(a*)*$", as + "b")); + EXPECT_TRUE(Search("^(a+)+$", as)); +} + +// Differential check against std::regex: on patterns std::regex handles +// without pathological backtracking, both engines must agree. +TEST(RegexTest, AgreesWithStdRegexOnRandomInputs) { + const char* patterns[] = { + "^[A-Za-z0-9 ]+$", "^\\d{1,3}(\\.\\d{1,3}){3}$", + "^(ab|cd)*e?$", "[a-f]+\\d*", + "^x.y$", "\\bword\\b", + "^-?\\d+$", "^[^0-9]*$", + }; + std::mt19937 rng(20260708); + std::uniform_int_distribution len(0, 12); + const std::string alphabet = "abcdexy01239. -_"; + std::uniform_int_distribution pick(0, alphabet.size() - 1); + for (const char* pattern : patterns) { + auto mine = smithy::Regex::Compile(pattern); + ASSERT_TRUE(mine.ok()) << pattern; + const std::regex theirs(pattern, std::regex::ECMAScript); + for (int i = 0; i < 500; ++i) { + std::string text; + const int n = len(rng); + for (int j = 0; j < n; ++j) text.push_back(alphabet[pick(rng)]); + EXPECT_EQ(mine->Search(text), std::regex_search(text, theirs)) + << "pattern: " << pattern << " text: '" << text << "'"; + } + } +} + +} // namespace