Testing & CI hardening: compile-the-output harness and coverage gaps (#48)

.github/workflows/ci.yml

@@ -87,7 +87,7 @@ jobs:
           CXX: clang++
         run: |
           set -euo pipefail
-          for target in json_decode cbor_decode uri server_dispatch; do
+          for target in json_decode cbor_decode uri server_dispatch regex; do
             echo "== fuzzing $target"
             bazelisk build --config=fuzz "//fuzz:${target}_fuzz"
             ./bazel-bin/fuzz/${target}_fuzz -max_total_time=30 -print_final_stats=1
@@ -181,8 +181,8 @@ jobs:
       # its shape is locked by the golden diff check in the codegen job.
       - name: clang-format
         run: |
-          find runtime examples \( -name '*.h' -o -name '*.cc' \) ! -path '*/generated/*' \
-            | xargs clang-format --dry-run --Werror
+          find runtime examples codegen/compile-tests \( -name '*.h' -o -name '*.cc' \) \
+            ! -path '*/generated/*' | xargs clang-format --dry-run --Werror
       # Excluded from tidy: src/json/json.cc includes the nlohmann backend,
       # which only exists inside the Bazel build graph; beast_src.cc is the
       # Boost implementation TU and only compiles with the BCR modules'

CHANGELOG.md

@@ -53,8 +53,31 @@ via `git_override` until then.
   size limits, graceful drain, TLS termination) and `BeastHttpClient`
   (keep-alive connection pool, per-request timeouts, TLS via BoringSSL with
   certificate + hostname verification on by default).
-- Fuzz harnesses (JSON, CBOR, URI, server dispatch) and a Google Benchmark
-  suite (serde, codecs, per-protocol request round trips, real-TCP transport
-  round trips incl. Beast and Beast TLS) run in CI.
+- Fuzz harnesses (JSON, CBOR, URI, server dispatch, regex) and a Google
+  Benchmark suite (serde, codecs, per-protocol request round trips, real-TCP
+  transport round trips incl. Beast and Beast TLS) run in CI.
+- CBOR decoder rejects additional-information 31 on integers and tags
+  (RFC 8949 §3.3 not-well-formed encodings previously decoded as 0 / -1 /
+  an ignored tag), found by the hostile corpus below.
+
+### Testing & CI (issue #48)
+
+- **Compile-the-output harness** (`codegen/compile-tests/`): the generator
+  runs inside the Bazel graph on a hostile gauntlet model — C++ keyword
+  member names, quote/backslash/newline enum values, raw-string delimiter
+  attacks, int64-extreme bounds/defaults, recursion, keyword union variants —
+  and CI compiles the result for every protocol, client and server mode both.
+  Issue #43's whole bug class now fails CI instead of a consumer's build.
+- Curated hostile CBOR corpus (`cbor_hostile_test.cc`): systematic
+  truncations, reserved encodings, indefinite-length abuse, depth bombs,
+  boundary integers/halves, and an every-strict-prefix-rejects property, as
+  the CBOR counterpart of the vendored JSONTestSuite bank.
+- Direct unit tests for `core/uuid.cc` (format, version/variant bits,
+  uniqueness, thread-local streams) and `client/observability.cc`
+  (attempt observations, trace-context propagation).
+- The regex ReDoS bound is a deterministic step-count assertion
+  (`Search(text, &steps)` instrumentation) instead of a wall-clock limit.
+- `make verify` / `make verify-full`: one-command local verification
+  mirroring the CI jobs one-to-one.
 
 [Unreleased]: https://github.com/aaylward/smithy-cpp/commits/main

Makefile

@@ -0,0 +1,79 @@
+# One-command verification (issue #48): `make verify` runs everything the CI
+# gate runs, in one place, so full local verification stops being eight
+# commands scattered across two build systems (docs/development.md#building-
+# and-testing has the background). Each aggregate target is also callable on
+# its own; the recipes deliberately mirror .github/workflows/ci.yml — when a
+# CI job changes, change the matching target here.
+
+BAZEL ?= bazelisk
+GRADLE ?= gradle
+
+# What CI gates a PR on: the bazel test matrix (one platform of it), the
+# gradle build + format check, golden freshness, and the format/starlark lint.
+.PHONY: verify
+verify: test codegen goldens lint
+	@echo "verify: OK"
+
+# verify plus the slower jobs: sanitizers, fuzzer smoke runs, the out-of-tree
+# consumer module, and clang-tidy.
+.PHONY: verify-full
+verify-full: verify sanitize fuzz-smoke consumer tidy
+	@echo "verify-full: OK"
+
+.PHONY: test
+test:
+	$(BAZEL) test //...
+
+.PHONY: codegen
+codegen:
+	cd codegen && $(GRADLE) build spotlessCheck
+
+# The checked-in generated code is the golden output; regeneration must be
+# byte-identical.
+.PHONY: goldens
+goldens:
+	cd codegen && $(GRADLE) generateFixtures generateProtocolTests
+	git diff --exit-code -- examples protocol-tests
+
+.PHONY: lint
+lint:
+	find runtime examples codegen/compile-tests \( -name '*.h' -o -name '*.cc' \) \
+		! -path '*/generated/*' | xargs clang-format --dry-run --Werror
+	buildifier --lint=warn --mode=check -r .
+
+.PHONY: tidy
+tidy:
+	find runtime/src examples -name '*.cc' ! -name '*_test.cc' ! -path '*/src/json/*' \
+		! -name 'beast_src.cc' ! -path '*/generated/*' -print0 \
+		| xargs -0 -I{} clang-tidy --quiet {} -- -Iruntime/include -I. -std=c++20
+
+.PHONY: sanitize
+sanitize:
+	CC=clang CXX=clang++ $(BAZEL) test //... --config=asan --config=ubsan
+
+.PHONY: fuzz-smoke
+fuzz-smoke:
+	set -e; for target in json_decode cbor_decode uri server_dispatch regex; do \
+		echo "== fuzzing $$target"; \
+		CC=clang CXX=clang++ $(BAZEL) build --config=fuzz "//fuzz:$${target}_fuzz"; \
+		./bazel-bin/fuzz/$${target}_fuzz -max_total_time=30 -print_final_stats=1; \
+	done
+
+.PHONY: consumer
+consumer:
+	cd examples/bazel-consumer && $(BAZEL) test //... && ./model-evolution-check.sh
+
+# Informational, never gates (PLAN Phase 7).
+.PHONY: benchmarks
+benchmarks:
+	$(BAZEL) run -c opt //benchmarks:serde_benchmark -- --benchmark_min_time=0.2s
+	$(BAZEL) run -c opt //benchmarks:request_benchmark -- --benchmark_min_time=0.2s
+	$(BAZEL) run -c opt //benchmarks:beast_benchmark -- --benchmark_min_time=0.2s
+
+# Rewrites instead of checking: the fix-it twin of `lint` + codegen's spotless.
+.PHONY: format
+format:
+	find runtime examples codegen/compile-tests \( -name '*.h' -o -name '*.cc' \) \
+		! -path '*/generated/*' | xargs clang-format -i
+	buildifier --lint=warn -r .
+	cd codegen && $(GRADLE) spotlessApply

codegen/compile-tests/BUILD.bazel

@@ -0,0 +1,73 @@
+load("@rules_cc//cc:defs.bzl", "cc_test")
+load("//bazel:defs.bzl", "smithy_cpp_client_library", "smithy_cpp_server_library")
+
+# The compile-the-output harness (issue #48): run the real generator on the
+# hostile gauntlet model and *compile* what comes out, for every protocol, in
+# both client and server mode. The Java unit suite asserts on source
+# substrings; this package is what turns "the generator emitted uncompilable
+# C++" (issue #43's whole bug class) into an ordinary CI failure. It is also
+# the one place SmithyCppGenerate executes for shapes no golden fixture pins.
+
+MODEL = ["model/gauntlet.smithy"]
+
+smithy_cpp_client_library(
+    name = "gauntlet_rest_client",
+    srcs = MODEL + ["model/bindings/simplerestjson.smithy"],
+    namespace = "compile::gauntlet::rest",
+    service = "compile.gauntlet#Gauntlet",
+)
+
+smithy_cpp_server_library(
+    name = "gauntlet_rest_server",
+    srcs = MODEL + ["model/bindings/simplerestjson.smithy"],
+    namespace = "compile::gauntlet::rest",
+    service = "compile.gauntlet#Gauntlet",
+)
+
+smithy_cpp_client_library(
+    name = "gauntlet_cbor_client",
+    srcs = MODEL + ["model/bindings/rpcv2cbor.smithy"],
+    namespace = "compile::gauntlet::cbor",
+    service = "compile.gauntlet#Gauntlet",
+)
+
+smithy_cpp_server_library(
+    name = "gauntlet_cbor_server",
+    srcs = MODEL + ["model/bindings/rpcv2cbor.smithy"],
+    namespace = "compile::gauntlet::cbor",
+    service = "compile.gauntlet#Gauntlet",
+)
+
+smithy_cpp_client_library(
+    name = "gauntlet_jsonrpc_client",
+    srcs = MODEL + ["model/bindings/jsonrpc2.smithy"],
+    namespace = "compile::gauntlet::jsonrpc",
+    service = "compile.gauntlet#Gauntlet",
+)
+
+smithy_cpp_server_library(
+    name = "gauntlet_jsonrpc_server",
+    srcs = MODEL + ["model/bindings/jsonrpc2.smithy"],
+    namespace = "compile::gauntlet::jsonrpc",
+    service = "compile.gauntlet#Gauntlet",
+)
+
+# Linking the test forces every generated TU above through the compiler; the
+# assertions then spot-check the escaping contract (keyword members get a
+# trailing underscore, hostile enum values round-trip, int64-min defaults).
+cc_test(
+    name = "gauntlet_compile_test",
+    size = "small",
+    srcs = ["gauntlet_compile_test.cc"],
+    deps = [
+        ":gauntlet_cbor_client",
+        ":gauntlet_cbor_server",
+        ":gauntlet_jsonrpc_client",
+        ":gauntlet_jsonrpc_server",
+        ":gauntlet_rest_client",
+        ":gauntlet_rest_server",
+        "//runtime:client",
+        "//runtime:http",
+        "@googletest//:gtest_main",
+    ],
+)

codegen/compile-tests/gauntlet_compile_test.cc

@@ -0,0 +1,130 @@
+// The compile-the-output harness (issue #48). Most of the test is the build:
+// including every generated header and linking all six gauntlet libraries
+// forces the generator's output for the hostile model through the compiler on
+// every platform CI runs. The assertions below then spot-check the escaping
+// contract itself — keyword members, hostile enum wire values, extreme
+// numeric bounds — so a silent change in the escaping scheme fails loudly
+// here rather than in a consumer's build.
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <limits>
+#include <string>
+
+#include "compile/gauntlet/cbor/client.h"
+#include "compile/gauntlet/cbor/server.h"
+#include "compile/gauntlet/jsonrpc/client.h"
+#include "compile/gauntlet/jsonrpc/server.h"
+#include "compile/gauntlet/rest/client.h"
+#include "compile/gauntlet/rest/server.h"
+
+namespace {
+
+namespace rest = compile::gauntlet::rest;
+
+TEST(GauntletCompileTest, KeywordMembersGetTrailingUnderscores) {
+  rest::RunGauntletInput input;
+  input.name = "escape me";
+  input.class_ = "keyword";
+  input.namespace_ = "keyword";
+  input.template_ = "keyword";
+  input.operator_ = true;
+  input.delete_ = false;
+  input.int_ = 7;
+  input.double_ = 1.5;
+  input.union_ = "keyword";
+  input.default_ = "keyword";
+  input.friend_ = "keyword";
+  input.this_ = "keyword";
+  input.auto_ = "keyword";
+  input.register_ = 9;
+  input.value = "not a keyword";
+  input.kind = "not a keyword";
+  input._leadingUnderscore = "kept verbatim";
+  EXPECT_EQ(input.int_, 7);
+  EXPECT_EQ(input, input);
+
+  rest::GetReportInput report;
+  report.class_ = "label";
+  report.switch_ = "query";
+  report.case_ = "header";
+  EXPECT_EQ(report.class_, "label");
+
+  rest::GauntletRejected rejected;
+  rejected.message = "still compiles";
+  rejected.class_ = "keyword";
+  EXPECT_EQ(rejected.class_, "keyword");
+}
+
+TEST(GauntletCompileTest, HostileEnumValuesRoundTrip) {
+  using Enum = rest::HostileEnum;
+  const struct {
+    Enum::Value value;
+    const char* wire;
+  } cases[] = {
+      {Enum::Value::kQuote, "he said \"more\""},     {Enum::Value::kBackslash, "C:\\temp\\new"},
+      {Enum::Value::kNewline, "line one\nline two"}, {Enum::Value::kTrickyRaw, ")__smithy\""},
+      {Enum::Value::kUnicodeValue, "caf\xc3\xa9"},
+  };
+  for (const auto& c : cases) {
+    const Enum parsed = Enum::FromString(c.wire);
+    EXPECT_EQ(parsed.value(), c.value) << c.wire;
+    EXPECT_EQ(parsed.ToString(), c.wire);
+  }
+  const Enum unknown = Enum::FromString("never modeled");
+  EXPECT_EQ(unknown.value(), Enum::Value::kUnknown);
+  EXPECT_EQ(unknown.ToString(), "never modeled");
+}
+
+TEST(GauntletCompileTest, IntEnumCoversInt32Extremes) {
+  EXPECT_EQ(static_cast<std::int32_t>(rest::HostileIntEnum::kBottom),
+            std::numeric_limits<std::int32_t>::min());
+  EXPECT_EQ(static_cast<std::int32_t>(rest::HostileIntEnum::kTop),
+            std::numeric_limits<std::int32_t>::max());
+  EXPECT_EQ(static_cast<std::int32_t>(rest::HostileIntEnum::kNothing), 0);
+}
+
+TEST(GauntletCompileTest, UnionKeywordVariantsWork) {
+  const auto number = rest::HostileUnion::FromInt(42);
+  ASSERT_TRUE(number.is_int_());
+  EXPECT_EQ(number.as_int_(), 42);
+
+  const auto text = rest::HostileUnion::FromClass("keyword variant");
+  ASSERT_TRUE(text.is_class_());
+  EXPECT_EQ(text.as_class_(), "keyword variant");
+  EXPECT_FALSE(text.is_int_());
+
+  rest::Node leaf;
+  leaf.label = "leaf";
+  const auto branch = rest::HostileUnion::FromNode(leaf);
+  ASSERT_TRUE(branch.is_node());
+  EXPECT_EQ(branch.as_node().label, "leaf");
+}
+
+TEST(GauntletCompileTest, RecursiveShapesUseValueSemanticBoxes) {
+  rest::Node root;
+  root.label = "root";
+  rest::Node child;
+  child.label = "child";
+  root.next = smithy::Boxed<rest::Node>(child);
+  root.children = std::vector<rest::Node>{child};
+  const rest::Node copy = root;  // deep copy through the box
+  EXPECT_EQ(copy, root);
+  EXPECT_EQ((*copy.next)->label, "child");
+}
+
+// The other two protocols generate the same shapes into their own
+// namespaces; touching one type from each keeps all their headers in the
+// build even if the includes above ever change.
+TEST(GauntletCompileTest, EveryProtocolEmitsTheGauntletShapes) {
+  compile::gauntlet::cbor::RunGauntletInput cbor_input;
+  cbor_input.class_ = "cbor";
+  EXPECT_EQ(cbor_input.class_, "cbor");
+
+  compile::gauntlet::jsonrpc::RunGauntletInput jsonrpc_input;
+  jsonrpc_input.class_ = "jsonrpc";
+  EXPECT_EQ(jsonrpc_input.class_, "jsonrpc");
+}
+
+}  // namespace

codegen/compile-tests/model/bindings/jsonrpc2.smithy

@@ -0,0 +1,10 @@
+// Protocol binding overlay: pairs with model/gauntlet.smithy to bind the
+// protocol-agnostic Gauntlet service to JSON-RPC 2.0. The @http traits in the
+// base model are simply ignored by this protocol.
+$version: "2.0"
+
+namespace compile.gauntlet
+
+use smithy.cpp.protocols#jsonRpc2
+
+apply Gauntlet @jsonRpc2

codegen/compile-tests/model/bindings/rpcv2cbor.smithy

@@ -0,0 +1,10 @@
+// Protocol binding overlay: pairs with model/gauntlet.smithy to bind the
+// protocol-agnostic Gauntlet service to rpcv2Cbor. The @http traits in the
+// base model are simply ignored by this protocol.
+$version: "2.0"
+
+namespace compile.gauntlet
+
+use smithy.protocols#rpcv2Cbor
+
+apply Gauntlet @rpcv2Cbor

codegen/compile-tests/model/bindings/simplerestjson.smithy

@@ -0,0 +1,9 @@
+// Protocol binding overlay: pairs with model/gauntlet.smithy to bind the
+// protocol-agnostic Gauntlet service to simpleRestJson.
+$version: "2.0"
+
+namespace compile.gauntlet
+
+use alloy#simpleRestJson
+
+apply Gauntlet @simpleRestJson