From c88f61a115795e894006603abf261586f8635098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Rubinos=20Rodr=C3=ADguez?= Date: Fri, 16 Oct 2026 10:04:58 +0200 Subject: [PATCH 1/2] test(h2): read the request body before the h2spec handler replies --- .github/workflows/ci.yml | 34 ++++++++++ .gitignore | 4 +- Makefile | 136 ++++++++------------------------------ rebar.config | 2 +- rebar.lock | 6 +- test/nhttp_compliance.erl | 21 +++++- 6 files changed, 86 insertions(+), 117 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc9e7de..dce9362 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,3 +52,37 @@ jobs: if: always() with: paths: '_build/test/logs/**/report.xml' + + compliance: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + - uses: erlef/setup-beam@v1.24.0 + with: + otp-version: '28.0' + rebar3-version: '3.25.0' + + - uses: actions/cache@v4 + with: + path: | + ~/.cache/rebar3 + _build + key: ${{ runner.os }}-${{ runner.arch }}-28.0-3.25.0-${{ hashFiles('rebar.config', 'rebar.lock') }} + + - run: make certs + + - run: make -k compliance + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: compliance-reports + path: compliance-reports/ + + - uses: test-summary/action@v2 + if: always() + with: + paths: 'compliance-reports/h2spec.xml' diff --git a/.gitignore b/.gitignore index fff4f80..0ae5df2 100644 --- a/.gitignore +++ b/.gitignore @@ -51,5 +51,5 @@ priv/h3spec test/conf/*.pem test/conf/*.key -# Autobahn WebSocket compliance reports -autobahn-reports/ +# Compliance reports (h2spec JUnit, Autobahn) +compliance-reports/ diff --git a/Makefile b/Makefile index 090dbb5..3a43522 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,12 @@ AUTOBAHN_IMAGE := nhttp/autobahn:$(AUTOBAHN_VERSION) AUTOBAHN_DOCKERFILE := priv/Dockerfile.autobahn AUTOBAHN_SPEC := priv/autobahn/fuzzingclient.json AUTOBAHN_GATE := priv/autobahn/gate.sh -AUTOBAHN_REPORTS := autobahn-reports +# Each suite starts its server through this script and exits with the +# status of the runner, so a failed suite fails the target. +COMPLIANCE_RUN := REBAR3=$(REBAR3) priv/compliance/run.sh +COMPLIANCE_REPORTS := compliance-reports +AUTOBAHN_REPORTS := $(COMPLIANCE_REPORTS)/autobahn +DOCKER_USER := --user $(shell id -u):$(shell id -g) # h3spec only ships an x86_64 Linux binary; force amd64 so arm64 hosts # (e.g. Apple Silicon, arm64 Linux with qemu-user) can still run it. DOCKER_PLATFORM := linux/amd64 @@ -44,10 +49,6 @@ TEST_CERTS_DIR := test/conf TEST_CERTS_MARKER := $(TEST_CERTS_DIR)/server.pem TEST_CERTS_SCRIPT := $(TEST_CERTS_DIR)/gen_test_certs.sh -# Temp files for server coordination -COMPLIANCE_PORT_FILE := .compliance_port -COMPLIANCE_PID_FILE := .compliance_pid - #============================================================================== # Core targets #============================================================================== @@ -59,8 +60,7 @@ compile: clean: $(REBAR3) clean - @rm -f .compliance_port .compliance_pid - @rm -rf $(AUTOBAHN_REPORTS) + @rm -rf $(COMPLIANCE_REPORTS) check: certs $(REBAR3) check @@ -103,30 +103,8 @@ binopt: h1-compliance: h1spec-image @echo "=== HTTP/1.1 Compliance Tests with h1spec (Docker) ===" @$(REBAR3) as test compile - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @echo "Starting HTTP/1.1 server (plaintext)..." - @( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_h1_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE) - @for _ in $$(seq 1 100); do \ - grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \ - sleep 0.1; \ - done - @PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \ - if [ -z "$$PORT" ]; then \ - echo "Error: Could not parse port from server output:"; \ - cat $(COMPLIANCE_PORT_FILE); \ - kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \ - rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \ - exit 1; \ - fi; \ - echo "Server running on port $$PORT"; \ - echo ""; \ - docker run --rm $(DOCKER_NET_ARGS) $(H1SPEC_IMAGE) \ - $(COMPLIANCE_TARGET) $$PORT || true - @-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null - @-pkill -f 'nhttp_h1_compliance:start' 2>/dev/null - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @echo "" - @echo "=== HTTP/1.1 Compliance Tests Complete ===" + @$(COMPLIANCE_RUN) nhttp_h1_compliance \ + 'docker run --rm $(DOCKER_NET_ARGS) $(H1SPEC_IMAGE) $(COMPLIANCE_TARGET) $$PORT' h1spec-image: @command -v docker >/dev/null 2>&1 || { \ @@ -141,30 +119,11 @@ h1spec-image: h2-compliance: certs h2spec-image @echo "=== HTTP/2 RFC 9113 Compliance Tests with h2spec (Docker) ===" @$(REBAR3) as test compile - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @echo "Starting HTTP/2 server (TLS)..." - @( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE) - @for _ in $$(seq 1 100); do \ - grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \ - sleep 0.1; \ - done - @PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \ - if [ -z "$$PORT" ]; then \ - echo "Error: Could not parse port from server output:"; \ - cat $(COMPLIANCE_PORT_FILE); \ - kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \ - rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \ - exit 1; \ - fi; \ - echo "Server running on port $$PORT"; \ - echo ""; \ - docker run --rm $(DOCKER_NET_ARGS) $(H2SPEC_IMAGE) \ - -t -k -h $(COMPLIANCE_TARGET) -p $$PORT -o 5 || true - @-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null - @-pkill -f 'nhttp_compliance:start' 2>/dev/null - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @echo "" - @echo "=== Compliance Tests Complete ===" + @mkdir -p $(COMPLIANCE_REPORTS) + @$(COMPLIANCE_RUN) nhttp_compliance \ + 'docker run --rm $(DOCKER_NET_ARGS) $(DOCKER_USER) \ + -v $(CURDIR)/$(COMPLIANCE_REPORTS):/reports $(H2SPEC_IMAGE) \ + -t -k -h $(COMPLIANCE_TARGET) -p $$PORT -o 5 -j /reports/h2spec.xml' h2spec-image: @command -v docker >/dev/null 2>&1 || { \ @@ -179,30 +138,9 @@ h2spec-image: h3-compliance: certs h3spec-image @echo "=== HTTP/3 Compliance Tests with h3spec (Docker) ===" @$(REBAR3) as test compile - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @echo "Starting HTTP/3 server (QUIC)..." - @( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_h3_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE) - @for _ in $$(seq 1 100); do \ - grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \ - sleep 0.1; \ - done - @PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \ - if [ -z "$$PORT" ]; then \ - echo "Error: Could not parse port from server output:"; \ - cat $(COMPLIANCE_PORT_FILE); \ - kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \ - rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \ - exit 1; \ - fi; \ - echo "Server running on port $$PORT"; \ - echo ""; \ - docker run --rm --platform $(DOCKER_PLATFORM) $(DOCKER_NET_ARGS) $(H3SPEC_IMAGE) \ - $(COMPLIANCE_TARGET) $$PORT -n -t 3000 || true - @-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null - @-pkill -f 'nhttp_h3_compliance:start' 2>/dev/null - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @echo "" - @echo "=== HTTP/3 Compliance Tests Complete ===" + @$(COMPLIANCE_RUN) nhttp_h3_compliance \ + 'docker run --rm --platform $(DOCKER_PLATFORM) $(DOCKER_NET_ARGS) $(H3SPEC_IMAGE) \ + $(COMPLIANCE_TARGET) $$PORT -n -t 3000' h3spec-image: @command -v docker >/dev/null 2>&1 || { \ @@ -220,34 +158,13 @@ ws-compliance: autobahn-image @$(REBAR3) as test compile @rm -rf $(AUTOBAHN_REPORTS) @mkdir -p $(AUTOBAHN_REPORTS) - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @echo "Starting WebSocket server (plaintext)..." - @( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_ws_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE) - @for _ in $$(seq 1 100); do \ - grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \ - sleep 0.1; \ - done - @PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \ - if [ -z "$$PORT" ]; then \ - echo "Error: Could not parse port from server output:"; \ - cat $(COMPLIANCE_PORT_FILE); \ - kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \ - rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \ - exit 1; \ - fi; \ - echo "Server running on port $$PORT"; \ - echo ""; \ - sed "s|ws://[^\"]*|ws://$(COMPLIANCE_TARGET):$$PORT/ws|" \ - $(AUTOBAHN_SPEC) > $(AUTOBAHN_REPORTS)/fuzzingclient.json; \ - docker run --rm $(DOCKER_NET_ARGS) --user $$(id -u):$$(id -g) \ - -v $(CURDIR)/$(AUTOBAHN_REPORTS):/reports $(AUTOBAHN_IMAGE) \ - wstest --mode fuzzingclient --spec /reports/fuzzingclient.json || true - @-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null - @-pkill -f 'nhttp_ws_compliance:start' 2>/dev/null - @rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE) - @./$(AUTOBAHN_GATE) $(AUTOBAHN_REPORTS) - @echo "" - @echo "=== WebSocket Compliance Tests Complete ===" + @$(COMPLIANCE_RUN) nhttp_ws_compliance \ + 'sed "s|ws://[^\"]*|ws://$(COMPLIANCE_TARGET):$$PORT/ws|" \ + $(AUTOBAHN_SPEC) > $(AUTOBAHN_REPORTS)/fuzzingclient.json \ + && docker run --rm $(DOCKER_NET_ARGS) $(DOCKER_USER) \ + -v $(CURDIR)/$(AUTOBAHN_REPORTS):/reports $(AUTOBAHN_IMAGE) \ + wstest --mode fuzzingclient --spec /reports/fuzzingclient.json; \ + $(AUTOBAHN_GATE) $(AUTOBAHN_REPORTS)' autobahn-image: @command -v docker >/dev/null 2>&1 || { \ @@ -291,8 +208,9 @@ help: @echo " make h3-compliance - Run HTTP/3 tests (Docker: $(H3SPEC_IMAGE))" @echo " make ws-compliance - Run WebSocket RFC 6455 tests (Docker: $(AUTOBAHN_IMAGE))" @echo "" - @echo " ws-compliance gates on the report it writes to $(AUTOBAHN_REPORTS)/;" - @echo " read the detail in $(AUTOBAHN_REPORTS)/index.html." + @echo " Each target exits with the status of its runner. Reports land in" + @echo " $(COMPLIANCE_REPORTS)/: h2spec.xml (JUnit) and autobahn/index.html." + @echo " make -k compliance runs every suite before it reports a failure." @echo "" @echo " Requirement: Docker daemon running. On macOS/Windows, Docker Desktop." @echo "" diff --git a/rebar.config b/rebar.config index 8bc62d9..71ede63 100644 --- a/rebar.config +++ b/rebar.config @@ -9,7 +9,7 @@ {deps, [ {opentelemetry_api, "1.4.0"}, {opentelemetry_api_experimental, "0.5.1"}, - {nhttp_lib, "1.3.0"} + {nhttp_lib, "1.3.1"} ]}. {project_plugins, [ diff --git a/rebar.lock b/rebar.lock index 636d797..cd59c77 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,5 +1,5 @@ {"1.2.0", -[{<<"nhttp_lib">>,{pkg,<<"nhttp_lib">>,<<"1.3.0">>},0}, +[{<<"nhttp_lib">>,{pkg,<<"nhttp_lib">>,<<"1.3.1">>},0}, {<<"nquic">>,{pkg,<<"nquic">>,<<"1.1.0">>},1}, {<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.4.0">>},0}, {<<"opentelemetry_api_experimental">>, @@ -7,12 +7,12 @@ 0}]}. [ {pkg_hash,[ - {<<"nhttp_lib">>, <<"C64A01263679D7FC02F0314E1B55DE4B834E80A244B2301BCF21A4FB815BD955">>}, + {<<"nhttp_lib">>, <<"BB7C74F3BBE827167788CE58996F622731D8374EE1BF0BF80B6A8E1B689F0B5A">>}, {<<"nquic">>, <<"26516E2D3FAC1A54EEDB8179ADCB30001F42D23602FD31C73E7F58D19E74559E">>}, {<<"opentelemetry_api">>, <<"63CA1742F92F00059298F478048DFB826F4B20D49534493D6919A0DB39B6DB04">>}, {<<"opentelemetry_api_experimental">>, <<"1B5AFACFCBD0834390336C845BC8AE08C8CF0D69BBED72EE53D178798B93E074">>}]}, {pkg_hash_ext,[ - {<<"nhttp_lib">>, <<"89EF54F897112E865D819E38162894E3E90F2C3A9FBE0A897F34CE53D5E141DE">>}, + {<<"nhttp_lib">>, <<"19C0419958A1F49F017C82ABEF1284D59C2CFC4C48107EEF4BD817E4DEE9F1AC">>}, {<<"nquic">>, <<"D03E339D97251BF0E0BDA97826D9FE116337B9A5A56A242E86ED9C0DCA3D1F5B">>}, {<<"opentelemetry_api">>, <<"3DFBBFAA2C2ED3121C5C483162836C4F9027DEF469C41578AF5EF32589FCFC58">>}, {<<"opentelemetry_api_experimental">>, <<"10297057EADA47267D4F832011BECEF07D25690E6BF91FEBCCFC4E740DBA1A6F">>}]} diff --git a/test/nhttp_compliance.erl b/test/nhttp_compliance.erl index 3e41b4f..770704d 100644 --- a/test/nhttp_compliance.erl +++ b/test/nhttp_compliance.erl @@ -20,7 +20,8 @@ -export([ init/1, - handle_request/2 + handle_request/2, + handle_request_body/3 ]). -define(DEFAULT_CERT_DIR, "test/conf"). @@ -71,6 +72,22 @@ start(Opts) -> init(_Args) -> {ok, #{}}. --spec handle_request(nhttp_lib:request(), #{}) -> {reply, nhttp_lib:response(), #{}}. +-doc """ +Consume a streaming request body before the reply. An early reply makes the +connection reset the stream with NO_ERROR (RFC 9113 Section 8.1), and the +codec then discards the frames that h2spec sends to provoke a stream error. +""". +-spec handle_request(nhttp_lib:request(), #{}) -> nhttp_handler:request_result(#{}). +handle_request(#{body := streaming}, State) -> + {accept_body, unread, State}; handle_request(_Req, State) -> {reply, nhttp_resp:ok(<<"hello">>), State}. + +-spec handle_request_body(nhttp_handler:body_event(), unread, #{}) -> + nhttp_handler:request_result(#{}). +handle_request_body({data, _Chunk}, unread, State) -> + {accept_body, unread, State}; +handle_request_body({fin, _Trailers}, unread, State) -> + {reply, nhttp_resp:ok(<<"hello">>), State}; +handle_request_body({abort, Reason}, unread, State) -> + {abort, Reason, State}. From 390f1e70a577f13b32065342c45fb87c085657e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Rubinos=20Rodr=C3=ADguez?= Date: Fri, 16 Oct 2026 10:42:37 +0200 Subject: [PATCH 2/2] chore: added missing file to run compliance --- priv/compliance/run.sh | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 priv/compliance/run.sh diff --git a/priv/compliance/run.sh b/priv/compliance/run.sh new file mode 100755 index 0000000..99617ba --- /dev/null +++ b/priv/compliance/run.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# +# Runs one compliance suite against a server started for it. +# +# Starts :start() in a rebar3 test shell, waits for the +# PORT: line it prints, runs with PORT in its environment, +# stops the server and its process group, and exits with the status of +# . That status is the receipt: a suite that fails here fails +# the make target and the CI job. +# +# Usage: run.sh +# +# is one string, run with `bash -c`, so it reads $PORT at run +# time. Example: +# run.sh nhttp_compliance 'docker run --rm --network=host h2spec -p $PORT' + +set -uo pipefail + +module=${1:?usage: run.sh } +command=${2:?usage: run.sh } +rebar3=${REBAR3:-rebar3} +start_timeout_ticks=300 + +log=$(mktemp) +server_pid="" + +cleanup() { + if [ -n "$server_pid" ]; then + pkill -TERM -P "$server_pid" 2>/dev/null + kill -TERM "$server_pid" 2>/dev/null + fi + pkill -TERM -f "${module}:start\(\)" 2>/dev/null + rm -f "$log" +} +trap cleanup EXIT + +( tail -f /dev/null | "$rebar3" as test shell --eval "${module}:start()." ) > "$log" 2>&1 & +server_pid=$! + +port="" +for _ in $(seq 1 "$start_timeout_ticks"); do + port=$(grep -oE 'PORT:[0-9]+' "$log" | head -1 | cut -d: -f2) + if [ -n "$port" ]; then + break + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + break + fi + sleep 0.1 +done + +if [ -z "$port" ]; then + echo "compliance: ${module} did not report a port. Server output:" >&2 + cat "$log" >&2 + exit 1 +fi + +echo "compliance: ${module} listening on port ${port}" +PORT=$port bash -c "$command"