test(h2): read the request body before the h2spec handler replies

.github/workflows/ci.yml

@@ -52,3 +52,37 @@ jobs:
         if: always()
         with:
           paths: '_build/test/logs/**/report.xml'
+
+  compliance:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: erlef/setup-beam@v1.24.0
+        with:
+          otp-version: '28.0'
+          rebar3-version: '3.25.0'
+
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/rebar3
+            _build
+          key: ${{ runner.os }}-${{ runner.arch }}-28.0-3.25.0-${{ hashFiles('rebar.config', 'rebar.lock') }}
+
+      - run: make certs
+
+      - run: make -k compliance
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: compliance-reports
+          path: compliance-reports/
+
+      - uses: test-summary/action@v2
+        if: always()
+        with:
+          paths: 'compliance-reports/h2spec.xml'

.gitignore

@@ -51,5 +51,5 @@ priv/h3spec
 test/conf/*.pem
 test/conf/*.key
 
-# Autobahn WebSocket compliance reports
-autobahn-reports/
+# Compliance reports (h2spec JUnit, Autobahn)
+compliance-reports/

Makefile

@@ -21,7 +21,12 @@ AUTOBAHN_IMAGE := nhttp/autobahn:$(AUTOBAHN_VERSION)
 AUTOBAHN_DOCKERFILE := priv/Dockerfile.autobahn
 AUTOBAHN_SPEC := priv/autobahn/fuzzingclient.json
 AUTOBAHN_GATE := priv/autobahn/gate.sh
-AUTOBAHN_REPORTS := autobahn-reports
+# Each suite starts its server through this script and exits with the
+# status of the runner, so a failed suite fails the target.
+COMPLIANCE_RUN := REBAR3=$(REBAR3) priv/compliance/run.sh
+COMPLIANCE_REPORTS := compliance-reports
+AUTOBAHN_REPORTS := $(COMPLIANCE_REPORTS)/autobahn
+DOCKER_USER := --user $(shell id -u):$(shell id -g)
 # h3spec only ships an x86_64 Linux binary; force amd64 so arm64 hosts
 # (e.g. Apple Silicon, arm64 Linux with qemu-user) can still run it.
 DOCKER_PLATFORM := linux/amd64
@@ -44,10 +49,6 @@ TEST_CERTS_DIR := test/conf
 TEST_CERTS_MARKER := $(TEST_CERTS_DIR)/server.pem
 TEST_CERTS_SCRIPT := $(TEST_CERTS_DIR)/gen_test_certs.sh
 
-# Temp files for server coordination
-COMPLIANCE_PORT_FILE := .compliance_port
-COMPLIANCE_PID_FILE := .compliance_pid
-
 #==============================================================================
 # Core targets
 #==============================================================================
@@ -59,8 +60,7 @@ compile:
 
 clean:
 	$(REBAR3) clean
-	@rm -f .compliance_port .compliance_pid
-	@rm -rf $(AUTOBAHN_REPORTS)
+	@rm -rf $(COMPLIANCE_REPORTS)
 
 check: certs
 	$(REBAR3) check
@@ -103,30 +103,8 @@ binopt:
 h1-compliance: h1spec-image
 	@echo "=== HTTP/1.1 Compliance Tests with h1spec (Docker) ==="
 	@$(REBAR3) as test compile
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@echo "Starting HTTP/1.1 server (plaintext)..."
-	@( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_h1_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE)
-	@for _ in $$(seq 1 100); do \
-		grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \
-		sleep 0.1; \
-	done
-	@PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \
-	if [ -z "$$PORT" ]; then \
-		echo "Error: Could not parse port from server output:"; \
-		cat $(COMPLIANCE_PORT_FILE); \
-		kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \
-		rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \
-		exit 1; \
-	fi; \
-	echo "Server running on port $$PORT"; \
-	echo ""; \
-	docker run --rm $(DOCKER_NET_ARGS) $(H1SPEC_IMAGE) \
-		$(COMPLIANCE_TARGET) $$PORT || true
-	@-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null
-	@-pkill -f 'nhttp_h1_compliance:start' 2>/dev/null
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@echo ""
-	@echo "=== HTTP/1.1 Compliance Tests Complete ==="
+	@$(COMPLIANCE_RUN) nhttp_h1_compliance \
+		'docker run --rm $(DOCKER_NET_ARGS) $(H1SPEC_IMAGE) $(COMPLIANCE_TARGET) $$PORT'
 
 h1spec-image:
 	@command -v docker >/dev/null 2>&1 || { \
@@ -141,30 +119,11 @@ h1spec-image:
 h2-compliance: certs h2spec-image
 	@echo "=== HTTP/2 RFC 9113 Compliance Tests with h2spec (Docker) ==="
 	@$(REBAR3) as test compile
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@echo "Starting HTTP/2 server (TLS)..."
-	@( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE)
-	@for _ in $$(seq 1 100); do \
-		grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \
-		sleep 0.1; \
-	done
-	@PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \
-	if [ -z "$$PORT" ]; then \
-		echo "Error: Could not parse port from server output:"; \
-		cat $(COMPLIANCE_PORT_FILE); \
-		kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \
-		rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \
-		exit 1; \
-	fi; \
-	echo "Server running on port $$PORT"; \
-	echo ""; \
-	docker run --rm $(DOCKER_NET_ARGS) $(H2SPEC_IMAGE) \
-		-t -k -h $(COMPLIANCE_TARGET) -p $$PORT -o 5 || true
-	@-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null
-	@-pkill -f 'nhttp_compliance:start' 2>/dev/null
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@echo ""
-	@echo "=== Compliance Tests Complete ==="
+	@mkdir -p $(COMPLIANCE_REPORTS)
+	@$(COMPLIANCE_RUN) nhttp_compliance \
+		'docker run --rm $(DOCKER_NET_ARGS) $(DOCKER_USER) \
+			-v $(CURDIR)/$(COMPLIANCE_REPORTS):/reports $(H2SPEC_IMAGE) \
+			-t -k -h $(COMPLIANCE_TARGET) -p $$PORT -o 5 -j /reports/h2spec.xml'
 
 h2spec-image:
 	@command -v docker >/dev/null 2>&1 || { \
@@ -179,30 +138,9 @@ h2spec-image:
 h3-compliance: certs h3spec-image
 	@echo "=== HTTP/3 Compliance Tests with h3spec (Docker) ==="
 	@$(REBAR3) as test compile
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@echo "Starting HTTP/3 server (QUIC)..."
-	@( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_h3_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE)
-	@for _ in $$(seq 1 100); do \
-		grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \
-		sleep 0.1; \
-	done
-	@PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \
-	if [ -z "$$PORT" ]; then \
-		echo "Error: Could not parse port from server output:"; \
-		cat $(COMPLIANCE_PORT_FILE); \
-		kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \
-		rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \
-		exit 1; \
-	fi; \
-	echo "Server running on port $$PORT"; \
-	echo ""; \
-	docker run --rm --platform $(DOCKER_PLATFORM) $(DOCKER_NET_ARGS) $(H3SPEC_IMAGE) \
-		$(COMPLIANCE_TARGET) $$PORT -n -t 3000 || true
-	@-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null
-	@-pkill -f 'nhttp_h3_compliance:start' 2>/dev/null
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@echo ""
-	@echo "=== HTTP/3 Compliance Tests Complete ==="
+	@$(COMPLIANCE_RUN) nhttp_h3_compliance \
+		'docker run --rm --platform $(DOCKER_PLATFORM) $(DOCKER_NET_ARGS) $(H3SPEC_IMAGE) \
+			$(COMPLIANCE_TARGET) $$PORT -n -t 3000'
 
 h3spec-image:
 	@command -v docker >/dev/null 2>&1 || { \
@@ -220,34 +158,13 @@ ws-compliance: autobahn-image
 	@$(REBAR3) as test compile
 	@rm -rf $(AUTOBAHN_REPORTS)
 	@mkdir -p $(AUTOBAHN_REPORTS)
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@echo "Starting WebSocket server (plaintext)..."
-	@( tail -f /dev/null | $(REBAR3) as test shell --eval 'nhttp_ws_compliance:start().' ) > $(COMPLIANCE_PORT_FILE) 2>&1 & echo $$! > $(COMPLIANCE_PID_FILE)
-	@for _ in $$(seq 1 100); do \
-		grep -qE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) 2>/dev/null && break; \
-		sleep 0.1; \
-	done
-	@PORT=$$(grep -oE 'PORT:[0-9]+' $(COMPLIANCE_PORT_FILE) | head -1 | cut -d: -f2); \
-	if [ -z "$$PORT" ]; then \
-		echo "Error: Could not parse port from server output:"; \
-		cat $(COMPLIANCE_PORT_FILE); \
-		kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null; \
-		rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE); \
-		exit 1; \
-	fi; \
-	echo "Server running on port $$PORT"; \
-	echo ""; \
-	sed "s|ws://[^\"]*|ws://$(COMPLIANCE_TARGET):$$PORT/ws|" \
-		$(AUTOBAHN_SPEC) > $(AUTOBAHN_REPORTS)/fuzzingclient.json; \
-	docker run --rm $(DOCKER_NET_ARGS) --user $$(id -u):$$(id -g) \
-		-v $(CURDIR)/$(AUTOBAHN_REPORTS):/reports $(AUTOBAHN_IMAGE) \
-		wstest --mode fuzzingclient --spec /reports/fuzzingclient.json || true
-	@-kill $$(cat $(COMPLIANCE_PID_FILE)) 2>/dev/null
-	@-pkill -f 'nhttp_ws_compliance:start' 2>/dev/null
-	@rm -f $(COMPLIANCE_PORT_FILE) $(COMPLIANCE_PID_FILE)
-	@./$(AUTOBAHN_GATE) $(AUTOBAHN_REPORTS)
-	@echo ""
-	@echo "=== WebSocket Compliance Tests Complete ==="
+	@$(COMPLIANCE_RUN) nhttp_ws_compliance \
+		'sed "s|ws://[^\"]*|ws://$(COMPLIANCE_TARGET):$$PORT/ws|" \
+			$(AUTOBAHN_SPEC) > $(AUTOBAHN_REPORTS)/fuzzingclient.json \
+		&& docker run --rm $(DOCKER_NET_ARGS) $(DOCKER_USER) \
+			-v $(CURDIR)/$(AUTOBAHN_REPORTS):/reports $(AUTOBAHN_IMAGE) \
+			wstest --mode fuzzingclient --spec /reports/fuzzingclient.json; \
+		$(AUTOBAHN_GATE) $(AUTOBAHN_REPORTS)'
 
 autobahn-image:
 	@command -v docker >/dev/null 2>&1 || { \
@@ -291,8 +208,9 @@ help:
 	@echo "    make h3-compliance   - Run HTTP/3 tests (Docker: $(H3SPEC_IMAGE))"
 	@echo "    make ws-compliance   - Run WebSocket RFC 6455 tests (Docker: $(AUTOBAHN_IMAGE))"
 	@echo ""
-	@echo "  ws-compliance gates on the report it writes to $(AUTOBAHN_REPORTS)/;"
-	@echo "  read the detail in $(AUTOBAHN_REPORTS)/index.html."
+	@echo "  Each target exits with the status of its runner. Reports land in"
+	@echo "  $(COMPLIANCE_REPORTS)/: h2spec.xml (JUnit) and autobahn/index.html."
+	@echo "  make -k compliance runs every suite before it reports a failure."
 	@echo ""
 	@echo "  Requirement: Docker daemon running. On macOS/Windows, Docker Desktop."
 	@echo ""

priv/compliance/run.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# Runs one compliance suite against a server started for it.
+#
+# Starts <server-module>:start() in a rebar3 test shell, waits for the
+# PORT:<n> line it prints, runs <command> with PORT in its environment,
+# stops the server and its process group, and exits with the status of
+# <command>. That status is the receipt: a suite that fails here fails
+# the make target and the CI job.
+#
+# Usage: run.sh <server-module> <command>
+#
+# <command> is one string, run with `bash -c`, so it reads $PORT at run
+# time. Example:
+#   run.sh nhttp_compliance 'docker run --rm --network=host h2spec -p $PORT'
+
+set -uo pipefail
+
+module=${1:?usage: run.sh <server-module> <command>}
+command=${2:?usage: run.sh <server-module> <command>}
+rebar3=${REBAR3:-rebar3}
+start_timeout_ticks=300
+
+log=$(mktemp)
+server_pid=""
+
+cleanup() {
+    if [ -n "$server_pid" ]; then
+        pkill -TERM -P "$server_pid" 2>/dev/null
+        kill -TERM "$server_pid" 2>/dev/null
+    fi
+    pkill -TERM -f "${module}:start\(\)" 2>/dev/null
+    rm -f "$log"
+}
+trap cleanup EXIT
+
+( tail -f /dev/null | "$rebar3" as test shell --eval "${module}:start()." ) > "$log" 2>&1 &
+server_pid=$!
+
+port=""
+for _ in $(seq 1 "$start_timeout_ticks"); do
+    port=$(grep -oE 'PORT:[0-9]+' "$log" | head -1 | cut -d: -f2)
+    if [ -n "$port" ]; then
+        break
+    fi
+    if ! kill -0 "$server_pid" 2>/dev/null; then
+        break
+    fi
+    sleep 0.1
+done
+
+if [ -z "$port" ]; then
+    echo "compliance: ${module} did not report a port. Server output:" >&2
+    cat "$log" >&2
+    exit 1
+fi
+
+echo "compliance: ${module} listening on port ${port}"
+PORT=$port bash -c "$command"

rebar.config

@@ -9,7 +9,7 @@
 {deps, [
     {opentelemetry_api, "1.4.0"},
     {opentelemetry_api_experimental, "0.5.1"},
-    {nhttp_lib, "1.3.0"}
+    {nhttp_lib, "1.3.1"}
 ]}.
 
 {project_plugins, [

rebar.lock

@@ -1,18 +1,18 @@
 {"1.2.0",
-[{<<"nhttp_lib">>,{pkg,<<"nhttp_lib">>,<<"1.3.0">>},0},
+[{<<"nhttp_lib">>,{pkg,<<"nhttp_lib">>,<<"1.3.1">>},0},
  {<<"nquic">>,{pkg,<<"nquic">>,<<"1.1.0">>},1},
  {<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.4.0">>},0},
  {<<"opentelemetry_api_experimental">>,
   {pkg,<<"opentelemetry_api_experimental">>,<<"0.5.1">>},
   0}]}.
 [
 {pkg_hash,[
- {<<"nhttp_lib">>, <<"C64A01263679D7FC02F0314E1B55DE4B834E80A244B2301BCF21A4FB815BD955">>},
+ {<<"nhttp_lib">>, <<"BB7C74F3BBE827167788CE58996F622731D8374EE1BF0BF80B6A8E1B689F0B5A">>},
  {<<"nquic">>, <<"26516E2D3FAC1A54EEDB8179ADCB30001F42D23602FD31C73E7F58D19E74559E">>},
  {<<"opentelemetry_api">>, <<"63CA1742F92F00059298F478048DFB826F4B20D49534493D6919A0DB39B6DB04">>},
  {<<"opentelemetry_api_experimental">>, <<"1B5AFACFCBD0834390336C845BC8AE08C8CF0D69BBED72EE53D178798B93E074">>}]},
 {pkg_hash_ext,[
- {<<"nhttp_lib">>, <<"89EF54F897112E865D819E38162894E3E90F2C3A9FBE0A897F34CE53D5E141DE">>},
+ {<<"nhttp_lib">>, <<"19C0419958A1F49F017C82ABEF1284D59C2CFC4C48107EEF4BD817E4DEE9F1AC">>},
  {<<"nquic">>, <<"D03E339D97251BF0E0BDA97826D9FE116337B9A5A56A242E86ED9C0DCA3D1F5B">>},
  {<<"opentelemetry_api">>, <<"3DFBBFAA2C2ED3121C5C483162836C4F9027DEF469C41578AF5EF32589FCFC58">>},
  {<<"opentelemetry_api_experimental">>, <<"10297057EADA47267D4F832011BECEF07D25690E6BF91FEBCCFC4E740DBA1A6F">>}]}

test/nhttp_compliance.erl

@@ -20,7 +20,8 @@
 
 -export([
     init/1,
-    handle_request/2
+    handle_request/2,
+    handle_request_body/3
 ]).
 
 -define(DEFAULT_CERT_DIR, "test/conf").
@@ -71,6 +72,22 @@ start(Opts) ->
 init(_Args) ->
     {ok, #{}}.
 
--spec handle_request(nhttp_lib:request(), #{}) -> {reply, nhttp_lib:response(), #{}}.
+-doc """
+Consume a streaming request body before the reply. An early reply makes the
+connection reset the stream with NO_ERROR (RFC 9113 Section 8.1), and the
+codec then discards the frames that h2spec sends to provoke a stream error.
+""".
+-spec handle_request(nhttp_lib:request(), #{}) -> nhttp_handler:request_result(#{}).
+handle_request(#{body := streaming}, State) ->
+    {accept_body, unread, State};
 handle_request(_Req, State) ->
     {reply, nhttp_resp:ok(<<"hello">>), State}.
+
+-spec handle_request_body(nhttp_handler:body_event(), unread, #{}) ->
+    nhttp_handler:request_result(#{}).
+handle_request_body({data, _Chunk}, unread, State) ->
+    {accept_body, unread, State};
+handle_request_body({fin, _Trailers}, unread, State) ->
+    {reply, nhttp_resp:ok(<<"hello">>), State};
+handle_request_body({abort, Reason}, unread, State) ->
+    {abort, Reason, State}.