From edc55163edee0e06a74956917da52e6a8255f504 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Wed, 16 Sep 2026 23:04:56 +0800 Subject: [PATCH 01/17] chore(deps): add ex_ssl dependency --- apps/http_core/mix.exs | 1 + mix.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/http_core/mix.exs b/apps/http_core/mix.exs index 485a432..87a459e 100644 --- a/apps/http_core/mix.exs +++ b/apps/http_core/mix.exs @@ -37,6 +37,7 @@ defmodule HttpCore.MixProject do defp deps do [ + {:ex_ssl, "~> 0.3"}, {:quic, "~> 1.6", runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} ] diff --git a/mix.lock b/mix.lock index b5fb157..2684afd 100644 --- a/mix.lock +++ b/mix.lock @@ -6,6 +6,7 @@ "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, + "ex_ssl": {:hex, :ex_ssl, "0.3.0", "2ecfd0dc4ab1e86418f469aa360560eecc8c94538717e9c022130f6cc121e86c", [:mix], [], "hexpm", "629ba6272dad27593784383aa7fda9e1c60a28a366e33aa235918fec994adabd"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, From f5c0fe60393bc8f0570b3f92e04fe45a0aa88ceb Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Sun, 20 Sep 2026 13:06:59 +0800 Subject: [PATCH 02/17] feat(tls): add selectable ssl backends Keep OTP :ssl as the default while supporting :ex_ssl for fetch, WebSocket, and EventSource TLS connections. --- .dialyzer_ignore.exs | 4 +- README.md | 55 +++- apps/http_core/lib/http/tls_backend.ex | 19 ++ apps/http_core/lib/http/transport.ex | 6 +- apps/http_core/lib/http/transport/ex_ssl.ex | 87 +++++ apps/http_core/lib/http/transport/ssl.ex | 4 + apps/http_core/lib/http/transport/tcp.ex | 6 + apps/http_core/lib/http/transport/unix.ex | 6 + apps/http_core/test/http/tls_backend_test.exs | 42 +++ .../test/http/tls_transport_test.exs | 274 ++++++++++++++++ .../lib/http/event_source.ex | 4 + .../lib/http/event_source/connection.ex | 10 +- .../lib/http/event_source/options.ex | 7 + .../test/http/event_source/options_test.exs | 35 +- .../test/http/event_source_test.exs | 105 +++++- .../test/support/event_source_server.ex | 147 +++++---- .../test/support/fixtures/localhost-ca.pem | 19 ++ .../test/support/fixtures/localhost.key | 28 ++ .../test/support/fixtures/localhost.pem | 19 ++ apps/http_fetch/lib/http.ex | 3 + apps/http_fetch/lib/http/fetch_options.ex | 36 ++- apps/http_fetch/lib/http/socket_client.ex | 40 ++- .../test/http/fetch_options_test.exs | 35 ++ .../test/http/socket_client_http2_test.exs | 129 ++++++++ .../test/http/ssl_transport_test.exs | 305 +++++++++++++++++- apps/http_web_socket/lib/http/web_socket.ex | 4 + .../lib/http/web_socket/connection.ex | 31 +- .../lib/http/web_socket/options.ex | 23 +- .../test/http/web_socket/options_test.exs | 37 ++- .../test/http/web_socket_test.exs | 30 ++ .../http/web_socket_tls_lifecycle_test.exs | 74 +++++ .../test/support/fixtures/localhost-ca.pem | 19 ++ .../test/support/fixtures/localhost.key | 28 ++ .../test/support/fixtures/localhost.pem | 19 ++ .../test/support/web_socket_server.ex | 192 +++++------ .../lib/http/web_transport.ex | 4 + .../lib/http/web_transport/options.ex | 10 +- .../test/http/web_transport/options_test.exs | 15 + .../http/web_transport/tls_config_test.exs | 24 ++ 39 files changed, 1731 insertions(+), 204 deletions(-) create mode 100644 apps/http_core/lib/http/tls_backend.ex create mode 100644 apps/http_core/lib/http/transport/ex_ssl.ex create mode 100644 apps/http_core/test/http/tls_backend_test.exs create mode 100644 apps/http_core/test/http/tls_transport_test.exs create mode 100644 apps/http_event_source/test/support/fixtures/localhost-ca.pem create mode 100644 apps/http_event_source/test/support/fixtures/localhost.key create mode 100644 apps/http_event_source/test/support/fixtures/localhost.pem create mode 100644 apps/http_web_socket/test/http/web_socket_tls_lifecycle_test.exs create mode 100644 apps/http_web_socket/test/support/fixtures/localhost-ca.pem create mode 100644 apps/http_web_socket/test/support/fixtures/localhost.key create mode 100644 apps/http_web_socket/test/support/fixtures/localhost.pem create mode 100644 apps/http_web_transport/test/http/web_transport/tls_config_test.exs diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index 4af2ded..25760b9 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -1,7 +1,7 @@ [ # Dialyzer loses the Task.Supervisor.async_nolink/4 return shape in HTTP.fetch/2 - ~r/(apps\/http_fetch\/)?lib\/http\.ex:267.*invalid_contract/, - ~r/(apps\/http_fetch\/)?lib\/http\.ex:268.*no_return/, + ~r/(apps\/http_fetch\/)?lib\/http\.ex:270.*invalid_contract/, + ~r/(apps\/http_fetch\/)?lib\/http\.ex:271.*no_return/, # HTTP.Promise.then/3 opaque type issue with Task struct - Task.Supervisor returns opaque Task ~r/(apps\/http_fetch\/)?lib\/http\/promise\.ex:96.*contract_with_opaque/ diff --git a/README.md b/README.md index 961a97e..af46a80 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ A modern HTTP client library for Elixir that provides a fetch API similar to web - **Browser-like API**: Familiar fetch interface with promises and async/await patterns - **Full HTTP support**: GET, POST, PUT, DELETE, PATCH, HEAD methods -- **Internal HTTP/1.1 transport**: Uses `:gen_tcp` for HTTP, `:ssl` for HTTPS, and Unix domain sockets +- **Internal HTTP/1.1 transport**: Uses `:gen_tcp` for HTTP, selectable TLS for HTTPS, and Unix domain sockets - **Unix Domain Sockets**: HTTP over Unix sockets for Docker daemon, systemd, and other local services - **Form data support**: HTTP.FormData for multipart/form-data and file uploads - **Streaming request bodies**: Fetch-style `duplex: "half"` uploads over HTTP/1.1 @@ -21,7 +21,7 @@ A modern HTTP client library for Elixir that provides a fetch API similar to web - **Promise-based**: Async operations with chaining support - **Request cancellation**: AbortController support for cancelling requests - **Automatic JSON parsing**: Built-in JSON response handling -- **Zero dependencies**: Uses only Erlang/OTP built-in modules +- **Selectable TLS**: OTP `:ssl` by default, with opt-in `:ex_ssl` for verified TLS 1.3 ## Browser Fetch API Compatibility @@ -119,6 +119,55 @@ response = IO.puts("Docker Version: #{docker_info["Version"]}") ``` +## TLS Backend Selection + +HTTPS fetch (HTTP/1.1 and HTTP/2), secure WebSocket, and HTTPS EventSource share +one TLS default. OTP `:ssl` remains the default when no configuration is set: + +```elixir +# config/config.exs or config/runtime.exs +config :http_core, tls_backend: :ex_ssl +``` + +A flat per-call option overrides that default: + +```elixir +HTTP.fetch("https://example.com", tls_backend: :ssl) + +HTTP.fetch("https://example.com", + tls_backend: :ex_ssl, + ssl: [cacertfile: "/path/to/ca.pem"] +) + +HTTP.WebSocket.new("wss://example.com/socket", [], tls_backend: :ex_ssl) +HTTP.EventSource.new("https://example.com/events", tls_backend: :ex_ssl) +``` + +`tls_backend` accepts `:ssl`, `:ex_ssl`, `"ssl"`, or `"ex_ssl"`. Maps also accept +`"tls_backend"` and `"tlsBackend"` keys. Omitted or `nil` values inherit the shared +configuration. The backend is captured when the request/client is created and +retained through redirects and EventSource reconnects; runtime configuration +changes affect new operations. Invalid selections fail explicitly. + +`ssl: [...]` supplies TLS settings to the selected backend. The `ex_ssl` 0.3.0 +backend uses the `SSL` module and requires peer verification and TLS 1.3. It uses +system CA certificates unless `cacerts` or `cacertfile` is supplied. DNS names +and IP addresses are verified against the peer certificate. TLS 1.2, +`verify: :verify_none`, client certificates, and arbitrary TCP socket options +are unsupported and return errors; connections never fall back to another +backend automatically. + +For `:ex_ssl`, `socket_opts` accepts `send_timeout` and +`send_timeout_close: true`. These override matching entries in `ssl`. Custom +ClientHello profiles can be passed through `ssl: [ex_ssl: [profile: profile]]`; +any ALPN list added by HTTP/2 selection must match the profile's ALPN list exactly. +See the [ex_ssl compatibility contract](https://github.com/gsmlg-dev/ex_ssl/blob/v0.3.0/docs/COMPATIBILITY.md). + +Plain HTTP, WS, and Unix sockets retain their existing transports. HTTP/3 and +WebTransport use QUIC's separate TLS implementation and ignore the shared +setting. An explicit non-`nil` `tls_backend` on either QUIC API returns +`{:error, :tls_backend_not_supported_for_quic}` (through the promise for fetch). + ## Form Data With File Upload ```elixir @@ -371,7 +420,7 @@ request = %HTTP.Request{ ``` **Transport Options:** -- `transport_options`: Socket transport options such as `timeout`, `connect_timeout`, `ssl`, +- `transport_options`: Socket transport options such as `timeout`, `connect_timeout`, `tls_backend`, `ssl`, `socket_opts`, and `redirect` `redirect` defaults to `:follow` with the socket transport. Pass `redirect: :manual` diff --git a/apps/http_core/lib/http/tls_backend.ex b/apps/http_core/lib/http/tls_backend.ex new file mode 100644 index 0000000..6ae8607 --- /dev/null +++ b/apps/http_core/lib/http/tls_backend.ex @@ -0,0 +1,19 @@ +defmodule HTTP.TLSBackend do + @moduledoc false + + @type t :: :ssl | :ex_ssl + + @spec resolve(term()) :: {:ok, t()} | {:error, :invalid_tls_backend} + def resolve(value \\ nil) + + def resolve(nil), do: normalize(Application.get_env(:http_core, :tls_backend, :ssl)) + def resolve(value), do: normalize(value) + + @spec transport(t()) :: HTTP.Transport.SSL | HTTP.Transport.ExSSL + def transport(:ssl), do: HTTP.Transport.SSL + def transport(:ex_ssl), do: HTTP.Transport.ExSSL + + defp normalize(value) when value in [:ssl, "ssl"], do: {:ok, :ssl} + defp normalize(value) when value in [:ex_ssl, "ex_ssl"], do: {:ok, :ex_ssl} + defp normalize(_value), do: {:error, :invalid_tls_backend} +end diff --git a/apps/http_core/lib/http/transport.ex b/apps/http_core/lib/http/transport.ex index 66db7b2..1084ca9 100644 --- a/apps/http_core/lib/http/transport.ex +++ b/apps/http_core/lib/http/transport.ex @@ -1,14 +1,16 @@ defmodule HTTP.Transport do @moduledoc false - @type socket :: port() | :ssl.sslsocket() + @type socket :: port() | :ssl.sslsocket() | SSL.Socket.t() @type message :: {:data, binary()} | :closed | {:error, term()} | :unknown @callback connect(String.t(), non_neg_integer(), keyword(), timeout()) :: {:ok, socket()} | {:error, term()} @callback controlling_process(socket(), pid()) :: :ok | {:error, term()} @callback send(socket(), iodata()) :: :ok | {:error, term()} + @callback recv(socket(), non_neg_integer(), timeout()) :: {:ok, binary()} | {:error, term()} + @callback negotiated_protocol(socket()) :: {:ok, binary() | nil} | {:error, term()} @callback setopts(socket(), keyword()) :: :ok | {:error, term()} - @callback close(socket()) :: :ok + @callback close(socket()) :: :ok | {:error, term()} @callback normalize_message(term(), socket()) :: message() end diff --git a/apps/http_core/lib/http/transport/ex_ssl.ex b/apps/http_core/lib/http/transport/ex_ssl.ex new file mode 100644 index 0000000..46e8cc4 --- /dev/null +++ b/apps/http_core/lib/http/transport/ex_ssl.ex @@ -0,0 +1,87 @@ +defmodule HTTP.Transport.ExSSL do + @moduledoc false + + @behaviour HTTP.Transport + + @impl true + def connect(host, port, opts, timeout) do + with {:ok, ssl_opts} <- tls_options(opts) do + SSL.connect(connect_host(host), port, ssl_opts, timeout) + end + end + + @impl true + def controlling_process(socket, pid), do: SSL.controlling_process(socket, pid) + + @impl true + def send(socket, iodata), do: SSL.send(socket, iodata) + + @impl true + def recv(socket, length, timeout), do: SSL.recv(socket, length, timeout) + + @impl true + def setopts(socket, opts), do: SSL.setopts(socket, opts) + + @impl true + def close(socket), do: SSL.close(socket) + + @impl true + def negotiated_protocol(socket) do + case SSL.negotiated_protocol(socket) do + {:error, :protocol_not_negotiated} -> {:ok, nil} + result -> result + end + end + + @impl true + def normalize_message({:ssl, socket, data}, socket), do: {:data, data} + def normalize_message({:ssl_closed, socket}, socket), do: :closed + def normalize_message({:ssl_error, socket, reason}, socket), do: {:error, reason} + def normalize_message(_, _), do: :unknown + + defp tls_options(opts) do + ssl_opts = Keyword.get(opts, :ssl, []) + socket_opts = Keyword.get(opts, :socket_opts, []) + + with :ok <- validate_keyword(ssl_opts), + :ok <- validate_keyword(socket_opts), + :ok <- validate_socket_options(socket_opts) do + # SSL supplies system trust only when neither caller CA option is present, + # and infers DNS SNI or IP identity from the connection host. + defaults = [ + mode: :binary, + packet: :raw, + active: false, + verify: :verify_peer, + versions: [:"tlsv1.3"], + depth: 4, + customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)] + ] + + {:ok, defaults |> Keyword.merge(ssl_opts) |> Keyword.merge(socket_opts)} + end + end + + defp validate_keyword(opts) do + if Keyword.keyword?(opts) and + length(Keyword.keys(opts)) == length(Enum.uniq(Keyword.keys(opts))) do + :ok + else + {:error, {:options, :invalid_options}} + end + end + + defp validate_socket_options(opts) do + case Enum.find(opts, fn {key, _} -> key not in [:send_timeout, :send_timeout_close] end) do + nil -> :ok + {key, _} -> {:error, {:options, {key, :unsupported_or_invalid}}} + end + end + + defp connect_host(host) do + case :inet.parse_address(String.to_charlist(host)) do + {:ok, address} -> address + {:error, :einval} -> host + end + end +end diff --git a/apps/http_core/lib/http/transport/ssl.ex b/apps/http_core/lib/http/transport/ssl.ex index f11130e..872a3ba 100644 --- a/apps/http_core/lib/http/transport/ssl.ex +++ b/apps/http_core/lib/http/transport/ssl.ex @@ -23,12 +23,16 @@ defmodule HTTP.Transport.SSL do @impl true def send(socket, iodata), do: :ssl.send(socket, iodata) + @impl true + def recv(socket, length, timeout), do: :ssl.recv(socket, length, timeout) + @impl true def setopts(socket, opts), do: :ssl.setopts(socket, opts) @impl true def close(socket), do: :ssl.close(socket) + @impl true @spec negotiated_protocol(:ssl.sslsocket()) :: {:ok, binary() | nil} | {:error, :closed} def negotiated_protocol(socket) do case :ssl.negotiated_protocol(socket) do diff --git a/apps/http_core/lib/http/transport/tcp.ex b/apps/http_core/lib/http/transport/tcp.ex index b6eea78..54ac28b 100644 --- a/apps/http_core/lib/http/transport/tcp.ex +++ b/apps/http_core/lib/http/transport/tcp.ex @@ -21,6 +21,12 @@ defmodule HTTP.Transport.TCP do @impl true def send(socket, iodata), do: :gen_tcp.send(socket, iodata) + @impl true + def recv(socket, length, timeout), do: :gen_tcp.recv(socket, length, timeout) + + @impl true + def negotiated_protocol(_socket), do: {:ok, nil} + @impl true def setopts(socket, opts), do: :inet.setopts(socket, opts) diff --git a/apps/http_core/lib/http/transport/unix.ex b/apps/http_core/lib/http/transport/unix.ex index 358130e..d9178a9 100644 --- a/apps/http_core/lib/http/transport/unix.ex +++ b/apps/http_core/lib/http/transport/unix.ex @@ -21,6 +21,12 @@ defmodule HTTP.Transport.Unix do @impl true def send(socket, iodata), do: :gen_tcp.send(socket, iodata) + @impl true + def recv(socket, length, timeout), do: :gen_tcp.recv(socket, length, timeout) + + @impl true + def negotiated_protocol(_socket), do: {:ok, nil} + @impl true def setopts(socket, opts), do: :inet.setopts(socket, opts) diff --git a/apps/http_core/test/http/tls_backend_test.exs b/apps/http_core/test/http/tls_backend_test.exs new file mode 100644 index 0000000..2404859 --- /dev/null +++ b/apps/http_core/test/http/tls_backend_test.exs @@ -0,0 +1,42 @@ +defmodule HTTP.TLSBackendTest do + use ExUnit.Case, async: false + + alias HTTP.TLSBackend + + setup do + previous = Application.fetch_env(:http_core, :tls_backend) + Application.delete_env(:http_core, :tls_backend) + + on_exit(fn -> + case previous do + {:ok, value} -> Application.put_env(:http_core, :tls_backend, value) + :error -> Application.delete_env(:http_core, :tls_backend) + end + end) + end + + test "defaults to OTP and resolves explicit atoms and strings" do + assert {:ok, :ssl} = TLSBackend.resolve() + assert {:ok, :ssl} = TLSBackend.resolve("ssl") + assert {:ok, :ex_ssl} = TLSBackend.resolve(:ex_ssl) + assert {:ok, :ex_ssl} = TLSBackend.resolve("ex_ssl") + end + + test "nil inherits the configured backend and explicit options override it" do + Application.put_env(:http_core, :tls_backend, :ex_ssl) + assert {:ok, :ex_ssl} = TLSBackend.resolve(nil) + assert {:ok, :ssl} = TLSBackend.resolve(:ssl) + Application.put_env(:http_core, :tls_backend, "ssl") + assert {:ok, :ssl} = TLSBackend.resolve() + end + + test "invalid configuration or selection fails instead of falling back" do + Application.put_env(:http_core, :tls_backend, :unknown) + assert {:error, :invalid_tls_backend} = TLSBackend.resolve() + assert {:ok, :ex_ssl} = TLSBackend.resolve(:ex_ssl) + + for value <- [:unknown, "SSL", false, 1, %{}] do + assert {:error, :invalid_tls_backend} = TLSBackend.resolve(value) + end + end +end diff --git a/apps/http_core/test/http/tls_transport_test.exs b/apps/http_core/test/http/tls_transport_test.exs new file mode 100644 index 0000000..754e22f --- /dev/null +++ b/apps/http_core/test/http/tls_transport_test.exs @@ -0,0 +1,274 @@ +defmodule HTTP.TLSTransportTest do + use ExUnit.Case, async: true + + alias HTTP.Transport.ExSSL + + @fixtures Path.expand("../../../http_fetch/test/support/fixtures", __DIR__) + @ca Path.join(@fixtures, "localhost-ca.pem") + @cert Path.join(@fixtures, "localhost.pem") + @key Path.join(@fixtures, "localhost.key") + + for transport <- [HTTP.Transport.SSL, ExSSL] do + @transport transport + + describe "#{inspect(transport)}" do + test "verified passive traffic and absent ALPN" do + port = + peer(fn socket -> + assert {:ok, "ping"} = :ssl.recv(socket, 4, 5_000) + :ok = :ssl.send(socket, "pong") + end) + + assert {:ok, socket} = + @transport.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + + assert {:ok, nil} = @transport.negotiated_protocol(socket) + assert :ok = @transport.send(socket, ["pi", "ng"]) + assert {:ok, "pong"} = @transport.recv(socket, 4, 5_000) + assert {:error, :closed} = @transport.recv(socket, 0, 5_000) + assert :ok = @transport.close(socket) + end + + test "worker ownership transfer, ALPN and active-once delivery survive worker exit" do + port = + peer( + fn socket -> + assert {:ok, "ping"} = :ssl.recv(socket, 4, 5_000) + :ok = :ssl.send(socket, "pong") + end, + alpn_preferred_protocols: ["h2"] + ) + + parent = self() + + {worker, monitor} = + spawn_monitor(fn -> + {:ok, socket} = + @transport.connect( + "localhost", + port, + [ssl: [cacertfile: @ca, alpn_advertised_protocols: ["h2"]]], + 5_000 + ) + + :ok = @transport.controlling_process(socket, parent) + send(parent, {:connected, socket}) + end) + + assert_receive {:connected, socket}, 5_000 + assert_receive {:DOWN, ^monitor, :process, ^worker, :normal}, 5_000 + assert {:ok, "h2"} = @transport.negotiated_protocol(socket) + assert :ok = @transport.setopts(socket, active: :once) + assert :ok = @transport.send(socket, "ping") + assert_receive {:ssl, ^socket, "pong"} = message, 5_000 + assert {:data, "pong"} = @transport.normalize_message(message, socket) + assert :ok = @transport.setopts(socket, active: :once) + assert_receive {:ssl_closed, ^socket} = message, 5_000 + assert :closed = @transport.normalize_message(message, socket) + assert :unknown = @transport.normalize_message({:ssl, :other_socket, "pong"}, socket) + assert :ok = @transport.close(socket) + end + + test "rejects an untrusted peer" do + port = peer(fn _socket -> :ok end) + assert {:error, _} = @transport.connect("localhost", port, [], 5_000) + end + + test "rejects a mismatched reference hostname" do + port = peer(fn _socket -> :ok end) + + assert {:error, _} = + @transport.connect( + "localhost", + port, + [ssl: [cacertfile: @ca, server_name_indication: ~c"wrong.example"]], + 5_000 + ) + end + end + end + + test "ex_ssl verifies IP literals without treating the IP as DNS SNI" do + port = peer(fn socket -> :ssl.send(socket, "ip") end) + assert {:ok, socket} = ExSSL.connect("127.0.0.1", port, [ssl: [cacertfile: @ca]], 5_000) + assert {:ok, "ip"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl rejects unsupported TLS and TCP options before connecting" do + for ssl <- [ + [verify: :verify_none], + [versions: [:"tlsv1.2"]], + [versions: [:"tlsv1.3", :"tlsv1.2"]], + [certfile: @cert] + ] do + assert {:error, {:options, _}} = ExSSL.connect("localhost", 0, [ssl: ssl], 100) + end + + assert {:error, {:options, {:nodelay, :unsupported_or_invalid}}} = + ExSSL.connect("localhost", 0, [socket_opts: [nodelay: true]], 100) + + assert {:error, {:options, {:send_timeout_close, :unsupported_or_invalid}}} = + ExSSL.connect("localhost", 0, [socket_opts: [send_timeout_close: false]], 100) + end + + test "ex_ssl rejects malformed and duplicate options" do + for opts <- [[ssl: nil], [socket_opts: [:binary]], [ssl: [depth: 1, depth: 2]]] do + assert {:error, {:options, :invalid_options}} = ExSSL.connect("localhost", 0, opts, 100) + end + end + + test "ex_ssl socket send options take precedence over matching TLS options" do + port = peer(fn socket -> :ssl.send(socket, "ok") end) + + assert {:ok, socket} = + ExSSL.connect( + "localhost", + port, + [ + ssl: [cacertfile: @ca, send_timeout_close: false], + socket_opts: [send_timeout: 1_000, send_timeout_close: true] + ], + 5_000 + ) + + assert {:ok, "ok"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl rejects TLS 1.2-only peers without backend fallback" do + port = peer(fn _socket -> :ok end, versions: [:"tlsv1.2"]) + assert {:error, _} = ExSSL.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + end + + test "ex_ssl keeps partial passive bytes across receive timeouts" do + parent = self() + + port = + peer(fn socket -> + :ok = :ssl.send(socket, "a") + send(parent, {:partial_sent, self()}) + + receive do + :finish -> :ssl.send(socket, "b") + after + 5_000 -> flunk("test did not finish the response") + end + end) + + assert {:ok, socket} = ExSSL.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + assert_receive {:partial_sent, server}, 5_000 + assert {:error, :timeout} = ExSSL.recv(socket, 2, 20) + send(server, :finish) + assert {:ok, "ab"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl closes a passive connection when its application owner exits" do + parent = self() + + port = + peer(fn socket -> + send(parent, {:peer_closed, :ssl.recv(socket, 0, 5_000)}) + end) + + {owner, monitor} = + spawn_monitor(fn -> + {:ok, socket} = ExSSL.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + send(parent, {:owned, socket}) + + receive do + :exit -> :ok + end + end) + + assert_receive {:owned, socket}, 5_000 + assert {:error, :not_owner} = ExSSL.controlling_process(socket, self()) + send(owner, :exit) + assert_receive {:DOWN, ^monitor, :process, ^owner, :normal}, 5_000 + assert_receive {:peer_closed, {:error, _}}, 5_000 + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl propagates custom-profile ALPN conflicts without connecting" do + profile = %SSL.ClientHello.WireProfile{extensions: [{:alpn, ["http/1.1"]}]} + + assert {:error, {:options, {:alpn_advertised_protocols, :profile_conflict}}} = + ExSSL.connect( + "localhost", + 0, + [ + ssl: [ + cacertfile: @ca, + alpn_advertised_protocols: ["h2"], + ex_ssl: [profile: profile] + ] + ], + 100 + ) + end + + test "ex_ssl accepts the default custom profile with HTTP ALPN" do + port = peer(fn socket -> :ssl.send(socket, "ok") end, alpn_preferred_protocols: ["http/1.1"]) + + assert {:ok, socket} = + ExSSL.connect( + "localhost", + port, + [ + ssl: [ + cacertfile: @ca, + alpn_advertised_protocols: ["http/1.1"], + ex_ssl: [profile: :default] + ] + ], + 5_000 + ) + + assert {:ok, "http/1.1"} = ExSSL.negotiated_protocol(socket) + assert {:ok, "ok"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + defp peer(handler, options \\ []) do + {:ok, listener} = + :ssl.listen( + 0, + [ + :binary, + packet: :raw, + active: false, + ip: {127, 0, 0, 1}, + certfile: @cert, + keyfile: @key + ] ++ Keyword.put_new(options, :versions, [:"tlsv1.3"]) + ) + + {:ok, {{127, 0, 0, 1}, port}} = :ssl.sockname(listener) + + pid = + spawn_link(fn -> + case :ssl.transport_accept(listener, 5_000) do + {:ok, tcp} -> + case :ssl.handshake(tcp, 5_000) do + {:ok, socket} -> + handler.(socket) + :ssl.close(socket) + + {:error, _} -> + :ok + end + + {:error, _} -> + :ok + end + end) + + on_exit(fn -> + :ssl.close(listener) + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + port + end +end diff --git a/apps/http_event_source/lib/http/event_source.ex b/apps/http_event_source/lib/http/event_source.ex index 8ba2237..26f009a 100644 --- a/apps/http_event_source/lib/http/event_source.ex +++ b/apps/http_event_source/lib/http/event_source.ex @@ -10,6 +10,10 @@ defmodule HTTP.EventSource do Custom server-sent event names are delivered through the message event's `type` field. + + For `https` connections, pass `tls_backend: :ssl | :ex_ssl` (or the equivalent + string in a map). When omitted, the shared `:http_core` TLS backend setting is + captured when the source is created and retained across reconnects. """ alias HTTP.EventSource.Connection diff --git a/apps/http_event_source/lib/http/event_source/connection.ex b/apps/http_event_source/lib/http/event_source/connection.ex index 81e20c3..3d43c2b 100644 --- a/apps/http_event_source/lib/http/event_source/connection.ex +++ b/apps/http_event_source/lib/http/event_source/connection.ex @@ -25,6 +25,7 @@ defmodule HTTP.EventSource.Connection do idle_timeout: :infinity, ssl: [], socket_opts: [], + tls_backend: :ssl, unix_socket: nil, max_line_size: 64 * 1024, transport: nil, @@ -74,6 +75,7 @@ defmodule HTTP.EventSource.Connection do idle_timeout: options.idle_timeout, ssl: options.ssl, socket_opts: options.socket_opts, + tls_backend: options.tls_backend, unix_socket: options.unix_socket, max_line_size: options.max_line_size, last_event_id: options.last_event_id, @@ -177,8 +179,11 @@ defmodule HTTP.EventSource.Connection do {:ok, HTTP.Transport.TCP, host, port || 80} end - defp select_transport(%{uri: %URI{scheme: "https", host: host, port: port}}) do - {:ok, HTTP.Transport.SSL, host, port || 443} + defp select_transport(%{ + uri: %URI{scheme: "https", host: host, port: port}, + tls_backend: backend + }) do + {:ok, HTTP.TLSBackend.transport(backend), host, port || 443} end defp select_transport(%{uri: %URI{scheme: scheme}}), do: {:error, {:unsupported_scheme, scheme}} @@ -350,6 +355,7 @@ defmodule HTTP.EventSource.Connection do defp rearm(%{transport: transport, socket: socket} = state) do case transport.setopts(socket, active: :once) do :ok -> {:noreply, state} + {:error, :closed} -> handle_transport_closed(state) {:error, reason} -> {:noreply, reconnect(state, reason)} end end diff --git a/apps/http_event_source/lib/http/event_source/options.ex b/apps/http_event_source/lib/http/event_source/options.ex index 57dc4f8..a5d1b83 100644 --- a/apps/http_event_source/lib/http/event_source/options.ex +++ b/apps/http_event_source/lib/http/event_source/options.ex @@ -24,6 +24,8 @@ defmodule HTTP.EventSource.Options do "socket_opts" => :socket_opts, "socketOpts" => :socket_opts, "ssl" => :ssl, + "tls_backend" => :tls_backend, + "tlsBackend" => :tls_backend, "unix_socket" => :unix_socket, "unixSocket" => :unix_socket, "with_credentials" => :with_credentials, @@ -42,6 +44,7 @@ defmodule HTTP.EventSource.Options do idle_timeout: :infinity, ssl: [], socket_opts: [], + tls_backend: :ssl, unix_socket: nil, max_line_size: @default_max_line_size, ref: nil @@ -59,6 +62,7 @@ defmodule HTTP.EventSource.Options do idle_timeout: timeout(), ssl: keyword(), socket_opts: keyword(), + tls_backend: HTTP.TLSBackend.t(), unix_socket: String.t() | nil, max_line_size: pos_integer(), ref: reference() @@ -82,6 +86,7 @@ defmodule HTTP.EventSource.Options do idle_timeout: Keyword.get(init, :idle_timeout, :infinity), ssl: Keyword.get(init, :ssl, []), socket_opts: Keyword.get(init, :socket_opts, []), + tls_backend: Keyword.fetch!(init, :tls_backend), unix_socket: Keyword.get(init, :unix_socket), max_line_size: Keyword.get(init, :max_line_size, @default_max_line_size), ref: Keyword.get(init, :ref, make_ref()) @@ -144,6 +149,7 @@ defmodule HTTP.EventSource.Options do {:ok, ssl} <- normalize_keyword(Keyword.get(init, :ssl, []), :invalid_ssl_options), {:ok, socket_opts} <- normalize_keyword(Keyword.get(init, :socket_opts, []), :invalid_socket_options), + {:ok, tls_backend} <- HTTP.TLSBackend.resolve(Keyword.get(init, :tls_backend)), {:ok, unix_socket} <- normalize_unix_socket(Keyword.get(init, :unix_socket)), {:ok, max_line_size} <- normalize_pos_integer( @@ -162,6 +168,7 @@ defmodule HTTP.EventSource.Options do |> Keyword.put(:idle_timeout, idle_timeout) |> Keyword.put(:ssl, ssl) |> Keyword.put(:socket_opts, socket_opts) + |> Keyword.put(:tls_backend, tls_backend) |> Keyword.put(:unix_socket, unix_socket) |> Keyword.put(:max_line_size, max_line_size)} end diff --git a/apps/http_event_source/test/http/event_source/options_test.exs b/apps/http_event_source/test/http/event_source/options_test.exs index 1ffa97c..83fdf8d 100644 --- a/apps/http_event_source/test/http/event_source/options_test.exs +++ b/apps/http_event_source/test/http/event_source/options_test.exs @@ -1,5 +1,5 @@ defmodule HTTP.EventSource.OptionsTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias HTTP.EventSource.Options @@ -55,6 +55,33 @@ defmodule HTTP.EventSource.OptionsTest do }) end + test "selects TLS backends from atom and string map options" do + assert {:ok, %{tls_backend: :ssl}} = + Options.new("https://example.com/events", tls_backend: :ssl) + + assert {:ok, %{tls_backend: :ex_ssl}} = + Options.new("https://example.com/events", %{"tlsBackend" => "ex_ssl"}) + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("https://example.com/events", %{"tls_backend" => "ssl"}) + end + + test "pins the configured TLS backend when options are constructed" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + + Application.put_env(:http_core, :tls_backend, :ex_ssl) + assert {:ok, options} = Options.new("https://example.com/events") + assert options.tls_backend == :ex_ssl + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("https://example.com/events", tls_backend: :ssl) + + Application.put_env(:http_core, :tls_backend, :ssl) + assert options.tls_backend == :ex_ssl + assert {:ok, %{tls_backend: :ssl}} = Options.new("https://example.com/events") + end + test "rejects invalid init options" do assert {:error, :invalid_owner} = Options.new("http://example.com/events", owner: :bad) @@ -63,5 +90,11 @@ defmodule HTTP.EventSource.OptionsTest do assert {:error, :invalid_reconnect_time} = Options.new("http://example.com/events", reconnect_time: -1) + + assert {:error, :invalid_tls_backend} = + Options.new("https://example.com/events", tls_backend: :unknown) end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(value), do: Application.put_env(:http_core, :tls_backend, value) end diff --git a/apps/http_event_source/test/http/event_source_test.exs b/apps/http_event_source/test/http/event_source_test.exs index e3bfb92..ec544a2 100644 --- a/apps/http_event_source/test/http/event_source_test.exs +++ b/apps/http_event_source/test/http/event_source_test.exs @@ -1,11 +1,15 @@ defmodule HTTP.EventSourceTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias HTTP.EventSource alias HTTP.EventSource.Event.Error alias HTTP.EventSource.Event.Message alias HTTP.EventSource.Event.Open + @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) + @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) + @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) + test "defines browser ready state constants" do assert EventSource.connecting() == 0 assert EventSource.open() == 1 @@ -104,7 +108,106 @@ defmodule HTTP.EventSourceTest do assert :ok = EventSource.close(source) end + test "delivers events over verified TLS 1.3 for each backend" do + for backend <- [:ssl, :ex_ssl] do + {:ok, _server, port} = + HTTPEventSource.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + body: "data: secure\n\n", + close: false + ) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + tls_backend: backend, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {EventSource, ^source, %Open{}}, 1_000 + assert_receive {EventSource, ^source, %Message{data: "secure"}}, 1_000 + assert :ok = EventSource.close(source) + assert_receive :event_source_server_closed, 1_000 + end + end + + test "retains the configured TLS backend across reconnects" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + Application.put_env(:http_core, :tls_backend, :ex_ssl) + + {:ok, server, port} = + HTTPEventSource.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + responses: [ + [body: "data: first\n\n", wait_for: :close_first_response], + [body: "data: second\n\n", close: false] + ] + ) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + ssl: [cacertfile: @cacertfile], + reconnect_time: 10 + ) + + assert_receive {:event_source_server_request, _first_request}, 1_000 + assert_receive {EventSource, ^source, %Message{data: "first"}}, 1_000 + Application.put_env(:http_core, :tls_backend, :invalid) + refute_receive {:event_source_server_request, _request}, 50 + send(server, :close_first_response) + assert_receive {EventSource, ^source, %Message{data: "second"}}, 1_000 + assert :ok = EventSource.close(source) + end + test "rejects invalid constructor input synchronously" do assert {:error, {:unsupported_scheme, "ftp"}} = EventSource.new("ftp://example.com/events") end + + test "finalizes events when TLS closes before a delayed consumer rearms" do + {:ok, server, port} = + HTTPEventSource.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + body: fn -> + receive do + :send_body -> "data: final\r\r" + after + 5_000 -> flunk("consumer did not release the server") + end + end + ) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile], + reconnect_time: 60_000 + ) + + assert_receive {EventSource, ^source, %Open{}}, 5_000 + %{socket: %{pid: tls_pid}} = :sys.get_state(source.pid) + monitor = Process.monitor(tls_pid) + :ok = :sys.suspend(source.pid) + + # Let TLS deliver data and its terminal notification while the consumer is + # paused. Rearming after it parses the data now necessarily returns :closed. + try do + send(server, :send_body) + assert_receive {:DOWN, ^monitor, :process, ^tls_pid, _}, 5_000 + after + :sys.resume(source.pid) + end + + assert_receive {EventSource, ^source, %Message{data: "final"}}, 5_000 + assert_receive {EventSource, ^source, %Error{reason: :eof}}, 5_000 + assert :ok = EventSource.close(source) + end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(value), do: Application.put_env(:http_core, :tls_backend, value) end diff --git a/apps/http_event_source/test/support/event_source_server.ex b/apps/http_event_source/test/support/event_source_server.ex index c82954c..7fda39f 100644 --- a/apps/http_event_source/test/support/event_source_server.ex +++ b/apps/http_event_source/test/support/event_source_server.ex @@ -4,66 +4,78 @@ defmodule HTTPEventSource.TestServer do def start_link(opts \\ []) do parent = self() responses = Keyword.get(opts, :responses, [opts]) + tls? = Keyword.get(opts, :tls, false) + transport = socket_module(tls?) + {:ok, listen_socket} = listen(tls?, opts) + {:ok, {{127, 0, 0, 1}, port}} = sockname(tls?, listen_socket) - {:ok, listen_socket} = - :gen_tcp.listen(0, [ - :binary, - packet: :raw, - active: false, - reuseaddr: true, - ip: {127, 0, 0, 1} - ]) - - {:ok, {{127, 0, 0, 1}, port}} = :inet.sockname(listen_socket) - - pid = - spawn_link(fn -> - accept_loop(listen_socket, parent, responses) - end) - + pid = spawn_link(fn -> accept_loop(tls?, transport, listen_socket, parent, responses) end) {:ok, pid, port} end - defp accept_loop(listen_socket, _parent, []) do - :gen_tcp.close(listen_socket) + defp listen(false, _opts) do + :gen_tcp.listen(0, [:binary, packet: :raw, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) end - defp accept_loop(listen_socket, parent, [response | rest]) do - case :gen_tcp.accept(listen_socket, 5_000) do + defp listen(true, opts) do + :ssl.listen(0, + mode: :binary, + packet: :raw, + active: false, + reuseaddr: true, + ip: {127, 0, 0, 1}, + versions: [:"tlsv1.3"], + certfile: Keyword.fetch!(opts, :certfile), + keyfile: Keyword.fetch!(opts, :keyfile) + ) + end + + defp accept_loop(_tls?, transport, listen_socket, _parent, []), + do: transport.close(listen_socket) + + defp accept_loop(tls?, transport, listen_socket, parent, [response | rest]) do + case accept(tls?, listen_socket) do {:ok, socket} -> - serve(socket, parent, response) - accept_loop(listen_socket, parent, rest) + serve(transport, socket, parent, response) + accept_loop(tls?, transport, listen_socket, parent, rest) {:error, reason} -> send(parent, {:event_source_server_error, reason}) - :gen_tcp.close(listen_socket) + transport.close(listen_socket) end end - defp serve(socket, parent, response) do - case recv_until(socket, "\r\n\r\n", <<>>) do + defp accept(false, listen_socket), do: :gen_tcp.accept(listen_socket, 5_000) + + defp accept(true, listen_socket) do + with {:ok, transport_socket} <- :ssl.transport_accept(listen_socket, 5_000), + do: :ssl.handshake(transport_socket) + end + + defp socket_module(false), do: :gen_tcp + defp socket_module(true), do: :ssl + defp sockname(false, listen_socket), do: :inet.sockname(listen_socket) + defp sockname(true, listen_socket), do: :ssl.sockname(listen_socket) + + defp serve(transport, socket, parent, response) do + case recv_until(transport, socket, "\r\n\r\n", <<>>) do {:ok, request} -> send(parent, {:event_source_server_request, request}) - :ok = :gen_tcp.send(socket, response_head(response)) - :ok = send_body(socket, Keyword.get(response, :body, "")) + :ok = transport.send(socket, response_head(response)) + :ok = send_body(transport, socket, Keyword.get(response, :body, "")) - if Keyword.get(response, :close, true) do - :gen_tcp.close(socket) - else - wait_for_close(socket, parent) - end + close_response(transport, socket, parent, response) {:error, reason} -> send(parent, {:event_source_server_error, reason}) - :gen_tcp.close(socket) + transport.close(socket) end end - defp recv_until(socket, marker, buffer) do + defp recv_until(transport, socket, marker, buffer) do if :binary.match(buffer, marker) == :nomatch do - with {:ok, data} <- :gen_tcp.recv(socket, 0, 1_000) do - recv_until(socket, marker, buffer <> data) - end + with {:ok, data} <- transport.recv(socket, 0, 1_000), + do: recv_until(transport, socket, marker, buffer <> data) else {:ok, buffer} end @@ -71,53 +83,56 @@ defmodule HTTPEventSource.TestServer do defp response_head(response) do status = Keyword.get(response, :status, 200) - reason = reason_phrase(status) headers = Keyword.get(response, :headers, []) content_type = Keyword.get(response, :content_type, "text/event-stream") - - content_type_header = - if content_type do - [{"Content-Type", content_type}] - else - [] - end + content_type_header = if content_type, do: [{"Content-Type", content_type}], else: [] [ "HTTP/1.1 ", Integer.to_string(status), " ", - reason, + reason_phrase(status), "\r\n", header_lines(content_type_header ++ headers), - "Connection: close\r\n", - "\r\n" + "Connection: close\r\n\r\n" ] end - defp header_lines(headers) do - Enum.map(headers, fn {name, value} -> [to_string(name), ": ", to_string(value), "\r\n"] end) - end + defp header_lines(headers), + do: + Enum.map(headers, fn {name, value} -> [to_string(name), ": ", to_string(value), "\r\n"] end) - defp send_body(_socket, ""), do: :ok + defp send_body(_transport, _socket, ""), do: :ok - defp send_body(socket, chunks) when is_list(chunks) do - Enum.each(chunks, fn chunk -> - :ok = :gen_tcp.send(socket, chunk) - end) - end + defp send_body(transport, socket, body) when is_function(body, 0), + do: send_body(transport, socket, body.()) - defp send_body(socket, body), do: :gen_tcp.send(socket, body) + defp send_body(transport, socket, chunks) when is_list(chunks), + do: Enum.each(chunks, &transport.send(socket, &1)) - defp wait_for_close(socket, parent) do - case :gen_tcp.recv(socket, 0, 2_000) do - {:error, :closed} -> - send(parent, :event_source_server_closed) + defp send_body(transport, socket, body), do: transport.send(socket, body) - {:error, reason} -> - send(parent, {:event_source_server_error, reason}) + defp close_response(transport, socket, parent, response) do + case Keyword.get(response, :wait_for) do + wait_for when is_atom(wait_for) and not is_nil(wait_for) -> + receive do + ^wait_for -> transport.close(socket) + end + + nil -> + if Keyword.get(response, :close, true) do + transport.close(socket) + else + wait_for_close(transport, socket, parent) + end + end + end - {:ok, _data} -> - wait_for_close(socket, parent) + defp wait_for_close(transport, socket, parent) do + case transport.recv(socket, 0, 2_000) do + {:error, :closed} -> send(parent, :event_source_server_closed) + {:error, reason} -> send(parent, {:event_source_server_error, reason}) + {:ok, _data} -> wait_for_close(transport, socket, parent) end end diff --git a/apps/http_event_source/test/support/fixtures/localhost-ca.pem b/apps/http_event_source/test/support/fixtures/localhost-ca.pem new file mode 100644 index 0000000..15227f1 --- /dev/null +++ b/apps/http_event_source/test/support/fixtures/localhost-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKzCCAhOgAwIBAgIUbWn02fvDN0zvFEGHFliwdXqASNIwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqaXND6GhQeV+tXwmYZYy +q+kFvpGWXCa49yG4/nJLfUYtpANqii88Y08J+6fAskHxoWYLidcnAvC5V51IcreX +g/N108fCvxTQ5jYfwIyj8oAEL4QZbnKklQrcLRFqN1Ef9f6cdNWL4pZZk7/ChbXP +ULHTNQF8FIoDUS4H5bjobU2lBER4H7Qa8oevkMGotgfjrzqWftTWt68dJiTsfoX1 +z+VH6t6t5vvT/L+0J7M9JAB++sckXSY9J+yIeG7Cu3sGtwTqIvuiiBH0latUYC9M +sJ0LS400sBonIZC4BdvVc9NCU8M1sx5yWV6eNRe7Cz3hrzQVcWMbAB73qXFMvpEO +OwIDAQABo2MwYTAdBgNVHQ4EFgQUUayz7IdQZHRIjmttS9hkKzG01M4wHwYDVR0j +BBgwFoAUUayz7IdQZHRIjmttS9hkKzG01M4wDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggEBAIrCD3yHZ+F/P/s0FXZRFFNB +AIqYjEwYr2OBN/ShHhceoqM55FbThtYPxUlVJH+uWkxpvnExDOauCapW7Cxlz9FF +3NvLjWf8fs80Uno5Dy6mik4rL6OsThYX8Ko2u6gkSiQm9yuT1iKvigGjU4q1qyft +ZJH8an7/egIIHxZya5oBuGKBnX241s6n9Osdtww/Eu5IOlZZeORJ0mAA7gTHaZyI +VquDo6Ml66LMyuIQkbtr19UHAEHrSfhmrujI5GruA/ejnma2O5sE1r1hVa91TV+L +TuaVxPvkdS+CIBvfwy90zmDbPhLt0kU5FiY2x6FlR1MrvVBn1uUQ59kYJ7yNI9k= +-----END CERTIFICATE----- diff --git a/apps/http_event_source/test/support/fixtures/localhost.key b/apps/http_event_source/test/support/fixtures/localhost.key new file mode 100644 index 0000000..d05ba0d --- /dev/null +++ b/apps/http_event_source/test/support/fixtures/localhost.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtsMy7LvUMU+H3 +27785rkCaSrkj5ljfUUPDA8+LM1ReN4ovqRaKjN8Tz88AOOwsZkXHLEJ5Uv8RbpS +8ruITF9moSXhVNFvVkBtf+SIBF2M5sDwsV/q8VFwUcvncDyLOsZwqsal9j+e9LjY +SlQtzzapBWfZm8O540RNpmpsa6k0M7jfp/m8P/l3RIlYzEuWPfCBd1ZmQMUN0T6H ++iYBBfNj6ioDUOUP5VfwA8cVXgblYbFKPF6uTtv95Ep4CFin0u6S0xVRlX0PERU5 +kyNLkmXUibClAY93IEkX5KDe0wb0WU4QBC9BAC8CegTFKd57MUlH0FtABCakLDCz +uHKFOc5bAgMBAAECggEAF95J5Bcp2Y3Vaagq8LoMBobJPCt3mrJmQREZLyQc+usv +LE42a6WM+bIyMF6261scfI6WzZNTA9HZLvjoSGymV4YhnHt5ZGFn7SQ8FWz0Jh63 +uNvHIJQU6uhyqtVDnKZxCp6NOdVDHOboV+Cu/LSTAFsb0E2VcgAHLNyHl2qcD+4p +qVDfyz/2BcAciQUTyQmq8tVPyJV226iZAkTJjah8H+MOXo6XqieNJ2WTz3H57PCE +tmm5YLeNl6S47O2369VRFmhYZZBS2tiBEfXNwwFCc6Q44esmJG1UtLUp6LMenNon +FJTod0QuZMjqfomzYzvJ9sCi8cvrABDVs8HQiw8RIQKBgQDY4ReBy63jdUJkDYQC +00SnrGywE7YNLYu+ZkHz+J14Xn/ylKFK0sz1WPfGZZQWtLWEXCYw3f2Z8d42Rd0Z +l/E7H2KiRq2oPrqc/mwfWtwjL0sqOuLswAzdL8AKLbl6vou1UjrJennuvaOh2C2s +sKJutX7CoM8GvXvWweNGY9xvuQKBgQDNBV8IkKuE4B8TIF7x49lkOFpWVgf6E1bS +My22JFIk8C0Jnxh8e/6mujeEJ2jMO/kpYPhWkDhKp4CTm+S6Ga6CSFzBjnA0dfaB +8Htyb4R5bJn7wD9+Zj+tHSG6gwVxJl/u2YfDAZL+zlKgVx7RoTGkF8mskHVmgrFT +DIJyFs0wswKBgFMSjR2DdfzNOnv2jV1DrWWIby1Wr4IGsyNgKd0YmsCzedDiS5HM +gwNra6UL3ZiA6ZJkdaB8N5qTAanKQvF9uMILuI0uA3CRbouaDLJJ7E5x3Bm16pwC +yCqlEqsTbptshzkR4UCxcCkZbKcelggytFUxofdM/1+2jsvpAnRA5fvJAoGAXS+/ +zkjTbQXhmfPws8l4mhDzHqLj5Uq8/7W7ZTqFC70O+3yQyKQjTuz9JtgyzgHEcoZc +2hubOnOAAZeuEthxdU4muuNfJLkpXk5MDeuaLwapxr/PHEilUK4ZEolTA+cJW6sM +BhrFEYP+ElsG6wl1YrxdMk5Gzl1A9BqPgAPVJ/kCgYEArpwxPGurJDQ3JszIWDJU +CX6KhwAncK9LALQT6EPB3Y54Ou4cq1XMSWlmqUUmN821J/mbR4f66F+nFy/q29Br +giwArg0Ngkt+uaKTfo6IeR7+b2vAjzZ+z3cEbNZpuyCAbQwytnXT8t7ZUonAQ7dO +TmHo1zxsuNSt2EYos+EP+VA= +-----END PRIVATE KEY----- diff --git a/apps/http_event_source/test/support/fixtures/localhost.pem b/apps/http_event_source/test/support/fixtures/localhost.pem new file mode 100644 index 0000000..178c710 --- /dev/null +++ b/apps/http_event_source/test/support/fixtures/localhost.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHTCCAgWgAwIBAgIUEJVRR6DfNeTUcvIJYQL9oVN8xyEwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbDMuy71DFPh99u+/Oa5Amkq5I+ZY31F +DwwPPizNUXjeKL6kWiozfE8/PADjsLGZFxyxCeVL/EW6UvK7iExfZqEl4VTRb1ZA +bX/kiARdjObA8LFf6vFRcFHL53A8izrGcKrGpfY/nvS42EpULc82qQVn2ZvDueNE +TaZqbGupNDO436f5vD/5d0SJWMxLlj3wgXdWZkDFDdE+h/omAQXzY+oqA1DlD+VX +8APHFV4G5WGxSjxerk7b/eRKeAhYp9LuktMVUZV9DxEVOZMjS5Jl1ImwpQGPdyBJ +F+Sg3tMG9FlOEAQvQQAvAnoExSneezFJR9BbQAQmpCwws7hyhTnOWwIDAQABo14w +XDAaBgNVHREEEzARhwR/AAABgglsb2NhbGhvc3QwHQYDVR0OBBYEFN1Qq2IrwIwg +nlCE50Q3jpSthaODMB8GA1UdIwQYMBaAFFGss+yHUGR0SI5rbUvYZCsxtNTOMA0G +CSqGSIb3DQEBCwUAA4IBAQBwO6NAaG2iwdO0iAaatt9cyP7ZvtJM0I0se1Dd1AtU +3iBlrEFNAWwph3ycD9Wr5HKswN7DhK+cOeklAeeBaxgB8egyHuJHKJLmoZQip+FQ +lCoMgPOBXH5IepiczMN4mUUCJn2ZNf0j/QkO0CjjefiYBYKunMQDNMtmhiB7LVIL +RAFo6kawJ49F3l/rTSKMMTGjlsW2wSM3pexevTvG/8dagT9ULQfLOuuHd+2paqKU +EoLAuaagqefwZaNASEvKP1Tl9I1GvZmjVTK8qJDmMJE+/D5HwscrDZmBvSDJdLLf +oZPdnbpZv0TXYvlpC5c1pHRPeA5LGa54GhM4Akc8ieDO +-----END CERTIFICATE----- diff --git a/apps/http_fetch/lib/http.ex b/apps/http_fetch/lib/http.ex index 95e5a4a..102e389 100644 --- a/apps/http_fetch/lib/http.ex +++ b/apps/http_fetch/lib/http.ex @@ -109,6 +109,9 @@ defmodule HTTP do - `:signal`: An `HTTP.AbortController` PID. If provided, the request can be aborted via this controller. - `:http_version`: `:http1`, `:http2`, `:http3`, `:h2c`, or `:auto`. + - `:tls_backend`: `:ssl` (default) or `:ex_ssl` for HTTPS requests. It may also be + configured globally with `config :http_core, :tls_backend, :ex_ssl`; an explicit + request value takes precedence. HTTP/3 does not support this option. - `:timeout`, `:connect_timeout`, `:ssl`, and `:socket_opts`: Elixir-specific transport extensions used by the socket or QUIC transport. - `:unix_socket`: Path to a Unix Domain Socket file (e.g., "/var/run/docker.sock"). diff --git a/apps/http_fetch/lib/http/fetch_options.ex b/apps/http_fetch/lib/http/fetch_options.ex index 9907abc..b6d67a7 100644 --- a/apps/http_fetch/lib/http/fetch_options.ex +++ b/apps/http_fetch/lib/http/fetch_options.ex @@ -19,7 +19,9 @@ defmodule HTTP.FetchOptions do - `connect_timeout` - connection timeout in milliseconds - `http_version` - protocol selection, one of `:http1`, `:http2`, `:http3`, `:h2c`, or `:auto`; defaults to `:http1` - - `ssl` - TLS options passed to `:ssl` + - `tls_backend` - TLS implementation, `:ssl` or `:ex_ssl`; defaults to the + shared `:http_core, :tls_backend` configuration + - `ssl` - TLS options passed to the selected TLS backend - `socket_opts` - socket options passed to the underlying transport - `unix_socket` - Unix Domain Socket path """ @@ -40,6 +42,8 @@ defmodule HTTP.FetchOptions do "socket_opts" => :socket_opts, "socketOpts" => :socket_opts, "ssl" => :ssl, + "tls_backend" => :tls_backend, + "tlsBackend" => :tls_backend, "timeout" => :timeout, "unix_socket" => :unix_socket, "unixSocket" => :unix_socket @@ -54,6 +58,7 @@ defmodule HTTP.FetchOptions do unix_socket: nil, redirect: :follow, http_version: :http1, + tls_backend: nil, timeout: nil, connect_timeout: nil, ssl: nil, @@ -61,6 +66,7 @@ defmodule HTTP.FetchOptions do @type redirect :: :follow | :manual | :error @type http_version :: :http1 | :http2 | :http3 | :h2c | :auto + @type tls_backend :: term() @type t :: %__MODULE__{ method: atom(), @@ -72,6 +78,7 @@ defmodule HTTP.FetchOptions do unix_socket: String.t() | nil, redirect: redirect(), http_version: http_version(), + tls_backend: tls_backend(), timeout: integer() | nil, connect_timeout: integer() | nil, ssl: list() | nil, @@ -109,6 +116,7 @@ defmodule HTTP.FetchOptions do |> maybe_add(:socket_opts, options.socket_opts) |> maybe_add(:redirect, options.redirect) |> maybe_add(:http_version, options.http_version) + |> maybe_add(:tls_backend, options.tls_backend) end @doc """ @@ -170,6 +178,9 @@ defmodule HTTP.FetchOptions do {:http_version, http_version}, acc -> %{acc | http_version: http_version} + {:tls_backend, tls_backend}, acc -> + %{acc | tls_backend: tls_backend} + {:timeout, timeout}, acc -> %{acc | timeout: timeout} @@ -196,12 +207,15 @@ defmodule HTTP.FetchOptions do defp normalize_headers(_), do: HTTP.Headers.new() defp normalize_options(%__MODULE__{} = options) do + http_version = normalize_http_version(options.http_version) + %{ options | method: normalize_method(options.method), redirect: normalize_redirect(options.redirect), duplex: normalize_duplex(options.duplex), - http_version: normalize_http_version(options.http_version) + http_version: http_version, + tls_backend: normalize_tls_backend(options.tls_backend, http_version) } end @@ -273,6 +287,24 @@ defmodule HTTP.FetchOptions do "unsupported http_version: #{inspect(http_version)}; expected :http1, :http2, :http3, :h2c, or :auto" end + defp normalize_tls_backend(nil, :http3), do: nil + defp normalize_tls_backend(tls_backend, :http3), do: tls_backend + defp normalize_tls_backend(tls_backend, _http_version), do: resolve_tls_backend(tls_backend) + + defp resolve_tls_backend(tls_backend) do + case HTTP.TLSBackend.resolve(tls_backend) do + {:ok, backend} -> + backend + + {:error, :invalid_tls_backend} -> + raise ArgumentError, tls_backend_error_message(tls_backend) + end + end + + defp tls_backend_error_message(tls_backend) do + "unsupported tls_backend: #{inspect(tls_backend)}; expected :ssl or :ex_ssl" + end + defp maybe_add(list, _key, nil), do: list defp maybe_add(list, key, value), do: Keyword.put(list, key, value) end diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index 0d28370..ba41ee9 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -10,6 +10,9 @@ defmodule HTTP.SocketClient do @spec request(Request.t(), pid() | nil, String.t() | nil) :: Response.t() | {:error, term()} def request(%Request{} = request, abort_controller_pid \\ nil, unix_socket_path \\ nil) do cond do + http_version(request) == :http3 and tls_backend(request) != nil -> + {:error, :tls_backend_not_supported_for_quic} + http_version(request) == :http3 and Request.streaming_body?(request) -> {:error, :streaming_request_body_unsupported_for_http3} @@ -17,7 +20,9 @@ defmodule HTTP.SocketClient do request_http3(request, abort_controller_pid, unix_socket_path) true -> - request_socket(request, abort_controller_pid, unix_socket_path) + with {:ok, request} <- pin_tls_backend(request) do + request_socket(request, abort_controller_pid, unix_socket_path) + end end end @@ -366,6 +371,7 @@ defmodule HTTP.SocketClient do defp rearm(state) do case state.transport.setopts(state.socket, active: :once) do :ok -> owner_loop(state) + {:error, :closed} -> handle_closed(state) {:error, reason} -> fail(state, reason) end end @@ -486,8 +492,9 @@ defmodule HTTP.SocketClient do defp connected_protocol(_transport, _socket, %{mode: :http1}), do: {:ok, :http1} defp connected_protocol(_transport, _socket, %{mode: :h2c}), do: {:ok, :http2} - defp connected_protocol(HTTP.Transport.SSL, socket, %{mode: :force_h2}) do - with {:ok, protocol} <- HTTP.Transport.SSL.negotiated_protocol(socket) do + defp connected_protocol(transport, socket, %{mode: :force_h2}) + when transport in [HTTP.Transport.SSL, HTTP.Transport.ExSSL] do + with {:ok, protocol} <- transport.negotiated_protocol(socket) do case normalize_alpn_protocol(protocol) do "h2" -> {:ok, :http2} other -> {:error, {:http2_not_negotiated, other}} @@ -495,8 +502,9 @@ defmodule HTTP.SocketClient do end end - defp connected_protocol(HTTP.Transport.SSL, socket, %{mode: :auto_https}) do - with {:ok, protocol} <- HTTP.Transport.SSL.negotiated_protocol(socket) do + defp connected_protocol(transport, socket, %{mode: :auto_https}) + when transport in [HTTP.Transport.SSL, HTTP.Transport.ExSSL] do + with {:ok, protocol} <- transport.negotiated_protocol(socket) do case normalize_alpn_protocol(protocol) do "h2" -> {:ok, :http2} _other -> {:ok, :http1} @@ -750,9 +758,12 @@ defmodule HTTP.SocketClient do {:ok, HTTP.Transport.TCP, host, uri.port || 80} end - defp select_transport(%Request{url: %URI{scheme: "https", host: host} = uri}, _socket_path) + defp select_transport( + %Request{url: %URI{scheme: "https", host: host} = uri} = request, + _socket_path + ) when is_binary(host) do - {:ok, HTTP.Transport.SSL, host, uri.port || 443} + {:ok, HTTP.TLSBackend.transport(tls_backend(request)), host, uri.port || 443} end defp select_transport(%Request{url: %URI{scheme: scheme}}, _socket_path) do @@ -782,7 +793,8 @@ defmodule HTTP.SocketClient do end end - defp protocol_selection(%Request{url: %URI{scheme: "https"}} = request, HTTP.Transport.SSL) do + defp protocol_selection(%Request{url: %URI{scheme: "https"}} = request, transport) + when transport in [HTTP.Transport.SSL, HTTP.Transport.ExSSL] do case http_version(request) do :http1 -> {:ok, %{mode: :http1, alpn_protocols: []}} @@ -802,6 +814,18 @@ defmodule HTTP.SocketClient do Keyword.get(request.transport_options, :http_version, :http1) end + defp tls_backend(%Request{} = request), do: Keyword.get(request.transport_options, :tls_backend) + + defp pin_tls_backend(%Request{} = request) do + with {:ok, backend} <- HTTP.TLSBackend.resolve(tls_backend(request)) do + {:ok, + %{ + request + | transport_options: Keyword.put(request.transport_options, :tls_backend, backend) + }} + end + end + defp request_timeout(%Request{} = request), do: Keyword.get(request.transport_options, :timeout, HTTP.Config.default_request_timeout()) diff --git a/apps/http_fetch/test/http/fetch_options_test.exs b/apps/http_fetch/test/http/fetch_options_test.exs index 7c55ae5..f4d1dd2 100644 --- a/apps/http_fetch/test/http/fetch_options_test.exs +++ b/apps/http_fetch/test/http/fetch_options_test.exs @@ -21,6 +21,7 @@ defmodule HTTP.FetchOptionsTest do "method" => "POST", "redirect" => "manual", "httpVersion" => "h2", + "tlsBackend" => "ex_ssl", "connectTimeout" => 2_000 }) @@ -28,6 +29,7 @@ defmodule HTTP.FetchOptionsTest do method: :post, redirect: :manual, http_version: :http2, + tls_backend: :ex_ssl, connect_timeout: 2_000 } = options @@ -91,6 +93,37 @@ defmodule HTTP.FetchOptionsTest do HTTP.FetchOptions.new(http_version: :spdy) end end + + test "normalizes TLS backend selection" do + assert %HTTP.FetchOptions{tls_backend: :ssl} = HTTP.FetchOptions.new([]) + assert %HTTP.FetchOptions{tls_backend: :ssl} = HTTP.FetchOptions.new(tls_backend: "ssl") + + assert %HTTP.FetchOptions{tls_backend: :ex_ssl} = + HTTP.FetchOptions.new(%{"tlsBackend" => "ex_ssl"}) + end + + test "rejects invalid TLS backend selection" do + assert_raise ArgumentError, ~r/unsupported tls_backend/, fn -> + HTTP.FetchOptions.new(tls_backend: :invalid) + end + + assert_raise ArgumentError, ~r/unsupported tls_backend/, fn -> + HTTP.FetchOptions.new(tls_backend: "SSL") + end + end + + test "does not resolve a nil TLS backend for HTTP/3" do + assert %HTTP.FetchOptions{http_version: :http3, tls_backend: nil} = + HTTP.FetchOptions.new(http_version: :http3) + end + + test "preserves explicit TLS backend markers for HTTP/3" do + assert %HTTP.FetchOptions{tls_backend: false} = + HTTP.FetchOptions.new(http_version: :http3, tls_backend: false) + + assert %HTTP.FetchOptions{tls_backend: :unknown} = + HTTP.FetchOptions.new(http_version: :http3, tls_backend: :unknown) + end end describe "to_transport_options/1" do @@ -101,6 +134,7 @@ defmodule HTTP.FetchOptionsTest do connect_timeout: 2_000, redirect: :manual, http_version: :http2, + tls_backend: :ex_ssl, ssl: [verify: :verify_none], socket_opts: [:inet6] ) @@ -110,6 +144,7 @@ defmodule HTTP.FetchOptionsTest do assert transport_options[:connect_timeout] == 2_000 assert transport_options[:redirect] == :manual assert transport_options[:http_version] == :http2 + assert transport_options[:tls_backend] == :ex_ssl assert transport_options[:ssl] == [verify: :verify_none] assert transport_options[:socket_opts] == [:inet6] end diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index a85a8ea..fd9680c 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -7,6 +7,7 @@ defmodule HTTP.SocketClientHTTP2Test do alias HTTP.HTTP2.HPACK @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) + @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) @ack 0x1 @@ -169,6 +170,123 @@ defmodule HTTP.SocketClientHTTP2Test do |> HTTP.Promise.await() end + test "auto over https negotiates h2 through ex_ssl" do + url = + start_https_h2_server!([<<"h2">>, <<"http/1.1">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response(socket, transport, "ex-ssl-h2") + assert_settings_ack(socket, transport, buffer) + end) + + response = + url + |> HTTP.fetch( + http_version: :auto, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "ex-ssl-h2" + end + + test "forced HTTPS HTTP/2 succeeds through ex_ssl" do + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response(socket, transport, "forced-ex-ssl-h2") + assert_settings_ack(socket, transport, buffer) + end) + + response = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert HTTP.Response.read_all(response) == "forced-ex-ssl-h2" + end + + test "auto HTTPS falls back to HTTP/1.1 through ex_ssl without ALPN" do + url = + start_https_h2_server!([], fn socket, _transport -> + assert {:ok, request} = recv_http1_headers(socket, <<>>) + assert request =~ "GET /test HTTP/1.1\r\n" + + :ok = + :ssl.send( + socket, + "HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nfallback" + ) + end) + + response = + url + |> HTTP.fetch( + http_version: :auto, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert HTTP.Response.read_all(response) == "fallback" + end + + test "streams large forced HTTPS HTTP/2 responses through ex_ssl" do + body = :binary.copy("h", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, body) + buffer = assert_settings_ack(socket, transport, buffer) + + chunks = chunk_binary(body, 16_384) + last_index = length(chunks) - 1 + + Enum.reduce(Enum.with_index(chunks), buffer, fn {chunk, index}, buffer -> + flags = if index == last_index, do: @end_stream, else: 0 + send_all(socket, transport, Frame.encode(:data, flags, 1, chunk)) + + buffer = assert_window_update(socket, transport, buffer, 0, byte_size(chunk)) + assert_window_update(socket, transport, buffer, 1, byte_size(chunk)) + end) + end) + + response = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert is_pid(response.stream) + assert HTTP.Response.read_all(response) == body + end + + test "forced HTTPS HTTP/2 through ex_ssl fails without h2 ALPN" do + url = + start_https_h2_server!([], fn socket, _transport -> + :timer.sleep(100) + :ssl.close(socket) + end) + + assert {:error, {:http2_not_negotiated, nil}} = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + end + defp start_h2c_server!(handler) do {:ok, listen_socket} = :gen_tcp.listen(0, [ @@ -263,6 +381,17 @@ defmodule HTTP.SocketClientHTTP2Test do {headers, buffer} end + defp recv_http1_headers(socket, acc) do + if String.contains?(acc, "\r\n\r\n") do + {:ok, acc} + else + case :ssl.recv(socket, 0, 5_000) do + {:ok, data} -> recv_http1_headers(socket, acc <> data) + {:error, reason} -> {:error, reason} + end + end + end + defp send_h2_response(socket, transport, body) do headers = response_headers(body) diff --git a/apps/http_fetch/test/http/ssl_transport_test.exs b/apps/http_fetch/test/http/ssl_transport_test.exs index 2e59fe7..ebc2a52 100644 --- a/apps/http_fetch/test/http/ssl_transport_test.exs +++ b/apps/http_fetch/test/http/ssl_transport_test.exs @@ -1,11 +1,29 @@ defmodule HTTP.SSLTransportTest do - use ExUnit.Case, async: true + use ExUnit.Case @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) describe "https transport" do + test "rejects explicit TLS backends for HTTP/3 through the promise" do + for tls_backend <- [false, :unknown, "ssl", "ex_ssl"] do + assert {:error, :tls_backend_not_supported_for_quic} = + "https://127.0.0.1:1/secure" + |> HTTP.fetch(http_version: :http3, tls_backend: tls_backend) + |> HTTP.Promise.await() + end + end + + test "does not resolve shared TLS configuration for HTTP/3" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + Application.put_env(:http_core, :tls_backend, :invalid) + + assert %HTTP.FetchOptions{http_version: :http3, tls_backend: nil} = + HTTP.FetchOptions.new(http_version: :http3) + end + test "rejects self-signed certificates by default" do url = start_https_server!(fn socket -> send_response(socket, "secure") end) @@ -48,6 +66,205 @@ defmodule HTTP.SSLTransportTest do assert HTTP.Response.read_all(response) == "trusted" end + test "fetches a verified response through ex_ssl" do + url = + start_https_server!(fn socket -> + assert {:ok, request} = recv_headers(socket, <<>>) + assert request =~ "GET /secure HTTP/1.1\r\n" + send_response(socket, "ex-ssl") + end) + + response = + url + |> HTTP.fetch(tls_backend: :ex_ssl, ssl: [cacertfile: @cacertfile]) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "ex-ssl" + end + + test "streams chunked responses through ex_ssl" do + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + "5\r\nchunk\r\n", + "2\r\ned\r\n", + "0\r\n\r\n" + ]) + end) + + response = + url + |> HTTP.fetch(tls_backend: :ex_ssl, ssl: [cacertfile: @cacertfile]) + |> HTTP.Promise.await() + + assert is_pid(response.stream) + assert HTTP.Response.read_all(response) == "chunked" + end + + test "streams close-delimited responses through ex_ssl" do + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + :ok = :ssl.send(socket, "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nclose-delimited") + end) + + controller = HTTP.AbortController.new() + + response = + url + |> HTTP.fetch( + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert is_pid(response.stream) + + owner = :sys.get_state(controller).request_id + assert {:ssl_closed, %SSL.Socket{pid: socket_pid}} = await_owner_close(owner) + + socket_monitor = Process.monitor(socket_pid) + assert_receive {:DOWN, ^socket_monitor, :process, ^socket_pid, _reason}, 2_000 + + assert HTTP.Response.read_all(response) == "close-delimited" + end + + test "uploads large buffered bodies through ex_ssl" do + body = :binary.copy("u", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_server!(fn socket -> + assert %{body: ^body, headers: %{"content-length" => content_length}} = + recv_request(socket) + + assert content_length == Integer.to_string(byte_size(body)) + send_response(socket, "uploaded") + end) + + response = + url + |> HTTP.fetch( + method: :post, + body: body, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert HTTP.Response.read_all(response) == "uploaded" + end + + test "cancels an in-flight ex_ssl request" do + test_pid = self() + + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send(test_pid, :request_received) + send(test_pid, {:server_recv_after_abort, :ssl.recv(socket, 0, 5_000)}) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive :request_received, 2_000 + :ok = HTTP.AbortController.abort(controller) + + assert {:error, :aborted} = HTTP.Promise.await(promise) + assert_receive {:server_recv_after_abort, {:error, :closed}}, 2_000 + end + + test "times out an ex_ssl request waiting for response headers" do + test_pid = self() + + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send(test_pid, :request_received) + send(test_pid, {:server_recv_after_timeout, :ssl.recv(socket, 0, 5_000)}) + end) + + promise = + HTTP.fetch(url, + timeout: 500, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive :request_received, 2_000 + assert {:error, :request_timeout} = HTTP.Promise.await(promise, 2_000) + assert_receive {:server_recv_after_timeout, {:error, :closed}}, 2_000 + end + + test "uses TLS backend transport options on direct requests" do + url = + start_https_server!(fn socket -> + assert {:ok, request} = recv_headers(socket, <<>>) + assert request =~ "GET /secure HTTP/1.1\r\n" + send_response(socket, "direct") + end) + + response = + HTTP.SocketClient.request(%HTTP.Request{ + url: URI.parse(url), + transport_options: [tls_backend: "ex_ssl", ssl: [cacertfile: @cacertfile]] + }) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "direct" + end + + test "pins the configured backend through HTTPS redirects" do + test_pid = self() + + destination = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send_response(socket, "redirected") + end) + + source = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send(test_pid, {:redirect_request_received, self()}) + + receive do + :send_redirect -> + :ok = + :ssl.send(socket, [ + "HTTP/1.1 302 Found\r\n", + "Location: ", + destination, + "\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ]) + end + end) + + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + Application.put_env(:http_core, :tls_backend, :ex_ssl) + + promise = HTTP.fetch(source, ssl: [cacertfile: @cacertfile]) + assert_receive {:redirect_request_received, redirect_server} + Application.put_env(:http_core, :tls_backend, :invalid) + send(redirect_server, :send_redirect) + + assert %HTTP.Response{status: 200} = response = HTTP.Promise.await(promise) + assert HTTP.Response.read_all(response) == "redirected" + end + test "streams large responses over ssl" do body = String.duplicate("s", HTTP.Config.streaming_threshold() + 1) @@ -67,6 +284,25 @@ defmodule HTTP.SSLTransportTest do assert response.body == response.stream assert HTTP.Response.read_all(response) == body end + + test "streams large verified responses through ex_ssl" do + body = String.duplicate("s", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send_response(socket, body) + end) + + response = + url + |> HTTP.fetch(tls_backend: "ex_ssl", ssl: [cacertfile: @cacertfile]) + |> HTTP.Promise.await() + + assert response.status == 200 + assert is_pid(response.stream) + assert HTTP.Response.read_all(response) == body + end end defp start_https_server!(handler) when is_function(handler, 1) do @@ -118,6 +354,70 @@ defmodule HTTP.SSLTransportTest do end end + defp await_owner_close(owner) do + deadline = System.monotonic_time(:millisecond) + 2_000 + wait_for_owner_close(owner, deadline) + end + + defp wait_for_owner_close(owner, deadline) do + case Process.info(owner, :messages) do + {:messages, messages} -> + case Enum.find(messages, &match?({:ssl_closed, %SSL.Socket{}}, &1)) do + nil -> + if System.monotonic_time(:millisecond) < deadline do + Process.sleep(10) + wait_for_owner_close(owner, deadline) + else + flunk("fetch owner did not queue an ExSSL close message") + end + + message -> + message + end + + nil -> + flunk("fetch owner exited before its queued ExSSL close could be observed") + end + end + + defp recv_request(socket) do + {head, rest} = recv_header_block(socket, <<>>) + [_request_line | header_lines] = String.split(head, "\r\n") + + headers = + header_lines + |> Enum.map(fn line -> + [name, value] = String.split(line, ":", parts: 2) + {String.downcase(name), String.trim(value)} + end) + |> Map.new() + + content_length = headers |> Map.fetch!("content-length") |> String.to_integer() + %{headers: headers, body: recv_body(socket, rest, content_length)} + end + + defp recv_header_block(socket, acc) do + case :binary.match(acc, "\r\n\r\n") do + {index, 4} -> + head = binary_part(acc, 0, index) + rest = binary_part(acc, index + 4, byte_size(acc) - index - 4) + {head, rest} + + :nomatch -> + {:ok, data} = :ssl.recv(socket, 0, 5_000) + recv_header_block(socket, acc <> data) + end + end + + defp recv_body(_socket, data, content_length) when byte_size(data) >= content_length do + binary_part(data, 0, content_length) + end + + defp recv_body(socket, data, content_length) do + {:ok, more} = :ssl.recv(socket, content_length - byte_size(data), 5_000) + recv_body(socket, data <> more, content_length) + end + defp send_response(socket, body) do :ok = :ssl.send(socket, [ @@ -132,4 +432,7 @@ defmodule HTTP.SSLTransportTest do :ssl.close(socket) end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(backend), do: Application.put_env(:http_core, :tls_backend, backend) end diff --git a/apps/http_web_socket/lib/http/web_socket.ex b/apps/http_web_socket/lib/http/web_socket.ex index 79864de..c683a1a 100644 --- a/apps/http_web_socket/lib/http/web_socket.ex +++ b/apps/http_web_socket/lib/http/web_socket.ex @@ -11,6 +11,10 @@ defmodule HTTP.WebSocket do Plain Elixir binaries are sent as text frames. Use `array_buffer/1` or `HTTP.Blob` for binary frames. + + For `wss` connections, pass `tls_backend: :ssl | :ex_ssl` (or the equivalent + string in a map). When omitted, the shared `:http_core` TLS backend setting is + captured when the socket is created. """ alias HTTP.WebSocket.ArrayBuffer diff --git a/apps/http_web_socket/lib/http/web_socket/connection.ex b/apps/http_web_socket/lib/http/web_socket/connection.ex index 620c59e..092d88a 100644 --- a/apps/http_web_socket/lib/http/web_socket/connection.ex +++ b/apps/http_web_socket/lib/http/web_socket/connection.ex @@ -30,6 +30,7 @@ defmodule HTTP.WebSocket.Connection do connect_timeout: 30_000, ssl: [], socket_opts: [], + tls_backend: :ssl, max_send_queue: 16 * 1024 * 1024, transport: nil, socket: nil, @@ -75,6 +76,7 @@ defmodule HTTP.WebSocket.Connection do connect_timeout: options.connect_timeout, ssl: options.ssl, socket_opts: options.socket_opts, + tls_backend: options.tls_backend, max_send_queue: options.max_send_queue, binary_type: options.binary_type, parser: Frame.new_parser(max_message_size: options.max_message_size) @@ -94,13 +96,15 @@ defmodule HTTP.WebSocket.Connection do Telemetry.connect_stop(state.uri, state.protocol, duration) emit(state, %Open{target: state.target}) - if extra != <<>> do - send(self(), {:websocket_data, extra}) - end + state = %{state | ready_state: @open, connect_started_at: started_at} - case rearm(state) do - :ok -> {:noreply, %{state | ready_state: @open, connect_started_at: started_at}} - {:error, reason} -> fail_connection(state, reason, started_at) + if extra == <<>> do + case rearm(state) do + :ok -> {:noreply, state} + {:error, reason} -> fail_connection(state, reason, started_at) + end + else + handle_socket_data(extra, state) end {:error, reason} -> @@ -146,8 +150,6 @@ defmodule HTTP.WebSocket.Connection do end @impl true - def handle_info({:websocket_data, data}, state), do: handle_socket_data(data, state) - def handle_info(:close_timeout, state) do {:stop, :normal, finish_close(state, state.close_code || 1006, state.close_reason, false)} end @@ -167,7 +169,7 @@ defmodule HTTP.WebSocket.Connection do defp connect_and_upgrade(state) do key = state |> Map.get(:uri) |> generate_key() - with {:ok, transport, host, port} <- select_transport(state.uri), + with {:ok, transport, host, port} <- select_transport(state), {:ok, request} <- Handshake.build_request(state.uri, state.protocols, state.headers, key), {:ok, socket} <- connect(transport, host, port, state), @@ -190,15 +192,15 @@ defmodule HTTP.WebSocket.Connection do defp generate_key(_uri), do: :crypto.strong_rand_bytes(16) |> Base.encode64() - defp select_transport(%URI{scheme: "ws", host: host, port: port}) do + defp select_transport(%{uri: %URI{scheme: "ws", host: host, port: port}}) do {:ok, HTTP.Transport.TCP, host, port || 80} end - defp select_transport(%URI{scheme: "wss", host: host, port: port}) do - {:ok, HTTP.Transport.SSL, host, port || 443} + defp select_transport(%{uri: %URI{scheme: "wss", host: host, port: port}, tls_backend: backend}) do + {:ok, HTTP.TLSBackend.transport(backend), host, port || 443} end - defp select_transport(%URI{scheme: scheme}), do: {:error, {:unsupported_scheme, scheme}} + defp select_transport(%{uri: %URI{scheme: scheme}}), do: {:error, {:unsupported_scheme, scheme}} defp connect(transport, host, port, state) do transport.connect( @@ -224,8 +226,7 @@ defmodule HTTP.WebSocket.Connection do end end - defp recv(HTTP.Transport.TCP, socket, timeout), do: :gen_tcp.recv(socket, 0, timeout) - defp recv(HTTP.Transport.SSL, socket, timeout), do: :ssl.recv(socket, 0, timeout) + defp recv(transport, socket, timeout), do: transport.recv(socket, 0, timeout) defp fail_connection(state, reason, started_at) do duration = System.monotonic_time(:microsecond) - started_at diff --git a/apps/http_web_socket/lib/http/web_socket/options.ex b/apps/http_web_socket/lib/http/web_socket/options.ex index 28281ed..3496346 100644 --- a/apps/http_web_socket/lib/http/web_socket/options.ex +++ b/apps/http_web_socket/lib/http/web_socket/options.ex @@ -6,6 +6,11 @@ defmodule HTTP.WebSocket.Options do @default_max_message_size 16 * 1024 * 1024 @default_max_send_queue 16 * 1024 * 1024 + @string_keys %{ + "tls_backend" => :tls_backend, + "tlsBackend" => :tls_backend + } + defstruct uri: nil, url: nil, protocols: [], @@ -16,6 +21,7 @@ defmodule HTTP.WebSocket.Options do connect_timeout: @default_connect_timeout, ssl: [], socket_opts: [], + tls_backend: :ssl, max_message_size: @default_max_message_size, max_send_queue: @default_max_send_queue, ref: nil @@ -31,6 +37,7 @@ defmodule HTTP.WebSocket.Options do connect_timeout: timeout(), ssl: keyword(), socket_opts: keyword(), + tls_backend: HTTP.TLSBackend.t(), max_message_size: pos_integer(), max_send_queue: pos_integer(), ref: reference() @@ -54,6 +61,7 @@ defmodule HTTP.WebSocket.Options do connect_timeout: Keyword.get(init, :connect_timeout, @default_connect_timeout), ssl: Keyword.get(init, :ssl, []), socket_opts: Keyword.get(init, :socket_opts, []), + tls_backend: Keyword.fetch!(init, :tls_backend), max_message_size: Keyword.get(init, :max_message_size, @default_max_message_size), max_send_queue: Keyword.get(init, :max_send_queue, @default_max_send_queue), ref: Keyword.get(init, :ref, make_ref()) @@ -129,7 +137,11 @@ defmodule HTTP.WebSocket.Options do end end - defp normalize_init(init) when is_map(init), do: init |> Map.to_list() |> normalize_init() + defp normalize_init(init) when is_map(init) do + init + |> Enum.map(fn {key, value} -> {normalize_key(key), value} end) + |> normalize_init() + end defp normalize_init(init) when is_list(init) do with {:ok, headers} <- normalize_headers(Keyword.get(init, :headers, [])), @@ -137,19 +149,24 @@ defmodule HTTP.WebSocket.Options do {:ok, owner} <- normalize_owner(Keyword.get(init, :owner, self())), {:ok, ssl} <- normalize_keyword(Keyword.get(init, :ssl, []), :invalid_ssl_options), {:ok, socket_opts} <- - normalize_keyword(Keyword.get(init, :socket_opts, []), :invalid_socket_options) do + normalize_keyword(Keyword.get(init, :socket_opts, []), :invalid_socket_options), + {:ok, tls_backend} <- HTTP.TLSBackend.resolve(Keyword.get(init, :tls_backend)) do {:ok, init |> Keyword.put(:headers, headers) |> Keyword.put(:binary_type, binary_type) |> Keyword.put(:owner, owner) |> Keyword.put(:ssl, ssl) - |> Keyword.put(:socket_opts, socket_opts)} + |> Keyword.put(:socket_opts, socket_opts) + |> Keyword.put(:tls_backend, tls_backend)} end end defp normalize_init(_init), do: {:error, :invalid_options} + defp normalize_key(key) when is_binary(key), do: Map.get(@string_keys, key, key) + defp normalize_key(key), do: key + defp normalize_headers(%HTTP.Headers{headers: headers}), do: normalize_headers(headers) defp normalize_headers(headers) when is_map(headers) do diff --git a/apps/http_web_socket/test/http/web_socket/options_test.exs b/apps/http_web_socket/test/http/web_socket/options_test.exs index dd936be..3029feb 100644 --- a/apps/http_web_socket/test/http/web_socket/options_test.exs +++ b/apps/http_web_socket/test/http/web_socket/options_test.exs @@ -1,5 +1,5 @@ defmodule HTTP.WebSocket.OptionsTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias HTTP.WebSocket.Options @@ -56,4 +56,39 @@ defmodule HTTP.WebSocket.OptionsTest do assert options.max_message_size == 32 assert options.max_send_queue == 64 end + + test "selects TLS backends from atom and string map options" do + assert {:ok, %{tls_backend: :ssl}} = + Options.new("wss://example.com/socket", [], tls_backend: :ssl) + + assert {:ok, %{tls_backend: :ex_ssl}} = + Options.new("wss://example.com/socket", [], %{"tlsBackend" => "ex_ssl"}) + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("wss://example.com/socket", [], %{"tls_backend" => "ssl"}) + end + + test "pins the configured TLS backend when options are constructed" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + + Application.put_env(:http_core, :tls_backend, :ex_ssl) + assert {:ok, options} = Options.new("wss://example.com/socket") + assert options.tls_backend == :ex_ssl + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("wss://example.com/socket", [], tls_backend: :ssl) + + Application.put_env(:http_core, :tls_backend, :ssl) + assert options.tls_backend == :ex_ssl + assert {:ok, %{tls_backend: :ssl}} = Options.new("wss://example.com/socket") + end + + test "rejects invalid TLS backend selections" do + assert {:error, :invalid_tls_backend} = + Options.new("wss://example.com/socket", [], tls_backend: :unknown) + end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(value), do: Application.put_env(:http_core, :tls_backend, value) end diff --git a/apps/http_web_socket/test/http/web_socket_test.exs b/apps/http_web_socket/test/http/web_socket_test.exs index 924a96f..6cc5b10 100644 --- a/apps/http_web_socket/test/http/web_socket_test.exs +++ b/apps/http_web_socket/test/http/web_socket_test.exs @@ -8,6 +8,10 @@ defmodule HTTP.WebSocketTest do alias HTTP.WebSocket.Event.Message alias HTTP.WebSocket.Event.Open + @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) + @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) + @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) + test "defines browser ready state constants" do assert WebSocket.connecting() == 0 assert WebSocket.open() == 1 @@ -81,6 +85,32 @@ defmodule HTTP.WebSocketTest do assert_receive {:websocket_server_received, :binary, <<4, 5, 6>>}, 1_000 end + test "upgrades, exchanges frames, and closes over verified TLS 1.3" do + for backend <- [:ssl, :ex_ssl] do + {:ok, _server, port} = + HTTPWebSocket.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + open_message: "welcome" + ) + + socket = + WebSocket.new("wss://127.0.0.1:#{port}/socket", [], + tls_backend: backend, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {WebSocket, ^socket, %Open{}}, 1_000 + assert_receive {WebSocket, ^socket, %Message{data: "welcome"}}, 1_000 + assert :ok = WebSocket.send(socket, "hello") + assert_receive {:websocket_server_received, :text, "hello"}, 1_000 + assert_receive {WebSocket, ^socket, %Message{data: "echo:hello"}}, 1_000 + assert :ok = WebSocket.close(socket, 1000, "done") + assert_receive {WebSocket, ^socket, %Close{code: 1000, was_clean: true}}, 1_000 + end + end + test "rejects invalid constructor input synchronously" do assert {:error, :fragment_not_allowed} = WebSocket.new("ws://example.com/socket#frag") end diff --git a/apps/http_web_socket/test/http/web_socket_tls_lifecycle_test.exs b/apps/http_web_socket/test/http/web_socket_tls_lifecycle_test.exs new file mode 100644 index 0000000..f32ed58 --- /dev/null +++ b/apps/http_web_socket/test/http/web_socket_tls_lifecycle_test.exs @@ -0,0 +1,74 @@ +defmodule HTTP.WebSocketTLSLifecycleTest do + use ExUnit.Case, async: false + + alias HTTP.WebSocket + alias HTTP.WebSocket.Event.Close + alias HTTP.WebSocket.Event.Message + alias HTTP.WebSocket.Event.Open + + @fixtures Path.expand("../support/fixtures", __DIR__) + + test "delivers upgrade-buffered frames after the TLS peer has closed" do + parent = self() + existing_connections = tls_connections() + + {:ok, server, port} = + HTTPWebSocket.TestServer.start_link( + tls: true, + certfile: Path.join(@fixtures, "localhost.pem"), + keyfile: Path.join(@fixtures, "localhost.key"), + # A complete text frame and close frame share the upgrade's TLS record. + upgrade_frames: <<0x81, 3, "bye", 0x88, 2, 1000::16>>, + close_after_upgrade: true + ) + + handler = {__MODULE__, make_ref()} + + :ok = + :telemetry.attach( + handler, + [:http_web_socket, :connect, :stop], + &__MODULE__.pause_after_upgrade/4, + {port, parent} + ) + + on_exit(fn -> :telemetry.detach(handler) end) + + socket = + WebSocket.new("wss://127.0.0.1:#{port}/socket", [], + tls_backend: :ex_ssl, + ssl: [cacertfile: Path.join(@fixtures, "localhost-ca.pem")] + ) + + assert_receive {:upgrade_received, consumer}, 5_000 + [tls_pid] = tls_connections() -- existing_connections + monitor = Process.monitor(tls_pid) + + try do + send(server, :close_tls) + assert_receive {:DOWN, ^monitor, :process, ^tls_pid, _}, 5_000 + after + send(consumer, :continue_upgrade) + end + + assert_receive {WebSocket, ^socket, %Open{}}, 5_000 + assert_receive {WebSocket, ^socket, %Message{data: "bye"}}, 5_000 + assert_receive {WebSocket, ^socket, %Close{code: 1000, was_clean: true}}, 5_000 + end + + def pause_after_upgrade(_event, _measurements, %{port: port}, {port, parent}) do + send(parent, {:upgrade_received, self()}) + + receive do + :continue_upgrade -> :ok + after + 5_000 -> :ok + end + end + + def pause_after_upgrade(_event, _measurements, _metadata, _config), do: :ok + + defp tls_connections do + for {_, pid, _, _} <- DynamicSupervisor.which_children(SSL.ConnectionSupervisor), do: pid + end +end diff --git a/apps/http_web_socket/test/support/fixtures/localhost-ca.pem b/apps/http_web_socket/test/support/fixtures/localhost-ca.pem new file mode 100644 index 0000000..15227f1 --- /dev/null +++ b/apps/http_web_socket/test/support/fixtures/localhost-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKzCCAhOgAwIBAgIUbWn02fvDN0zvFEGHFliwdXqASNIwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqaXND6GhQeV+tXwmYZYy +q+kFvpGWXCa49yG4/nJLfUYtpANqii88Y08J+6fAskHxoWYLidcnAvC5V51IcreX +g/N108fCvxTQ5jYfwIyj8oAEL4QZbnKklQrcLRFqN1Ef9f6cdNWL4pZZk7/ChbXP +ULHTNQF8FIoDUS4H5bjobU2lBER4H7Qa8oevkMGotgfjrzqWftTWt68dJiTsfoX1 +z+VH6t6t5vvT/L+0J7M9JAB++sckXSY9J+yIeG7Cu3sGtwTqIvuiiBH0latUYC9M +sJ0LS400sBonIZC4BdvVc9NCU8M1sx5yWV6eNRe7Cz3hrzQVcWMbAB73qXFMvpEO +OwIDAQABo2MwYTAdBgNVHQ4EFgQUUayz7IdQZHRIjmttS9hkKzG01M4wHwYDVR0j +BBgwFoAUUayz7IdQZHRIjmttS9hkKzG01M4wDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggEBAIrCD3yHZ+F/P/s0FXZRFFNB +AIqYjEwYr2OBN/ShHhceoqM55FbThtYPxUlVJH+uWkxpvnExDOauCapW7Cxlz9FF +3NvLjWf8fs80Uno5Dy6mik4rL6OsThYX8Ko2u6gkSiQm9yuT1iKvigGjU4q1qyft +ZJH8an7/egIIHxZya5oBuGKBnX241s6n9Osdtww/Eu5IOlZZeORJ0mAA7gTHaZyI +VquDo6Ml66LMyuIQkbtr19UHAEHrSfhmrujI5GruA/ejnma2O5sE1r1hVa91TV+L +TuaVxPvkdS+CIBvfwy90zmDbPhLt0kU5FiY2x6FlR1MrvVBn1uUQ59kYJ7yNI9k= +-----END CERTIFICATE----- diff --git a/apps/http_web_socket/test/support/fixtures/localhost.key b/apps/http_web_socket/test/support/fixtures/localhost.key new file mode 100644 index 0000000..d05ba0d --- /dev/null +++ b/apps/http_web_socket/test/support/fixtures/localhost.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtsMy7LvUMU+H3 +27785rkCaSrkj5ljfUUPDA8+LM1ReN4ovqRaKjN8Tz88AOOwsZkXHLEJ5Uv8RbpS +8ruITF9moSXhVNFvVkBtf+SIBF2M5sDwsV/q8VFwUcvncDyLOsZwqsal9j+e9LjY +SlQtzzapBWfZm8O540RNpmpsa6k0M7jfp/m8P/l3RIlYzEuWPfCBd1ZmQMUN0T6H ++iYBBfNj6ioDUOUP5VfwA8cVXgblYbFKPF6uTtv95Ep4CFin0u6S0xVRlX0PERU5 +kyNLkmXUibClAY93IEkX5KDe0wb0WU4QBC9BAC8CegTFKd57MUlH0FtABCakLDCz +uHKFOc5bAgMBAAECggEAF95J5Bcp2Y3Vaagq8LoMBobJPCt3mrJmQREZLyQc+usv +LE42a6WM+bIyMF6261scfI6WzZNTA9HZLvjoSGymV4YhnHt5ZGFn7SQ8FWz0Jh63 +uNvHIJQU6uhyqtVDnKZxCp6NOdVDHOboV+Cu/LSTAFsb0E2VcgAHLNyHl2qcD+4p +qVDfyz/2BcAciQUTyQmq8tVPyJV226iZAkTJjah8H+MOXo6XqieNJ2WTz3H57PCE +tmm5YLeNl6S47O2369VRFmhYZZBS2tiBEfXNwwFCc6Q44esmJG1UtLUp6LMenNon +FJTod0QuZMjqfomzYzvJ9sCi8cvrABDVs8HQiw8RIQKBgQDY4ReBy63jdUJkDYQC +00SnrGywE7YNLYu+ZkHz+J14Xn/ylKFK0sz1WPfGZZQWtLWEXCYw3f2Z8d42Rd0Z +l/E7H2KiRq2oPrqc/mwfWtwjL0sqOuLswAzdL8AKLbl6vou1UjrJennuvaOh2C2s +sKJutX7CoM8GvXvWweNGY9xvuQKBgQDNBV8IkKuE4B8TIF7x49lkOFpWVgf6E1bS +My22JFIk8C0Jnxh8e/6mujeEJ2jMO/kpYPhWkDhKp4CTm+S6Ga6CSFzBjnA0dfaB +8Htyb4R5bJn7wD9+Zj+tHSG6gwVxJl/u2YfDAZL+zlKgVx7RoTGkF8mskHVmgrFT +DIJyFs0wswKBgFMSjR2DdfzNOnv2jV1DrWWIby1Wr4IGsyNgKd0YmsCzedDiS5HM +gwNra6UL3ZiA6ZJkdaB8N5qTAanKQvF9uMILuI0uA3CRbouaDLJJ7E5x3Bm16pwC +yCqlEqsTbptshzkR4UCxcCkZbKcelggytFUxofdM/1+2jsvpAnRA5fvJAoGAXS+/ +zkjTbQXhmfPws8l4mhDzHqLj5Uq8/7W7ZTqFC70O+3yQyKQjTuz9JtgyzgHEcoZc +2hubOnOAAZeuEthxdU4muuNfJLkpXk5MDeuaLwapxr/PHEilUK4ZEolTA+cJW6sM +BhrFEYP+ElsG6wl1YrxdMk5Gzl1A9BqPgAPVJ/kCgYEArpwxPGurJDQ3JszIWDJU +CX6KhwAncK9LALQT6EPB3Y54Ou4cq1XMSWlmqUUmN821J/mbR4f66F+nFy/q29Br +giwArg0Ngkt+uaKTfo6IeR7+b2vAjzZ+z3cEbNZpuyCAbQwytnXT8t7ZUonAQ7dO +TmHo1zxsuNSt2EYos+EP+VA= +-----END PRIVATE KEY----- diff --git a/apps/http_web_socket/test/support/fixtures/localhost.pem b/apps/http_web_socket/test/support/fixtures/localhost.pem new file mode 100644 index 0000000..178c710 --- /dev/null +++ b/apps/http_web_socket/test/support/fixtures/localhost.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHTCCAgWgAwIBAgIUEJVRR6DfNeTUcvIJYQL9oVN8xyEwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbDMuy71DFPh99u+/Oa5Amkq5I+ZY31F +DwwPPizNUXjeKL6kWiozfE8/PADjsLGZFxyxCeVL/EW6UvK7iExfZqEl4VTRb1ZA +bX/kiARdjObA8LFf6vFRcFHL53A8izrGcKrGpfY/nvS42EpULc82qQVn2ZvDueNE +TaZqbGupNDO436f5vD/5d0SJWMxLlj3wgXdWZkDFDdE+h/omAQXzY+oqA1DlD+VX +8APHFV4G5WGxSjxerk7b/eRKeAhYp9LuktMVUZV9DxEVOZMjS5Jl1ImwpQGPdyBJ +F+Sg3tMG9FlOEAQvQQAvAnoExSneezFJR9BbQAQmpCwws7hyhTnOWwIDAQABo14w +XDAaBgNVHREEEzARhwR/AAABgglsb2NhbGhvc3QwHQYDVR0OBBYEFN1Qq2IrwIwg +nlCE50Q3jpSthaODMB8GA1UdIwQYMBaAFFGss+yHUGR0SI5rbUvYZCsxtNTOMA0G +CSqGSIb3DQEBCwUAA4IBAQBwO6NAaG2iwdO0iAaatt9cyP7ZvtJM0I0se1Dd1AtU +3iBlrEFNAWwph3ycD9Wr5HKswN7DhK+cOeklAeeBaxgB8egyHuJHKJLmoZQip+FQ +lCoMgPOBXH5IepiczMN4mUUCJn2ZNf0j/QkO0CjjefiYBYKunMQDNMtmhiB7LVIL +RAFo6kawJ49F3l/rTSKMMTGjlsW2wSM3pexevTvG/8dagT9ULQfLOuuHd+2paqKU +EoLAuaagqefwZaNASEvKP1Tl9I1GvZmjVTK8qJDmMJE+/D5HwscrDZmBvSDJdLLf +oZPdnbpZv0TXYvlpC5c1pHRPeA5LGa54GhM4Akc8ieDO +-----END CERTIFICATE----- diff --git a/apps/http_web_socket/test/support/web_socket_server.ex b/apps/http_web_socket/test/support/web_socket_server.ex index 7fd8f36..04a4a5b 100644 --- a/apps/http_web_socket/test/support/web_socket_server.ex +++ b/apps/http_web_socket/test/support/web_socket_server.ex @@ -7,110 +7,137 @@ defmodule HTTPWebSocket.TestServer do def start_link(opts \\ []) do parent = self() - - {:ok, listen_socket} = - :gen_tcp.listen(0, [ - :binary, - packet: :raw, - active: false, - reuseaddr: true, - ip: {127, 0, 0, 1} - ]) - - {:ok, {{127, 0, 0, 1}, port}} = :inet.sockname(listen_socket) + tls? = Keyword.get(opts, :tls, false) + transport = socket_module(tls?) + {:ok, listen_socket} = listen(tls?, opts) + {:ok, {{127, 0, 0, 1}, port}} = sockname(tls?, listen_socket) pid = spawn_link(fn -> - {:ok, socket} = :gen_tcp.accept(listen_socket) - :ok = :gen_tcp.close(listen_socket) - serve(socket, parent, opts) + with {:ok, socket} <- accept(tls?, listen_socket), + :ok <- transport.close(listen_socket) do + serve(socket, parent, opts, transport) + else + {:error, reason} -> send(parent, {:websocket_server_error, reason}) + end end) {:ok, pid, port} end - defp serve(socket, parent, opts) do - with {:ok, request} <- recv_until(socket, "\r\n\r\n", <<>>), + defp listen(false, _opts) do + :gen_tcp.listen(0, [:binary, packet: :raw, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) + end + + defp listen(true, opts) do + :ssl.listen(0, + mode: :binary, + packet: :raw, + active: false, + reuseaddr: true, + ip: {127, 0, 0, 1}, + versions: [:"tlsv1.3"], + certfile: Keyword.fetch!(opts, :certfile), + keyfile: Keyword.fetch!(opts, :keyfile) + ) + end + + defp accept(false, listen_socket), do: :gen_tcp.accept(listen_socket) + + defp accept(true, listen_socket) do + with {:ok, transport_socket} <- :ssl.transport_accept(listen_socket), + do: :ssl.handshake(transport_socket) + end + + defp socket_module(false), do: :gen_tcp + defp socket_module(true), do: :ssl + defp sockname(false, listen_socket), do: :inet.sockname(listen_socket) + defp sockname(true, listen_socket), do: :ssl.sockname(listen_socket) + + defp serve(socket, parent, opts, transport) do + with {:ok, request} <- recv_until(transport, socket, "\r\n\r\n", <<>>), {:ok, key} <- request_header(request, "sec-websocket-key") do - protocol = Keyword.get(opts, :protocol) - :ok = :gen_tcp.send(socket, handshake_response(key, protocol)) + :ok = + transport.send(socket, [ + handshake_response(key, Keyword.get(opts, :protocol)), + Keyword.get(opts, :upgrade_frames, <<>>) + ]) + send(parent, {:websocket_server_handshake, request}) - maybe_send_open_message(socket, Keyword.get(opts, :open_message)) + if Keyword.get(opts, :close_after_upgrade, false) do + receive do + :close_tls -> transport.close(socket) + after + 5_000 -> transport.close(socket) + end + else + maybe_send_open_message(transport, socket, Keyword.get(opts, :open_message)) - case maybe_send_close(socket, Keyword.get(opts, :close_after_open)) do - :closed -> :ok - :open -> loop(socket, parent, <<>>) + case maybe_send_close(transport, socket, Keyword.get(opts, :close_after_open)) do + :closed -> :ok + :open -> loop(transport, socket, parent, <<>>) + end end else {:error, reason} -> send(parent, {:websocket_server_error, reason}) - :gen_tcp.close(socket) + transport.close(socket) end end - defp loop(socket, parent, buffer) do + defp loop(transport, socket, parent, buffer) do case take_client_frame(buffer) do {:ok, opcode, payload, rest} -> - handle_frame(socket, parent, opcode, payload) - loop(socket, parent, rest) + handle_frame(transport, socket, parent, opcode, payload) + loop(transport, socket, parent, rest) :more -> - case :gen_tcp.recv(socket, 0, 1_000) do - {:ok, data} -> - loop(socket, parent, buffer <> data) - - {:error, :closed} -> - send(parent, :websocket_server_closed) - - {:error, reason} -> - send(parent, {:websocket_server_error, reason}) + case transport.recv(socket, 0, 1_000) do + {:ok, data} -> loop(transport, socket, parent, buffer <> data) + {:error, :closed} -> send(parent, :websocket_server_closed) + {:error, reason} -> send(parent, {:websocket_server_error, reason}) end end end - defp handle_frame(socket, parent, 0x1, payload) do + defp handle_frame(transport, socket, parent, 0x1, payload) do send(parent, {:websocket_server_received, :text, payload}) - :ok = send_server_frame(socket, 0x1, "echo:" <> payload) + :ok = send_server_frame(transport, socket, 0x1, "echo:" <> payload) end - defp handle_frame(_socket, parent, 0x2, payload) do - send(parent, {:websocket_server_received, :binary, payload}) - end + defp handle_frame(_transport, _socket, parent, 0x2, payload), + do: send(parent, {:websocket_server_received, :binary, payload}) - defp handle_frame(socket, parent, 0x8, payload) do + defp handle_frame(transport, socket, parent, 0x8, payload) do send(parent, {:websocket_server_received, :close, payload}) - :ok = send_server_frame(socket, 0x8, payload) - :gen_tcp.close(socket) + :ok = send_server_frame(transport, socket, 0x8, payload) + transport.close(socket) end - defp handle_frame(_socket, parent, opcode, payload) do - send(parent, {:websocket_server_received, opcode, payload}) - end + defp handle_frame(_transport, _socket, parent, opcode, payload), + do: send(parent, {:websocket_server_received, opcode, payload}) - defp maybe_send_open_message(_socket, nil), do: :ok + defp maybe_send_open_message(_transport, _socket, nil), do: :ok - defp maybe_send_open_message(socket, {:binary, payload}) do - send_server_frame(socket, 0x2, payload) - end + defp maybe_send_open_message(transport, socket, {:binary, payload}), + do: send_server_frame(transport, socket, 0x2, payload) - defp maybe_send_open_message(socket, message) when is_binary(message) do - send_server_frame(socket, 0x1, message) - end + defp maybe_send_open_message(transport, socket, message) when is_binary(message), + do: send_server_frame(transport, socket, 0x1, message) - defp maybe_send_close(_socket, nil), do: :open + defp maybe_send_close(_transport, _socket, nil), do: :open - defp maybe_send_close(socket, {code, reason}) do - :ok = send_server_frame(socket, 0x8, <>) - :gen_tcp.close(socket) + defp maybe_send_close(transport, socket, {code, reason}) do + :ok = send_server_frame(transport, socket, 0x8, <>) + transport.close(socket) :closed end - defp recv_until(socket, marker, buffer) do + defp recv_until(transport, socket, marker, buffer) do if :binary.match(buffer, marker) == :nomatch do - with {:ok, data} <- :gen_tcp.recv(socket, 0, 1_000) do - recv_until(socket, marker, buffer <> data) - end + with {:ok, data} <- transport.recv(socket, 0, 1_000), + do: recv_until(transport, socket, marker, buffer <> data) else {:ok, buffer} end @@ -121,11 +148,8 @@ defmodule HTTPWebSocket.TestServer do |> String.split("\r\n") |> Enum.find_value(fn line -> case String.split(line, ":", parts: 2) do - [name, value] -> - if String.downcase(name) == wanted_name, do: {:ok, String.trim(value)} - - _ -> - nil + [name, value] -> if String.downcase(name) == wanted_name, do: {:ok, String.trim(value)} + _ -> nil end end) |> case do @@ -134,33 +158,16 @@ defmodule HTTPWebSocket.TestServer do end end - defp handshake_response(key, nil) do - [ - "HTTP/1.1 101 Switching Protocols\r\n", - "Upgrade: websocket\r\n", - "Connection: Upgrade\r\n", - "Sec-WebSocket-Accept: ", - accept_key(key), - "\r\n\r\n" - ] - end - defp handshake_response(key, protocol) do [ - "HTTP/1.1 101 Switching Protocols\r\n", - "Upgrade: websocket\r\n", - "Connection: Upgrade\r\n", - "Sec-WebSocket-Accept: ", + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ", accept_key(key), - "\r\n", - "Sec-WebSocket-Protocol: ", - protocol, + if(protocol, do: ["\r\nSec-WebSocket-Protocol: ", protocol], else: []), "\r\n\r\n" ] end defp accept_key(key), do: :crypto.hash(:sha, key <> @guid) |> Base.encode64() - defp take_client_frame(buffer) when byte_size(buffer) < 6, do: :more defp take_client_frame(<>) do @@ -183,7 +190,6 @@ defmodule HTTPWebSocket.TestServer do defp take_length(126, _rest), do: :more defp take_length(127, <>), do: {:ok, length, rest} defp take_length(127, _rest), do: :more - defp take_masked_payload(rest, length) when byte_size(rest) < 4 + length, do: :more defp take_masked_payload(<>, length) do @@ -191,21 +197,17 @@ defmodule HTTPWebSocket.TestServer do {:ok, mask_key, payload, remaining} end - defp send_server_frame(socket, opcode, payload) do - :gen_tcp.send(socket, server_frame(opcode, payload)) - end + defp send_server_frame(transport, socket, opcode, payload), + do: transport.send(socket, server_frame(opcode, payload)) - defp server_frame(opcode, payload) when byte_size(payload) <= 125 do - <<0x80 ||| opcode, byte_size(payload), payload::binary>> - end + defp server_frame(opcode, payload) when byte_size(payload) <= 125, + do: <<0x80 ||| opcode, byte_size(payload), payload::binary>> defp unmask(payload, <>) do payload |> :binary.bin_to_list() |> Enum.with_index() - |> Enum.map(fn {byte, index} -> - bxor(byte, Enum.at([a, b, c, d], rem(index, 4))) - end) + |> Enum.map(fn {byte, index} -> bxor(byte, Enum.at([a, b, c, d], rem(index, 4))) end) |> :binary.list_to_bin() end end diff --git a/apps/http_web_transport/lib/http/web_transport.ex b/apps/http_web_transport/lib/http/web_transport.ex index ce3b9bd..45125a3 100644 --- a/apps/http_web_transport/lib/http/web_transport.ex +++ b/apps/http_web_transport/lib/http/web_transport.ex @@ -5,6 +5,10 @@ defmodule HTTP.WebTransport do This module implements the public API shape and lifecycle management for WebTransport sessions. The default backend speaks WebTransport extended CONNECT over HTTP/3 using the shared QUIC transport in `:http_core`. + + QUIC uses its own TLS implementation. The shared `:http_core, :tls_backend` + configuration does not apply here; an explicit non-nil `tls_backend` option + returns `{:error, :tls_backend_not_supported_for_quic}`. """ alias HTTP.WebTransport.CloseInfo diff --git a/apps/http_web_transport/lib/http/web_transport/options.ex b/apps/http_web_transport/lib/http/web_transport/options.ex index 44b4beb..3b73688 100644 --- a/apps/http_web_transport/lib/http/web_transport/options.ex +++ b/apps/http_web_transport/lib/http/web_transport/options.ex @@ -37,7 +37,9 @@ defmodule HTTP.WebTransport.Options do "server_certificate_hashes" => :server_certificate_hashes, "socketOpts" => :socket_opts, "socket_opts" => :socket_opts, - "ssl" => :ssl + "ssl" => :ssl, + "tlsBackend" => :tls_backend, + "tls_backend" => :tls_backend } defstruct uri: nil, @@ -149,7 +151,8 @@ defmodule HTTP.WebTransport.Options do end defp normalize_init(init) when is_list(init) do - with {:ok, headers} <- normalize_headers(Keyword.get(init, :headers, [])), + with :ok <- validate_tls_backend(Keyword.get(init, :tls_backend)), + {:ok, headers} <- normalize_headers(Keyword.get(init, :headers, [])), {:ok, owner} <- normalize_owner(Keyword.get(init, :owner, self())), {:ok, allow_pooling} <- normalize_boolean(Keyword.get(init, :allow_pooling, false), :invalid_allow_pooling), @@ -230,6 +233,9 @@ defmodule HTTP.WebTransport.Options do defp normalize_init(_init), do: {:error, :invalid_options} + defp validate_tls_backend(nil), do: :ok + defp validate_tls_backend(_backend), do: {:error, :tls_backend_not_supported_for_quic} + defp default_backend, do: HTTP.WebTransport.Transport.QUIC defp normalize_key(key) when is_binary(key), do: Map.get(@string_keys, key, key) diff --git a/apps/http_web_transport/test/http/web_transport/options_test.exs b/apps/http_web_transport/test/http/web_transport/options_test.exs index dfc87b5..b90cea5 100644 --- a/apps/http_web_transport/test/http/web_transport/options_test.exs +++ b/apps/http_web_transport/test/http/web_transport/options_test.exs @@ -78,4 +78,19 @@ defmodule HTTP.WebTransport.OptionsTest do server_certificate_hashes: [%{algorithm: "sha-256", value: <<0>>}] ) end + + test "rejects explicit TLS backends because WebTransport uses QUIC TLS" do + for backend <- [:ssl, :ex_ssl, "ssl", "ex_ssl", false] do + for options <- [ + [tls_backend: backend], + %{"tls_backend" => backend}, + %{"tlsBackend" => backend} + ] do + assert {:error, :tls_backend_not_supported_for_quic} = + Options.new("https://example.com/transport", options) + end + end + + assert {:ok, _} = Options.new("https://example.com/transport", tls_backend: nil) + end end diff --git a/apps/http_web_transport/test/http/web_transport/tls_config_test.exs b/apps/http_web_transport/test/http/web_transport/tls_config_test.exs new file mode 100644 index 0000000..94008c5 --- /dev/null +++ b/apps/http_web_transport/test/http/web_transport/tls_config_test.exs @@ -0,0 +1,24 @@ +defmodule HTTP.WebTransport.TLSConfigTest do + use ExUnit.Case, async: false + + alias HTTP.WebTransport.Options + alias HTTP.WebTransport.Transport.QUIC + + test "shared TCP TLS configuration does not affect QUIC options" do + previous = Application.fetch_env(:http_core, :tls_backend) + + on_exit(fn -> + case previous do + {:ok, value} -> Application.put_env(:http_core, :tls_backend, value) + :error -> Application.delete_env(:http_core, :tls_backend) + end + end) + + for backend <- [:ex_ssl, :invalid] do + Application.put_env(:http_core, :tls_backend, backend) + + assert {:ok, %{backend: QUIC}} = + Options.new("https://example.com/transport") + end + end +end From 010b0b850745b43faab73093849d3fb6a1dc6650 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Sun, 20 Sep 2026 19:33:21 +0800 Subject: [PATCH 03/17] fix(tls): preserve completed responses and validate packaged clients --- .formatter.exs | 2 +- .github/workflows/ci.yml | 29 ++ .github/workflows/e2e.yml | 14 +- .github/workflows/test.yml | 12 +- AGENTS.md | 16 + CHANGELOG.md | 16 + CLAUDE.md | 8 + README.md | 42 +++ apps/http_core/lib/http/http2.ex | 17 + apps/http_core/mix.exs | 2 +- apps/http_core/test/http/http2_test.exs | 22 ++ apps/http_fetch/lib/http/socket_client.ex | 21 +- .../test/http/socket_client_http2_test.exs | 290 ++++++++++++++++- docs/pr-14-validation.md | 189 +++++++++++ mix.exs | 4 +- scripts/external_consumer_smoke.exs | 308 ++++++++++++++++++ scripts/external_consumer_smoke.sh | 61 ++++ 17 files changed, 1039 insertions(+), 14 deletions(-) create mode 100644 docs/pr-14-validation.md create mode 100644 scripts/external_consumer_smoke.exs create mode 100644 scripts/external_consumer_smoke.sh diff --git a/.formatter.exs b/.formatter.exs index 11dbefe..98af6c6 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,5 +1,5 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs"], + inputs: ["{mix,.formatter}.exs", "scripts/*.exs"], subdirectories: ["apps/*"] ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 328557f..9ed056e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,13 @@ jobs: - name: Compile with warnings as errors run: mix compile --warnings-as-errors + - name: Compile test environment with warnings as errors + env: + MIX_ENV: test + run: | + mix deps.get + mix compile --warnings-as-errors + format: name: Format Check runs-on: ubuntu-latest @@ -214,3 +221,25 @@ jobs: - name: Build package working-directory: apps/${{ matrix.app }} run: mix hex.build --unpack -o /tmp/${{ matrix.app }}_pkg + + external-consumer: + name: External consumer smoke + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Elixir + uses: erlef/setup-beam@v1 + with: + elixir-version: ${{ env.ELIXIR_VERSION }} + otp-version: ${{ env.OTP_VERSION }} + + - name: Install dependencies + env: + MIX_ENV: prod + run: mix deps.get + + - name: Run external consumer smoke test + run: bash scripts/external_consumer_smoke.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ee3a8de..84b3560 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -62,8 +62,15 @@ jobs: ${{ runner.os }}-mix- - name: Install Elixir dependencies + env: + MIX_ENV: test run: mix deps.get + - name: Compile umbrella dependencies + env: + MIX_ENV: test + run: mix compile --warnings-as-errors + - name: Set up Go if: matrix.app == 'http_fetch' uses: actions/setup-go@v5 @@ -112,15 +119,14 @@ jobs: env: MIX_ENV: test E2E_BASE_URL: ${{ env.E2E_BASE_URL }} - run: mix test.e2e + run: mix test apps/http_fetch/e2e - name: Run app E2E test suite if present if: matrix.app != 'http_fetch' - working-directory: apps/${{ matrix.app }} run: | set -euo pipefail - if [ -d e2e ]; then - MIX_ENV=test mix test e2e/ + if [ -d "apps/${{ matrix.app }}/e2e" ]; then + MIX_ENV=test mix test "apps/${{ matrix.app }}/e2e" else echo "No e2e suite for ${{ matrix.app }}" fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7668146..813b0f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,8 +53,16 @@ jobs: ${{ runner.os }}-mix- - name: Install dependencies + env: + MIX_ENV: test run: mix deps.get + - name: Compile umbrella dependencies + env: + MIX_ENV: test + run: mix compile --warnings-as-errors + - name: Run tests - working-directory: apps/${{ matrix.app }} - run: mix test + env: + MIX_ENV: test + run: mix test apps/${{ matrix.app }}/test diff --git a/AGENTS.md b/AGENTS.md index 4c8141a..4d4a9ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,22 @@ mix docs # ExDoc HTML Run a single test file or line: `mix test apps/http_fetch/test/http/response_test.exs:42`. First-time Dialyzer setup: `mix dialyzer --plt` (2-3 min, cached in `apps/http_fetch/priv/plts/`). +### Testing an individual umbrella app + +The child applications share the umbrella's `_build`, `deps`, and lockfile, but +a child Mix project does not put runtime applications of an `in_umbrella` +dependency on its own code path. Run scoped tests through the root Mix project +after the root preparation step: + +```bash +MIX_ENV=test mix deps.get +MIX_ENV=test mix compile --warnings-as-errors +MIX_ENV=test mix test apps/http_fetch/test +``` + +Use the same root preparation and replace the path for any other app under +`apps/`. The E2E workflow uses the same root-scoped form for `apps/*/e2e`. + ## Project layout This is a Mix umbrella with independent child apps under `apps/`. Shared HTTP diff --git a/CHANGELOG.md b/CHANGELOG.md index 92bfaa7..802452e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Changed +- Keep OTP `:ssl` as the default TCP TLS backend while allowing verified TLS + 1.3 through the optional `:ex_ssl` backend (`~> 0.3.0`). HTTP/3 and + WebTransport continue to use QUIC's independent TLS implementation. +- Run individual app tests and E2E suites from the umbrella root after explicit + test-environment preparation, so transitive runtime applications are compiled + and on the code path even on cold checkouts. +- Validate all five built packages in an isolated external consumer, including + transitive dependencies and verified local TLS requests. + +### Fixed +- Deliver complete HTTP/2 responses when the peer closes immediately after an + END_STREAM DATA frame and a non-essential WINDOW_UPDATE or acknowledgement + returns `:closed`. Incomplete responses, required request writes, and other + errors still fail normally. + ## [0.11.0] - 2026-07-04 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 48db8f7..49850c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,6 +84,14 @@ mix docs MIX_ENV=prod mix compile ``` +For individual app tests, stay at the umbrella root: run `MIX_ENV=test mix +deps.get`, `MIX_ENV=test mix compile --warnings-as-errors`, then `mix test +apps//test`. Running Mix inside a child app does not traverse the runtime +dependencies of its `in_umbrella` dependencies. E2E uses the same root preparation +and `mix test apps//e2e`, or `mix test.e2e` for all E2E suites. +Run `bash scripts/external_consumer_smoke.sh` to build all five packages and +verify an isolated non-umbrella consumer, including transitive TLS dependencies. + ## Important Implementation Details ### Request Options Mapping diff --git a/README.md b/README.md index af46a80..dc84902 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ A modern HTTP client library for Elixir that provides a fetch API similar to web browsers, built on Erlang's built-in socket modules. +For development, prepare the umbrella before running a scoped app test: +`MIX_ENV=test mix deps.get && MIX_ENV=test mix compile --warnings-as-errors` +followed by `MIX_ENV=test mix test apps/http_fetch/test`. Running the test from +the root keeps runtime applications of `in_umbrella` dependencies, including +`ex_ssl`, on the code path without adding duplicate child dependencies. + ## Features - **Browser-like API**: Familiar fetch interface with promises and async/await patterns @@ -149,6 +155,8 @@ configuration. The backend is captured when the request/client is created and retained through redirects and EventSource reconnects; runtime configuration changes affect new operations. Invalid selections fail explicitly. +`http_core` declares `ex_ssl ~> 0.3.0` as a transitive runtime dependency, +allowing 0.3.x patch updates. Consumers do not need to add it separately. `ssl: [...]` supplies TLS settings to the selected backend. The `ex_ssl` 0.3.0 backend uses the `SSL` module and requires peer verification and TLS 1.3. It uses system CA certificates unless `cacerts` or `cacertfile` is supplied. DNS names @@ -157,6 +165,12 @@ and IP addresses are verified against the peer certificate. TLS 1.2, are unsupported and return errors; connections never fall back to another backend automatically. +A complete HTTP/2 response remains deliverable if the peer closes before the +client can write its remaining WINDOW_UPDATE or acknowledgement frames. This +only applies to `:closed` after the parser reports completion and the pending +writes contain no request DATA. Truncated responses and other write failures +remain errors; flow-control and streaming delivery order are unchanged. + For `:ex_ssl`, `socket_opts` accepts `send_timeout` and `send_timeout_close: true`. These override matching entries in `ssl`. Custom ClientHello profiles can be passed through `ssl: [ex_ssl: [profile: profile]]`; @@ -505,10 +519,22 @@ open doc/index.html ### Running Tests +Run these commands from the umbrella root, including when testing one app. +The root dependency graph includes the runtime dependencies of every umbrella +app; invoking Mix inside a child app does not traverse its `in_umbrella` +dependencies in the same way. + ```bash +# Prepare dependencies, including on a cold checkout +MIX_ENV=test mix deps.get +MIX_ENV=test mix compile --warnings-as-errors + # Run all unit tests mix test +# Run one app (replace the app name as needed) +mix test apps/http_fetch/test + # Run specific test file mix test apps/http_fetch/test/http/response_test.exs @@ -535,6 +561,22 @@ MIX_ENV=test mix test.e2e ``` In CI, the `e2e.yml` workflow handles all of this automatically. +`mix test.e2e` keeps execution at the umbrella root. To run one suite, use +`MIX_ENV=test mix test apps/http_web_socket/e2e` (or another app's `e2e` +directory) after the same preparation as the unit tests. + +### Testing Packaged Consumers + +```bash +bash scripts/external_consumer_smoke.sh +``` + +This builds all five current Hex packages and installs their unpacked contents +into a temporary project outside the umbrella, with independent dependencies +and build output and no repository lockfile. Local paths resolve the unpublished +internal packages; `ex_ssl` is resolved only through `http_core`. The smoke +checks runtime application startup, verified local TLS 1.3 requests with both +TCP TLS backends, and the separate WebTransport QUIC boundary. ### Code Formatting diff --git a/apps/http_core/lib/http/http2.ex b/apps/http_core/lib/http/http2.ex index 644a77a..0ce3900 100644 --- a/apps/http_core/lib/http/http2.ex +++ b/apps/http_core/lib/http/http2.ex @@ -98,6 +98,14 @@ defmodule HTTP.HTTP2 do {%{conn | outbound: []}, Enum.reverse(outbound)} end + @spec complete_response?(t()) :: boolean() + def complete_response?(%__MODULE__{done?: done?}), do: done? + + @spec outbound_control_only?(t()) :: boolean() + def outbound_control_only?(%__MODULE__{outbound: outbound}) do + Enum.all?(outbound, &control_frame?/1) + end + defp append_buffer(%__MODULE__{buffer: buffer} = conn, data) do %{conn | buffer: buffer <> data} end @@ -474,6 +482,15 @@ defmodule HTTP.HTTP2 do %{conn | outbound: [iodata | outbound]} end + defp control_frame?(iodata) do + case Frame.decode(IO.iodata_to_binary(iodata)) do + {:ok, %Frame{type: :window_update}, ""} -> true + {:ok, %Frame{type: :settings, flags: flags}, ""} -> Frame.flag?(flags, @flag_ack) + {:ok, %Frame{type: :ping, flags: flags}, ""} -> Frame.flag?(flags, @flag_ack) + _ -> false + end + end + defp flush_pending_body(%__MODULE__{pending_body: ""} = conn), do: conn defp flush_pending_body(%__MODULE__{} = conn) do diff --git a/apps/http_core/mix.exs b/apps/http_core/mix.exs index 87a459e..fb258c2 100644 --- a/apps/http_core/mix.exs +++ b/apps/http_core/mix.exs @@ -37,7 +37,7 @@ defmodule HttpCore.MixProject do defp deps do [ - {:ex_ssl, "~> 0.3"}, + {:ex_ssl, "~> 0.3.0"}, {:quic, "~> 1.6", runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} ] diff --git a/apps/http_core/test/http/http2_test.exs b/apps/http_core/test/http/http2_test.exs index bc539c3..8355d53 100644 --- a/apps/http_core/test/http/http2_test.exs +++ b/apps/http_core/test/http/http2_test.exs @@ -244,6 +244,28 @@ defmodule HTTP.HTTP2Test do HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) end + test "does not discard a reset following END_STREAM in the same batch" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, @end_stream, 1, "ok"), + Frame.encode(:rst_stream, 0, 1, <<0x8::32>>) + ] + + assert {:error, {:stream_reset, :cancel}} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + + test "does not discard a protocol error following END_STREAM in the same batch" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, @end_stream, 1, "ok"), + Frame.encode(:window_update, 0, 0, <<0::32>>) + ] + + assert {:error, :invalid_window_update_increment} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + test "rejects data before final response headers" do assert {:error, :data_before_response_headers} = HTTP.HTTP2.stream( diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index ba41ee9..864295a 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -266,10 +266,18 @@ defmodule HTTP.SocketClient do case state.protocol_module.stream(state.protocol, data) do {:ok, protocol, events} -> state = %{state | protocol: protocol} + response_complete? = completed_response_control_writes?(state) case flush_protocol_writes(state) do - {:ok, state} -> handle_events(state, events) - {:error, reason} -> fail(state, reason) + {:ok, state} -> + handle_events(state, events) + + {:error, reason} -> + if reason == :closed and response_complete? and :done in events do + handle_events(state, events) + else + fail(state, reason) + end end {:error, reason} -> @@ -729,6 +737,15 @@ defmodule HTTP.SocketClient do defp flush_protocol_writes(state), do: {:ok, state} + defp completed_response_control_writes?(%{ + protocol_module: HTTP.HTTP2, + protocol: protocol + }) do + HTTP.HTTP2.complete_response?(protocol) and HTTP.HTTP2.outbound_control_only?(protocol) + end + + defp completed_response_control_writes?(_state), do: false + defp flush_protocol_write(state, data) do timeout = remaining_timeout(state.deadline_at) diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index fd9680c..4dfaf2c 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -211,6 +211,229 @@ defmodule HTTP.SocketClientHTTP2Test do assert HTTP.Response.read_all(response) == "forced-ex-ssl-h2" end + test "delivers a complete ex_ssl HTTP/2 response queued before a peer close" do + test_pid = self() + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + receive do + :send_response_and_close -> + send_h2_response(socket, transport, "queued-before-close") + :ok = :ssl.close(socket) + send(test_pid, :server_closed) + end + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) + if Process.alive?(controller), do: HTTP.AbortController.abort(controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_response_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "queued-before-close" + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do + test "fails when a closed ex_ssl HTTP/2 connection still needs request body writes for a #{completion} response" do + test_pid = self() + body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + receive do + :send_response_and_close -> + send_all(socket, transport, [ + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + Frame.encode(:headers, @end_headers, 1, response_headers("complete")), + Frame.encode(:data, unquote(end_stream_flag), 1, "complete") + ]) + + :ssl.close(socket) + send(test_pid, :server_closed) + end + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + if Process.info(owner, :status) == {:status, :suspended}, + do: :erlang.resume_process(owner) + + if Process.alive?(controller), do: HTTP.AbortController.abort(controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_response_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, :closed} = HTTP.Promise.await(promise) + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + end + + test "delivers a complete ex_ssl HTTP/2 response when the peer closes immediately" do + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send_h2_response(socket, transport, "closed-after-response") + end) + + response = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "closed-after-response" + end + + test "delivers a complete large ex_ssl HTTP/2 response without a final window update" do + test_pid = self() + body = :binary.copy("z", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, body) + buffer = assert_settings_ack(socket, transport, buffer) + chunks = chunk_binary(body, 16_384) + last_index = length(chunks) - 1 + + {final_chunk, preceding_chunks} = List.pop_at(chunks, last_index) + + Enum.reduce(preceding_chunks, buffer, fn chunk, buffer -> + send_all(socket, transport, Frame.encode(:data, 0, 1, chunk)) + buffer = assert_window_update(socket, transport, buffer, 0, byte_size(chunk)) + assert_window_update(socket, transport, buffer, 1, byte_size(chunk)) + end) + + send(test_pid, {:server_ready_to_finish, self()}) + + receive do + :send_final_chunk_and_close -> + send_all(socket, transport, Frame.encode(:data, @end_stream, 1, final_chunk)) + :ok = :ssl.close(socket) + send(test_pid, :server_closed) + end + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + response = + HTTP.Promise.await(promise) + + stream_monitor = Process.monitor(response.stream) + reader = Task.async(fn -> HTTP.Response.read_all(response) end) + assert_receive {:server_ready_to_finish, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) + if Process.alive?(controller), do: HTTP.AbortController.abort(controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_final_chunk_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert response.status == 200 + assert Task.await(reader, 5_000) == body + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + test "rejects an ex_ssl HTTP/2 response that closes without END_STREAM" do + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, "truncated") + send_all(socket, transport, Frame.encode(:data, 0, 1, "truncated")) + end) + + assert {:error, :closed} = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + end + test "auto HTTPS falls back to HTTP/1.1 through ex_ssl without ALPN" do url = start_https_h2_server!([], fn socket, _transport -> @@ -346,7 +569,7 @@ defmodule HTTP.SocketClientHTTP2Test do case :ssl.handshake(transport_socket) do {:ok, socket} -> handler.(socket, :ssl) - :ssl.close(socket) + :ok = :ssl.close(socket) {:error, _reason} -> :ok @@ -494,6 +717,71 @@ defmodule HTTP.SocketClientHTTP2Test do :ok = apply(transport, :send, [socket, iodata]) end + defp await_owner(controller) do + case :sys.get_state(controller).request_id do + owner when is_pid(owner) -> owner + nil -> flunk("socket owner was not registered before the request reached the server") + end + end + + defp await_owner_tls_data_and_close(owner) do + await_owner_tls_data_and_close(owner, nil, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_owner_loop(owner) do + await_owner_loop(owner, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_owner_loop(owner, deadline_at) do + if Process.info(owner, :current_function) == + {:current_function, {HTTP.SocketClient, :owner_loop, 1}} and + Process.info(owner, :status) == {:status, :waiting} do + :ok + else + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("socket owner did not return to its receive loop before the TLS close gate") + else + receive do + after + min(10, remaining) -> await_owner_loop(owner, deadline_at) + end + end + end + end + + defp await_owner_tls_data_and_close(owner, tls_pid, deadline_at) do + case Process.info(owner, :messages) do + {:messages, messages} -> + tls_pid = + tls_pid || + Enum.find_value(messages, fn + {:ssl, %SSL.Socket{pid: pid}, _data} -> pid + _message -> nil + end) + + if is_pid(tls_pid) and + Enum.any?(messages, &match?({:ssl_closed, %SSL.Socket{pid: ^tls_pid}}, &1)) do + tls_pid + else + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("socket owner did not receive the queued TLS response and close") + else + receive do + after + min(10, remaining) -> await_owner_tls_data_and_close(owner, tls_pid, deadline_at) + end + end + end + + nil -> + flunk("socket owner exited before receiving the queued TLS response and close") + end + end + defp chunk_binary(binary, size) when byte_size(binary) <= size, do: [binary] defp chunk_binary(binary, size) do diff --git a/docs/pr-14-validation.md b/docs/pr-14-validation.md new file mode 100644 index 0000000..889e366 --- /dev/null +++ b/docs/pr-14-validation.md @@ -0,0 +1,189 @@ +# PR #14 validation + +Base reviewed and fetched: `f5c0fe60393bc8f0570b3f92e04fe45a0aa88ceb` +(`codex/replace-ssl-with-ex-ssl`). The remote PR had no later commits when work +started. Existing uncommitted fixes were preserved and completed. + +Environment: Elixir 1.18.5, Erlang/OTP 28 (ERTS 16.4.0.5), Linux; Go 1.26.6 +for the vendored E2E server (the workflow requires Go 1.22+). + +## Application build failure + +The failure is the child Mix dependency graph and code path, not a stale cache. +The child projects share the umbrella's `../../deps`, `../../_build`, and lock, +but an `in_umbrella` dependency is not recursively traversed in the same way +when running Mix from a child. `http_core` declares `ex_ssl`, while the four +client applications declare `http_core`; their child `mix deps` output lacks +`ex_ssl`. `http_core.app` still lists `ex_ssl` as a runtime application. + +In an isolated detached worktree at the reviewed HEAD, `http_core` child tests +pass but all four client child tests fail with: + +```text +** (Mix) Could not start application ex_ssl: +could not find application file: ex_ssl.app +``` + +Even after the root test-environment compile generates +`_build/test/lib/ex_ssl/ebin/ex_ssl.app`, child execution still fails. Thus +precompiling alone or clearing a cache does not fix the missing code path. +The original Test run 35490860631 and E2E run 35490860636 both show the same +failure, with undefined `SSL` module warnings during child compilation. + +Test and E2E workflows now explicitly prepare `MIX_ENV=test` at the root and +run `mix test apps//test` or `mix test apps//e2e` from the root. +The root `test.e2e` alias also stays at the root. No client duplicates the +`ex_ssl` dependency, disables runtime startup, or suppresses compiler warnings. + +## Dependency and transport contract + +`http_core` now requires `ex_ssl ~> 0.3.0`; the existing lock remains 0.3.0 and +no unrelated dependency was upgraded. The external smoke uses freshly built +package contents in a separate temporary consumer, without the umbrella lock, +deps or build output. Only unpublished internal package resolution uses local +paths; the consumer does not declare `ex_ssl` itself. + +OTP `:ssl` stays the default. `:ex_ssl` is explicitly selected or inherited +from `:http_core` configuration and remains captured across redirects and +EventSource reconnects. No backend fallback was added. The supported paths +are HTTP/1.1, HTTP/2, secure WebSocket and HTTPS EventSource. HTTP/3 and +WebTransport retain QUIC TLS and reject an explicit TCP TLS backend. + +The ex_ssl 0.3.0 integration still requires verified TLS 1.3. TLS 1.2, +`verify_none`, client certificates and unsupported socket options are rejected; +custom profile ALPN must match the requested ALPN list. + +## HTTP/2 response/close ordering + +The parser emits response events and queues receive-window updates for nonempty +DATA, including END_STREAM DATA. SocketClient previously flushed those writes +before delivering the events. After a normal TLS shutdown, the write returned +`:closed` and the client discarded the already-complete response. + +The change preserves that order. A failed write is non-fatal only when all four +conditions hold: HTTP/2 reports completion, this batch contains `:done`, all +queued frames are WINDOW_UPDATE / SETTINGS ACK / PING ACK, and the write error +is exactly `:closed`. Pending request DATA makes the failure fatal, including +when a coalesced WINDOW_UPDATE releases the final upload bytes. Reset and +protocol errors are returned by the parser before this exception is considered. +No `:einval`, `:econnreset`, certificate, cancellation or timeout errors are +ignored. Cleanup and stream delivery use the existing event handlers. + +The new real TLS 1.3/ex_ssl tests establish the following order: + +1. The local server waits on a message gate after receiving the request. +2. The test waits until the socket owner is waiting in `owner_loop/1`, then + suspends it; the separate TLS process keeps running. +3. The server sends complete response frames and successfully closes TLS. +4. The test observes both data and `ssl_closed` queued for the suspended owner + and monitors the TLS process exiting before resuming the owner. +5. The exact body is delivered, and the owner exits normally. For the streaming + case, a concurrent reader handles normal backpressure, the final DATA is + gated separately after earlier frames, and the stream also exits normally. + +The buffered case coalesces SETTINGS/HEADERS/DATA. The 5,000,001-byte streaming +case separates response headers and DATA; its server does not wait for the +final WINDOW_UPDATE before closing. Waiting predicates use bounded polling of +actual process/message state; elapsed sleeps do not establish the ordering. + +With these tests copied into the detached original-HEAD worktree, leaving both +production modules unchanged, the buffered test fails with `{:error, :closed}` +and the streaming test raises `stream read failed: :closed`. The final combined +line-selected baseline run executes both +regressions (17 discovered, 2 failures, 15 excluded by line selection). Both pass +with the fix. The final HTTP/2 file contains 17 tests, including +required upload-write failures with and without END_STREAM, truncation, and +both existing TLS backends. Two additional parser tests preserve errors when +RST_STREAM or an invalid WINDOW_UPDATE follows END_STREAM in the same batch. +The full 17-test HTTP/2 file was also repeated with seeds 1 through 5: all 85 +executions passed, with no excluded tests. Earlier HTTP/1.1, EventSource and +WebSocket close-order regression tests remain unchanged and pass in the full +suite. + +## Final local validation + +All commands below were actually executed from the root unless indicated. +Unit and E2E suites have zero failures and zero skipped tests. Per-app rows +repeat the corresponding root-suite coverage; they are not additional unique +tests. Line-selected red/green runs intentionally exclude other tests. + +| Command | Result | +| --- | --- | +| `mix deps.get`; `MIX_ENV=test mix deps.get` | Passed; lock unchanged | +| `mix format --check-formatted` | Passed, including the new smoke script | +| `mix compile --warnings-as-errors` | Passed | +| `MIX_ENV=test mix compile --warnings-as-errors` | Passed | +| `mix test` | 401 tests + 20 doctests; 0 failures, 0 skipped | +| `MIX_ENV=test mix test apps/http_core/test` | 161 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test` | 157 tests + 20 doctests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/test` | 33 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/test` | 26 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/test` | 24 tests; 0 failures | +| `mix credo` | No issues | +| `mix dialyzer --format github` | Passed; 4 existing intentional ignores, 0 new warnings | +| `MIX_ENV=prod mix hex.build --unpack -o ` in each app | All five packages built, also exercised by smoke | +| `bash scripts/external_consumer_smoke.sh` | Passed; five current packages, eight verified TLS client exchanges | +| `bash -n scripts/external_consumer_smoke.sh` | Passed | +| `go build -o ../test_server/server .` in `apps/http_fetch/priv/test_server` | Passed | +| `MIX_ENV=test mix test apps/http_fetch/e2e` | 50 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/e2e` | 2 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test.e2e` with the Go server running | All 58 E2E tests; 0 failures | + +`http_core` has no E2E directory; the existing workflow conditional reports that +fact rather than skipping an existing suite. The root-scoped commands above +are the commands used by the modified Test and E2E workflows. CI retains the +development compile and also checks test-environment compilation. + +Cold-start validation used detached worktrees under `.trees/` with independent +`deps` and `_build`, without deleting the user's existing build output. The +original root `mix deps.get` followed by child `mix test` reproduced undefined +SSL-module warnings and missing `ex_ssl.app`; cold root test preparation and +root-scoped execution succeeded. The external consumer likewise starts with +an empty temporary dependency/build tree and no repository lockfile on every +invocation. Three consecutive complete consumer smoke runs passed after its +server synchronization was finalized, followed by the final full-validation run. + +The smoke checks actual `hex_metadata.config` requirements, including +`optional: false`, and executes HTTP/1.1, HTTP/2 with ALPN, WSS upgrade/message, +and HTTPS EventSource open/message with both the omitted/default backend and +explicit ex_ssl. WebTransport explicitly rejects a TCP TLS backend and retains +QUIC options under shared ex_ssl configuration. The existing real QUIC E2E +suite also passed; the consumer smoke itself does not create a QUIC session. + +## Remaining limitation and diagnostic failures + +The first external smoke server closed its HTTP/2 TLS connection immediately +without coordinating with the client. Repetition exposed an OTP `:einval` error. +A separate diagnostic loaded `HTTP.SocketClient` from the original f5c0fe6 +worktree (the code path was printed and checked): 100 immediate-close OTP TLS +1.3 requests yielded 69 successes, 26 `:closed`, and 5 `:einval` errors. Thus +this risk predates this fix. This diagnostic is not a substitute for the +synchronized ex_ssl regression tests. + +The general consumer smoke now waits for a SETTINGS ACK before server closure; +it tests installed-package operation. The dedicated close-order regression +still closes before the final client acknowledgement/window update. The +production fix deliberately does not treat `:einval` as a normal close; the +pre-existing OTP immediate-close limitation remains outside this narrow fix. +No dependency source was modified, and no ex_ssl upstream defect was needed +to explain either integration fix. + +Local evidence logs are under `/tmp/http_fetch-pr14-validation/`, with original +workflow logs `/tmp/http_fetch-pr14-old-{test,e2e}.log`, cold reproduction logs +`/tmp/http_fetch-pr14-original-workflow-{deps,child}.log`, and the OTP baseline +diagnostic `/tmp/http_fetch-pr14-baseline-h2-immediate-close.log`. These local +logs are not shipped in the packages. + +The final red/green commands were: + +```bash +# In the detached original-HEAD worktree, after copying only the final test file +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs:214 apps/http_fetch/test/http/socket_client_http2_test.exs:350 --trace +# Expected: 2 failures, 15 excluded (the two close-order regressions) + +# In the fixed working tree; also repeated with --seed 1 through --seed 5 +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs +# 17 tests, 0 failures, 0 excluded +``` diff --git a/mix.exs b/mix.exs index 2319747..67fa1c5 100644 --- a/mix.exs +++ b/mix.exs @@ -54,9 +54,7 @@ defmodule HttpFetch.Umbrella.MixProject do end defp run_e2e_tests([]) do - args = Enum.flat_map(@e2e_apps, &["--app", &1]) ++ ["cmd", "mix", "test.e2e"] - - Mix.Task.run("do", args) + Mix.Task.run("test", Enum.map(@e2e_apps, &"apps/#{&1}/e2e")) end defp run_e2e_tests(args) do diff --git a/scripts/external_consumer_smoke.exs b/scripts/external_consumer_smoke.exs new file mode 100644 index 0000000..3e21bb3 --- /dev/null +++ b/scripts/external_consumer_smoke.exs @@ -0,0 +1,308 @@ +defmodule ExternalConsumerSmoke do + alias HTTP.HTTP2.Frame + alias HTTP.HTTP2.HPACK + + @http1_response "HTTP/1.1 200 OK\r\nContent-Length: 11\r\nConnection: close\r\n\r\nconsumer-ok" + @end_stream 0x1 + @end_headers 0x4 + @guid "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + def run do + for app <- [:http_fetch, :http_web_socket, :http_event_source, :http_web_transport] do + {:ok, _} = Application.ensure_all_started(app) + end + + assert Enum.any?(Application.started_applications(), fn {app, _, _} -> app == :ex_ssl end), + ":ex_ssl was not started through http_core's package dependency" + + certfile = System.fetch_env!("HTTP_FETCH_CERTFILE") + cacertfile = System.fetch_env!("HTTP_FETCH_CACERTFILE") + keyfile = System.fetch_env!("HTTP_FETCH_KEYFILE") + + assert_package_metadata!() + + {:ok, :ssl} = HTTP.TLSBackend.resolve() + fetch_http1!(certfile, keyfile, cacertfile) + fetch_h2!(certfile, keyfile, cacertfile) + web_socket!(certfile, keyfile, cacertfile) + event_source!(certfile, keyfile, cacertfile) + + {:error, :tls_backend_not_supported_for_quic} = + HTTP.WebTransport.new("https://localhost:443/", tls_backend: :ex_ssl) + + Application.put_env(:http_core, :tls_backend, :ex_ssl) + + {:ok, %{backend: HTTP.WebTransport.Transport.QUIC}} = + HTTP.WebTransport.Options.new("https://localhost:443/") + + Application.delete_env(:http_core, :tls_backend) + + IO.puts("external consumer smoke passed") + end + + defp fetch_http1!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, []) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {:ok, _} = recv_headers(socket) + :ok = :ssl.send(socket, @http1_response) + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [ssl: [cacertfile: cacertfile]], + ex_ssl: [tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + response = HTTP.fetch("https://localhost:#{port}/", options) |> HTTP.Promise.await() + + assert response.status == 200 and HTTP.Response.read_all(response) == "consumer-ok", + "HTTP/1.1 #{label} failed" + end + end + + defp assert_package_metadata! do + package_dir = System.fetch_env!("HTTP_FETCH_PACKAGE_DIR") + core = metadata!(package_dir, "http_core") + core_version = Map.fetch!(core, <<"version">>) + + assert requirement!(core, <<"ex_ssl">>) == <<"~> 0.3.0">>, + "http_core package must require ex_ssl ~> 0.3.0" + + for app <- ["http_fetch", "http_web_socket", "http_event_source", "http_web_transport"] do + assert requirement!(metadata!(package_dir, app), <<"http_core">>) == "~> " <> core_version, + "#{app} package must require the built http_core version" + end + end + + defp metadata!(package_dir, app) do + {:ok, entries} = + :file.consult(String.to_charlist(Path.join([package_dir, app, "hex_metadata.config"]))) + + Map.new(entries) + end + + defp requirement!(metadata, name) do + metadata + |> Map.fetch!(<<"requirements">>) + |> Enum.map(&Map.new/1) + |> Enum.find_value(fn requirement -> + if Map.fetch!(requirement, <<"name">>) == name do + assert Map.fetch!(requirement, <<"optional">>) == false, + "#{name} package dependency must not be optional" + + Map.fetch!(requirement, <<"requirement">>) + end + end) + |> case do + nil -> raise "package is missing #{name} requirement" + requirement -> requirement + end + end + + defp fetch_h2!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, alpn_preferred_protocols: [<<"h2">>]) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {preface, buffer} = recv_exact(socket, byte_size(HTTP.HTTP2.connection_preface()), <<>>) + assert preface == HTTP.HTTP2.connection_preface(), "missing HTTP/2 preface" + {_settings, buffer} = recv_frame(socket, buffer) + {%Frame{type: :headers, stream_id: 1}, _buffer} = recv_frame(socket, buffer) + headers = HPACK.encode_headers([{":status", "200"}, {"content-length", "2"}]) + + :ok = + :ssl.send(socket, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, headers), + Frame.encode(:data, @end_stream, 1, "h2") + ]) + + {%Frame{type: :settings, flags: 1, stream_id: 0, payload: ""}, _buffer} = + recv_frame(socket, "") + + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [http_version: :http2, ssl: [cacertfile: cacertfile]], + ex_ssl: [http_version: :http2, tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + response = HTTP.fetch("https://localhost:#{port}/", options) |> HTTP.Promise.await() + + assert response.status == 200 and HTTP.Response.read_all(response) == "h2", + "HTTP/2 #{label} failed" + end + end + + defp web_socket!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, []) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {:ok, request} = recv_headers(socket) + [_, key] = Regex.run(~r/sec-websocket-key:\s*([^\r\n]+)/i, request) + accept = :crypto.hash(:sha, String.trim(key) <> @guid) |> Base.encode64() + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ", + accept, + "\r\n\r\n", + <<0x81, 11, "consumer-ws">> + ]) + + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [ssl: [cacertfile: cacertfile]], + ex_ssl: [tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + socket = HTTP.WebSocket.new("wss://localhost:#{port}/", [], options) + assert match?(%HTTP.WebSocket{}, socket), "WebSocket #{label} did not start" + await_web_socket_open(socket) + await_web_socket_message(socket, "consumer-ws") + end + end + + defp event_source!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, []) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {:ok, _} = recv_headers(socket) + + :ok = + :ssl.send( + socket, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\ndata: consumer-sse\n\n" + ) + + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [ssl: [cacertfile: cacertfile]], + ex_ssl: [tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + source = HTTP.EventSource.new("https://localhost:#{port}/", options) + assert match?(%HTTP.EventSource{}, source), "EventSource #{label} did not start" + await_event_source_open(source) + await_event_source_message(source, "consumer-sse") + :ok = HTTP.EventSource.close(source) + end + end + + defp listen!(certfile, keyfile, extra) do + {:ok, listen} = + :ssl.listen( + 0, + [ + :binary, + active: false, + ip: {127, 0, 0, 1}, + reuseaddr: true, + versions: [:"tlsv1.3"], + certfile: certfile, + keyfile: keyfile + ] ++ extra + ) + + {:ok, {{127, 0, 0, 1}, port}} = :ssl.sockname(listen) + {listen, port} + end + + defp accept!(listen) do + with {:ok, transport} <- :ssl.transport_accept(listen, 5_000), + do: :ssl.handshake(transport, 5_000) + end + + defp recv_headers(socket, buffer \\ "") do + if String.contains?(buffer, "\r\n\r\n"), + do: {:ok, buffer}, + else: + with({:ok, data} <- :ssl.recv(socket, 0, 5_000), do: recv_headers(socket, buffer <> data)) + end + + defp recv_exact(_socket, 0, buffer), do: {"", buffer} + + defp recv_exact(_socket, size, buffer) when byte_size(buffer) >= size do + <> = buffer + {data, rest} + end + + defp recv_exact(socket, size, buffer) do + {:ok, data} = :ssl.recv(socket, 0, 5_000) + recv_exact(socket, size, buffer <> data) + end + + defp recv_frame(socket, buffer) do + case Frame.decode(buffer) do + {:ok, frame, rest} -> + {frame, rest} + + :more -> + {:ok, data} = :ssl.recv(socket, 0, 5_000) + recv_frame(socket, buffer <> data) + end + end + + defp await_web_socket_open(socket) do + receive do + {HTTP.WebSocket, ^socket, %HTTP.WebSocket.Event.Open{}} -> :ok + _ -> await_web_socket_open(socket) + after + 5_000 -> raise "WebSocket did not open" + end + end + + defp await_web_socket_message(socket, expected) do + receive do + {HTTP.WebSocket, ^socket, %HTTP.WebSocket.Event.Message{data: ^expected}} -> :ok + _ -> await_web_socket_message(socket, expected) + after + 5_000 -> raise "WebSocket did not deliver #{expected}" + end + end + + defp await_event_source_open(source) do + receive do + {HTTP.EventSource, ^source, %HTTP.EventSource.Event.Open{}} -> :ok + _ -> await_event_source_open(source) + after + 5_000 -> raise "EventSource did not open" + end + end + + defp await_event_source_message(source, expected) do + receive do + {HTTP.EventSource, ^source, %HTTP.EventSource.Event.Message{data: ^expected}} -> :ok + _ -> await_event_source_message(source, expected) + after + 5_000 -> raise "EventSource did not deliver #{expected}" + end + end + + defp assert(value, message) + defp assert(true, _message), do: :ok + defp assert(false, message), do: raise(message) +end + +ExternalConsumerSmoke.run() diff --git a/scripts/external_consumer_smoke.sh b/scripts/external_consumer_smoke.sh new file mode 100644 index 0000000..d45ecbc --- /dev/null +++ b/scripts/external_consumer_smoke.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT + +package_dir="$work_dir/packages" +consumer_dir="$work_dir/consumer" +mkdir -p "$package_dir" + +apps=(http_core http_fetch http_web_socket http_event_source http_web_transport) + +for app in "${apps[@]}"; do + ( + cd "$repo_root/apps/$app" + MIX_ENV=prod mix hex.build --unpack -o "$package_dir/$app" + ) +done + +mix new "$consumer_dir" --sup >/dev/null + +cat >"$consumer_dir/mix.exs" <<'EOF' +defmodule ExternalConsumer.MixProject do + use Mix.Project + + def project do + [app: :external_consumer, version: "0.1.0", deps: deps()] + end + + def application do + [extra_applications: [:logger, :public_key, :ssl]] + end + + defp deps do + package_dir = System.fetch_env!("HTTP_FETCH_PACKAGE_DIR") + + [ + {:http_core, path: Path.join(package_dir, "http_core")}, + {:http_fetch, path: Path.join(package_dir, "http_fetch")}, + {:http_web_socket, path: Path.join(package_dir, "http_web_socket")}, + {:http_event_source, path: Path.join(package_dir, "http_event_source")}, + {:http_web_transport, path: Path.join(package_dir, "http_web_transport")} + ] + end +end +EOF + +cp "$repo_root/scripts/external_consumer_smoke.exs" "$consumer_dir/smoke.exs" + +( + cd "$consumer_dir" + export HTTP_FETCH_PACKAGE_DIR="$package_dir" + export HTTP_FETCH_CERTFILE="$repo_root/apps/http_fetch/test/support/fixtures/localhost.pem" + export HTTP_FETCH_CACERTFILE="$repo_root/apps/http_fetch/test/support/fixtures/localhost-ca.pem" + export HTTP_FETCH_KEYFILE="$repo_root/apps/http_fetch/test/support/fixtures/localhost.key" + MIX_ENV=prod mix deps.get + MIX_ENV=prod mix deps.tree --only runtime + MIX_ENV=prod mix compile --warnings-as-errors + MIX_ENV=prod mix run smoke.exs +) From a1312cc40c8d6aad2cb60e750bfba84f9b3ac1cf Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Sun, 20 Sep 2026 22:45:40 +0800 Subject: [PATCH 04/17] fix(http2): drain ex_ssl receive buffers after peer close --- CHANGELOG.md | 4 + README.md | 9 +- apps/http_core/lib/http/http2.ex | 4 +- apps/http_core/test/http/http2_test.exs | 22 + apps/http_fetch/lib/http/socket_client.ex | 53 +- .../test/http/socket_client_http2_test.exs | 771 +++++++++++++++++- docs/pr-14-validation.md | 183 ++++- 7 files changed, 1010 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 802452e..d8c5856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 END_STREAM DATA frame and a non-essential WINDOW_UPDATE or acknowledgement returns `:closed`. Incomplete responses, required request writes, and other errors still fail normally. +- Preserve unread ex_ssl TLS records when an HTTP/2 control write fails after + normal peer closure. Drain them through the existing active-once receiver and + require a complete HTTP/2 response, without extending deadlines or accepting + failed uploads, truncated responses, resets, or protocol errors. ## [0.11.0] - 2026-07-04 diff --git a/README.md b/README.md index dc84902..7d3cf7e 100644 --- a/README.md +++ b/README.md @@ -167,9 +167,12 @@ backend automatically. A complete HTTP/2 response remains deliverable if the peer closes before the client can write its remaining WINDOW_UPDATE or acknowledgement frames. This -only applies to `:closed` after the parser reports completion and the pending -writes contain no request DATA. Truncated responses and other write failures -remain errors; flow-control and streaming delivery order are unchanged. +also covers responses buffered across multiple TLS records by `ex_ssl` after a +normal peer shutdown: the client drains the receive side before deciding whether +the response completed. Only `:closed` on optional control writes qualifies, +and the request body must already be fully sent. END_STREAM is still required; +truncation, required request writes, abnormal closure, cancellation and timeout +remain errors. The original deadline and streaming backpressure are preserved. For `:ex_ssl`, `socket_opts` accepts `send_timeout` and `send_timeout_close: true`. These override matching entries in `ssl`. Custom diff --git a/apps/http_core/lib/http/http2.ex b/apps/http_core/lib/http/http2.ex index 0ce3900..9b6c0af 100644 --- a/apps/http_core/lib/http/http2.ex +++ b/apps/http_core/lib/http/http2.ex @@ -102,10 +102,12 @@ defmodule HTTP.HTTP2 do def complete_response?(%__MODULE__{done?: done?}), do: done? @spec outbound_control_only?(t()) :: boolean() - def outbound_control_only?(%__MODULE__{outbound: outbound}) do + def outbound_control_only?(%__MODULE__{outbound: outbound, pending_body: ""}) do Enum.all?(outbound, &control_frame?/1) end + def outbound_control_only?(%__MODULE__{}), do: false + defp append_buffer(%__MODULE__{buffer: buffer} = conn, data) do %{conn | buffer: buffer <> data} end diff --git a/apps/http_core/test/http/http2_test.exs b/apps/http_core/test/http/http2_test.exs index 8355d53..8aa63a5 100644 --- a/apps/http_core/test/http/http2_test.exs +++ b/apps/http_core/test/http/http2_test.exs @@ -12,6 +12,28 @@ defmodule HTTP.HTTP2Test do @initial_window_size 65_535 @max_window_size 2_147_483_647 + describe "outbound_control_only?/1" do + test "only permits acknowledgements and window updates after the upload is sent" do + controls = [ + Frame.encode(:settings, @ack, 0, ""), + Frame.encode(:ping, @ack, 0, "12345678"), + Frame.encode(:window_update, 0, 1, <<0::1, 1::31>>) + ] + + conn = %HTTP.HTTP2{outbound: controls} + assert HTTP.HTTP2.outbound_control_only?(conn) + refute HTTP.HTTP2.outbound_control_only?(%{conn | pending_body: "unsent"}) + + for required <- [ + Frame.encode(:data, @end_stream, 1, "upload"), + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:ping, 0, 0, "12345678") + ] do + refute HTTP.HTTP2.outbound_control_only?(%{conn | outbound: [required | controls]}) + end + end + end + describe "serialize_request/1" do test "serializes the connection preface, settings, and request headers" do request = %HTTP.Request{ diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index 864295a..aea64bf 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -266,18 +266,14 @@ defmodule HTTP.SocketClient do case state.protocol_module.stream(state.protocol, data) do {:ok, protocol, events} -> state = %{state | protocol: protocol} - response_complete? = completed_response_control_writes?(state) + discard_closed_controls? = discard_closed_control_writes?(state, events) - case flush_protocol_writes(state) do + case flush_protocol_writes(state, discard_closed_controls?) do {:ok, state} -> handle_events(state, events) {:error, reason} -> - if reason == :closed and response_complete? and :done in events do - handle_events(state, events) - else - fail(state, reason) - end + fail(state, reason) end {:error, reason} -> @@ -627,7 +623,9 @@ defmodule HTTP.SocketClient do {:ok, pid} -> receive do {:send_result, ^ref, result} -> - close_on_error(transport, socket, result) + # The caller owns cleanup. A failed control write can leave readable + # TLS data behind, so sending must not destroy the receive side. + result :abort -> transport.close(socket) @@ -651,13 +649,6 @@ defmodule HTTP.SocketClient do end end - defp close_on_error(_transport, _socket, :ok), do: :ok - - defp close_on_error(transport, socket, {:error, reason}) do - transport.close(socket) - {:error, reason} - end - defp send_prepared_request(transport, socket, {:buffer, iodata}, deadline_at) do send_request(transport, socket, iodata, remaining_timeout(deadline_at)) end @@ -725,32 +716,44 @@ defmodule HTTP.SocketClient do defp ack_stream_chunk(_stream, nil), do: :ok defp ack_stream_chunk(stream, ack_ref), do: send(stream, {:stream_chunk_ack, ack_ref}) - defp flush_protocol_writes(%{protocol_module: HTTP.HTTP2, protocol: protocol} = state) do + defp flush_protocol_writes( + %{protocol_module: HTTP.HTTP2, protocol: protocol} = state, + discard_closed_controls? + ) do {protocol, iodata} = HTTP.HTTP2.take_outbound(protocol) state = %{state | protocol: protocol} case IO.iodata_to_binary(iodata) do "" -> {:ok, state} - data -> flush_protocol_write(state, data) + data -> flush_protocol_write(state, data, discard_closed_controls?) end end - defp flush_protocol_writes(state), do: {:ok, state} + defp flush_protocol_writes(state, _discard_closed_controls?), do: {:ok, state} - defp completed_response_control_writes?(%{ - protocol_module: HTTP.HTTP2, - protocol: protocol - }) do - HTTP.HTTP2.complete_response?(protocol) and HTTP.HTTP2.outbound_control_only?(protocol) + defp discard_closed_control_writes?( + %{protocol_module: HTTP.HTTP2, protocol: protocol, transport: transport}, + events + ) do + # Classify before take_outbound/1 clears the queue, including any request + # body still waiting for flow-control credit. In ex_ssl 0.3.0 an established + # socket's peer close_notify rejects writes with :closed while retaining + # unread plaintext; abnormal TCP closure returns :econnreset instead. + # This owner has not closed the socket locally. Rearming it drains that + # plaintext (or reports EOF); only the HTTP parser can complete a response. + HTTP.HTTP2.outbound_control_only?(protocol) and + (transport == HTTP.Transport.ExSSL or + (HTTP.HTTP2.complete_response?(protocol) and :done in events)) end - defp completed_response_control_writes?(_state), do: false + defp discard_closed_control_writes?(_state, _events), do: false - defp flush_protocol_write(state, data) do + defp flush_protocol_write(state, data, discard_closed_controls?) do timeout = remaining_timeout(state.deadline_at) case send_request(state.transport, state.socket, data, timeout) do :ok -> {:ok, state} + {:error, :closed} when discard_closed_controls? -> {:ok, state} {:error, reason} -> {:error, reason} end end diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index 4dfaf2c..be0f882 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -242,8 +242,7 @@ defmodule HTTP.SocketClientHTTP2Test do await_owner_loop(owner) on_exit(fn -> - if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) - if Process.alive?(controller), do: HTTP.AbortController.abort(controller) + cleanup_owner(owner, controller) end) true = :erlang.suspend_process(owner) @@ -261,6 +260,539 @@ defmodule HTTP.SocketClientHTTP2Test do assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 end + @tag :cross_record + test "drains a second ex_ssl TLS record after control writes fail following peer close" do + test_pid = self() + body = "second-record-body" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "drains a split HTTP/2 frame from the ex_ssl receive buffer after peer close" do + test_pid = self() + body = "split-frame-body" + headers = Frame.encode(:headers, @end_headers, 1, response_headers(body)) + split_at = 5 + <> = headers + + first_record = [Frame.encode(:settings, 0, 0, ""), headers_start] + first_record_binary = IO.iodata_to_binary(first_record) + second_record = [headers_rest, Frame.encode(:data, @end_stream, 1, body)] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "rejects a truncated second ex_ssl TLS record after a control write fails" do + test_pid = self() + body = "truncated-second-record" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, 0, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, :closed} = HTTP.Promise.await(promise) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "does not turn a reset buffered after peer close into a successful response" do + test_pid = self() + body = "reset-after-drain" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + + second_record = [ + Frame.encode(:data, @end_stream, 1, body), + Frame.encode(:rst_stream, 0, 1, <<0x8::32>>) + ] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, {:stream_reset, :cancel}} = HTTP.Promise.await(promise) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "fails when SETTINGS acknowledgement cannot flush a pending HTTP/2 request body" do + test_pid = self() + body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_response_and_close) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers("complete")), + Frame.encode(:data, @end_stream, 1, "complete") + ]) + + :ok = :ssl.close(socket) + send(test_pid, :server_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_response_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, :closed} = + HTTP.Promise.await(promise) + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "drains a final streaming HTTP/2 frame buffered by ex_ssl after peer close" do + test_pid = self() + body = :binary.copy("q", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, body) + buffer = assert_settings_ack(socket, transport, buffer) + chunks = chunk_binary(body, 16_384) + preceding_chunks = Enum.drop(chunks, -3) + [first_final_chunk, second_final_chunk, third_final_chunk] = Enum.take(chunks, -3) + + Enum.reduce(preceding_chunks, buffer, fn chunk, current_buffer -> + send_all(socket, transport, Frame.encode(:data, 0, 1, chunk)) + + current_buffer = + assert_window_update(socket, transport, current_buffer, 0, byte_size(chunk)) + + assert_window_update(socket, transport, current_buffer, 1, byte_size(chunk)) + end) + + first_final_frame = Frame.encode(:data, 0, 1, first_final_chunk) + split_at = 8_000 + <> = first_final_frame + + first_record = [Frame.encode(:ping, 0, 0, "final123"), first_frame_start] + + second_record = [ + first_frame_rest, + Frame.encode(:data, 0, 1, second_final_chunk), + Frame.encode(:data, @end_stream, 1, third_final_chunk) + ] + + send( + test_pid, + {:server_ready_to_finish, self(), IO.iodata_to_binary(first_record), second_record} + ) + + await_test_gate(:send_final_first_record) + send_all(socket, transport, first_record) + send(test_pid, :final_first_record_sent) + + await_test_gate(:send_final_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :final_second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + stream_monitor = monitor_test_process(response.stream) + reader = Task.async(fn -> HTTP.Response.read_all(response) end) + monitor_test_process(reader.pid) + + assert_receive {:server_ready_to_finish, server_pid, first_record, second_record}, 10_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_final_first_record) + assert_receive :final_first_record_sent, 5_000 + + {tls_pid, ^first_record} = await_owner_tls_data(owner, first_record) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_final_second_record_and_close) + assert_receive :final_second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert Task.await(reader, 10_000) == body + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "aborts an ex_ssl cross-record drain while stream backpressure holds the final body" do + {server_pid, controller, promise, owner, first_record, second_record} = + cross_record_stream_drain_fixture(self(), 5_000) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + {tls_pid, owner_monitor, tls_monitor} = + queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream = response.stream + stream_monitor = monitor_test_process(stream) + + holder = hold_cross_record_stream_chunk(stream, self()) + monitor_test_process(holder) + + assert_receive {:held_cross_record_stream_chunk, ^holder, ^stream, _chunk, _ack_ref}, + 5_000 + + await_cross_record_stream_backpressure(owner) + + :ok = HTTP.AbortController.abort(controller) + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + + reader = + Task.Supervisor.async_nolink(:http_fetch_task_supervisor, fn -> + assert_raise RuntimeError, "stream read failed: :aborted", fn -> + HTTP.Response.read_all(response) + end + end) + + monitor_test_process(reader.pid) + await_cross_record_read_all(reader.pid) + send(holder, :release_cross_record_stream_chunk) + + assert %RuntimeError{message: "stream read failed: :aborted"} = Task.await(reader, 5_000) + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + end + + @tag :cross_record + test "keeps the original deadline while an ex_ssl cross-record drain is backpressured" do + timeout = 2_000 + started_at = System.monotonic_time(:millisecond) + + {server_pid, controller, promise, owner, first_record, second_record} = + cross_record_stream_drain_fixture(self(), timeout) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + # This explicit timer gate consumes half the request budget before entering + # the drain. A drain that reset the request timeout would outlive the + # original absolute deadline asserted below. + Process.send_after(self(), :begin_cross_record_deadline_drain, div(timeout, 2)) + assert_receive :begin_cross_record_deadline_drain, div(timeout, 2) + 250 + + {tls_pid, owner_monitor, tls_monitor} = + queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream = response.stream + stream_monitor = monitor_test_process(stream) + + holder = hold_cross_record_stream_chunk(stream, self()) + monitor_test_process(holder) + + assert_receive {:held_cross_record_stream_chunk, ^holder, ^stream, _chunk, _ack_ref}, + 5_000 + + await_cross_record_stream_backpressure(owner) + + deadline_at = started_at + timeout + remaining = deadline_at - System.monotonic_time(:millisecond) + assert remaining > 0 + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, remaining + 400 + + reader = + Task.Supervisor.async_nolink(:http_fetch_task_supervisor, fn -> + assert_raise RuntimeError, ~r/^stream read failed: :(request_timeout|timeout)$/, fn -> + HTTP.Response.read_all(response) + end + end) + + monitor_test_process(reader.pid) + await_cross_record_read_all(reader.pid) + send(holder, :release_cross_record_stream_chunk) + + assert %RuntimeError{message: message} = Task.await(reader, 5_000) + assert message in ["stream read failed: :request_timeout", "stream read failed: :timeout"] + + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + end + for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do test "fails when a closed ex_ssl HTTP/2 connection still needs request body writes for a #{completion} response" do test_pid = self() @@ -717,6 +1249,32 @@ defmodule HTTP.SocketClientHTTP2Test do :ok = apply(transport, :send, [socket, iodata]) end + defp await_test_gate(gate) do + receive do + ^gate -> :ok + after + 5_000 -> exit({:test_gate_timeout, gate}) + end + end + + defp monitor_test_process(pid) do + on_exit(fn -> + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + Process.monitor(pid) + end + + defp cleanup_owner(owner, controller) do + if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) + + if Process.alive?(controller) do + HTTP.AbortController.abort(controller) + else + if Process.alive?(owner), do: send(owner, :abort) + end + end + defp await_owner(controller) do case :sys.get_state(controller).request_id do owner when is_pid(owner) -> owner @@ -728,6 +1286,75 @@ defmodule HTTP.SocketClientHTTP2Test do await_owner_tls_data_and_close(owner, nil, System.monotonic_time(:millisecond) + 5_000) end + defp await_owner_tls_data(owner, expected_data) do + await_owner_tls_data(owner, expected_data, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_owner_tls_data(owner, expected_data, deadline_at) do + case Process.info(owner, :messages) do + {:messages, messages} -> + case Enum.find(messages, fn + {:ssl, %SSL.Socket{}, ^expected_data} -> true + _message -> false + end) do + {:ssl, %SSL.Socket{pid: tls_pid}, ^expected_data} -> {tls_pid, expected_data} + nil -> await_owner_tls_data_or_fail(owner, expected_data, deadline_at) + end + + nil -> + flunk("socket owner exited before receiving the first TLS record") + end + end + + defp await_owner_tls_data_or_fail(owner, expected_data, deadline_at) do + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("socket owner did not receive the first TLS record as one active-once delivery") + else + receive do + after + min(10, remaining) -> await_owner_tls_data(owner, expected_data, deadline_at) + end + end + end + + defp assert_owner_has_only_tls_data(owner, tls_pid, expected_count) do + {:messages, messages} = Process.info(owner, :messages) + + assert messages + |> Enum.count(&match?({:ssl, %SSL.Socket{pid: ^tls_pid}, _data}, &1)) == expected_count + end + + defp assert_tls_buffered_after_peer_close(tls_pid, minimum_size) do + assert_tls_buffered_after_peer_close( + tls_pid, + minimum_size, + System.monotonic_time(:millisecond) + 5_000 + ) + end + + defp assert_tls_buffered_after_peer_close(tls_pid, minimum_size, deadline_at) do + {_phase, state} = :sys.get_state(tls_pid) + + if state.closed and state.size >= minimum_size do + assert state.active == false + :ok + else + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("second TLS record was not retained in the TLS receive buffer after peer close") + else + receive do + after + min(10, remaining) -> + assert_tls_buffered_after_peer_close(tls_pid, minimum_size, deadline_at) + end + end + end + end + defp await_owner_loop(owner) do await_owner_loop(owner, System.monotonic_time(:millisecond) + 5_000) end @@ -782,6 +1409,146 @@ defmodule HTTP.SocketClientHTTP2Test do end end + defp cross_record_stream_drain_fixture(test_pid, timeout) do + body = "backpressured-final-record" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode( + :headers, + @end_headers, + 1, + HPACK.encode_headers([{":status", "200"}, {"x-protocol", "h2"}]) + ) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:cross_record_stream_server_ready, self()}) + + await_test_gate(:send_cross_record_stream_first) + send_all(socket, transport, first_record) + send(test_pid, :cross_record_stream_first_sent) + + await_test_gate(:send_cross_record_stream_second) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :cross_record_stream_second_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + timeout: timeout, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:cross_record_stream_server_ready, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + {server_pid, controller, promise, owner, first_record_binary, second_record} + end + + defp queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) do + true = :erlang.suspend_process(owner) + send(server_pid, :send_cross_record_stream_first) + assert_receive :cross_record_stream_first_sent, 5_000 + + {tls_pid, ^first_record} = await_owner_tls_data(owner, first_record) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_cross_record_stream_second) + assert_receive :cross_record_stream_second_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + owner_monitor = Process.monitor(owner) + tls_monitor = Process.monitor(tls_pid) + true = :erlang.resume_process(owner) + {tls_pid, owner_monitor, tls_monitor} + end + + defp hold_cross_record_stream_chunk(stream, test_pid) do + spawn(fn -> + send(stream, {:read_chunk, self(), :ack}) + + receive do + {:stream_chunk, ^stream, chunk, ack_ref} -> + send(test_pid, {:held_cross_record_stream_chunk, self(), stream, chunk, ack_ref}) + + receive do + :release_cross_record_stream_chunk -> + send(stream, {:stream_chunk_ack, ack_ref}) + after + 5_000 -> + send(test_pid, {:held_cross_record_stream_chunk_timeout, self()}) + end + after + 5_000 -> + send(test_pid, {:held_cross_record_stream_chunk_timeout, self()}) + end + end) + end + + defp await_cross_record_stream_backpressure(owner) do + await_cross_record_stream_backpressure(owner, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_cross_record_stream_backpressure(owner, deadline_at) do + if Process.info(owner, :current_function) == {:current_function, {HTTP.Stream, :chunk, 3}} and + Process.info(owner, :status) == {:status, :waiting} do + :ok + else + await_cross_record_condition( + owner, + deadline_at, + "socket owner did not block on stream backpressure", + fn -> await_cross_record_stream_backpressure(owner, deadline_at) end + ) + end + end + + defp await_cross_record_read_all(reader) do + await_cross_record_read_all(reader, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_cross_record_read_all(reader, deadline_at) do + if Process.info(reader, :current_function) == + {:current_function, {HTTP.Response, :collect_stream, 2}} and + Process.info(reader, :status) == {:status, :waiting} do + :ok + else + await_cross_record_condition( + reader, + deadline_at, + "response reader did not wait for stream error", + fn -> await_cross_record_read_all(reader, deadline_at) end + ) + end + end + + defp await_cross_record_condition(_pid, deadline_at, message, fun) do + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk(message) + else + receive do + after + min(10, remaining) -> fun.() + end + end + end + defp chunk_binary(binary, size) when byte_size(binary) <= size, do: [binary] defp chunk_binary(binary, size) do diff --git a/docs/pr-14-validation.md b/docs/pr-14-validation.md index 889e366..ebf9356 100644 --- a/docs/pr-14-validation.md +++ b/docs/pr-14-validation.md @@ -1,5 +1,178 @@ # PR #14 validation +## Follow-up: response data across TLS records + +Reviewed baseline and remote PR HEAD: `010b0b850745b43faab73093849d3fb6a1dc6650`. +The branch had no later commits when this follow-up started. The earlier fixes +below remain intact. Validation uses Elixir 1.18.5 / OTP 28, as required by CI. + +### Reproduction and cause + +The new test runs through `HTTP.fetch` with a trusted local TLS 1.3 server and +locked ex_ssl 0.3.0. It suspends the HTTP owner only after it reaches its receive +loop. The server sends SETTINGS/HEADERS and waits at a second message gate. +The test verifies that exactly this plaintext is the owner's sole TLS data +message, consuming `active: :once`, before releasing the second server write. +The server then sends DATA/END_STREAM and closes TLS without waiting for an ACK. +A bounded test-only probe checks only `closed`, `active`, and plaintext buffer +size in the TLS process: closed is true, active is false, and the second batch +is still buffered. The owner still has only the first data message. TLS process +exit and `ssl_closed` are deliberately **not** prerequisites for resuming it. + +The test was copied into `.trees/r2-baseline`, a detached checkout at exactly +`010b0b8` with independently downloaded dependencies and compiled test output. +Production files there were unchanged. This actual command failed: + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs:264 --seed 0 +# 18 tests discovered, 1 failure, 17 excluded by line selection +# HTTP.Promise.await returned {:error, :closed}, not the expected response. +``` + +With the fix, the same initial test and command passed: 18 discovered, 0 failures, +17 excluded. A separate control-classification regression, copied unchanged into +the same baseline and selected with `apps/http_core/test/http/http2_test.exs:15`, +also failed there (19 discovered, 1 failure, 18 excluded): an unsent `pending_body` +was previously classified as optional controls. The fixed full parser file passed +all 19 tests. Evidence: `/tmp/http_fetch-pr14-r2/010b0b8-cross-record-red.log` +and `cross-record-first-green.log`. Later coverage is listed separately below. + +The first batch creates a SETTINGS ACK but no `:done`. After peer close_notify, +`SSL.send` returns `:closed`; `send_request/4 -> close_on_error/3` then explicitly +closed the TLS socket before the caller could classify the error. That destroyed +the still-unconsumed second record. The previous complete-batch exception could +not help: the HTTP parser had not yet seen END_STREAM. + +Read-only inspection of the locked dependency confirms its contract: +`active: :once` consumes one delivery; peer close_notify rejects new writes but +retains plaintext until subsequent `setopts(active: :once)` or `recv` drains it. +A raw TCP close without close_notify is `:econnreset`; fatal TLS alerts remain +TLS errors. Explicit `SSL.close` terminates the buffered receive side. The +public API has no connection-state query, and production uses no private TLS +state. Test probes never print TLS keys or complete internal state. No dependency +source or lockfile was changed; this is an http_fetch integration bug. + +### Fix and safety boundaries + +Send results now reach the caller without implicit socket destruction. Initial +request/upload failures still return to the owner's cleanup; failed required +protocol writes still use `fail -> finish -> cleanup`. Cancellation, deadlines, +worker-start failure and send timeout retain their termination paths. + +Before removing outbound frames, the client checks the actual queue for only +WINDOW_UPDATE, SETTINGS ACK or PING ACK, **and** checks that `pending_body` is +empty. Thus both already-queued request DATA and uploads waiting for more window +credit prevent the exception. Only ex_ssl's established, exclusively owned +socket can use `:closed` to continue an incomplete response: this owner has not +locally closed it, and that backend distinguishes abnormal closure. Other +transports retain the previous completed-response/current-`:done` restriction. + +An optional-control `:closed` clears that attempted queue once and runs the +existing event handlers and active-once rearm. It does not report success. +Each further attempt requires newly received data; there is no retry loop or +new timer. Stream chunk acknowledgements retain backpressure and the original +absolute request deadline remains in force. EOF without END_STREAM fails through +`HTTP2.close/1`; resets and protocol errors still fail in the parser. No `:einval`, +`:econnreset`, TLS error, required write error, cancellation or timeout is ignored. + +### Final regression coverage and baseline comparison + +Eight new integration tests use the `:cross_record` tag. They cover: + +- Small buffered response with SETTINGS/HEADERS in the first delivery and + DATA/END_STREAM still inside ex_ssl after peer close. +- A HEADERS frame header split across deliveries while SETTINGS ACK is pending. +- The same receive-buffer sequence without END_STREAM: still `:closed`. +- DATA/END_STREAM followed by RST_STREAM in the later buffered batch: still + `{:stream_reset, :cancel}`, never a successful response. +- A complete response with only optional frames queued but an upload still + waiting for window credit: still fails. Its fixture now proves closure before + resuming the owner; an initial version exposed a fixture race in seed 202 and + was corrected rather than weakening the expected error. +- A 5,000,001-byte stream whose final three DATA frames have not been consumed + when the owner pauses. PING plus part of a DATA payload is the first delivery; + the rest and two further DATA frames remain in TLS buffers before resume. + Exact body equality and normal stream/TLS/owner exits are checked. +- Cancellation while the drained final chunk is held by a reader acknowledgement + barrier and the owner is demonstrably waiting in `HTTP.Stream.chunk/3`. +- The original deadline under the same backpressure. A timer gate consumes half + a 2-second request budget before entering drain; owner termination must occur + by the original monotonic deadline plus 400ms, not a restarted deadline. + +Server gates and state probes have timeouts. Failure cleanup resumes/aborts the +owner and releases test stream/reader processes. Successful and error scenarios +monitor the owner and TLS process; streaming cases also monitor the stream. +The existing complete-batch regressions and required-upload DATA failures with +and without END_STREAM remain in the file, alongside both TLS backends. + +After completing the tests, the final test file was copied back into the same +`010b0b8` checkout, still without any production changes: + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --only cross_record --seed 1 +# Baseline: 25 discovered, 8 executed, 7 failures, 17 excluded. +# The truncated-response rejection passes; the other seven regressions fail. +``` + +The full log is `010b0b8-final-cross-record-red.log`. The fixed tree passes all +8 selected tests with seeds 1–5 (40 executions). The **entire** HTTP/2 file was +also run with seeds 1–5, with no filtering: 25 tests per run, 125 executions, +0 failures, 0 skipped or excluded. Each full result is in `http2-seed-N.log`. + +### Follow-up local validation results + +All commands below ran from the umbrella root unless stated. These results are +from the follow-up working tree, not the previous commit's CI. Unit and E2E +suites report 0 failures and 0 skipped tests. Scoped suites repeat root coverage. + +| Command | Actual result | +| --- | --- | +| `mix deps.get`; `MIX_ENV=test mix deps.get` | Passed; `mix.lock` unchanged | +| `mix format --check-formatted` | Passed | +| `mix compile --warnings-as-errors` | Passed | +| `MIX_ENV=test mix compile --warnings-as-errors` | Passed | +| `mix test` | 410 tests + 20 doctests; 0 failures | +| `MIX_ENV=test mix test apps/http_core/test` | 162 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test` | 165 tests + 20 doctests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/test` | 33 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/test` | 26 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/test` | 24 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --seed N`, N=1..5 | Each: 25 tests, 0 failures | +| `mix credo` | No issues | +| `mix dialyzer --format github` | Passed; 4 existing intentional ignores, no new warnings | +| `bash scripts/external_consumer_smoke.sh` | Passed; five newly built packages, isolated dependency/build tree, eight trusted TLS exchanges and QUIC boundary checks | +| `go build -o ../test_server/server .` in `apps/http_fetch/priv/test_server` | Passed | +| `MIX_ENV=test mix test apps/http_fetch/e2e` | 50 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/e2e` | 2 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test.e2e` | 58 tests; 0 failures | + +For E2E, the built Go server was started, its reported port set in +`E2E_BASE_URL=http://127.0.0.1:`, and the server terminated afterward. +`http_core` has no E2E directory. The package smoke builds all five packages +with `mix hex.build` before starting a consumer that shares neither repository +`deps`/`_build` nor its lockfile and declares no extra ex_ssl dependency. +The detached baseline also started with independent empty build/dependency +folders; no user build files were deleted. + +No requested local check remains unexecuted. Evidence and the command/exit-code +manifest are under `/tmp/http_fetch-pr14-r2/` (`final-results.json`, `final-*.log`). +Remote checks for `010b0b8` are historical and are not follow-up validation. + +### Preserved limitations + +The pre-existing OTP immediate-close `:einval` limitation described below is +unchanged; no new error whitelist or TLS fallback was introduced. Existing +stream deadline expiry can report either `:request_timeout` (owner timer) or +`:timeout` (the chunk acknowledgement wait reaches the same absolute deadline). +The deadline regression accepts exactly these existing timeout errors, never +`:closed` or success, and does not normalize production error semantics. +The bounded test-only buffer probe intentionally targets the locked ex_ssl +0.3.0 state layout; production code uses only its public transport operations. + +## Earlier round (historical evidence at 010b0b8) + Base reviewed and fetched: `f5c0fe60393bc8f0570b3f92e04fe45a0aa88ceb` (`codex/replace-ssl-with-ex-ssl`). The remote PR had no later commits when work started. Existing uncommitted fixes were preserved and completed. @@ -7,7 +180,7 @@ started. Existing uncommitted fixes were preserved and completed. Environment: Elixir 1.18.5, Erlang/OTP 28 (ERTS 16.4.0.5), Linux; Go 1.26.6 for the vendored E2E server (the workflow requires Go 1.22+). -## Application build failure +### Application build failure The failure is the child Mix dependency graph and code path, not a stale cache. The child projects share the umbrella's `../../deps`, `../../_build`, and lock, @@ -35,7 +208,7 @@ run `mix test apps//test` or `mix test apps//e2e` from the root. The root `test.e2e` alias also stays at the root. No client duplicates the `ex_ssl` dependency, disables runtime startup, or suppresses compiler warnings. -## Dependency and transport contract +### Dependency and transport contract `http_core` now requires `ex_ssl ~> 0.3.0`; the existing lock remains 0.3.0 and no unrelated dependency was upgraded. The external smoke uses freshly built @@ -53,7 +226,7 @@ The ex_ssl 0.3.0 integration still requires verified TLS 1.3. TLS 1.2, `verify_none`, client certificates and unsupported socket options are rejected; custom profile ALPN must match the requested ALPN list. -## HTTP/2 response/close ordering +### HTTP/2 response/close ordering The parser emits response events and queues receive-window updates for nonempty DATA, including END_STREAM DATA. SocketClient previously flushed those writes @@ -100,7 +273,7 @@ executions passed, with no excluded tests. Earlier HTTP/1.1, EventSource and WebSocket close-order regression tests remain unchanged and pass in the full suite. -## Final local validation +### Final local validation All commands below were actually executed from the root unless indicated. Unit and E2E suites have zero failures and zero skipped tests. Per-app rows @@ -152,7 +325,7 @@ explicit ex_ssl. WebTransport explicitly rejects a TCP TLS backend and retains QUIC options under shared ex_ssl configuration. The existing real QUIC E2E suite also passed; the consumer smoke itself does not create a QUIC session. -## Remaining limitation and diagnostic failures +### Remaining limitation and diagnostic failures The first external smoke server closed its HTTP/2 TLS connection immediately without coordinating with the client. Repetition exposed an OTP `:einval` error. From 690258ac38e50b0d1a968d9d5e510c560f45f5d4 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Mon, 21 Sep 2026 00:23:54 +0800 Subject: [PATCH 05/17] fix(http2): preserve complete early responses during uploads --- CHANGELOG.md | 10 +- README.md | 14 +- apps/http_core/lib/http/http2.ex | 50 ++- apps/http_core/test/http/http2_test.exs | 202 +++++++++- apps/http_fetch/lib/http/socket_client.ex | 21 +- .../test/http/socket_client_http2_test.exs | 378 +++++++++++++++++- docs/pr-14-validation.md | 188 ++++++++- 7 files changed, 813 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8c5856..79f6eb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,12 +20,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Deliver complete HTTP/2 responses when the peer closes immediately after an END_STREAM DATA frame and a non-essential WINDOW_UPDATE or acknowledgement - returns `:closed`. Incomplete responses, required request writes, and other - errors still fail normally. + returns `:closed`. Incomplete responses and required request writes before + response completion still fail normally. - Preserve unread ex_ssl TLS records when an HTTP/2 control write fails after normal peer closure. Drain them through the existing active-once receiver and require a complete HTTP/2 response, without extending deadlines or accepting - failed uploads, truncated responses, resets, or protocol errors. + truncated responses, error resets, or protocol errors. +- Preserve valid HTTP/2 early final responses while cancelling remaining upload + DATA, including queued DATA released by WINDOW_UPDATE. Accept NO_ERROR resets + only after response completion; require END_STREAM and a complete field block + even for responses with no body. ## [0.11.0] - 2026-07-04 diff --git a/README.md b/README.md index 7d3cf7e..62f25ac 100644 --- a/README.md +++ b/README.md @@ -169,10 +169,16 @@ A complete HTTP/2 response remains deliverable if the peer closes before the client can write its remaining WINDOW_UPDATE or acknowledgement frames. This also covers responses buffered across multiple TLS records by `ex_ssl` after a normal peer shutdown: the client drains the receive side before deciding whether -the response completed. Only `:closed` on optional control writes qualifies, -and the request body must already be fully sent. END_STREAM is still required; -truncation, required request writes, abnormal closure, cancellation and timeout -remain errors. The original deadline and streaming backpressure are preserved. +the response completed. Only `:closed` on optional control writes qualifies. +A complete early response (such as 413) stops the remaining upload, including +request DATA queued by WINDOW_UPDATE in the same batch. It also survives a +subsequent RST_STREAM(NO_ERROR), as required by +[RFC 9113 §8.1](https://www.rfc-editor.org/rfc/rfc9113.html#section-8.1). +Completion requires END_STREAM and the complete HEADERS/CONTINUATION field +block; an unfinished upload neither proves nor prevents response completion. +Truncation, required writes before completion, abnormal closure, cancellation +and timeout remain errors. The original deadline and streaming backpressure +are preserved. For `:ex_ssl`, `socket_opts` accepts `send_timeout` and `send_timeout_close: true`. These override matching entries in `ssl`. Custom diff --git a/apps/http_core/lib/http/http2.ex b/apps/http_core/lib/http/http2.ex index 9b6c0af..0a0f29e 100644 --- a/apps/http_core/lib/http/http2.ex +++ b/apps/http_core/lib/http/http2.ex @@ -34,6 +34,7 @@ defmodule HTTP.HTTP2 do max_frame_size: @initial_max_frame_size, pending_body: "", outbound: [], + request_stopped?: false, done?: false @type event :: {:headers, non_neg_integer(), Headers.t()} | {:body, binary()} | :done @@ -101,13 +102,24 @@ defmodule HTTP.HTTP2 do @spec complete_response?(t()) :: boolean() def complete_response?(%__MODULE__{done?: done?}), do: done? + @spec request_stopped?(t()) :: boolean() + def request_stopped?(%__MODULE__{request_stopped?: stopped?}), do: stopped? + + @spec stop_request(t()) :: t() + def stop_request(%__MODULE__{} = conn) do + %{ + conn + | pending_body: "", + outbound: Enum.reject(conn.outbound, &request_data_frame?/1), + request_stopped?: true + } + end + @spec outbound_control_only?(t()) :: boolean() - def outbound_control_only?(%__MODULE__{outbound: outbound, pending_body: ""}) do + def outbound_control_only?(%__MODULE__{outbound: outbound}) do Enum.all?(outbound, &control_frame?/1) end - def outbound_control_only?(%__MODULE__{}), do: false - defp append_buffer(%__MODULE__{buffer: buffer} = conn, data) do %{conn | buffer: buffer <> data} end @@ -206,7 +218,7 @@ defmodule HTTP.HTTP2 do {:ok, conn} <- consume_receive_window(conn, flow_controlled_size) do end_stream? = Frame.flag?(frame.flags, @flag_end_stream) forbidden? = response_body_forbidden?(conn) - conn = if end_stream?, do: %{conn | done?: true}, else: conn + conn = if end_stream?, do: complete_response(conn), else: conn events = [] @@ -219,6 +231,14 @@ defmodule HTTP.HTTP2 do defp handle_frame(_conn, %Frame{type: :data}), do: {:error, :invalid_data_stream} + defp handle_frame(%__MODULE__{done?: true} = conn, %Frame{ + type: :rst_stream, + stream_id: @client_stream_id, + payload: <<0::32>> + }) do + {:ok, conn, []} + end + defp handle_frame(_conn, %Frame{ type: :rst_stream, stream_id: @client_stream_id, @@ -288,7 +308,7 @@ defmodule HTTP.HTTP2 do with true <- end_stream? || {:error, :invalid_response_trailers}, {:ok, hpack, headers} <- HPACK.decode(conn.hpack, header_block), :ok <- validate_response_trailers(headers) do - {:ok, %{conn | hpack: hpack, done?: true}, [:done]} + {:ok, conn |> Map.put(:hpack, hpack) |> complete_response(), [:done]} end end @@ -301,9 +321,9 @@ defmodule HTTP.HTTP2 do {:ok, conn, []} else headers = Headers.new(regular_headers) - done? = end_stream? or HTTP.HTTP1.body_forbidden?(conn.method, status) - conn = %{conn | status: status, done?: done?} - events = [{:headers, status, headers}] |> maybe_done_event(done?) + conn = %{conn | status: status} + conn = if end_stream?, do: complete_response(conn), else: conn + events = [{:headers, status, headers}] |> maybe_done_event(end_stream?) {:ok, conn, events} end @@ -493,7 +513,15 @@ defmodule HTTP.HTTP2 do end end + defp request_data_frame?(iodata) do + case Frame.decode(IO.iodata_to_binary(iodata)) do + {:ok, %Frame{type: :data, stream_id: @client_stream_id}, ""} -> true + _ -> false + end + end + defp flush_pending_body(%__MODULE__{pending_body: ""} = conn), do: conn + defp flush_pending_body(%__MODULE__{request_stopped?: true} = conn), do: conn defp flush_pending_body(%__MODULE__{} = conn) do writable = min(conn.connection_send_window, conn.stream_send_window) @@ -514,6 +542,12 @@ defmodule HTTP.HTTP2 do end end + defp complete_response(conn) do + conn + |> Map.put(:done?, true) + |> stop_request() + end + defp pseudo_headers(%Request{} = request) do [ {":method", Request.method_token(request.method)}, diff --git a/apps/http_core/test/http/http2_test.exs b/apps/http_core/test/http/http2_test.exs index 8aa63a5..777593a 100644 --- a/apps/http_core/test/http/http2_test.exs +++ b/apps/http_core/test/http/http2_test.exs @@ -13,7 +13,7 @@ defmodule HTTP.HTTP2Test do @max_window_size 2_147_483_647 describe "outbound_control_only?/1" do - test "only permits acknowledgements and window updates after the upload is sent" do + test "classifies only acknowledgements and window updates" do controls = [ Frame.encode(:settings, @ack, 0, ""), Frame.encode(:ping, @ack, 0, "12345678"), @@ -22,7 +22,7 @@ defmodule HTTP.HTTP2Test do conn = %HTTP.HTTP2{outbound: controls} assert HTTP.HTTP2.outbound_control_only?(conn) - refute HTTP.HTTP2.outbound_control_only?(%{conn | pending_body: "unsent"}) + assert HTTP.HTTP2.outbound_control_only?(%{conn | pending_body: "unsent"}) for required <- [ Frame.encode(:data, @end_stream, 1, "upload"), @@ -203,21 +203,209 @@ defmodule HTTP.HTTP2Test do end test "combines HEADERS and CONTINUATION before decoding" do - conn = HTTP.HTTP2.new(:get) + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) header_block = HPACK.encode_headers([{":status", "204"}, {"x-test", "split"}]) size = div(IO.iodata_length(header_block), 2) header_block = IO.iodata_to_binary(header_block) <> = header_block + assert {:ok, conn, []} = + HTTP.HTTP2.stream(conn, Frame.encode(:headers, @end_stream, 1, first)) + + refute HTTP.HTTP2.complete_response?(conn) + refute HTTP.HTTP2.request_stopped?(conn) + assert conn.pending_body == "xxxxx" + + assert {:ok, conn, [{:headers, 204, headers}, :done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:continuation, @end_headers, 1, second)) + + assert HTTP.Headers.get(headers, "x-test") == "split" + assert HTTP.HTTP2.complete_response?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + end + + test "does not complete a bodyless response until END_STREAM arrives" do + assert {:ok, conn, [{:headers, 204, _headers}]} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([{":status", "204"}]) + ) + + refute HTTP.HTTP2.complete_response?(conn) + assert {:error, :closed} = HTTP.HTTP2.close(conn) + + assert {:ok, conn, [:done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:data, @end_stream, 1, "")) + + assert HTTP.HTTP2.complete_response?(conn) + end + + test "stops an upload explicitly without discarding non-upload frames" do + conn = %HTTP.HTTP2{ + pending_body: "unsent", + outbound: [ + Frame.encode(:data, @end_stream, 1, "upload"), + Frame.encode(:settings, @ack, 0, ""), + Frame.encode(:settings, 0, 0, "") + ] + } + + conn = HTTP.HTTP2.stop_request(conn) + {conn, outbound} = HTTP.HTTP2.take_outbound(conn) + assert [] = conn.outbound + + assert {:ok, %Frame{type: :settings, flags: 0, payload: ""}, outbound} = + outbound |> IO.iodata_to_binary() |> Frame.decode() + + assert {:ok, %Frame{type: :settings, flags: @ack, payload: ""}, ""} = + Frame.decode(outbound) + + refute HTTP.HTTP2.outbound_control_only?(%{ + conn + | outbound: [Frame.encode(:settings, 0, 0, "")] + }) + + refute HTTP.HTTP2.complete_response?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + + assert {:ok, conn, []} = + HTTP.HTTP2.stream( + conn, + Frame.encode(:window_update, 0, 1, <<0::1, 1::31>>) + ) + + assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) + end + + test "stops a pending upload after an early final response and preserves control frames" do + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) + + final_response = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + frames = [ - Frame.encode(:headers, @end_stream, 1, first), - Frame.encode(:continuation, @end_headers, 1, second) + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + final_response ] - assert {:ok, _conn, [{:headers, 204, headers}, :done]} = + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = HTTP.HTTP2.stream(conn, IO.iodata_to_binary(frames)) - assert HTTP.Headers.get(headers, "x-test") == "split" + assert HTTP.HTTP2.complete_response?(conn) + + {conn, outbound} = HTTP.HTTP2.take_outbound(conn) + + assert {:ok, %Frame{type: :settings, flags: flags, payload: ""}, ""} = + outbound |> IO.iodata_to_binary() |> Frame.decode() + + assert (flags &&& @ack) == @ack + assert HTTP.HTTP2.outbound_control_only?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + + assert {:ok, conn, []} = + HTTP.HTTP2.stream( + conn, + IO.iodata_to_binary([ + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>) + ]) + ) + + assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) + end + + test "does not restart an upload when WINDOW_UPDATE follows an early final response" do + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) + + final_response = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + frames = [ + final_response, + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>) + ] + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(conn, IO.iodata_to_binary(frames)) + + assert {_conn, []} = HTTP.HTTP2.take_outbound(conn) + assert HTTP.HTTP2.request_stopped?(conn) + end + + test "accepts NO_ERROR reset after a complete response without duplicate events" do + final_headers = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + reset = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), final_headers) + + assert HTTP.HTTP2.complete_response?(conn) + assert {:ok, ^conn, []} = HTTP.HTTP2.stream(conn, reset) + assert {:ok, ^conn, []} = HTTP.HTTP2.stream(conn, reset) + + assert {:ok, _conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), final_headers <> reset) + + assert {:error, {:stream_reset, :no_error}} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + Frame.encode(:rst_stream, 0, 1, <<0::32>>) + ) + end + + test "keeps an incomplete response reset as an error" do + headers = response_headers_frame([{":status", "200"}]) + reset = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + assert {:ok, conn, [{:headers, 200, _headers}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers) + + refute HTTP.HTTP2.complete_response?(conn) + assert {:error, {:stream_reset, :no_error}} = HTTP.HTTP2.stream(conn, reset) end test "decodes hpack static, dynamic, and huffman response headers" do diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index aea64bf..d5a4cfc 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -735,9 +735,10 @@ defmodule HTTP.SocketClient do %{protocol_module: HTTP.HTTP2, protocol: protocol, transport: transport}, events ) do - # Classify before take_outbound/1 clears the queue, including any request - # body still waiting for flow-control credit. In ex_ssl 0.3.0 an established - # socket's peer close_notify rejects writes with :closed while retaining + # Classify queued frames independently from any upload waiting for credit. + # A complete early response stops that upload in the protocol layer. + # In ex_ssl 0.3.0 an established socket's peer close_notify rejects writes + # with :closed while retaining # unread plaintext; abnormal TCP closure returns :econnreset instead. # This owner has not closed the socket locally. Rearming it drains that # plaintext (or reports EOF); only the HTTP parser can complete a response. @@ -752,9 +753,17 @@ defmodule HTTP.SocketClient do timeout = remaining_timeout(state.deadline_at) case send_request(state.transport, state.socket, data, timeout) do - :ok -> {:ok, state} - {:error, :closed} when discard_closed_controls? -> {:ok, state} - {:error, reason} -> {:error, reason} + :ok -> + {:ok, state} + + {:error, :closed} when discard_closed_controls? -> + # Sending is over, but receiving is not necessarily over. In particular, + # buffered WINDOW_UPDATE frames must not restart the abandoned upload + # while we drain the response through the original receive/deadline loop. + {:ok, %{state | protocol: HTTP.HTTP2.stop_request(state.protocol)}} + + {:error, reason} -> + {:error, reason} end end diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index be0f882..0cac81e 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -532,7 +532,8 @@ defmodule HTTP.SocketClientHTTP2Test do end @tag :cross_record - test "fails when SETTINGS acknowledgement cannot flush a pending HTTP/2 request body" do + @tag :early_response + test "delivers a complete 413 response when the HTTP/2 request body is still pending" do test_pid = self() body = :binary.copy("p", @initial_window_size + 5) @@ -548,10 +549,12 @@ defmodule HTTP.SocketClientHTTP2Test do await_test_gate(:send_response_and_close) + response_body = "payload-too-large" + send_all(socket, transport, [ Frame.encode(:settings, 0, 0, ""), - Frame.encode(:headers, @end_headers, 1, response_headers("complete")), - Frame.encode(:data, @end_stream, 1, "complete") + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)), + Frame.encode(:data, @end_stream, 1, response_body) ]) :ok = :ssl.close(socket) @@ -587,12 +590,330 @@ defmodule HTTP.SocketClientHTTP2Test do owner_monitor = Process.monitor(owner) true = :erlang.resume_process(owner) - assert {:error, :closed} = - HTTP.Promise.await(promise) + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == "payload-too-large" + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :early_response + test "drains a cross-record 413 response without resuming its pending HTTP/2 upload" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + response_body = "cross-record-payload-too-large" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + + second_record = [ + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + Frame.encode(:data, @end_stream, 1, response_body) + ] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_early_response_first_record) + send_all(socket, transport, first_record) + send(test_pid, :early_response_first_record_sent) + + await_test_gate(:send_early_response_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :early_response_second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_early_response_first_record) + assert_receive :early_response_first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_early_response_second_record_and_close) + assert_receive :early_response_second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == response_body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :early_response + test "delivers a no-body 413 response through ex_ssl while the HTTP/2 upload is pending" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + await_test_gate(:send_no_body_early_response_and_close) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers ||| @end_stream, 1, response_headers(413, "")) + ]) + + :ok = :ssl.close(socket) + send(test_pid, :no_body_early_response_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + true = :erlang.suspend_process(owner) + send(server_pid, :send_no_body_early_response_and_close) + assert_receive :no_body_early_response_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + + assert response.status == 413 + assert HTTP.Response.read_all(response) == "" assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 end + @tag :early_response + test "delivers an early 413 response through OTP TLS while the peer stays open for SETTINGS ACK" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + response_body = "otp-payload-too-large" + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)), + Frame.encode(:data, @end_stream, 1, response_body) + ]) + + buffer = assert_settings_ack(socket, transport, buffer) + buffer = assert_window_update(socket, transport, buffer, 0, byte_size(response_body)) + buffer = assert_window_update(socket, transport, buffer, 1, byte_size(response_body)) + assert buffer == "" + send(test_pid, {:otp_early_response_settings_acknowledged, self()}) + await_test_gate(:close_otp_early_response) + end) + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + ssl: [verify: :verify_none] + ) + + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == response_body + assert_receive {:otp_early_response_settings_acknowledged, server_pid}, 5_000 + send(server_pid, :close_otp_early_response) + end + + @tag :early_response + test "delivers one complete response when a later buffered NO_ERROR reset follows END_STREAM" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + body = "no-error-reset-after-end-stream" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)), + Frame.encode(:data, @end_stream, 1, body) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_complete_response_record) + send_all(socket, transport, first_record) + send(test_pid, :complete_response_record_sent) + + await_test_gate(:send_no_error_reset_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :no_error_reset_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_complete_response_record) + assert_receive :complete_response_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_no_error_reset_and_close) + assert_receive :no_error_reset_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do + @tag :early_response + test "handles a #{completion} NO_ERROR reset in the same HTTP/2 response batch" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + body = "no-error-reset" + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)), + Frame.encode(:data, unquote(end_stream_flag), 1, body), + Frame.encode(:rst_stream, 0, 1, <<0::32>>) + ]) + + send(test_pid, {:no_error_response_sent, self()}) + await_test_gate(:close_no_error_response) + end) + + result = + url + |> HTTP.fetch( + method: :post, + body: request_body, + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + case unquote(completion) do + :complete -> + assert result.status == 200 + assert HTTP.Response.read_all(result) == body + + :incomplete -> + assert result == {:error, {:stream_reset, :no_error}} + end + + assert_receive {:no_error_response_sent, server_pid}, 5_000 + send(server_pid, :close_no_error_response) + end + end + @tag :cross_record test "drains a final streaming HTTP/2 frame buffered by ex_ssl after peer close" do test_pid = self() @@ -794,7 +1115,8 @@ defmodule HTTP.SocketClientHTTP2Test do end for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do - test "fails when a closed ex_ssl HTTP/2 connection still needs request body writes for a #{completion} response" do + @tag :early_response + test "handles a closed ex_ssl HTTP/2 connection with request body writes for a #{completion} response" do test_pid = self() body = :binary.copy("p", @initial_window_size + 5) @@ -808,18 +1130,20 @@ defmodule HTTP.SocketClientHTTP2Test do assert initial_body == binary_part(body, 0, @initial_window_size) send(test_pid, {:server_received_request, self()}) - receive do - :send_response_and_close -> - send_all(socket, transport, [ - Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), - Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), - Frame.encode(:headers, @end_headers, 1, response_headers("complete")), - Frame.encode(:data, unquote(end_stream_flag), 1, "complete") - ]) - - :ssl.close(socket) - send(test_pid, :server_closed) - end + await_test_gate(:send_response_and_close) + + frames = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + Frame.encode(:headers, @end_headers, 1, response_headers("complete")), + Frame.encode(:data, unquote(end_stream_flag), 1, "complete") + ] + + send_all(socket, transport, frames) + + :ssl.close(socket) + send(test_pid, :server_closed) end) controller = HTTP.AbortController.new() @@ -854,7 +1178,17 @@ defmodule HTTP.SocketClientHTTP2Test do owner_monitor = Process.monitor(owner) true = :erlang.resume_process(owner) - assert {:error, :closed} = HTTP.Promise.await(promise) + result = HTTP.Promise.await(promise) + + case unquote(completion) do + :complete -> + assert result.status == 200 + assert HTTP.Response.read_all(result) == "complete" + + :incomplete -> + assert result == {:error, :closed} + end + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 end end @@ -1164,9 +1498,11 @@ defmodule HTTP.SocketClientHTTP2Test do ]) end - defp response_headers(body) do + defp response_headers(body), do: response_headers(200, body) + + defp response_headers(status, body) do HPACK.encode_headers([ - {":status", "200"}, + {":status", Integer.to_string(status)}, {"content-length", Integer.to_string(byte_size(body))}, {"x-protocol", "h2"} ]) diff --git a/docs/pr-14-validation.md b/docs/pr-14-validation.md index ebf9356..a8e4013 100644 --- a/docs/pr-14-validation.md +++ b/docs/pr-14-validation.md @@ -1,6 +1,192 @@ # PR #14 validation -## Follow-up: response data across TLS records +## Early final responses (baseline a1312cc) + +The current round started at `a1312cc40c8d6aad2cb60e750bfba84f9b3ac1cf`, +which was also the remote PR HEAD; the working tree was clean. The previous +cross-record drain remains in place. This round corrects its overly broad +assumption that an unfinished request upload invalidates a completed response. +The earlier records below are historical; their blanket pending-upload and +NO_ERROR-reset failure rules are superseded by this section. + +### Protocol basis and root cause + +[RFC 9113 §8.1](https://www.rfc-editor.org/rfc/rfc9113.html#section-8.1) defines +response completion independently of request transmission. A server may finish +its response before receiving the entire request, then use RST_STREAM(NO_ERROR) +to stop the upload without invalidating that response. Completion requires +END_STREAM and completion of any associated HEADERS/CONTINUATION field block. +A final status, Content-Length, or body-forbidden response alone is insufficient. + +Three decisions had been conflated: + +1. `outbound_control_only?/1` classified the queued frames **and** required an + empty `pending_body`. This prevented normal-close draining and discarded + complete early responses even when only an ACK needed writing. +2. Parsing WINDOW_UPDATE could queue more request DATA before the final response + was recognized, and response completion did not remove that obsolete upload. +3. Every target-stream RST_STREAM was returned as an error, including NO_ERROR + after a complete response. Body-forbidden responses were also incorrectly + considered complete without END_STREAM. + +The implementation separates frame classification, stopped request transmission +and response completion. Valid completion stops the remaining upload and removes +queued request DATA while preserving control frames. A normal ex_ssl close +reported by an optional control write stops transmission without marking the +response complete; the original active-once/deadline loop drains and validates +what remains. Subsequent buffered WINDOW_UPDATE cannot restart that upload. +Actual request DATA write failures before this transition remain failures. +NO_ERROR reset is harmless only after verified response completion; CANCEL, +protocol/TLS errors, truncation, cancellation and deadline retain their errors. + +### Initial baseline evidence + +A fresh detached worktree at `.trees/r3-baseline` was prepared with independent +`deps` and `_build` using `MIX_ENV=test mix deps.get` and +`MIX_ENV=test mix compile --warnings-as-errors`. Only the regression test file +was copied in; production code remained at a1312cc. + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --only early_response --seed 0 +# Initial baseline: 25 discovered, 1 executed, 1 failure, 24 excluded. +``` + +The client POSTs 65,535 + 5 bytes. The server reads only the initial window, +then waits for the test to suspend the HTTP owner before sending a valid 413 +response with exact body `payload-too-large` and END_STREAM. The test confirms +normal TLS closure and the queued response before resuming the owner. +`HTTP.Promise.await` actually returns `{:error, :closed}`, so the expected 413 +response assertion fails. Log: `/tmp/http_fetch-pr14-r3/a1312cc-first-red.log`. + +### Corrected expectations and additional baseline evidence + +The old test named `fails when SETTINGS acknowledgement cannot flush a pending +HTTP/2 request body` is corrected to expect a 413 and exact response body after +the same deterministic peer-close barrier. The completed branch of the existing +WINDOW_UPDATE/upload test is likewise corrected to preserve the response and +discard unsent DATA. Its incomplete branch retains the actual DATA-write failure +expectation; NO_ERROR-reset rejection is separate, so a parser reset cannot hide +loss of required-write coverage. + +The queue-classification test now permits an ACK-only queue even when +`pending_body` is nonempty. Separate tests verify explicit upload stopping, +retention of non-upload frames, continued ordinary upload flow control, and no +false response completion. The old bodyless HEADERS/CONTINUATION fixture is made +protocol-valid by carrying END_STREAM on HEADERS and waiting for END_HEADERS on +CONTINUATION; the first fragment alone neither completes the response nor stops +the pending upload. + +The final core test file was copied into the unchanged a1312cc worktree and run +with these selections: + +```bash +MIX_ENV=test mix test apps/http_core/test/http/http2_test.exs:235 apps/http_core/test/http/http2_test.exs:288 apps/http_core/test/http/http2_test.exs:372 --seed 0 +# 25 discovered, 3 executed, 3 failures, 22 excluded. +``` + +These are actual behavior failures, not missing-new-API errors: premature `:done` +for 204 without END_STREAM; request DATA still queued after a complete response; +and `{:stream_reset, :no_error}` after a complete response. Log: +`/tmp/http_fetch-pr14-r3/a1312cc-core-final-red.log`. The fixed full core HTTP/2 +file passes all 25 tests. Same-batch and subsequent-call NO_ERROR resets are +covered, including repeated reset with no duplicate events, and reset before a +complete response remains an error. + +### Final integration regression evidence + +The final integration file was copied unchanged into the a1312cc worktree: + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --only early_response --seed 0 +# Baseline: 31 discovered, 9 executed, 6 failures, 22 excluded. +``` + +Five failures returned `{:error, :closed}` instead of the complete response: +413 with body, bodyless 413, cross-record 413, a completed response followed by +a buffered NO_ERROR reset, and WINDOW_UPDATE plus a completed response. The +sixth returned `{:error, {:stream_reset, :no_error}}` for a complete response +and reset parsed in the same batch. The three passing controls were OTP's early +response, incomplete-response NO_ERROR reset, and the actual required DATA-write +failure. Log: `/tmp/http_fetch-pr14-r3/a1312cc-final-integration-red.log`. + +All early-response fixtures use a 65,540-byte POST, with the peer reading the +initial 65,535-byte window. The simple 413 and bodyless variants provide no +additional upload credit. The cross-record variant adds WINDOW_UPDATE in its +buffered second batch to verify that draining cannot restart a stopped upload. +Message gates prove that the first plaintext batch is the paused owner's sole +TLS message before releasing the second write. A bounded probe then verifies +normal peer closure, inactive delivery, and the exact remaining plaintext buffer +size. Only these flags and sizes are observed; no TLS keys or state dumps are +printed. The owner resumes while the TLS receive buffer still exists. + +The separate-record NO_ERROR integration test permits the owner to finish after +the complete first batch and release the buffered reset during cleanup. Protocol +unit tests additionally parse the reset in a subsequent call and prove that it +produces no duplicate events. Same-batch integration exercises actual reset +parsing. NO_ERROR before completion, CANCEL, truncation, fragmented frames, +required writes, streaming backpressure, cancellation and deadline keep their +negative coverage. Owner and TLS process monitors verify release in the gated +close tests. Existing cross-record/frame-fragment and streaming tests are kept. + +The complete final integration file passed seeds 1, 2, 3, 4 and 5: **31 tests, +0 failures per run**, no skips or exclusions. The core HTTP/2 file passed seeds +101, 202, 303, 404 and 505: **25 tests, 0 failures per run**. These repetitions +check stability; the record/buffer barriers establish the timing itself. + +### Current local acceptance results + +Executed on Elixir **1.18.5**, Erlang/OTP **28** (ERTS 16.4.0.5), Go **1.26.6**. +All commands below ran from the umbrella root, except the explicitly noted Go +build and isolated baseline reproduction. Logs and exit codes are in +`/tmp/http_fetch-pr14-r3/final-*.log` and `final-results.json`. + +| Actual command | Final result | +| --- | --- | +| `mix deps.get`; `MIX_ENV=test mix deps.get` | Both passed; lockfile unchanged | +| `mix format --check-formatted` | Passed | +| `mix compile --warnings-as-errors` | Passed | +| `MIX_ENV=test mix compile --warnings-as-errors` | Passed | +| `mix test` | **422 tests + 20 doctests, 0 failures, 0 skipped** | +| `MIX_ENV=test mix test apps/http_core/test` | 168 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test` | 171 tests + 20 doctests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/test` | 33 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/test` | 26 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/test` | 24 tests, 0 failures | +| `mix test apps/http_fetch/test/http/socket_client_http2_test.exs --seed N` for N=1..5 | 31 tests/run, 0 failures | +| `mix test apps/http_core/test/http/http2_test.exs --seed N` for N=101,202,303,404,505 | 25 tests/run, 0 failures | +| `mix credo` | Passed, no issues | +| `mix dialyzer --format github` | Passed; 4 existing ignored diagnostics, 0 new diagnostics | +| `bash scripts/external_consumer_smoke.sh` | Passed: five packages built from current source, metadata checked, isolated dependency resolution/compilation/startup and eight local TLS exchanges | +| `MIX_ENV=test mix test apps/http_fetch/e2e` | 50 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/e2e` | 3 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/e2e` | 2 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/e2e` | 3 tests, 0 failures | +| `MIX_ENV=test mix test.e2e` | Same 58 E2E tests, 0 failures | + +The Go fixture was rebuilt with `go build -o ../test_server/server .` from +`apps/http_fetch/priv/test_server`, started before E2E, and stopped afterward. +`E2E_BASE_URL` pointed at its reported local port. `http_core` has no E2E suite; +its root-scoped unit suite covers that workflow matrix entry. All final selected +app and E2E runs had zero skipped/excluded tests. Baseline line/tag exclusions +above are deliberate selection, not skipped failing tests. + +Cold compilation was exercised in the independent a1312cc worktree; the final +source was also cold-compiled by the external consumer with its own `deps`, +`_build`, and newly resolved lockfile. The consumer obtains ex_ssl transitively +from the built http_core package, with no manually added ex_ssl dependency. +WebTransport retains its existing QUIC boundary check; it is not a TCP TLS test. + +No requested local checks remain unexecuted. These results are local, not proof +of remote CI on a future commit; new remote results belong to their exact SHA. +Known limits remain: tests probe private ex_ssl 0.3.0 buffer flags only for +synchronization; production uses public transport calls. OTP's previously +recorded immediate-close `:einval` is still an error, not a new exception. The +OTP early-response test keeps the peer open for control acknowledgements; the +ex_ssl regressions prove close-before-control-write behavior. No backend +fallback, POST retry, certificate-policy relaxation, new timeout, dependency +patch, or dependency upgrade was introduced. + +## Historical: cross-record drain at a1312cc Reviewed baseline and remote PR HEAD: `010b0b850745b43faab73093849d3fb6a1dc6650`. The branch had no later commits when this follow-up started. The earlier fixes From b4414db7d9061c8892483e3bd632aa384a03ac70 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 13:15:38 +0800 Subject: [PATCH 06/17] fix(http2): validate bounded response completion before close drainage --- README.md | 11 + apps/http_core/lib/http/http2.ex | 189 +++++++++++++++--- .../http_core/test/http/http2_limits_test.exs | 65 ++++++ apps/http_core/test/http/http2_test.exs | 174 ++++++++++++++++ .../test/http/socket_client_http2_test.exs | 114 ++++++++++- docs/ex-ssl-consumer-contract.md | 47 +++++ docs/pr-14-validation.md | 25 +++ 7 files changed, 595 insertions(+), 30 deletions(-) create mode 100644 apps/http_core/test/http/http2_limits_test.exs create mode 100644 docs/ex-ssl-consumer-contract.md diff --git a/README.md b/README.md index 62f25ac..e9f6212 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,15 @@ subsequent RST_STREAM(NO_ERROR), as required by [RFC 9113 §8.1](https://www.rfc-editor.org/rfc/rfc9113.html#section-8.1). Completion requires END_STREAM and the complete HEADERS/CONTINUATION field block; an unfinished upload neither proves nor prevents response completion. +HTTP/2 also validates Content-Length against unpadded DATA bytes before +completion, rejects body overruns immediately, and reports mismatches as +`:content_length_mismatch`. Valid HEAD/304 representation lengths do not require +a body. Malformed/conflicting lengths, values longer than 20 decimal digits, +and values outside the unsigned 64-bit bound return `:invalid_content_length`. +Inbound frames are limited to the advertised 16,384-byte payload size and +compressed header blocks to 65,536 bytes, including CONTINUATION fragments. +Content-Length is forbidden on informational/204 responses and in trailers; +DATA or HEADERS after END_STREAM is rejected rather than completed again. Truncation, required writes before completion, abnormal closure, cancellation and timeout remain errors. The original deadline and streaming backpressure are preserved. @@ -185,6 +194,8 @@ For `:ex_ssl`, `socket_opts` accepts `send_timeout` and ClientHello profiles can be passed through `ssl: [ex_ssl: [profile: profile]]`; any ALPN list added by HTTP/2 selection must match the profile's ALPN list exactly. See the [ex_ssl compatibility contract](https://github.com/gsmlg-dev/ex_ssl/blob/v0.3.0/docs/COMPATIBILITY.md). +The [consumer contract inventory](docs/ex-ssl-consumer-contract.md) maps the +implemented subset and intentional restrictions to its tests. Plain HTTP, WS, and Unix sockets retain their existing transports. HTTP/3 and WebTransport use QUIC's separate TLS implementation and ignore the shared diff --git a/apps/http_core/lib/http/http2.ex b/apps/http_core/lib/http/http2.ex index 0a0f29e..0eec59b 100644 --- a/apps/http_core/lib/http/http2.ex +++ b/apps/http_core/lib/http/http2.ex @@ -12,8 +12,10 @@ defmodule HTTP.HTTP2 do @client_stream_id 1 @initial_window_size 65_535 @initial_max_frame_size 16_384 + @max_header_block_size 65_536 @max_window_size 2_147_483_647 @max_max_frame_size 16_777_215 + @max_content_length "18446744073709551615" @flag_end_stream 0x1 @flag_ack 0x1 @@ -35,6 +37,8 @@ defmodule HTTP.HTTP2 do pending_body: "", outbound: [], request_stopped?: false, + expected_content_length: nil, + received_content_length: 0, done?: false @type event :: {:headers, non_neg_integer(), Headers.t()} | {:body, binary()} | :done @@ -124,6 +128,12 @@ defmodule HTTP.HTTP2 do %{conn | buffer: buffer <> data} end + # We do not advertise a larger receive frame size. Peer SETTINGS only change + # the size of frames we send, not this inbound bound. + defp parse(%__MODULE__{buffer: <>}, _events) + when length > @initial_max_frame_size, + do: {:error, :frame_size_error} + defp parse(%__MODULE__{} = conn, events) do case Frame.decode(conn.buffer) do :more -> @@ -170,6 +180,9 @@ defmodule HTTP.HTTP2 do defp handle_frame(_conn, %Frame{type: :settings}), do: {:error, :invalid_settings_stream} + defp handle_frame(%__MODULE__{done?: true}, %Frame{type: :headers, stream_id: @client_stream_id}), + do: {:error, :stream_closed} + defp handle_frame(conn, %Frame{type: :headers, stream_id: @client_stream_id} = frame) do with {:ok, fragment} <- headers_fragment(frame) do end_stream? = Frame.flag?(frame.flags, @flag_end_stream) @@ -180,6 +193,7 @@ defmodule HTTP.HTTP2 do continuation = %{ stream_id: frame.stream_id, fragments: [fragment], + size: byte_size(fragment), end_stream?: end_stream? } @@ -194,38 +208,53 @@ defmodule HTTP.HTTP2 do %__MODULE__{continuation: %{stream_id: stream_id} = continuation} = conn, %Frame{type: :continuation, stream_id: stream_id} = frame ) do - fragments = [frame.payload | continuation.fragments] + size = continuation.size + byte_size(frame.payload) - if Frame.flag?(frame.flags, @flag_end_headers) do - header_block = fragments |> Enum.reverse() |> IO.iodata_to_binary() - - conn - |> Map.put(:continuation, nil) - |> decode_response_headers(header_block, continuation.end_stream?) + if size > @max_header_block_size do + {:error, :header_block_too_large} else - {:ok, %{conn | continuation: %{continuation | fragments: fragments}}, []} + fragments = + if frame.payload == "", + do: continuation.fragments, + else: [frame.payload | continuation.fragments] + + if Frame.flag?(frame.flags, @flag_end_headers) do + header_block = fragments |> Enum.reverse() |> IO.iodata_to_binary() + + conn + |> Map.put(:continuation, nil) + |> decode_response_headers(header_block, continuation.end_stream?) + else + {:ok, %{conn | continuation: %{continuation | fragments: fragments, size: size}}, []} + end end end defp handle_frame(_conn, %Frame{type: :continuation}), do: {:error, :unexpected_continuation} + defp handle_frame(%__MODULE__{done?: true}, %Frame{type: :data, stream_id: @client_stream_id}), + do: {:error, :stream_closed} + defp handle_frame(%__MODULE__{status: nil}, %Frame{type: :data, stream_id: @client_stream_id}) do {:error, :data_before_response_headers} end defp handle_frame(conn, %Frame{type: :data, stream_id: @client_stream_id} = frame) do with {:ok, data, flow_controlled_size} <- data_payload(frame), - {:ok, conn} <- consume_receive_window(conn, flow_controlled_size) do + {:ok, conn} <- consume_receive_window(conn, flow_controlled_size), + :ok <- validate_response_body(conn, data), + {:ok, conn} <- count_response_body(conn, data) do end_stream? = Frame.flag?(frame.flags, @flag_end_stream) forbidden? = response_body_forbidden?(conn) - conn = if end_stream?, do: complete_response(conn), else: conn - events = - [] - |> maybe_body_event(data, forbidden?) - |> maybe_done_event(end_stream?) + with {:ok, conn} <- maybe_complete_response(conn, end_stream?) do + events = + [] + |> maybe_body_event(data, forbidden?) + |> maybe_done_event(end_stream?) - {:ok, conn, events} + {:ok, conn, events} + end end end @@ -307,8 +336,9 @@ defmodule HTTP.HTTP2 do when is_integer(status) do with true <- end_stream? || {:error, :invalid_response_trailers}, {:ok, hpack, headers} <- HPACK.decode(conn.hpack, header_block), - :ok <- validate_response_trailers(headers) do - {:ok, conn |> Map.put(:hpack, hpack) |> complete_response(), [:done]} + :ok <- validate_response_trailers(headers), + {:ok, conn} <- complete_response(conn) do + {:ok, %{conn | hpack: hpack}, [:done]} end end @@ -318,14 +348,19 @@ defmodule HTTP.HTTP2 do conn = %{conn | hpack: hpack} if status in 100..199 do - {:ok, conn, []} + with :ok <- validate_informational_response_headers(regular_headers) do + {:ok, conn, []} + end else headers = Headers.new(regular_headers) conn = %{conn | status: status} - conn = if end_stream?, do: complete_response(conn), else: conn - events = [{:headers, status, headers}] |> maybe_done_event(end_stream?) - {:ok, conn, events} + with {:ok, conn} <- set_expected_content_length(conn, regular_headers), + {:ok, conn} <- maybe_complete_response(conn, end_stream?) do + events = [{:headers, status, headers}] |> maybe_done_event(end_stream?) + + {:ok, conn, events} + end end end end @@ -345,7 +380,9 @@ defmodule HTTP.HTTP2 do end defp validate_response_trailers(headers) do - if Enum.any?(headers, fn {name, _value} -> String.starts_with?(name, ":") end) do + if Enum.any?(headers, fn {name, _value} -> + String.starts_with?(name, ":") or name == "content-length" + end) do {:error, :invalid_response_trailers} else :ok @@ -494,6 +531,103 @@ defmodule HTTP.HTTP2 do defp maybe_done_event(events, false), do: events defp maybe_done_event(events, true), do: events ++ [:done] + defp set_expected_content_length(conn, headers) do + headers + |> response_content_lengths() + |> parse_content_lengths() + |> case do + {:ok, nil} -> + {:ok, conn} + + {:ok, length} -> + cond do + conn.status in 100..199 or conn.status == 204 -> + {:error, :invalid_content_length} + + response_body_forbidden?(conn) -> + {:ok, conn} + + true -> + {:ok, %{conn | expected_content_length: length}} + end + + {:error, _reason} = error -> + error + end + end + + defp validate_informational_response_headers(headers) do + case headers |> response_content_lengths() |> parse_content_lengths() do + {:ok, nil} -> :ok + _ -> {:error, :invalid_content_length} + end + end + + defp response_content_lengths(headers) do + headers + |> Enum.filter(fn {name, _value} -> name == "content-length" end) + |> Enum.map(fn {_name, value} -> value end) + end + + defp parse_content_lengths([]), do: {:ok, nil} + + defp parse_content_lengths(values) do + if Enum.any?(values, &invalid_content_length_value?/1) do + {:error, :invalid_content_length} + else + lengths = Enum.map(values, &String.to_integer/1) + + case Enum.uniq(lengths) do + [length] -> {:ok, length} + _ -> {:error, :invalid_content_length} + end + end + end + + defp invalid_content_length_value?(value) do + byte_size(value) not in 1..20 or + (byte_size(value) == 20 and value > @max_content_length) or + not Enum.all?(:binary.bin_to_list(value), &(&1 in ?0..?9)) + end + + defp validate_response_body(conn, data) do + if response_body_forbidden?(conn) and data != "" do + {:error, :invalid_response_body} + else + :ok + end + end + + defp count_response_body(conn, data) do + conn = Map.update!(conn, :received_content_length, &(&1 + byte_size(data))) + + if response_content_length_valid_or_pending?(conn) do + {:ok, conn} + else + {:error, :content_length_mismatch} + end + end + + defp maybe_complete_response(conn, false), do: {:ok, conn} + defp maybe_complete_response(conn, true), do: complete_response(conn) + + defp response_content_length_valid?(%__MODULE__{expected_content_length: nil}), do: true + + defp response_content_length_valid?(%__MODULE__{ + expected_content_length: expected, + received_content_length: received + }), + do: expected == received + + defp response_content_length_valid_or_pending?(%__MODULE__{expected_content_length: nil}), + do: true + + defp response_content_length_valid_or_pending?(%__MODULE__{ + expected_content_length: expected, + received_content_length: received + }), + do: received <= expected + defp response_body_forbidden?(%__MODULE__{status: nil}), do: false defp response_body_forbidden?(%__MODULE__{method: method, status: status}) do @@ -543,9 +677,14 @@ defmodule HTTP.HTTP2 do end defp complete_response(conn) do - conn - |> Map.put(:done?, true) - |> stop_request() + if response_content_length_valid?(conn) do + {:ok, + conn + |> Map.put(:done?, true) + |> stop_request()} + else + {:error, :content_length_mismatch} + end end defp pseudo_headers(%Request{} = request) do diff --git a/apps/http_core/test/http/http2_limits_test.exs b/apps/http_core/test/http/http2_limits_test.exs new file mode 100644 index 0000000..5179f04 --- /dev/null +++ b/apps/http_core/test/http/http2_limits_test.exs @@ -0,0 +1,65 @@ +defmodule HTTP.HTTP2LimitsTest do + use ExUnit.Case, async: true + + alias HTTP.HTTP2 + alias HTTP.HTTP2.{Frame, HPACK} + + test "rejects an oversized advertised frame before retaining its payload at every header split" do + header = <<16_385::24, 0, 0, 1::32>> + + for split <- 1..8 do + <> = header + assert {:ok, conn, []} = HTTP2.stream(HTTP2.new(:get), first) + assert {:error, :frame_size_error} = HTTP2.stream(conn, rest) + end + end + + test "bounds accumulated CONTINUATION fragments even without END_HEADERS" do + fragment = :binary.copy(<<0>>, 16_384) + assert {:ok, conn, []} = HTTP2.stream(HTTP2.new(:get), Frame.encode(:headers, 0, 1, fragment)) + + conn = + Enum.reduce(1..3, conn, fn _, conn -> + assert {:ok, conn, []} = HTTP2.stream(conn, Frame.encode(:continuation, 0, 1, fragment)) + conn + end) + + assert {:error, :header_block_too_large} = + HTTP2.stream(conn, Frame.encode(:continuation, 0, 1, <<0>>)) + end + + test "accepts a bounded header block split over legal frames and TCP chunks" do + value = :binary.copy("v", 20_000) + + block = + HPACK.encode_headers([{":status", "200"}, {"x-large", value}]) |> IO.iodata_to_binary() + + <> = block + wire = Frame.encode(:headers, 1, 1, first) <> Frame.encode(:continuation, 4, 1, rest) + + {conn, events} = + for <>, reduce: {HTTP2.new(:get), []} do + {conn, events} -> + assert {:ok, conn, next} = HTTP2.stream(conn, chunk) + {conn, events ++ next} + end + + assert HTTP2.complete_response?(conn) + assert [{:headers, 200, headers}, :done] = events + assert HTTP.Headers.get(headers, "x-large") == value + end + + test "rejects whitespace around HTTP/2 Content-Length" do + for value <- [" 2", "2 ", "\t2"] do + frame = + Frame.encode( + :headers, + 4, + 1, + HPACK.encode_headers([{":status", "200"}, {"content-length", value}]) + ) + + assert {:error, :invalid_content_length} = HTTP2.stream(HTTP2.new(:get), frame) + end + end +end diff --git a/apps/http_core/test/http/http2_test.exs b/apps/http_core/test/http/http2_test.exs index 777593a..0eeb63b 100644 --- a/apps/http_core/test/http/http2_test.exs +++ b/apps/http_core/test/http/http2_test.exs @@ -9,6 +9,7 @@ defmodule HTTP.HTTP2Test do @end_stream 0x1 @ack 0x1 @end_headers 0x4 + @padded 0x8 @initial_window_size 65_535 @max_window_size 2_147_483_647 @@ -202,6 +203,179 @@ defmodule HTTP.HTTP2Test do assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) end + test "rejects a final DATA frame shorter or longer than Content-Length" do + for {declared_length, body} <- [{3, "ok"}, {2, "too"}] do + frames = [ + response_headers_frame([ + {":status", "200"}, + {"content-length", Integer.to_string(declared_length)} + ]), + Frame.encode(:data, @end_stream, 1, body) + ] + + assert {:error, :content_length_mismatch} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + end + + test "rejects DATA exceeding Content-Length before END_STREAM" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, 0, 1, "too") + ] + + assert {:error, :content_length_mismatch} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + + test "rejects malformed and conflicting Content-Length response headers" do + for headers <- [ + [{":status", "200"}, {"content-length", "two"}], + [{":status", "200"}, {"content-length", "2"}, {"content-length", "3"}], + [{":status", "200"}, {"content-length", String.duplicate("9", 21)}], + [{":status", "200"}, {"content-length", "18446744073709551616"}] + ] do + assert {:error, :invalid_content_length} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), response_headers_frame(headers)) + end + end + + test "accepts the largest unsigned 64-bit Content-Length" do + assert {:ok, %{expected_content_length: 18_446_744_073_709_551_615}, + [{:headers, 200, _headers}]} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([ + {":status", "200"}, + {"content-length", "18446744073709551615"} + ]) + ) + end + + test "rejects Content-Length on informational response headers" do + assert {:error, :invalid_content_length} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([{":status", "100"}, {"content-length", "0"}]) + ) + end + + test "counts DATA payload without padding toward Content-Length" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, @padded ||| @end_stream, 1, <<2, "ok", 0, 0>>) + ] + + assert {:ok, _conn, [{:headers, 200, _headers}, {:body, "ok"}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + + test "validates Content-Length when trailers complete a streamed response" do + headers = response_headers_frame([{":status", "200"}, {"content-length", "4"}]) + + trailers = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{"x-checksum", "ok"}]) + ) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "he"}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers <> Frame.encode(:data, 0, 1, "he")) + + assert {:error, :content_length_mismatch} = + HTTP.HTTP2.stream(conn, trailers) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "four"}]} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + headers <> Frame.encode(:data, 0, 1, "four") + ) + + assert {:ok, _conn, [:done]} = HTTP.HTTP2.stream(conn, trailers) + end + + test "rejects Content-Length in response trailers" do + headers = response_headers_frame([{":status", "200"}, {"content-length", "2"}]) + + trailers = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{"content-length", "2"}]) + ) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "ok"}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers <> Frame.encode(:data, 0, 1, "ok")) + + assert {:error, :invalid_response_trailers} = HTTP.HTTP2.stream(conn, trailers) + end + + test "completes a Content-Length response streamed across calls" do + headers = response_headers_frame([{":status", "200"}, {"content-length", "4"}]) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "he"}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers <> Frame.encode(:data, 0, 1, "he")) + + assert {:ok, _conn, [{:body, "ll"}, :done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:data, @end_stream, 1, "ll")) + end + + test "allows HEAD and 304 representation lengths without response content" do + for {method, status} <- [{:head, "200"}, {:get, "304"}] do + frame = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", status}, {"content-length", "10"}]) + ) + + assert {:ok, _conn, [{:headers, _status, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(method), frame) + end + end + + test "rejects Content-Length on a 204 response" do + assert {:error, :invalid_content_length} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([{":status", "204"}, {"content-length", "0"}]) + ) + end + + test "rejects nonempty DATA on a body-forbidden response" do + headers = response_headers_frame([{":status", "204"}]) + + assert {:ok, conn, [{:headers, 204, _headers}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers) + + assert {:error, :invalid_response_body} = + HTTP.HTTP2.stream(conn, Frame.encode(:data, @end_stream, 1, "nope")) + end + + test "rejects DATA and HEADERS after response completion" do + complete = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), complete) + + for frame <- [ + Frame.encode(:data, @end_stream, 1, ""), + response_headers_frame([{":status", "200"}]) + ] do + assert {:error, :stream_closed} = HTTP.HTTP2.stream(conn, frame) + end + end + test "combines HEADERS and CONTINUATION before decoding" do body = :binary.copy("x", @initial_window_size + 5) diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index 0cac81e..fe47ea3 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -397,6 +397,106 @@ defmodule HTTP.SocketClientHTTP2Test do assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 end + for mode <- [:buffered_short, :buffered_long, :streamed_short] do + @tag :cross_record + @tag :content_length_gate + @tag length_mode: mode + test "rejects #{mode} Content-Length mismatch after cross-record peer close", %{ + length_mode: mode + } do + test_pid = self() + body = "invalid-length-body" + + declared_size = + case mode do + :buffered_short -> byte_size(body) + 1 + :buffered_long -> byte_size(body) - 1 + :streamed_short -> HTTP.Config.streaming_threshold() + 1 + end + + headers = + HPACK.encode_headers([ + {":status", "200"}, + {"content-length", Integer.to_string(declared_size)} + ]) + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, headers) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + case mode do + :streamed_short -> + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream_monitor = Process.monitor(response.stream) + + assert_raise RuntimeError, "stream read failed: :content_length_mismatch", fn -> + HTTP.Response.read_all(response) + end + + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + + _ -> + assert {:error, :content_length_mismatch} = HTTP.Promise.await(promise) + end + + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + end + @tag :cross_record test "rejects a truncated second ex_ssl TLS record after a control write fails" do test_pid = self() @@ -1662,18 +1762,22 @@ defmodule HTTP.SocketClientHTTP2Test do |> Enum.count(&match?({:ssl, %SSL.Socket{pid: ^tls_pid}, _data}, &1)) == expected_count end - defp assert_tls_buffered_after_peer_close(tls_pid, minimum_size) do + defp assert_tls_buffered_after_peer_close(tls_pid, expected_size) do + assert Application.spec(:ex_ssl, :vsn) == ~c"0.3.0", + "revalidate this private buffer probe before testing another ex_ssl version" + assert_tls_buffered_after_peer_close( tls_pid, - minimum_size, + expected_size, System.monotonic_time(:millisecond) + 5_000 ) end - defp assert_tls_buffered_after_peer_close(tls_pid, minimum_size, deadline_at) do + defp assert_tls_buffered_after_peer_close(tls_pid, expected_size, deadline_at) do {_phase, state} = :sys.get_state(tls_pid) - if state.closed and state.size >= minimum_size do + if state.closed do + assert state.size == expected_size assert state.active == false :ok else @@ -1685,7 +1789,7 @@ defmodule HTTP.SocketClientHTTP2Test do receive do after min(10, remaining) -> - assert_tls_buffered_after_peer_close(tls_pid, minimum_size, deadline_at) + assert_tls_buffered_after_peer_close(tls_pid, expected_size, deadline_at) end end end diff --git a/docs/ex-ssl-consumer-contract.md b/docs/ex-ssl-consumer-contract.md new file mode 100644 index 0000000..703c73a --- /dev/null +++ b/docs/ex-ssl-consumer-contract.md @@ -0,0 +1,47 @@ +# TCP TLS consumer contract + +Audited against PR #14 head `690258ac38e50b0d1a968d9d5e510c560f45f5d4` +on 2026-09-22. OTP `:ssl` stays the default; ex_ssl is explicitly selected. +This inventory separates implemented consumer requirements from planned library +features. It does not claim full OTP compatibility. + +| Requirement | Status | Production boundary | Executable coverage | +| --- | --- | --- | --- | +| Connect, ownership, send, passive recv, active-once, ALPN, close | implemented | `http_core` transport behaviour and SSL/ExSSL adapters | `apps/http_core/test/http/tls_transport_test.exs` | +| Shared default and explicit per-call backend | implemented | `HTTP.TLSBackend`, fetch/WSS/EventSource option normalization | `apps/http_core/test/http/tls_backend_test.exs` and each client's option tests | +| Verified HTTP/1.1 fixed/chunked/close-delimited responses, large uploads, cancellation/deadline | implemented | `HTTP.SocketClient` | `apps/http_fetch/test/http/ssl_transport_test.exs` | +| Verified ALPN HTTP/2, streaming/flow control, early response with pending upload | implemented | shared HTTP/2 parser and socket owner | `apps/http_core/test/http/http2_test.exs`, `apps/http_fetch/test/http/socket_client_http2_test.exs` | +| Authenticated close during optional control writes | implemented | ex_ssl-only `:closed` classification, existing receive loop and deadline | deterministic cross-record tests in `socket_client_http2_test.exs` | +| HTTP/2 Content-Length completion and input bounds | implemented in this continuation | counted unpadded DATA, u64 decimal length, 16 KiB frames, 64 KiB compressed header blocks | core HTTP/2 and limits tests; real TLS cross-record buffered/streamed mismatch regressions | +| Redirect backend pinning | implemented | backend resolved before redirect lifecycle | `ssl_transport_test.exs` | +| Passive WSS Upgrade followed by active-once frames | implemented | `HTTP.WebSocket.Connection`, shared transport recv | `apps/http_web_socket/test/http/web_socket_test.exs`, `web_socket_tls_lifecycle_test.exs` | +| EventSource reconnect with pinned backend | implemented | `HTTP.EventSource.Connection` | `apps/http_event_source/test/http/event_source_test.exs` | +| Wrong CA/reference hostname and profile/ALPN conflicts | implemented | adapter forwards to verified ex_ssl options | `tls_transport_test.exs` (adapter-level evidence) | +| No automatic backend fallback | implemented | fixed adapter, explicit option/handshake errors | `tls_backend_test.exs`, TLS-1.2-only peer negative in `tls_transport_test.exs` | +| `socket_opts: [send_timeout: ..., send_timeout_close: true]` | implemented | ExSSL translates these two options; socket values override matching `ssl` values | `tls_transport_test.exs` | +| Arbitrary TCP options, verify-none, client certificates, TLS 1.2/mixed versions | deliberately unsupported in ex_ssl 0.3.0 | explicit redacted option errors | `tls_transport_test.exs` | +| Introspection beyond negotiated ALPN, active-N, packet modes | not required by audited consumer | no production callsites | library roadmap; not a Phase 0 blocker | +| HTTP/3 and WebTransport | separate QUIC implementation | explicit TCP backend rejected; shared default ignored | fetch SSL transport tests and WebTransport option/TLS config tests | + +Only `HTTP.Transport.SSL` calls OTP `:ssl` in production. TLS listen, accept, +handshake, and peer traffic in test support are reference-server operations. +QUIC calls in HTTP/3/WebTransport are outside this TCP TLS contract. + +Public `ssl:` and `socket_opts:` containers remain backend-specific. The OTP +adapter preserves its existing option behavior. The ex_ssl adapter validates +keyword shape and duplicates and rejects unsupported socket options. No option +is dropped to make a connection succeed. + +The cross-record fixture suspends the HTTP owner after it enters its receive +loop, proves its sole first plaintext delivery, releases later peer output, and +checks authenticated closure with retained plaintext before resuming. Its private +ex_ssl state probe is test-only, guarded to version 0.3.0, and checks exact byte +count plus passive mode. Larger streaming responses drain incrementally through +flow control before the final gated records; they do not require an oversized +passive TLS buffer. + +Current validation is tracked in the ex_ssl worktree's +`docs/EX_SSL_HTTP_FETCH_PROGRESS.md`. Historical PR validation remains in +[pr-14-validation.md](pr-14-validation.md). Commands must run from the umbrella +root. Adapter-level security tests do not establish per-client-family coverage +of every invalid option, and local OTP peers do not prove every server or runtime. diff --git a/docs/pr-14-validation.md b/docs/pr-14-validation.md index a8e4013..e2629d1 100644 --- a/docs/pr-14-validation.md +++ b/docs/pr-14-validation.md @@ -1,5 +1,30 @@ # PR #14 validation +## Phase 0 continuation (2026-09-22) + +Current PR head remains `690258ac38e50b0d1a968d9d5e510c560f45f5d4`. +The new work is isolated on `codex/tls-backend-plan`. Baseline rerun passed +422 tests plus 20 doctests on Elixir 1.18.5 / OTP 28 with seed 22. +The cross-record closure and early-response fixes are preserved. + +The Phase 0 audit found a separate response-completion gap: Content-Length was +not checked against HTTP/2 DATA. Pure regressions initially failed in four +cases. The change counts unpadded DATA, validates all completion paths and +rejects malformed or conflicting lengths. Real TLS regressions use the existing +owner/peer barriers to cover short buffered, oversized buffered, and short +streamed bodies after authenticated close. The private ex_ssl state probe now +checks exact retained bytes and explicitly guards the tested 0.3.0 version. + +Final root-scoped suites at seed 25 passed **442 tests plus 20 doctests**, +zero failures, no exclusions. Core HTTP/2 plus bounds tests passed 42 tests, +and the HTTP/2 consumer suite passed 34 tests (seed 24). These include the +original deterministic closure/early-response tests. Formatting, dev/test +warnings-as-errors compilation and Credo passed. + +Current execution commands, outcomes and remaining phases are tracked in the +ex_ssl companion worktree at `docs/EX_SSL_HTTP_FETCH_PROGRESS.md`; historical +results below are not results for this continuation. + ## Early final responses (baseline a1312cc) The current round started at `a1312cc40c8d6aad2cb60e750bfba84f9b3ac1cf`, From 93efee08b509e4c2d06c572a88853e5ef133900c Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 13:40:01 +0800 Subject: [PATCH 07/17] test(tls): verify expanded ex_ssl algorithms through packaged HTTP clients --- README.md | 6 + docs/ex-ssl-consumer-contract.md | 20 +++ scripts/ex_ssl_algorithms_test.exs | 211 +++++++++++++++++++++++++++++ scripts/ex_ssl_source_smoke.sh | 31 +++++ 4 files changed, 268 insertions(+) create mode 100644 scripts/ex_ssl_algorithms_test.exs create mode 100755 scripts/ex_ssl_source_smoke.sh diff --git a/README.md b/README.md index e9f6212..18a93be 100644 --- a/README.md +++ b/README.md @@ -616,3 +616,9 @@ mix format --check-formatted ## License MIT License + +For unreleased ex_ssl algorithm candidates, the source-only integration gate is +`EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh`. +It validates new algorithms against fresh HTTP package artifacts with a temporary +source override. Published ex_ssl 0.3.0 retains its documented algorithm subset; +see [the consumer contract](docs/ex-ssl-consumer-contract.md). diff --git a/docs/ex-ssl-consumer-contract.md b/docs/ex-ssl-consumer-contract.md index 703c73a..aef8492 100644 --- a/docs/ex-ssl-consumer-contract.md +++ b/docs/ex-ssl-consumer-contract.md @@ -45,3 +45,23 @@ Current validation is tracked in the ex_ssl worktree's [pr-14-validation.md](pr-14-validation.md). Commands must run from the umbrella root. Adapter-level security tests do not establish per-client-family coverage of every invalid option, and local OTP peers do not prove every server or runtime. + +## Phase 1 source candidate + +The unreleased ex_ssl candidate adds P-384 ECDHE/ECDSA, Ed25519 and +RSA-PSS-PSS SHA-256/384/512. Consumer package metadata still uses ex_ssl 0.3.0; +these new algorithms require the candidate source until a separate release. + +Run `EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh` +from the umbrella root. This builds fresh http_core/http_fetch artifacts into a +temporary consumer and explicitly overrides ex_ssl there; repository manifests, +lockfiles and installed sources are unchanged. It is separate from the existing +five-package released-dependency smoke, which has no ex_ssl override. + +Seed 36 on OTP 28 / Elixir 1.18.5: 12 tests, zero failures (ten positive exchanges +cover each new signature over HTTP/1.1+P-384 HRR and HTTP/2+direct P-384; five +hostname-negative scenarios in one test; one OTP-default assertion). Both +transport modes retain peer verification. New fixture setup initially omitted +`http_version: :http2`, so five HTTP/2 cases failed; the corrected fixture sets +both HTTP mode and exact profile ALPN and waits for the SETTINGS acknowledgement. +No production workaround was added. Log: `/tmp/http-fetch-tls-plan-algorithms.log`. diff --git a/scripts/ex_ssl_algorithms_test.exs b/scripts/ex_ssl_algorithms_test.exs new file mode 100644 index 0000000..4869f05 --- /dev/null +++ b/scripts/ex_ssl_algorithms_test.exs @@ -0,0 +1,211 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +Code.require_file( + Path.join(System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support/signature_fixtures.ex") +) + +defmodule CandidateAlgorithmsTest do + use ExUnit.Case, async: false + alias ExSSL.TestSupport.SignatureFixtures + alias HTTP.HTTP2.{Frame, HPACK} + alias SSL.ClientHello.WireProfile + + setup_all do + directory = + Path.join(System.tmp_dir!(), "http-algorithms-#{System.unique_integer([:positive])}") + + File.mkdir_p!(directory) + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: SignatureFixtures.create(directory)} + end + + for {scheme, name} <- [ + {0x0503, :ecdsa_secp384r1_sha384}, + {0x0807, :eddsa_ed25519}, + {0x0809, :rsa_pss_pss_sha256}, + {0x080A, :rsa_pss_pss_sha384}, + {0x080B, :rsa_pss_pss_sha512} + ], + protocol <- ["http/1.1", "h2"] do + test "verified #{name} HTTP response over #{protocol} with P384", context do + fixture = Map.fetch!(context.fixtures, unquote(scheme)) + body = "verified-#{unquote(name)}" + + with_peer( + fixture, + unquote(name), + unquote(protocol), + fn port -> + response = + HTTP.fetch("https://127.0.0.1:#{port}/algorithm", + http_version: if(unquote(protocol) == "h2", do: :http2, else: :http1), + tls_backend: :ex_ssl, + ssl: [ + cacerts: [fixture.der], + server_name_indication: ~c"exssl.test", + ex_ssl: [profile: profile(unquote(scheme), unquote(protocol))] + ] + ) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + end, + body + ) + end + end + + test "wrong reference identity still fails for every expanded signature", context do + for {scheme, name} <- [ + {0x0503, :ecdsa_secp384r1_sha384}, + {0x0807, :eddsa_ed25519}, + {0x0809, :rsa_pss_pss_sha256}, + {0x080A, :rsa_pss_pss_sha384}, + {0x080B, :rsa_pss_pss_sha512} + ] do + fixture = Map.fetch!(context.fixtures, scheme) + + with_peer( + fixture, + name, + "http/1.1", + fn port -> + assert {:error, _} = + HTTP.fetch("https://127.0.0.1:#{port}/algorithm", + tls_backend: :ex_ssl, + ssl: [ + cacerts: [fixture.der], + server_name_indication: ~c"wrong.test", + ex_ssl: [profile: profile(scheme, "http/1.1")] + ] + ) + |> HTTP.Promise.await() + end, + :reject + ) + end + end + + test "default backend remains OTP" do + assert HTTP.FetchOptions.new([]).tls_backend == :ssl + end + + defp profile(scheme, protocol) do + # HTTP/1.1 forces HRR; HTTP/2 sends P384 in the first ClientHello. + initial = if protocol == "http/1.1", do: 0x001D, else: 0x0018 + + %WireProfile{ + name: :candidate_algorithm, + cipher_suites: [0x1301], + extensions: [ + {:server_name, :from_connection}, + {:supported_versions, [0x0304]}, + {:supported_groups, [initial, 0x0018] |> Enum.uniq()}, + {:signature_algorithms, [scheme]}, + {:key_share, [initial]}, + {:alpn, [protocol]} + ] + } + end + + defp with_peer(fixture, scheme, protocol, client, body) do + {:ok, listener} = + :ssl.listen(0, + certfile: String.to_charlist(fixture.certificate), + keyfile: String.to_charlist(fixture.key), + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true, + signature_algs: [scheme], + supported_groups: [:secp384r1], + alpn_preferred_protocols: [protocol] + ) + + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, socket} = :ssl.transport_accept(listener, 5_000) + + case :ssl.handshake(socket, 5_000) do + {:error, _} when body == :reject -> + :ok + + {:ok, socket} -> + try do + refute body == :reject + assert {:ok, ^protocol} = :ssl.negotiated_protocol(socket) + respond(socket, protocol, body) + after + :ssl.close(socket) + end + end + end) + + try do + client.(port) + assert :ok = Task.await(task, 10_000) + after + :ssl.close(listener) + Task.shutdown(task, :brutal_kill) + end + + assert {:error, :econnrefused} = + :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 1_000) + end + + defp respond(socket, "http/1.1", body) do + request = headers(socket, "") + assert String.starts_with?(request, "GET /algorithm HTTP/1.1\r\n") + + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: #{byte_size(body)}\r\n\r\n", + body + ]) + end + + defp respond(socket, "h2", body) do + preface = HTTP.HTTP2.connection_preface() + assert {:ok, ^preface} = :ssl.recv(socket, byte_size(preface), 5_000) + assert %Frame{type: :settings} = frame(socket) + assert %Frame{type: :headers, stream_id: 1, payload: block} = frame(socket) + assert {:ok, _, headers} = HPACK.decode(HPACK.new_decoder(), block) + assert {":path", "/algorithm"} in headers + + :ok = + :ssl.send(socket, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode( + :headers, + 4, + 1, + HPACK.encode_headers([ + {":status", "200"}, + {"content-length", Integer.to_string(byte_size(body))} + ]) + ), + Frame.encode(:data, 1, 1, body) + ]) + + assert %Frame{type: :settings, flags: 1, stream_id: 0, payload: ""} = frame(socket) + :ok + end + + defp frame(socket) do + assert {:ok, <> = header} = :ssl.recv(socket, 9, 5_000) + payload = if size == 0, do: "", else: elem(:ssl.recv(socket, size, 5_000), 1) + assert {:ok, frame, ""} = Frame.decode(header <> payload) + frame + end + + defp headers(socket, buffer) when byte_size(buffer) < 16_384 do + if String.contains?(buffer, "\r\n\r\n"), + do: buffer, + else: + ( + {:ok, bytes} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> bytes) + ) + end +end diff --git a/scripts/ex_ssl_source_smoke.sh b/scripts/ex_ssl_source_smoke.sh new file mode 100755 index 0000000..e4d8b76 --- /dev/null +++ b/scripts/ex_ssl_source_smoke.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Cross-repository candidate validation. Release smoke remains external_consumer_smoke.sh. +set -euo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +: "${EX_SSL_SOURCE_DIR:?Set EX_SSL_SOURCE_DIR to the candidate ex_ssl checkout}" +export EX_SSL_SOURCE_DIR=$(cd "$EX_SSL_SOURCE_DIR" && pwd) +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT +export HTTP_FETCH_PACKAGE_DIR="$work_dir/packages" +mkdir -p "$HTTP_FETCH_PACKAGE_DIR" "$work_dir/consumer/test" +for app in http_core http_fetch; do + (cd "$repo_root/apps/$app" && MIX_ENV=prod mix hex.build --unpack -o "$HTTP_FETCH_PACKAGE_DIR/$app") +done +cat > "$work_dir/consumer/mix.exs" <<'MIX' +defmodule CandidateConsumer.MixProject do + use Mix.Project + def project do + packages = System.fetch_env!("HTTP_FETCH_PACKAGE_DIR") + [app: :candidate_consumer, version: "0.0.0", deps: [ + {:http_core, path: Path.join(packages, "http_core")}, + {:http_fetch, path: Path.join(packages, "http_fetch")}, + {:ex_ssl, path: System.fetch_env!("EX_SSL_SOURCE_DIR"), override: true} + ]] + end + def application, do: [extra_applications: [:logger, :ssl, :public_key]] +end +MIX +cp "$repo_root/mix.lock" "$work_dir/consumer/mix.lock" +cp "$repo_root/scripts/ex_ssl_algorithms_test.exs" "$work_dir/consumer/test/algorithms_test.exs" +printf 'ExUnit.start()\n' > "$work_dir/consumer/test/test_helper.exs" +(cd "$work_dir/consumer" && mix deps.get && mix compile --warnings-as-errors && mix test --seed 36) From cbbc2f661f63614c9a2d2992aad9788fe15824c0 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 14:20:54 +0800 Subject: [PATCH 08/17] feat(tls): scope client identities across HTTP redirects --- README.md | 12 +- apps/http_fetch/lib/http/socket_client.ex | 25 +- docs/ex-ssl-consumer-contract.md | 29 +- scripts/ex_ssl_mtls_redirects_test.exs | 167 +++++++++++ scripts/ex_ssl_mtls_streams_test.exs | 321 ++++++++++++++++++++++ scripts/ex_ssl_mtls_test.exs | 253 +++++++++++++++++ scripts/ex_ssl_source_smoke.sh | 11 +- 7 files changed, 811 insertions(+), 7 deletions(-) create mode 100644 scripts/ex_ssl_mtls_redirects_test.exs create mode 100644 scripts/ex_ssl_mtls_streams_test.exs create mode 100644 scripts/ex_ssl_mtls_test.exs diff --git a/README.md b/README.md index 18a93be..de4722e 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,14 @@ For `:ex_ssl`, `socket_opts` accepts `send_timeout` and `send_timeout_close: true`. These override matching entries in `ssl`. Custom ClientHello profiles can be passed through `ssl: [ex_ssl: [profile: profile]]`; any ALPN list added by HTTP/2 selection must match the profile's ALPN list exactly. +For a source candidate that implements client authentication, ex_ssl credentials +stay within the initial request origin during automatic redirects. A scheme, +hostname or effective-port change returns +`{:error, :client_identity_cross_origin_redirect}`. To authorize another origin, +use `redirect: :manual` and explicitly make a new request with that identity. +The OTP backend retains its existing redirect behavior. Published ex_ssl 0.3.0 +does not yet support client identity options. + See the [ex_ssl compatibility contract](https://github.com/gsmlg-dev/ex_ssl/blob/v0.3.0/docs/COMPATIBILITY.md). The [consumer contract inventory](docs/ex-ssl-consumer-contract.md) maps the implemented subset and intentional restrictions to its tests. @@ -617,8 +625,8 @@ mix format --check-formatted MIT License -For unreleased ex_ssl algorithm candidates, the source-only integration gate is +For unreleased ex_ssl algorithm and mTLS candidates, the source-only integration gate is `EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh`. -It validates new algorithms against fresh HTTP package artifacts with a temporary +It validates new algorithms and mTLS against all five fresh package artifacts with a temporary source override. Published ex_ssl 0.3.0 retains its documented algorithm subset; see [the consumer contract](docs/ex-ssl-consumer-contract.md). diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index d5a4cfc..e82db37 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -397,6 +397,9 @@ defmodule HTTP.SocketClient do :done + {:error, :client_identity_cross_origin_redirect = reason} -> + fail(state, reason) + {:error, _reason} -> send_response(state.parent, state.ref, response) finish(state) @@ -959,7 +962,8 @@ defmodule HTTP.SocketClient do defp redirect_request(request, response) do with location when is_binary(location) <- Headers.get(response.headers, "location"), - %URI{} = uri <- URI.merge(request.url, location) do + %URI{} = uri <- URI.merge(request.url, location), + :ok <- validate_client_identity_redirect(request, uri) do request = request |> rewrite_redirect_method(response.status) @@ -967,10 +971,29 @@ defmodule HTTP.SocketClient do {:ok, %{request | url: uri}} else + {:error, _} = error -> error _ -> {:error, :invalid_redirect} end end + defp validate_client_identity_redirect(request, uri) do + ssl_options = Keyword.get(request.transport_options, :ssl, []) + + if tls_backend(request) == :ex_ssl and + Enum.any?([:cert, :certfile, :key, :keyfile], &Keyword.has_key?(ssl_options, &1)) and + client_identity_origin(request.url) != client_identity_origin(uri) do + {:error, :client_identity_cross_origin_redirect} + else + :ok + end + end + + defp client_identity_origin(uri) do + scheme = String.downcase(uri.scheme || "") + + {scheme, String.downcase(uri.host || ""), uri.port || HTTP.HTTP1.default_port(scheme)} + end + defp rewrite_redirect_method(%{method: :post} = request, status) when status in [301, 302], do: drop_redirect_body(request) diff --git a/docs/ex-ssl-consumer-contract.md b/docs/ex-ssl-consumer-contract.md index aef8492..f879782 100644 --- a/docs/ex-ssl-consumer-contract.md +++ b/docs/ex-ssl-consumer-contract.md @@ -53,7 +53,7 @@ RSA-PSS-PSS SHA-256/384/512. Consumer package metadata still uses ex_ssl 0.3.0; these new algorithms require the candidate source until a separate release. Run `EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh` -from the umbrella root. This builds fresh http_core/http_fetch artifacts into a +from the umbrella root. This builds all five fresh package artifacts into a temporary consumer and explicitly overrides ex_ssl there; repository manifests, lockfiles and installed sources are unchanged. It is separate from the existing five-package released-dependency smoke, which has no ex_ssl override. @@ -65,3 +65,30 @@ transport modes retain peer verification. New fixture setup initially omitted `http_version: :http2`, so five HTTP/2 cases failed; the corrected fixture sets both HTTP mode and exact profile ALPN and waits for the SETTINGS acknowledgement. No production workaround was added. Log: `/tmp/http-fetch-tls-plan-algorithms.log`. + + +## Phase 2 source candidate + +Candidate ex_ssl `fc1319d` supports one bounded initial-handshake client identity +through `ssl: [certfile: ..., keyfile: ...]` or the documented in-memory forms. +It remains separate from the server's `cacerts`/`cacertfile` trust. Encrypted keys, +hardware signing and multiple identities remain unsupported. The library's +compatibility matrix documents CertificateRequest selection limits. + +For `:ex_ssl` with any configured `cert`, `certfile`, `key` or `keyfile`, an +automatic redirect that changes scheme, case-insensitive hostname or effective +port returns `{:error, :client_identity_cross_origin_redirect}` before opening +the next connection. Same-origin redirects retain the identity. Use +`redirect: :manual` and issue a separate, deliberate request if another origin +is authorized to receive those credentials. OTP backend behavior is unchanged. +This policy also rejects a downgrade to plain HTTP. HTTP/3/WebTransport remain +on QUIC and do not use these credentials. + +The source smoke includes required RSA/EC/large-chain HTTP/1.1 and HTTP/2 +requests, exact server-observed client DER, optional auth, missing/wrong-CA/ +expired/wrong-purpose/incompatible credentials, pre-I/O key mismatch and bad +server hostname. WSS verifies passive Upgrade, active-once frames and close; +EventSource verifies the identity across same-origin reconnects. Redirect tests +cover all three origin components, DNS casing, manual reuse and unchanged OTP. +These are source-candidate tests; released ex_ssl 0.3.0 still rejects client +identity options. No release manifest or installed dependency has been changed. diff --git a/scripts/ex_ssl_mtls_redirects_test.exs b/scripts/ex_ssl_mtls_redirects_test.exs new file mode 100644 index 0000000..23d6164 --- /dev/null +++ b/scripts/ex_ssl_mtls_redirects_test.exs @@ -0,0 +1,167 @@ +# Candidate source checkout only; run through ex_ssl_source_smoke.sh. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file(Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support", fixture])) +end + +defmodule CandidateMTLSRedirectsTest do + use ExUnit.Case, async: false + alias ExSSL.TestSupport.ClientAuthFixtures + + setup_all do + directory = + Path.join(System.tmp_dir!(), "mtls-redirects-#{System.unique_integer([:positive])}") + + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "automatic ex_ssl redirects cannot move a client identity to another origin", %{ + fixtures: f + } do + for change <- [:port, :host, :scheme] do + {source, port} = listener(f) + {target, target_port} = listener(f) + + destination = + case change do + :port -> "https://127.0.0.1:#{target_port}/next" + :host -> "https://localhost:#{port}/next" + :scheme -> "http://127.0.0.1:#{port}/next" + end + + peer = serve(source, f.rsa.der, [redirect(destination)]) + assert {:error, :client_identity_cross_origin_redirect} = fetch(port, f) + assert :ok = Task.await(peer, 5_000) + assert {:error, :timeout} = :ssl.transport_accept(target, 0) + assert {:error, :timeout} = :ssl.transport_accept(source, 0) + :ssl.close(source) + :ssl.close(target) + end + end + + test "same-origin redirects retain the exact configured client identity", %{fixtures: f} do + {source, port} = listener(f) + peer = serve(source, f.rsa.der, [redirect("/next"), ok()]) + response = fetch(port, f) + assert response.status == 200 + assert response.redirected + assert HTTP.Response.read_all(response) == "mtls" + assert :ok = Task.await(peer, 5_000) + end + + test "DNS host casing does not change client identity origin", %{fixtures: f} do + {source, port} = listener(f) + peer = serve(source, f.rsa.der, [redirect("https://LOCALHOST:#{port}/next"), ok()]) + response = fetch_url("https://localhost:#{port}/start", f, []) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "mtls" + assert :ok = Task.await(peer, 5_000) + end + + test "manual redirect lets the caller deliberately reuse an identity in a new request", %{ + fixtures: f + } do + {source, port} = listener(f) + {target, target_port} = listener(f) + source_peer = serve(source, f.rsa.der, [redirect("https://127.0.0.1:#{target_port}/next")]) + response = fetch(port, f, redirect: :manual) + assert response.status == 302 + assert :ok = Task.await(source_peer, 5_000) + assert {:error, :timeout} = :ssl.transport_accept(target, 0) + target_peer = serve(target, f.rsa.der, [ok()]) + result = fetch(target_port, f) + assert result.status == 200 + assert HTTP.Response.read_all(result) == "mtls" + assert :ok = Task.await(target_peer, 5_000) + end + + test "OTP backend keeps its existing cross-origin client-identity behavior", %{fixtures: f} do + {source, port} = listener(f) + {target, target_port} = listener(f) + source_peer = serve(source, f.rsa.der, [redirect("https://127.0.0.1:#{target_port}/next")]) + target_peer = serve(target, f.rsa.der, [ok()]) + response = fetch(port, f, tls_backend: :ssl) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "mtls" + assert :ok = Task.await(source_peer, 5_000) + assert :ok = Task.await(target_peer, 5_000) + end + + defp listener(f) do + {:ok, socket} = + :ssl.listen(0, + certfile: f.server.certificate, + keyfile: f.server.key, + cacerts: [f.ca.der], + verify: :verify_peer, + fail_if_no_peer_cert: true, + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true + ) + + on_exit(fn -> :ssl.close(socket) end) + {:ok, {_, port}} = :ssl.sockname(socket) + {socket, port} + end + + defp serve(listener, expected, responses) do + task = + Task.async(fn -> + Enum.each(responses, fn response -> + {:ok, transport} = :ssl.transport_accept(listener, 5_000) + {:ok, socket} = :ssl.handshake(transport, 5_000) + + try do + assert {:ok, ^expected} = :ssl.peercert(socket) + headers(socket, <<>>) + assert :ok = :ssl.send(socket, response) + after + :ssl.close(socket) + end + end) + end) + + on_exit(fn -> if Process.alive?(task.pid), do: Process.exit(task.pid, :kill) end) + task + end + + defp headers(socket, buffer) do + if :binary.match(buffer, "\r\n\r\n") == :nomatch do + assert {:ok, data} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> data) + else + buffer + end + end + + defp fetch(port, f, extra \\ []) do + fetch_url("https://127.0.0.1:#{port}/start", f, extra) + end + + defp fetch_url(url, f, extra) do + HTTP.fetch( + url, + Keyword.merge( + [ + tls_backend: :ex_ssl, + timeout: 5_000, + ssl: [ + cacerts: [f.ca.der], + server_name_indication: ~c"exssl.test", + certfile: f.rsa.certificate, + keyfile: f.rsa.key + ] + ], + extra + ) + ) + |> HTTP.Promise.await(5_000) + end + + defp redirect(url), + do: "HTTP/1.1 302 Found\r\nLocation: #{url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + + defp ok, do: "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nmtls" +end diff --git a/scripts/ex_ssl_mtls_streams_test.exs b/scripts/ex_ssl_mtls_streams_test.exs new file mode 100644 index 0000000..5664daa --- /dev/null +++ b/scripts/ex_ssl_mtls_streams_test.exs @@ -0,0 +1,321 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file( + Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test", "support", fixture]) + ) +end + +defmodule CandidateMtlsStreamsTest do + use ExUnit.Case, async: false + import Bitwise + + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.EventSource + alias HTTP.EventSource.Event.Error + alias HTTP.EventSource.Event.Message, as: SSEMessage + alias HTTP.EventSource.Event.Open, as: SSEOpen + alias HTTP.WebSocket + alias HTTP.WebSocket.Event.Close + alias HTTP.WebSocket.Event.Message, as: WSMessage + alias HTTP.WebSocket.Event.Open, as: WSOpen + + @guid "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + @timeout 5_000 + + setup_all do + directory = + Path.join(System.tmp_dir!(), "http-mtls-streams-#{System.unique_integer([:positive])}") + + File.mkdir_p!(directory) + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "packaged WSS uses ex_ssl mTLS through passive upgrade and active-once echo close", %{ + fixtures: fixtures + } do + ref = make_ref() + %{listener: listener, port: port, task: task} = start_web_socket_peer(self(), ref, fixtures) + + socket = + WebSocket.new("wss://127.0.0.1:#{port}/socket", [], + tls_backend: :ex_ssl, + ssl: client_ssl_options(fixtures) + ) + + try do + assert_receive {:mtls_wss, ^ref, :authenticated, client_der}, @timeout + assert client_der == fixtures.rsa.der + assert_receive {:mtls_wss, ^ref, :upgrade}, @timeout + assert_receive {WebSocket, ^socket, %WSOpen{}}, @timeout + + assert :ok = WebSocket.send(socket, "mtls-echo") + assert_receive {:mtls_wss, ^ref, {:text, "mtls-echo"}}, @timeout + assert_receive {WebSocket, ^socket, %WSMessage{data: "echo:mtls-echo"}}, @timeout + + assert :ok = WebSocket.close(socket, 1000, "done") + assert_receive {:mtls_wss, ^ref, {:close, <<1000::16, "done">>}}, @timeout + + assert_receive {WebSocket, ^socket, %Close{code: 1000, reason: "done", was_clean: true}}, + @timeout + + assert :ok = Task.await(task, @timeout) + after + _ = WebSocket.close(socket) + stop_peer(listener, task) + end + end + + test "packaged EventSource reconnects to the same mTLS endpoint with its pinned backend", %{ + fixtures: fixtures + } do + ref = make_ref() + %{listener: listener, port: port, task: task} = start_event_source_peer(self(), ref, fixtures) + previous_backend = Application.get_env(:http_core, :tls_backend, :unset) + + on_exit(fn -> restore_backend(previous_backend) end) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + tls_backend: :ex_ssl, + reconnect_time: 10, + ssl: client_ssl_options(fixtures) + ) + + try do + assert_receive {:mtls_sse, ^ref, :authenticated, 1, client_der}, @timeout + assert client_der == fixtures.rsa.der + assert_receive {:mtls_sse, ^ref, {:request, 1, first_request}}, @timeout + assert String.starts_with?(first_request, "GET /events HTTP/1.1\r\n") + assert_receive {EventSource, ^source, %SSEOpen{}}, @timeout + + assert_receive {EventSource, ^source, %SSEMessage{data: "first", last_event_id: "41"}}, + @timeout + + assert_receive {:mtls_sse, ^ref, :first_ready}, @timeout + + Application.put_env(:http_core, :tls_backend, :invalid) + assert %{tls_backend: :ex_ssl} = :sys.get_state(source.pid) + send(task.pid, {:close_first, ref}) + assert_receive {:mtls_sse, ^ref, :first_closed}, @timeout + assert_receive {EventSource, ^source, %Error{reason: :eof}}, @timeout + + assert_receive {:mtls_sse, ^ref, :authenticated, 2, second_der}, @timeout + assert second_der == fixtures.rsa.der + assert_receive {:mtls_sse, ^ref, {:request, 2, second_request}}, @timeout + assert String.starts_with?(second_request, "GET /events HTTP/1.1\r\n") + assert second_request =~ "Last-Event-ID: 41\r\n" + assert_receive {EventSource, ^source, %SSEOpen{}}, @timeout + + assert_receive {EventSource, ^source, %SSEMessage{data: "second", last_event_id: "41"}}, + @timeout + + assert :ok = EventSource.close(source) + assert :ok = Task.await(task, @timeout) + after + _ = EventSource.close(source) + stop_peer(listener, task) + end + end + + defp start_web_socket_peer(parent, ref, fixtures) do + {:ok, listener} = listen(fixtures) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, socket} = accept_authenticated(listener, fixtures.rsa.der) + send(parent, {:mtls_wss, ref, :authenticated, fixtures.rsa.der}) + request = recv_headers(socket, <<>>) + key = websocket_key(request) + :ok = :ssl.send(socket, websocket_upgrade_response(key)) + send(parent, {:mtls_wss, ref, :upgrade}) + + {:text, "mtls-echo", buffer} = recv_client_frame(socket, <<>>) + send(parent, {:mtls_wss, ref, {:text, "mtls-echo"}}) + :ok = :ssl.send(socket, frame(:text, "echo:mtls-echo")) + + {:close, payload, <<>>} = recv_client_frame(socket, buffer) + send(parent, {:mtls_wss, ref, {:close, payload}}) + :ok = :ssl.send(socket, frame(:close, payload)) + :ssl.close(socket) + :ssl.close(listener) + end) + + %{listener: listener, port: port, task: task} + end + + defp start_event_source_peer(parent, ref, fixtures) do + {:ok, listener} = listen(fixtures) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + serve_event_source_connection( + listener, + parent, + ref, + fixtures.rsa.der, + 1, + "id: 41\ndata: first\n\n", + true + ) + + serve_event_source_connection( + listener, + parent, + ref, + fixtures.rsa.der, + 2, + "data: second\n\n", + false + ) + + :ssl.close(listener) + end) + + %{listener: listener, port: port, task: task} + end + + defp serve_event_source_connection(listener, parent, ref, expected_der, index, body, close?) do + {:ok, socket} = accept_authenticated(listener, expected_der) + send(parent, {:mtls_sse, ref, :authenticated, index, expected_der}) + request = recv_headers(socket, <<>>) + send(parent, {:mtls_sse, ref, {:request, index, request}}) + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: ", + if(close?, do: "close", else: "keep-alive"), + "\r\n\r\n", + body + ]) + + if close? do + send(parent, {:mtls_sse, ref, :first_ready}) + + receive do + {:close_first, ^ref} -> :ok + after + @timeout -> raise "first EventSource connection was not explicitly released" + end + + :ssl.close(socket) + send(parent, {:mtls_sse, ref, :first_closed}) + else + assert {:error, :closed} = :ssl.recv(socket, 0, @timeout) + end + end + + defp listen(fixtures) do + :ssl.listen(0, [ + :binary, + active: false, + mode: :binary, + packet: :raw, + reuseaddr: true, + ip: {127, 0, 0, 1}, + versions: [:"tlsv1.3"], + certfile: String.to_charlist(fixtures.server.certificate), + keyfile: String.to_charlist(fixtures.server.key), + cacerts: [fixtures.ca.der], + verify: :verify_peer, + fail_if_no_peer_cert: true + ]) + end + + defp accept_authenticated(listener, expected_der) do + with {:ok, transport} <- :ssl.transport_accept(listener, @timeout), + {:ok, socket} <- :ssl.handshake(transport, @timeout), + {:ok, ^expected_der} <- :ssl.peercert(socket) do + {:ok, socket} + end + end + + defp client_ssl_options(fixtures) do + [ + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + certfile: fixtures.rsa.certificate, + keyfile: fixtures.rsa.key + ] + end + + defp recv_headers(socket, buffer) when byte_size(buffer) < 16_384 do + if :binary.match(buffer, "\r\n\r\n") == :nomatch do + {:ok, bytes} = :ssl.recv(socket, 0, @timeout) + recv_headers(socket, buffer <> bytes) + else + buffer + end + end + + defp websocket_key(request) do + [_, key] = Regex.run(~r/sec-websocket-key:\s*([^\r\n]+)/i, request) + String.trim(key) + end + + defp websocket_upgrade_response(key) do + accept = :crypto.hash(:sha, key <> @guid) |> Base.encode64() + + [ + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ", + accept, + "\r\n\r\n" + ] + end + + defp recv_client_frame(socket, buffer) do + case parse_client_frame(buffer) do + {:ok, frame, rest} -> + {frame_type(frame), frame.payload, rest} + + :more -> + {:ok, bytes} = :ssl.recv(socket, 0, @timeout) + recv_client_frame(socket, buffer <> bytes) + end + end + + defp parse_client_frame(<>) do + opcode = first &&& 0x0F + masked? = (second &&& 0x80) != 0 + length_code = second &&& 0x7F + + with true <- masked?, + {:ok, length, rest} <- frame_length(length_code, rest), + <> <- rest do + {:ok, %{opcode: opcode, payload: unmask(encrypted, mask)}, remainder} + else + _ -> :more + end + end + + defp parse_client_frame(_), do: :more + defp frame_length(length, rest) when length <= 125, do: {:ok, length, rest} + defp frame_length(126, <>), do: {:ok, length, rest} + defp frame_length(127, <>), do: {:ok, length, rest} + defp frame_length(_, _), do: :more + defp frame_type(%{opcode: 0x1}), do: :text + defp frame_type(%{opcode: 0x8}), do: :close + + defp frame(:text, payload) when byte_size(payload) <= 125, + do: <<0x81, byte_size(payload), payload::binary>> + + defp frame(:close, payload) when byte_size(payload) <= 125, + do: <<0x88, byte_size(payload), payload::binary>> + + defp unmask(payload, mask) do + payload + |> :binary.bin_to_list() + |> Enum.with_index() + |> Enum.map_join(fn {byte, index} -> <> end) + end + + defp stop_peer(listener, task) do + _ = :ssl.close(listener) + if Process.alive?(task.pid), do: Task.shutdown(task, :brutal_kill) + end + + defp restore_backend(:unset), do: Application.delete_env(:http_core, :tls_backend) + defp restore_backend(value), do: Application.put_env(:http_core, :tls_backend, value) +end diff --git a/scripts/ex_ssl_mtls_test.exs b/scripts/ex_ssl_mtls_test.exs new file mode 100644 index 0000000..73e5052 --- /dev/null +++ b/scripts/ex_ssl_mtls_test.exs @@ -0,0 +1,253 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +source = System.fetch_env!("EX_SSL_SOURCE_DIR") +Code.require_file(Path.join(source, "test/support/signature_fixtures.ex")) +Code.require_file(Path.join(source, "test/support/client_auth_fixtures.ex")) + +defmodule CandidateMTLSTest do + use ExUnit.Case, async: false + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.HTTP2.{Frame, HPACK} + + @body "packaged-mtls" + @path "/mtls" + + setup_all do + directory = Path.join(System.tmp_dir!(), "http-mtls-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + for identity <- [:rsa, :ec, :large], protocol <- ["http/1.1", "h2"] do + test "required #{identity} identity returns verified #{protocol} HTTP response", context do + fixture = context.fixtures[unquote(identity)] + if unquote(identity) == :large, do: assert(byte_size(fixture.der) > 16_384) + exchange(context.fixtures, fixture, :required, unquote(protocol)) + end + end + + test "optional client auth without an identity returns an HTTP response", context do + exchange(context.fixtures, nil, :optional, "http/1.1") + end + + test "missing, wrong-CA, expired, and wrong-purpose identities cannot fetch", context do + for identity <- [ + nil, + context.fixtures.wrong, + context.fixtures.expired, + context.fixtures.server + ] do + with_peer( + context.fixtures, + :required, + "http/1.1", + fn port -> + assert {:error, _} = fetch(port, context.fixtures, identity, "http/1.1") + end, + :reject + ) + end + end + + test "mismatched client key fails before network I/O", context do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, ip: {127, 0, 0, 1}]) + {:ok, {_, port}} = :inet.sockname(listener) + + try do + assert {:error, _} = + fetch(port, context.fixtures, context.fixtures.rsa, "http/1.1", + keyfile: context.fixtures.ec.key + ) + + assert {:error, :timeout} = :gen_tcp.accept(listener, 200) + after + :gen_tcp.close(listener) + end + end + + test "incompatible requested signature does not authenticate", context do + with_peer( + %{context.fixtures | server: context.fixtures.ec_server}, + :required, + "http/1.1", + fn port -> + assert {:error, _} = fetch(port, context.fixtures, context.fixtures.rsa, "http/1.1") + end, + :reject, + signature_algs: [:ecdsa_secp256r1_sha256] + ) + end + + test "wrong server hostname fails even when client credentials are valid", context do + with_peer( + context.fixtures, + :required, + "http/1.1", + fn port -> + assert {:error, _} = + fetch(port, context.fixtures, context.fixtures.rsa, "http/1.1", + server_name_indication: ~c"wrong.test" + ) + end, + :reject + ) + end + + defp exchange(fixtures, identity, mode, protocol) do + expected = if identity, do: identity.der, else: nil + + response = + with_peer( + fixtures, + mode, + protocol, + fn port -> fetch(port, fixtures, identity, protocol) end, + expected + ) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + defp fetch(port, fixtures, identity, protocol, overrides \\ []) do + ssl = [ + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + verify: :verify_peer, + alpn_advertised_protocols: [protocol] + ] + + credentials = + if identity, do: [certfile: identity.certificate, keyfile: identity.key], else: [] + + ssl = Keyword.merge(ssl ++ credentials, overrides) + + HTTP.fetch("https://127.0.0.1:#{port}#{@path}", + http_version: if(protocol == "h2", do: :http2, else: :http1), + tls_backend: :ex_ssl, + ssl: ssl, + timeout: 5_000, + connect_timeout: 5_000 + ) + |> HTTP.Promise.await(10_000) + end + + defp with_peer(fixtures, mode, protocol, client, expected, extra \\ []) do + ssl_options = + [ + certfile: String.to_charlist(fixtures.server.certificate), + keyfile: String.to_charlist(fixtures.server.key), + cacerts: [fixtures.ca.der], + verify: :verify_peer, + fail_if_no_peer_cert: mode == :required, + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true, + alpn_preferred_protocols: [protocol] + ] + |> Keyword.merge(extra) + + {:ok, listener} = :ssl.listen(0, ssl_options) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, transport} = :ssl.transport_accept(listener, 5_000) + + case :ssl.handshake(transport, 5_000) do + {:ok, socket} -> + try do + if expected == :reject do + case :ssl.recv(socket, 0, 5_000) do + {:error, _} -> :ok + {:ok, bytes} -> flunk("rejected peer sent HTTP bytes: #{byte_size(bytes)}") + end + else + assert {:ok, ^protocol} = :ssl.negotiated_protocol(socket) + + case expected do + nil -> assert {:error, :no_peercert} = :ssl.peercert(socket) + der -> assert {:ok, ^der} = :ssl.peercert(socket) + end + + respond(socket, protocol) + end + after + :ssl.close(socket) + end + + {:error, _} when expected == :reject -> + :ok + end + end) + + result = + try do + result = client.(port) + assert :ok = Task.await(task, 10_000) + result + after + :ssl.close(listener) + Task.shutdown(task, :brutal_kill) + end + + assert {:error, :econnrefused} = + :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 1_000) + + result + end + + defp respond(socket, "http/1.1") do + request = headers(socket, "") + assert String.starts_with?(request, "GET #{@path} HTTP/1.1\r\n") + + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: #{byte_size(@body)}\r\n\r\n", + @body + ]) + end + + defp respond(socket, "h2") do + preface = HTTP.HTTP2.connection_preface() + assert {:ok, ^preface} = :ssl.recv(socket, byte_size(preface), 5_000) + assert %Frame{type: :settings} = frame(socket) + assert %Frame{type: :headers, stream_id: 1, payload: block} = frame(socket) + assert {:ok, _, headers} = HPACK.decode(HPACK.new_decoder(), block) + assert {":path", @path} in headers + + :ok = + :ssl.send(socket, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode( + :headers, + 4, + 1, + HPACK.encode_headers([ + {":status", "200"}, + {"content-length", Integer.to_string(byte_size(@body))} + ]) + ), + Frame.encode(:data, 1, 1, @body) + ]) + + assert %Frame{type: :settings, flags: 1, stream_id: 0, payload: ""} = frame(socket) + :ok + end + + defp frame(socket) do + assert {:ok, <> = header} = :ssl.recv(socket, 9, 5_000) + payload = if size == 0, do: "", else: elem(:ssl.recv(socket, size, 5_000), 1) + assert {:ok, frame, ""} = Frame.decode(header <> payload) + frame + end + + defp headers(socket, buffer) when byte_size(buffer) < 16_384 do + if String.contains?(buffer, "\r\n\r\n"), + do: buffer, + else: + ( + {:ok, bytes} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> bytes) + ) + end +end diff --git a/scripts/ex_ssl_source_smoke.sh b/scripts/ex_ssl_source_smoke.sh index e4d8b76..ee74bd8 100755 --- a/scripts/ex_ssl_source_smoke.sh +++ b/scripts/ex_ssl_source_smoke.sh @@ -8,7 +8,7 @@ work_dir=$(mktemp -d) trap 'rm -rf "$work_dir"' EXIT export HTTP_FETCH_PACKAGE_DIR="$work_dir/packages" mkdir -p "$HTTP_FETCH_PACKAGE_DIR" "$work_dir/consumer/test" -for app in http_core http_fetch; do +for app in http_core http_fetch http_web_socket http_event_source http_web_transport; do (cd "$repo_root/apps/$app" && MIX_ENV=prod mix hex.build --unpack -o "$HTTP_FETCH_PACKAGE_DIR/$app") done cat > "$work_dir/consumer/mix.exs" <<'MIX' @@ -19,6 +19,9 @@ defmodule CandidateConsumer.MixProject do [app: :candidate_consumer, version: "0.0.0", deps: [ {:http_core, path: Path.join(packages, "http_core")}, {:http_fetch, path: Path.join(packages, "http_fetch")}, + {:http_web_socket, path: Path.join(packages, "http_web_socket")}, + {:http_event_source, path: Path.join(packages, "http_event_source")}, + {:http_web_transport, path: Path.join(packages, "http_web_transport")}, {:ex_ssl, path: System.fetch_env!("EX_SSL_SOURCE_DIR"), override: true} ]] end @@ -26,6 +29,8 @@ defmodule CandidateConsumer.MixProject do end MIX cp "$repo_root/mix.lock" "$work_dir/consumer/mix.lock" -cp "$repo_root/scripts/ex_ssl_algorithms_test.exs" "$work_dir/consumer/test/algorithms_test.exs" +for test_file in "$repo_root"/scripts/ex_ssl_*_test.exs; do + cp "$test_file" "$work_dir/consumer/test/$(basename "$test_file")" +done printf 'ExUnit.start()\n' > "$work_dir/consumer/test/test_helper.exs" -(cd "$work_dir/consumer" && mix deps.get && mix compile --warnings-as-errors && mix test --seed 36) +(cd "$work_dir/consumer" && mix deps.get && mix compile --warnings-as-errors && mix test "$@" --seed 36) From f482322dec1934d26724ace8ed5c1a49219c6ceb Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 14:49:44 +0800 Subject: [PATCH 09/17] feat(tls): forward validated candidate policy and TCP options --- README.md | 6 +- apps/http_core/lib/http/transport/ex_ssl.ex | 13 +- apps/http_fetch/lib/http/socket_client.ex | 31 +- docs/ex-ssl-consumer-contract.md | 21 ++ scripts/ex_ssl_options_test.exs | 340 ++++++++++++++++++++ 5 files changed, 402 insertions(+), 9 deletions(-) create mode 100644 scripts/ex_ssl_options_test.exs diff --git a/README.md b/README.md index de4722e..740e70c 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,11 @@ and timeout remain errors. The original deadline and streaming backpressure are preserved. For `:ex_ssl`, `socket_opts` accepts `send_timeout` and -`send_timeout_close: true`. These override matching entries in `ssl`. Custom +`send_timeout_close: true`. The unreleased source candidate also implements +`nodelay`, `keepalive`, `sndbuf`, `recbuf`, and local `ip`/`port`; the adapter +forwards only this allowlist and ex_ssl validates values. IPv6 literals infer the +family; an IPv6 local `ip` tuple selects IPv6 DNS resolution. Both option +containers must be keyword lists. These override matching entries in `ssl`. Custom ClientHello profiles can be passed through `ssl: [ex_ssl: [profile: profile]]`; any ALPN list added by HTTP/2 selection must match the profile's ALPN list exactly. For a source candidate that implements client authentication, ex_ssl credentials diff --git a/apps/http_core/lib/http/transport/ex_ssl.ex b/apps/http_core/lib/http/transport/ex_ssl.ex index 46e8cc4..0304ad2 100644 --- a/apps/http_core/lib/http/transport/ex_ssl.ex +++ b/apps/http_core/lib/http/transport/ex_ssl.ex @@ -72,7 +72,18 @@ defmodule HTTP.Transport.ExSSL do end defp validate_socket_options(opts) do - case Enum.find(opts, fn {key, _} -> key not in [:send_timeout, :send_timeout_close] end) do + allowed = [ + :send_timeout, + :send_timeout_close, + :nodelay, + :keepalive, + :sndbuf, + :recbuf, + :ip, + :port + ] + + case Enum.find(opts, fn {key, _} -> key not in allowed end) do nil -> :ok {key, _} -> {:error, {:options, {key, :unsupported_or_invalid}}} end diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index e82db37..654e972 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -525,15 +525,32 @@ defmodule HTTP.SocketClient do defp connect(transport, host, port, request, selection, timeout) do connect_timeout = min(connect_timeout(request), timeout) - interruptible_connect( - transport, - host, - port, - transport_opts(request, selection, timeout), - connect_timeout - ) + with :ok <- validate_transport_option_lists(transport, request) do + interruptible_connect( + transport, + host, + port, + transport_opts(request, selection, timeout), + connect_timeout + ) + end end + defp validate_transport_option_lists(HTTP.Transport.ExSSL, request) do + if Enum.all?([:ssl, :socket_opts], fn key -> + opts = Keyword.get(request.transport_options, key, []) + + Keyword.keyword?(opts) and + length(Keyword.keys(opts)) == length(Enum.uniq(Keyword.keys(opts))) + end) do + :ok + else + {:error, {:options, :invalid_options}} + end + end + + defp validate_transport_option_lists(_transport, _request), do: :ok + defp interruptible_connect(transport, host, port, opts, timeout) do parent = self() ref = make_ref() diff --git a/docs/ex-ssl-consumer-contract.md b/docs/ex-ssl-consumer-contract.md index f879782..92a4af1 100644 --- a/docs/ex-ssl-consumer-contract.md +++ b/docs/ex-ssl-consumer-contract.md @@ -92,3 +92,24 @@ EventSource verifies the identity across same-origin reconnects. Redirect tests cover all three origin components, DNS casing, manual reuse and unchanged OTP. These are source-candidate tests; released ex_ssl 0.3.0 still rejects client identity options. No release manifest or installed dependency has been changed. + +## Phase 3 source candidate + +The adapter forwards the safe TCP allowlist: `nodelay`, `keepalive`, `sndbuf`, +`recbuf`, local `ip`/`port`, plus the existing send deadline options. The source +candidate validates values and supports mutable driver options; released ex_ssl +0.3.0 continues to reject newly unsupported keys. Keyword containers and duplicate +keys reject before fetch adds deadlines or ALPN, including improper lists. +Raw active/packet controls, linger and arbitrary socket backends remain rejected. + +The candidate accepts ordered TLS1.3 `ciphers`, `signature_algs`, +`signature_algs_cert` and `supported_groups` through `ssl`. Generated profiles +preserve order; explicit profile conflicts fail before I/O. The certificate +signature policy is separate from handshake CertificateVerify. No supplied +`signature_algs_cert` preserves the earlier chain policy. See the library matrix +for supported names/maps and deliberate differences from OTP. + +`scripts/ex_ssl_options_test.exs` exercises real packaged HTTP policy and TCP +selection, IPv6 local binding/DNS, raw option precedence/mutation, authentication +failures and pre-I/O rejection of malformed/unsafe/conflicting options. The same +shared adapter serves WSS and SSE. No TLS1.2 or default change is introduced here. diff --git a/scripts/ex_ssl_options_test.exs b/scripts/ex_ssl_options_test.exs new file mode 100644 index 0000000..aa4f793 --- /dev/null +++ b/scripts/ex_ssl_options_test.exs @@ -0,0 +1,340 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +source = System.fetch_env!("EX_SSL_SOURCE_DIR") +Code.require_file(Path.join(source, "test/support/signature_fixtures.ex")) +Code.require_file(Path.join(source, "test/support/client_auth_fixtures.ex")) + +defmodule CandidateOptionsTest do + use ExUnit.Case, async: false + + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.Transport.ExSSL + alias SSL.ClientHello.WireProfile + + @body "ordered-options" + + setup_all do + directory = Path.join(System.tmp_dir!(), "http-options-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "HTTP.fetch honors ordered cipher, group, CertificateVerify, and issuer policies", + context do + response = + with_peer( + context.fixtures, + fn port -> + fetch(port, context.fixtures, + ciphers: ["TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256"], + supported_groups: [:secp256r1, :x25519], + signature_algs: [:rsa_pss_rsae_sha256, :ecdsa_secp256r1_sha256], + signature_algs_cert: [:rsa_pkcs1_sha256] + ) + end, + @body, + ciphers: [ + %{key_exchange: :any, cipher: :aes_256_gcm, mac: :aead, prf: :sha384} + ], + supported_groups: [:secp256r1] + ) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + test "an incompatible CertificateVerify or issuer policy fails authentication", context do + for policy <- [ + [signature_algs: [:ecdsa_secp256r1_sha256]], + [signature_algs: [:rsa_pss_rsae_sha256], signature_algs_cert: [:ecdsa_secp256r1_sha256]] + ] do + with_peer( + context.fixtures, + fn port -> + assert {:error, _} = fetch(port, context.fixtures, policy) + end, + :reject + ) + end + end + + test "a conflicting explicit profile fails before TCP I/O" do + profile = %WireProfile{ + name: :candidate_option_conflict, + cipher_suites: [0x1301], + extensions: [ + {:supported_versions, [0x0304]}, + {:supported_groups, [0x001D]}, + {:signature_algorithms, [0x0804]}, + {:key_share, [0x001D]}, + {:alpn, ["http/1.1"]} + ] + } + + assert_before_io(fn port -> + assert {:error, {:options, _}} = + fetch(port, nil, + ex_ssl: [profile: profile], + ciphers: ["TLS_AES_256_GCM_SHA384"] + ) + end) + end + + test "adapter applies safe TCP options and socket_opts override matching ssl entries", + context do + with_peer( + context.fixtures, + fn port -> + ssl = tls_options(context.fixtures) ++ [nodelay: false, keepalive: false] + + socket_opts = [ + nodelay: true, + keepalive: true, + sndbuf: 16_384, + recbuf: 16_384, + ip: {127, 0, 0, 1}, + port: 0 + ] + + {:ok, socket} = + ExSSL.connect("127.0.0.1", port, [ssl: ssl, socket_opts: socket_opts], 5_000) + + try do + {:connected, state} = :sys.get_state(socket.pid) + tcp = state.tcp + assert {:ok, values} = :inet.getopts(tcp, [:nodelay, :keepalive, :sndbuf, :recbuf]) + assert Keyword.fetch!(values, :nodelay) + assert Keyword.fetch!(values, :keepalive) + assert Keyword.fetch!(values, :sndbuf) >= 16_384 + assert Keyword.fetch!(values, :recbuf) >= 16_384 + assert {:ok, {{127, 0, 0, 1}, local_port}} = :inet.sockname(tcp) + assert local_port > 0 + after + ExSSL.close(socket) + end + end, + :no_http + ) + end + + test "HTTP.fetch forwards the safe TCP allowlist", context do + response = + with_peer(context.fixtures, fn port -> + fetch(port, context.fixtures, [], + socket_opts: [ + nodelay: true, + keepalive: true, + sndbuf: 16_384, + recbuf: 16_384, + ip: {127, 0, 0, 1}, + port: 0 + ] + ) + end) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + test "mutable TCP options apply, but an invalid combined setopts changes nothing", context do + with_peer( + context.fixtures, + fn port -> + {:ok, socket} = + ExSSL.connect("127.0.0.1", port, [ssl: tls_options(context.fixtures)], 5_000) + + try do + {:connected, state} = :sys.get_state(socket.pid) + tcp = state.tcp + + assert :ok = + ExSSL.setopts(socket, + nodelay: true, + keepalive: true, + sndbuf: 16_384, + recbuf: 16_384 + ) + + assert {:ok, values} = :inet.getopts(tcp, [:nodelay, :keepalive, :sndbuf, :recbuf]) + assert Keyword.fetch!(values, :nodelay) + assert Keyword.fetch!(values, :keepalive) + assert Keyword.fetch!(values, :sndbuf) >= 16_384 + assert Keyword.fetch!(values, :recbuf) >= 16_384 + + assert {:error, {:options, _}} = + ExSSL.setopts(socket, nodelay: false, ip: {127, 0, 0, 1}) + + assert {:ok, [nodelay: true]} = :inet.getopts(tcp, [:nodelay]) + after + ExSSL.close(socket) + end + end, + :no_http + ) + end + + test "IPv6 local bind selects the DNS address family with certificate verification", context do + response = + with_peer( + context.fixtures, + fn port -> + HTTP.fetch("https://localhost:#{port}/options", + tls_backend: :ex_ssl, + ssl: tls_options(context.fixtures), + socket_opts: [ip: {0, 0, 0, 0, 0, 0, 0, 1}], + timeout: 5_000, + connect_timeout: 5_000 + ) + |> HTTP.Promise.await(10_000) + end, + @body, + ip: {0, 0, 0, 0, 0, 0, 0, 1} + ) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + test "malformed and unsafe socket options fail before network I/O" do + for socket_opts <- [ + [linger: {true, 0}], + [active: true], + [packet: 4], + [nodelay: :invalid], + [nodelay: true, nodelay: false], + [ip: {999, 0, 0, 1}], + [port: -1], + [send_timeout_close: false, linger: {true, 0}], + [{:nodelay, true} | :improper] + ] do + assert_before_io(fn port -> + assert {:error, {:options, _}} = + HTTP.fetch("https://127.0.0.1:#{port}/options", + tls_backend: :ex_ssl, + socket_opts: socket_opts, + timeout: 2_000, + connect_timeout: 2_000 + ) + |> HTTP.Promise.await(3_000) + end) + end + end + + test "malformed ssl options fail before HTTP/2 ALPN construction and network I/O" do + assert_before_io(fn port -> + assert {:error, {:options, _}} = + HTTP.fetch("https://127.0.0.1:#{port}/options", + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [{:verify, :verify_peer} | :improper], + timeout: 2_000, + connect_timeout: 2_000 + ) + |> HTTP.Promise.await(3_000) + end) + end + + defp fetch(port, fixtures, ssl_extra, opts \\ []) do + ssl = if(fixtures, do: tls_options(fixtures), else: []) ++ ssl_extra + + HTTP.fetch( + "https://127.0.0.1:#{port}/options", + [ + tls_backend: :ex_ssl, + ssl: ssl, + timeout: 5_000, + connect_timeout: 5_000 + ] ++ opts + ) + |> HTTP.Promise.await(10_000) + end + + defp tls_options(fixtures) do + [ + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + verify: :verify_peer, + alpn_advertised_protocols: ["http/1.1"] + ] + end + + defp with_peer(fixtures, client, response \\ @body, extra \\ []) do + options = + [ + certfile: String.to_charlist(fixtures.server.certificate), + keyfile: String.to_charlist(fixtures.server.key), + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true, + alpn_preferred_protocols: ["http/1.1"] + ] + |> Keyword.merge(extra) + + {:ok, listener} = :ssl.listen(0, options) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, transport} = :ssl.transport_accept(listener, 5_000) + + case :ssl.handshake(transport, 5_000) do + {:ok, socket} -> + try do + case response do + :reject -> + assert {:error, _} = :ssl.recv(socket, 0, 5_000) + + :no_http -> + assert {:error, _} = :ssl.recv(socket, 0, 5_000) + :ok + + body -> + assert {:ok, "http/1.1"} = :ssl.negotiated_protocol(socket) + assert String.starts_with?(headers(socket, ""), "GET /options HTTP/1.1\r\n") + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: #{byte_size(body)}\r\n\r\n", + body + ]) + end + after + :ssl.close(socket) + end + + {:error, _} when response == :reject -> + :ok + end + end) + + try do + result = client.(port) + assert :ok = Task.await(task, 10_000) + result + after + :ssl.close(listener) + Task.shutdown(task, :brutal_kill) + end + end + + defp headers(socket, buffer) when byte_size(buffer) < 16_384 do + if String.contains?(buffer, "\r\n\r\n") do + buffer + else + {:ok, bytes} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> bytes) + end + end + + defp assert_before_io(client) do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, ip: {127, 0, 0, 1}]) + {:ok, {_, port}} = :inet.sockname(listener) + + try do + client.(port) + assert {:error, :timeout} = :gen_tcp.accept(listener, 200) + after + :gen_tcp.close(listener) + end + end +end From 4aa8855e782971dce7a87a3ac2575323b95ac9c7 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 15:52:04 +0800 Subject: [PATCH 10/17] refactor(tls): keep option validation clear to configured Credo --- apps/http_fetch/lib/http/socket_client.ex | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index 654e972..8ffc6ab 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -537,12 +537,15 @@ defmodule HTTP.SocketClient do end defp validate_transport_option_lists(HTTP.Transport.ExSSL, request) do - if Enum.all?([:ssl, :socket_opts], fn key -> - opts = Keyword.get(request.transport_options, key, []) + valid? = + Enum.all?([:ssl, :socket_opts], fn key -> + opts = Keyword.get(request.transport_options, key, []) - Keyword.keyword?(opts) and - length(Keyword.keys(opts)) == length(Enum.uniq(Keyword.keys(opts))) - end) do + Keyword.keyword?(opts) and + length(Keyword.keys(opts)) == length(Enum.uniq(Keyword.keys(opts))) + end) + + if valid? do :ok else {:error, {:options, :invalid_options}} From 381f198f498bb11cec4ff7879536b10211bc69e7 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 15:52:04 +0800 Subject: [PATCH 11/17] test(tls): verify packaged TLS 1.2 flows and TLS 1.3 resumption --- README.md | 16 ++ docs/ex-ssl-consumer-contract.md | 28 +++ scripts/ex_ssl_resumption_test.exs | 137 +++++++++++++++ scripts/ex_ssl_source_smoke.sh | 1 + scripts/ex_ssl_tls12_peer.py | 253 +++++++++++++++++++++++++++ scripts/ex_ssl_tls12_test.exs | 269 +++++++++++++++++++++++++++++ 6 files changed, 704 insertions(+) create mode 100644 scripts/ex_ssl_resumption_test.exs create mode 100644 scripts/ex_ssl_tls12_peer.py create mode 100644 scripts/ex_ssl_tls12_test.exs diff --git a/README.md b/README.md index 740e70c..92d2445 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,22 @@ use `redirect: :manual` and explicitly make a new request with that identity. The OTP backend retains its existing redirect behavior. Published ex_ssl 0.3.0 does not yet support client identity options. +The unreleased ex_ssl source candidate also supports verified TLS 1.2 for +HTTP/1.1, HTTP/2, WSS and EventSource. Select it with `ssl: [versions: +[:"tlsv1.2"]]`; a mixed TLS 1.3/TLS 1.2 offer selects the peer's supported +version. Its independent OpenSSL source gate includes 262,144-byte HTTP/2 responses +with observed connection and stream WINDOW_UPDATE frames. This does not change +the published 0.3.0 contract or the OTP default. + +TLS 1.3 session resumption in that candidate is explicit: +`ssl: [versions: [:"tlsv1.3"], session_tickets: :auto]`. Tickets are disabled +by default. Auto mode currently rejects client identities and mixed/TLS 1.2 +version offers; early data and PSK-only exchange are unsupported. When a server +declines a ticket, a full handshake continues on the same connection without +replaying request bytes. The source-only package test checks two fresh HTTP/1.1 connections +against an independent OpenSSL peer and requires server-observed session reuse. +This is a bounded subset, not full OTP `:ssl` parity. + See the [ex_ssl compatibility contract](https://github.com/gsmlg-dev/ex_ssl/blob/v0.3.0/docs/COMPATIBILITY.md). The [consumer contract inventory](docs/ex-ssl-consumer-contract.md) maps the implemented subset and intentional restrictions to its tests. diff --git a/docs/ex-ssl-consumer-contract.md b/docs/ex-ssl-consumer-contract.md index 92a4af1..2202d9c 100644 --- a/docs/ex-ssl-consumer-contract.md +++ b/docs/ex-ssl-consumer-contract.md @@ -113,3 +113,31 @@ for supported names/maps and deliberate differences from OTP. selection, IPv6 local binding/DNS, raw option precedence/mutation, authentication failures and pre-I/O rejection of malformed/unsafe/conflicting options. The same shared adapter serves WSS and SSE. No TLS1.2 or default change is introduced here. + +## Phase 4 source candidate + +The source candidate adds verified TLS 1.2 ECDHE AES-GCM to the shared ex_ssl +adapter. The independent OpenSSL packaged-source gate covers HTTP/1.1 and HTTP/2 +with both TLS 1.2-only and mixed offers, mixed-offer TLS 1.3 selection, WSS +Upgrade/frame/close, and EventSource reconnect with pinned backend and +Last-Event-ID. All seven scenarios pass in the final 47-test source gate. +The expanded HTTP/2 fixture sends 262,144 bytes, honors connection/stream +flow control, and requires observed WINDOW_UPDATE frames; its seven-test gate +also passes. The released ex_ssl 0.3.0 remains TLS 1.3-only. TLS 1.2 session resumption and full +OTP option parity are not claimed. + +## Phase 5 source candidate + +`ssl: [versions: [:"tlsv1.3"], session_tickets: :auto]` enables bounded TLS 1.3 +ticket reuse for a fresh connection to the same authenticated context. The +default is `:disabled`; auto rejects TLS 1.2/mixed offers and configured client +identity. Early data, persistent tickets, PSK-only key exchange, and automatic +reconnect/replay are unsupported. An unaccepted ticket follows normal full +handshake processing on the same socket. + +`scripts/ex_ssl_resumption_test.exs` uses two packaged HTTP/1.1 fetches against +one Python/OpenSSL context and checks the peer's `session_reused` value is false +then true. It is a source-candidate test and does not extend the released +dependency contract. Cache policy isolation and measured performance are +recorded in the library readiness report. This consumer check proves only the +HTTP/1.1 adapter path; HTTP/2/WSS/SSE resumption is not separately verified. diff --git a/scripts/ex_ssl_resumption_test.exs b/scripts/ex_ssl_resumption_test.exs new file mode 100644 index 0000000..e0b7d9e --- /dev/null +++ b/scripts/ex_ssl_resumption_test.exs @@ -0,0 +1,137 @@ +# Source-candidate package test; run through scripts/ex_ssl_source_smoke.sh. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file(Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support", fixture])) +end + +defmodule CandidateResumptionTest do + use ExUnit.Case, async: false + + alias ExSSL.TestSupport.ClientAuthFixtures + + @timeout 10_000 + @python Path.join(__DIR__, "ex_ssl_tls12_peer.py") + + setup_all do + directory = + Path.join(System.tmp_dir!(), "http-resumption-#{System.unique_integer([:positive])}") + + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "packaged HTTP/1.1 forwards TLS 1.3 auto tickets across fresh connections", %{ + fixtures: fixtures + } do + peer = start_peer(fixtures) + + try do + for {index, resumed} <- [{1, false}, {2, true}] do + promise = + HTTP.fetch("https://127.0.0.1:#{peer.port}/resumption", + tls_backend: :ex_ssl, + ssl: [ + versions: [:"tlsv1.3"], + verify: :verify_peer, + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + alpn_advertised_protocols: ["http/1.1"], + session_tickets: :auto + ], + timeout: @timeout, + connect_timeout: @timeout + ) + + assert {:ok, + %{ + "index" => ^index, + "version" => "TLSv1.3", + "alpn" => "http/1.1", + "session_reused" => ^resumed, + "client_der_b64" => :null + }} = event(peer, "handshake") + + assert {:ok, %{"bytes" => 6}} = event(peer, "exchange") + response = HTTP.Promise.await(promise, @timeout) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "BBBBBB" + end + after + stop_peer(peer) + end + end + + defp start_peer(fixtures) do + executable = System.find_executable("python3") || raise "python3 unavailable" + assert File.regular?(@python) + + handle = + Port.open({:spawn_executable, executable}, [ + :binary, + :exit_status, + :stderr_to_stdout, + {:line, 65_536}, + args: [ + "-B", + "-u", + @python, + "--certfile", + fixtures.server.certificate, + "--keyfile", + fixtures.server.key, + "--cafile", + fixtures.ca.certificate, + "--mode", + "resumption" + ] + ]) + + {:os_pid, os_pid} = Port.info(handle, :os_pid) + peer = %{handle: handle, os_pid: os_pid, port: nil} + + case event(peer, "ready") do + {:ok, %{"port" => port}} -> + %{peer | port: port} + + error -> + stop_peer(peer) + raise "resumption peer startup failed: #{inspect(error)}" + end + end + + defp event(%{handle: handle}, expected) do + receive do + {^handle, {:data, {:eol, line}}} -> + case :json.decode(line) do + %{"kind" => ^expected} = value -> {:ok, value} + %{"kind" => "failure"} = value -> {:error, value} + other -> {:error, {:unexpected_peer_event, other}} + end + + {^handle, {:data, {:noeol, _}}} -> + {:error, :peer_line_too_long} + + {^handle, {:exit_status, status}} -> + {:error, {:peer_exit, status}} + after + @timeout -> {:error, :peer_timeout} + end + rescue + _ -> {:error, :invalid_peer_event} + end + + defp stop_peer(%{handle: handle, os_pid: os_pid}) do + case Port.info(handle, :os_pid) do + {:os_pid, ^os_pid} -> + _ = System.cmd("kill", ["-TERM", Integer.to_string(os_pid)], stderr_to_stdout: true) + + receive do + {^handle, {:exit_status, _}} -> :ok + after + 2_000 -> raise "resumption peer did not stop" + end + + nil -> + :ok + end + end +end diff --git a/scripts/ex_ssl_source_smoke.sh b/scripts/ex_ssl_source_smoke.sh index ee74bd8..230246a 100755 --- a/scripts/ex_ssl_source_smoke.sh +++ b/scripts/ex_ssl_source_smoke.sh @@ -32,5 +32,6 @@ cp "$repo_root/mix.lock" "$work_dir/consumer/mix.lock" for test_file in "$repo_root"/scripts/ex_ssl_*_test.exs; do cp "$test_file" "$work_dir/consumer/test/$(basename "$test_file")" done +cp "$repo_root/scripts/ex_ssl_tls12_peer.py" "$work_dir/consumer/test/ex_ssl_tls12_peer.py" printf 'ExUnit.start()\n' > "$work_dir/consumer/test/test_helper.exs" (cd "$work_dir/consumer" && mix deps.get && mix compile --warnings-as-errors && mix test "$@" --seed 36) diff --git a/scripts/ex_ssl_tls12_peer.py b/scripts/ex_ssl_tls12_peer.py new file mode 100644 index 0000000..bb455a3 --- /dev/null +++ b/scripts/ex_ssl_tls12_peer.py @@ -0,0 +1,253 @@ +"""Bounded independent OpenSSL peer for packaged ex_ssl consumer tests.""" + +import argparse +import base64 +import hashlib +import json +import socket +import ssl +import struct +import sys + + +def event(kind, **fields): + print(json.dumps({"kind": kind, **fields}), flush=True) + + +def exact(connection, length): + result = bytearray() + while len(result) < length: + part = connection.recv(length - len(result)) + if not part: + raise EOFError("truncated") + result.extend(part) + return bytes(result) + + +def headers(connection): + result = bytearray() + while b"\r\n\r\n" not in result: + if len(result) >= 16384: + raise ValueError("headers_too_large") + part = connection.recv(4096) + if not part: + raise EOFError("headers_truncated") + result.extend(part) + if len(result) > 16384: + raise ValueError("headers_too_large") + return bytes(result) + + +def frame(connection): + head = exact(connection, 9) + length = int.from_bytes(head[:3], "big") + if length > 16384: + raise ValueError("h2_frame_too_large") + return head[3], head[4], int.from_bytes(head[5:], "big") & 0x7FFFFFFF, exact(connection, length) + + +def send_frame(connection, kind, flags, stream, payload): + if len(payload) > 16384: + raise ValueError("h2_frame_too_large") + connection.sendall(len(payload).to_bytes(3, "big") + bytes([kind, flags]) + struct.pack("!I", stream) + payload) + + +def http1(connection, large, path=b"/tls12"): + request = headers(connection) + if not request.startswith(b"GET " + path + b" HTTP/1.1\r\n"): + raise ValueError("wrong_http1_request") + body = b"B" * (262144 if large else 6) + connection.sendall(b"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: " + str(len(body)).encode() + b"\r\n\r\n" + body) + event("exchange", bytes=len(body)) + + +def http2(connection, large): + if exact(connection, 24) != b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n": + raise ValueError("wrong_h2_preface") + reads = 0 + + def next_frame(): + nonlocal reads + if reads >= 256: + raise ValueError("too_many_h2_control_frames") + reads += 1 + return frame(connection) + + kind, _, stream, settings = next_frame() + if kind != 4 or stream != 0: + raise ValueError("missing_h2_settings") + if len(settings) % 6: + raise ValueError("invalid_h2_settings") + stream_window = 65535 + for offset in range(0, len(settings), 6): + setting, value = struct.unpack("!HI", settings[offset:offset + 6]) + if setting == 4: + if value > 0x7FFFFFFF: + raise ValueError("invalid_h2_initial_window") + stream_window = value + connection_window = 65535 + saw_headers = False + acknowledged = False + window_updates = 0 + + def control(kind, flags, stream, payload): + nonlocal acknowledged, connection_window, stream_window, window_updates + if kind == 4 and flags == 1 and stream == 0 and not payload: + acknowledged = True + elif kind == 8 and stream in (0, 1) and len(payload) == 4: + increment = struct.unpack("!I", payload)[0] & 0x7FFFFFFF + if increment == 0: + raise ValueError("invalid_h2_window_update") + if stream == 0: + connection_window += increment + else: + stream_window += increment + if connection_window > 0x7FFFFFFF or stream_window > 0x7FFFFFFF: + raise ValueError("h2_window_overflow") + window_updates += 1 + elif kind in (3, 7): + raise ValueError("h2_stream_or_connection_reset") + + for _ in range(8): + kind, flags, stream, payload = next_frame() + control(kind, flags, stream, payload) + if kind == 1 and stream == 1: + saw_headers = True + break + if not saw_headers: + raise ValueError("missing_h2_headers") + body = b"B" * (262144 if large else 6) + send_frame(connection, 4, 0, 0, b"") + # HPACK indexed :status 200 (8); literal content-length with indexed name (28). + digits = str(len(body)).encode() + block = b"\x88\x0f\x0d" + bytes([len(digits)]) + digits + send_frame(connection, 1, 4, 1, block) + offset = 0 + while offset < len(body): + available = min(16384, connection_window, stream_window, len(body) - offset) + if available == 0: + control(*next_frame()) + continue + chunk = body[offset:offset + available] + flags = 1 if offset + available == len(body) else 0 + send_frame(connection, 0, flags, 1, chunk) + offset += available + connection_window -= available + stream_window -= available + if not acknowledged: + while not acknowledged: + control(*next_frame()) + if not acknowledged: + raise ValueError("missing_h2_settings_ack") + event("exchange", bytes=len(body), window_updates=window_updates) + + +def websocket(connection): + request = headers(connection) + if not request.startswith(b"GET /socket HTTP/1.1\r\n"): + raise ValueError("wrong_wss_request") + fields = request.split(b"\r\n") + keys = [line.split(b":", 1)[1].strip() for line in fields if line.lower().startswith(b"sec-websocket-key:")] + if len(keys) != 1: + raise ValueError("missing_wss_key") + accept = base64.b64encode(hashlib.sha1(keys[0] + b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest()) + connection.sendall(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + accept + b"\r\n\r\n") + event("upgrade") + head = exact(connection, 2) + if head[0] != 0x81 or head[1] != 0x80 + 10: + raise ValueError("wrong_wss_frame") + mask = exact(connection, 4) + payload = exact(connection, 10) + decoded = bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) + if decoded != b"tls12-echo": + raise ValueError("wrong_wss_payload") + echoed = b"echo:" + decoded + connection.sendall(bytes([0x81, len(echoed)]) + echoed) + head = exact(connection, 2) + if head[0] != 0x88 or not (head[1] & 0x80): + raise ValueError("missing_wss_close") + size = head[1] & 0x7F + if size > 125: + raise ValueError("wss_close_too_large") + mask = exact(connection, 4) + payload = exact(connection, size) + decoded = bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) + connection.sendall(bytes([0x88, len(decoded)]) + decoded) + event("exchange", bytes=10) + + +def sse(connection, index): + request = headers(connection) + if not request.startswith(b"GET /events HTTP/1.1\r\n"): + raise ValueError("wrong_sse_request") + last_id_ok = b"Last-Event-ID: 41\r\n" in request + event("request", index=index, last_id_ok=last_id_ok) + if index == 1: + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\nid: 41\ndata: first\n\n") + event("first_ready") + if sys.stdin.buffer.readline() != b"go\n": + raise ValueError("missing_release") + else: + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: keep-alive\r\n\r\ndata: second\n\n") + event("second_ready") + sys.stdin.buffer.readline() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--certfile", required=True) + parser.add_argument("--keyfile", required=True) + parser.add_argument("--cafile", required=True) + parser.add_argument("--mode", choices=["http1", "h2", "wss", "sse", "resumption"], required=True) + parser.add_argument("--max-version", choices=["tls12", "tls13"], default="tls12") + parser.add_argument("--large", action="store_true") + args = parser.parse_args() + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_3 if args.mode == "resumption" else ssl.TLSVersion.TLSv1_2 + context.maximum_version = ssl.TLSVersion.TLSv1_2 if args.max_version == "tls12" and args.mode != "resumption" else ssl.TLSVersion.TLSv1_3 + context.options |= ssl.OP_NO_COMPRESSION | ssl.OP_NO_RENEGOTIATION + context.load_cert_chain(args.certfile, args.keyfile) + context.load_verify_locations(cafile=args.cafile) + context.verify_mode = ssl.CERT_NONE if args.mode == "resumption" else ssl.CERT_REQUIRED + context.set_ciphers("ECDHE-RSA-AES128-GCM-SHA256") + context.set_ecdh_curve("prime256v1") + context.set_alpn_protocols(["h2" if args.mode == "h2" else "http/1.1"]) + + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(4) + listener.settimeout(10) + event("ready", port=listener.getsockname()[1], openssl=ssl.OPENSSL_VERSION) + count = 2 if args.mode in ("sse", "resumption") else 1 + for index in range(1, count + 1): + raw, _ = listener.accept() + raw.settimeout(10) + try: + with context.wrap_socket(raw, server_side=True) as connection: + der = connection.getpeercert(binary_form=True) + event("handshake", index=index, version=connection.version(), cipher=connection.cipher()[0], alpn=connection.selected_alpn_protocol(), client_der_b64=base64.b64encode(der).decode() if der else None, session_reused=connection.session_reused) + if args.mode == "http1": + http1(connection, args.large) + elif args.mode == "resumption": + http1(connection, False, b"/resumption") + elif args.mode == "h2": + http2(connection, args.large) + elif args.mode == "wss": + websocket(connection) + else: + sse(connection, index) + if args.mode != "sse": + try: + connection.unwrap().close() + except (ssl.SSLError, OSError, EOFError): + pass + except (ssl.SSLError, OSError, EOFError, ValueError) as error: + event("failure", kind_name=type(error).__name__) + raw.close() + listener.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/ex_ssl_tls12_test.exs b/scripts/ex_ssl_tls12_test.exs new file mode 100644 index 0000000..ce99248 --- /dev/null +++ b/scripts/ex_ssl_tls12_test.exs @@ -0,0 +1,269 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file(Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support", fixture])) +end + +defmodule CandidateTLS12Test do + use ExUnit.Case, async: false + + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.EventSource + alias HTTP.WebSocket + + @timeout 10_000 + @python Path.join(__DIR__, "ex_ssl_tls12_peer.py") + + setup_all do + directory = Path.join(System.tmp_dir!(), "http-tls12-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + for protocol <- [:http1, :http2], versions <- [[:"tlsv1.2"], [:"tlsv1.3", :"tlsv1.2"]] do + test "packaged #{protocol} fetch uses required mTLS over #{inspect(versions)} and returns large response", + %{ + fixtures: fixtures + } do + mode = if unquote(protocol) == :http2, do: "h2", else: "http1" + + with_peer(fixtures, mode, ["--large"], fn peer -> + promise = + HTTP.fetch("https://127.0.0.1:#{peer.port}/tls12", + tls_backend: :ex_ssl, + http_version: unquote(protocol), + ssl: + client_ssl( + fixtures, + unquote(versions), + if(mode == "h2", do: "h2", else: "http/1.1") + ), + timeout: @timeout, + connect_timeout: @timeout + ) + + expected_bytes = 262_144 + assert_handshake(peer, fixtures, "TLSv1.2", if(mode == "h2", do: "h2", else: "http/1.1")) + assert {:ok, %{"bytes" => ^expected_bytes} = exchange} = event(peer, "exchange") + if mode == "h2", do: assert(exchange["window_updates"] > 0) + response = HTTP.Promise.await(promise, @timeout) + assert response.status == 200 + assert HTTP.Response.read_all(response) == :binary.copy("B", expected_bytes) + end) + end + end + + test "mixed offer selects TLS 1.3 on a capable peer without retry", %{fixtures: fixtures} do + with_peer(fixtures, "http1", ["--max-version", "tls13"], fn peer -> + response = + HTTP.fetch("https://127.0.0.1:#{peer.port}/tls12", + tls_backend: :ex_ssl, + ssl: client_ssl(fixtures, [:"tlsv1.3", :"tlsv1.2"], "http/1.1"), + timeout: @timeout, + connect_timeout: @timeout + ) + |> HTTP.Promise.await(@timeout) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "BBBBBB" + assert_handshake(peer, fixtures, "TLSv1.3", "http/1.1") + assert {:ok, %{"bytes" => 6}} = event(peer, "exchange") + end) + end + + test "packaged WSS upgrades through TLS 1.2 mTLS and echoes an active-once frame", %{ + fixtures: fixtures + } do + with_peer(fixtures, "wss", [], fn peer -> + socket = + WebSocket.new("wss://127.0.0.1:#{peer.port}/socket", [], + tls_backend: :ex_ssl, + ssl: client_ssl(fixtures, [:"tlsv1.2"], "http/1.1") + ) + + try do + assert_handshake(peer, fixtures, "TLSv1.2", "http/1.1") + assert {:ok, _} = event(peer, "upgrade") + assert_receive {WebSocket, ^socket, %HTTP.WebSocket.Event.Open{}}, @timeout + assert :ok = WebSocket.send(socket, "tls12-echo") + + assert_receive {WebSocket, ^socket, + %HTTP.WebSocket.Event.Message{data: "echo:tls12-echo"}}, + @timeout + + assert :ok = WebSocket.close(socket, 1000, "done") + + assert_receive {WebSocket, ^socket, + %HTTP.WebSocket.Event.Close{code: 1000, was_clean: true}}, + @timeout + + assert {:ok, %{"bytes" => 10}} = event(peer, "exchange") + after + _ = WebSocket.close(socket) + end + end) + end + + test "packaged SSE reconnects to same TLS 1.2 mTLS origin with backend and Last-Event-ID pinned", + %{ + fixtures: fixtures + } do + previous = Application.get_env(:http_core, :tls_backend, :unset) + on_exit(fn -> restore_backend(previous) end) + + with_peer(fixtures, "sse", [], fn peer -> + source = + EventSource.new("https://127.0.0.1:#{peer.port}/events", + tls_backend: :ex_ssl, + reconnect_time: 10, + ssl: client_ssl(fixtures, [:"tlsv1.2"], "http/1.1") + ) + + try do + assert_handshake(peer, fixtures, "TLSv1.2", "http/1.1", 1) + assert {:ok, %{"index" => 1, "last_id_ok" => false}} = event(peer, "request") + assert_receive {EventSource, ^source, %HTTP.EventSource.Event.Open{}}, @timeout + + assert_receive {EventSource, ^source, + %HTTP.EventSource.Event.Message{data: "first", last_event_id: "41"}}, + @timeout + + assert {:ok, _} = event(peer, "first_ready") + Application.put_env(:http_core, :tls_backend, :invalid) + assert %{tls_backend: :ex_ssl} = :sys.get_state(source.pid) + true = Port.command(peer.handle, "go\n") + + assert_handshake(peer, fixtures, "TLSv1.2", "http/1.1", 2) + assert {:ok, %{"index" => 2, "last_id_ok" => true}} = event(peer, "request") + assert_receive {EventSource, ^source, %HTTP.EventSource.Event.Open{}}, @timeout + + assert_receive {EventSource, ^source, + %HTTP.EventSource.Event.Message{data: "second", last_event_id: "41"}}, + @timeout + + assert {:ok, _} = event(peer, "second_ready") + after + _ = EventSource.close(source) + end + end) + end + + defp client_ssl(fixtures, versions, protocol) do + [ + versions: versions, + verify: :verify_peer, + alpn_advertised_protocols: [protocol], + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + certfile: fixtures.rsa.certificate, + keyfile: fixtures.rsa.key + ] + end + + defp with_peer(fixtures, mode, extra, function) do + peer = start_peer(fixtures, mode, extra) + + try do + function.(peer) + after + stop_peer(peer) + end + + assert {:error, :econnrefused} = + :gen_tcp.connect(~c"127.0.0.1", peer.port, [:binary, active: false], 1_000) + end + + defp start_peer(fixtures, mode, extra) do + assert File.regular?(@python) + executable = System.find_executable("python3") || raise "python3 unavailable" + + args = + [ + "-B", + "-u", + @python, + "--certfile", + fixtures.server.certificate, + "--keyfile", + fixtures.server.key, + "--cafile", + fixtures.ca.certificate, + "--mode", + mode + ] ++ extra + + handle = + Port.open({:spawn_executable, executable}, [ + :binary, + :exit_status, + :stderr_to_stdout, + {:line, 65_536}, + args: args + ]) + + {:os_pid, os_pid} = Port.info(handle, :os_pid) + peer = %{handle: handle, os_pid: os_pid, port: nil} + + case event(peer, "ready") do + {:ok, %{"port" => port}} -> + %{peer | port: port} + + error -> + stop_peer(peer) + raise "TLS 1.2 peer startup failed: #{inspect(error)}" + end + end + + defp assert_handshake(peer, fixtures, version, alpn, index \\ 1) do + assert {:ok, + %{ + "index" => ^index, + "version" => ^version, + "alpn" => ^alpn, + "client_der_b64" => client_der, + "cipher" => cipher + }} = event(peer, "handshake") + + assert Base.decode64!(client_der) == fixtures.rsa.der + if version == "TLSv1.2", do: assert(cipher == "ECDHE-RSA-AES128-GCM-SHA256") + end + + defp event(%{handle: handle}, expected) do + receive do + {^handle, {:data, {:eol, line}}} -> + case :json.decode(line) do + %{"kind" => ^expected} = value -> {:ok, value} + %{"kind" => "failure"} = value -> {:error, value} + other -> {:error, {:unexpected_peer_event, other}} + end + + {^handle, {:data, {:noeol, _}}} -> + {:error, :peer_line_too_long} + + {^handle, {:exit_status, status}} -> + {:error, {:peer_exit, status}} + after + @timeout -> {:error, :peer_timeout} + end + rescue + _ -> {:error, :invalid_peer_event} + end + + defp stop_peer(%{handle: handle, os_pid: os_pid}) do + case Port.info(handle, :os_pid) do + {:os_pid, ^os_pid} -> + _ = System.cmd("kill", ["-TERM", Integer.to_string(os_pid)], stderr_to_stdout: true) + + receive do + {^handle, {:exit_status, _}} -> :ok + after + 2_000 -> raise "TLS 1.2 peer did not stop" + end + + nil -> + :ok + end + end + + defp restore_backend(:unset), do: Application.delete_env(:http_core, :tls_backend) + defp restore_backend(value), do: Application.put_env(:http_core, :tls_backend, value) +end From 03b487db7847dcf8ba77d0ac4737e1bbf3fe65d0 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 17:26:20 +0800 Subject: [PATCH 12/17] test(http2): preserve cross-record baseline regressions --- apps/http_core/test/http/http2_test.exs | 22 + .../test/http/socket_client_http2_test.exs | 771 +++++++++++++++++- 2 files changed, 791 insertions(+), 2 deletions(-) diff --git a/apps/http_core/test/http/http2_test.exs b/apps/http_core/test/http/http2_test.exs index 8355d53..8aa63a5 100644 --- a/apps/http_core/test/http/http2_test.exs +++ b/apps/http_core/test/http/http2_test.exs @@ -12,6 +12,28 @@ defmodule HTTP.HTTP2Test do @initial_window_size 65_535 @max_window_size 2_147_483_647 + describe "outbound_control_only?/1" do + test "only permits acknowledgements and window updates after the upload is sent" do + controls = [ + Frame.encode(:settings, @ack, 0, ""), + Frame.encode(:ping, @ack, 0, "12345678"), + Frame.encode(:window_update, 0, 1, <<0::1, 1::31>>) + ] + + conn = %HTTP.HTTP2{outbound: controls} + assert HTTP.HTTP2.outbound_control_only?(conn) + refute HTTP.HTTP2.outbound_control_only?(%{conn | pending_body: "unsent"}) + + for required <- [ + Frame.encode(:data, @end_stream, 1, "upload"), + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:ping, 0, 0, "12345678") + ] do + refute HTTP.HTTP2.outbound_control_only?(%{conn | outbound: [required | controls]}) + end + end + end + describe "serialize_request/1" do test "serializes the connection preface, settings, and request headers" do request = %HTTP.Request{ diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index 4dfaf2c..be0f882 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -242,8 +242,7 @@ defmodule HTTP.SocketClientHTTP2Test do await_owner_loop(owner) on_exit(fn -> - if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) - if Process.alive?(controller), do: HTTP.AbortController.abort(controller) + cleanup_owner(owner, controller) end) true = :erlang.suspend_process(owner) @@ -261,6 +260,539 @@ defmodule HTTP.SocketClientHTTP2Test do assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 end + @tag :cross_record + test "drains a second ex_ssl TLS record after control writes fail following peer close" do + test_pid = self() + body = "second-record-body" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "drains a split HTTP/2 frame from the ex_ssl receive buffer after peer close" do + test_pid = self() + body = "split-frame-body" + headers = Frame.encode(:headers, @end_headers, 1, response_headers(body)) + split_at = 5 + <> = headers + + first_record = [Frame.encode(:settings, 0, 0, ""), headers_start] + first_record_binary = IO.iodata_to_binary(first_record) + second_record = [headers_rest, Frame.encode(:data, @end_stream, 1, body)] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "rejects a truncated second ex_ssl TLS record after a control write fails" do + test_pid = self() + body = "truncated-second-record" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, 0, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, :closed} = HTTP.Promise.await(promise) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "does not turn a reset buffered after peer close into a successful response" do + test_pid = self() + body = "reset-after-drain" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + + second_record = [ + Frame.encode(:data, @end_stream, 1, body), + Frame.encode(:rst_stream, 0, 1, <<0x8::32>>) + ] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, {:stream_reset, :cancel}} = HTTP.Promise.await(promise) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "fails when SETTINGS acknowledgement cannot flush a pending HTTP/2 request body" do + test_pid = self() + body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_response_and_close) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers("complete")), + Frame.encode(:data, @end_stream, 1, "complete") + ]) + + :ok = :ssl.close(socket) + send(test_pid, :server_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_response_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, :closed} = + HTTP.Promise.await(promise) + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "drains a final streaming HTTP/2 frame buffered by ex_ssl after peer close" do + test_pid = self() + body = :binary.copy("q", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, body) + buffer = assert_settings_ack(socket, transport, buffer) + chunks = chunk_binary(body, 16_384) + preceding_chunks = Enum.drop(chunks, -3) + [first_final_chunk, second_final_chunk, third_final_chunk] = Enum.take(chunks, -3) + + Enum.reduce(preceding_chunks, buffer, fn chunk, current_buffer -> + send_all(socket, transport, Frame.encode(:data, 0, 1, chunk)) + + current_buffer = + assert_window_update(socket, transport, current_buffer, 0, byte_size(chunk)) + + assert_window_update(socket, transport, current_buffer, 1, byte_size(chunk)) + end) + + first_final_frame = Frame.encode(:data, 0, 1, first_final_chunk) + split_at = 8_000 + <> = first_final_frame + + first_record = [Frame.encode(:ping, 0, 0, "final123"), first_frame_start] + + second_record = [ + first_frame_rest, + Frame.encode(:data, 0, 1, second_final_chunk), + Frame.encode(:data, @end_stream, 1, third_final_chunk) + ] + + send( + test_pid, + {:server_ready_to_finish, self(), IO.iodata_to_binary(first_record), second_record} + ) + + await_test_gate(:send_final_first_record) + send_all(socket, transport, first_record) + send(test_pid, :final_first_record_sent) + + await_test_gate(:send_final_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :final_second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + stream_monitor = monitor_test_process(response.stream) + reader = Task.async(fn -> HTTP.Response.read_all(response) end) + monitor_test_process(reader.pid) + + assert_receive {:server_ready_to_finish, server_pid, first_record, second_record}, 10_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_final_first_record) + assert_receive :final_first_record_sent, 5_000 + + {tls_pid, ^first_record} = await_owner_tls_data(owner, first_record) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_final_second_record_and_close) + assert_receive :final_second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert Task.await(reader, 10_000) == body + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "aborts an ex_ssl cross-record drain while stream backpressure holds the final body" do + {server_pid, controller, promise, owner, first_record, second_record} = + cross_record_stream_drain_fixture(self(), 5_000) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + {tls_pid, owner_monitor, tls_monitor} = + queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream = response.stream + stream_monitor = monitor_test_process(stream) + + holder = hold_cross_record_stream_chunk(stream, self()) + monitor_test_process(holder) + + assert_receive {:held_cross_record_stream_chunk, ^holder, ^stream, _chunk, _ack_ref}, + 5_000 + + await_cross_record_stream_backpressure(owner) + + :ok = HTTP.AbortController.abort(controller) + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + + reader = + Task.Supervisor.async_nolink(:http_fetch_task_supervisor, fn -> + assert_raise RuntimeError, "stream read failed: :aborted", fn -> + HTTP.Response.read_all(response) + end + end) + + monitor_test_process(reader.pid) + await_cross_record_read_all(reader.pid) + send(holder, :release_cross_record_stream_chunk) + + assert %RuntimeError{message: "stream read failed: :aborted"} = Task.await(reader, 5_000) + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + end + + @tag :cross_record + test "keeps the original deadline while an ex_ssl cross-record drain is backpressured" do + timeout = 2_000 + started_at = System.monotonic_time(:millisecond) + + {server_pid, controller, promise, owner, first_record, second_record} = + cross_record_stream_drain_fixture(self(), timeout) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + # This explicit timer gate consumes half the request budget before entering + # the drain. A drain that reset the request timeout would outlive the + # original absolute deadline asserted below. + Process.send_after(self(), :begin_cross_record_deadline_drain, div(timeout, 2)) + assert_receive :begin_cross_record_deadline_drain, div(timeout, 2) + 250 + + {tls_pid, owner_monitor, tls_monitor} = + queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream = response.stream + stream_monitor = monitor_test_process(stream) + + holder = hold_cross_record_stream_chunk(stream, self()) + monitor_test_process(holder) + + assert_receive {:held_cross_record_stream_chunk, ^holder, ^stream, _chunk, _ack_ref}, + 5_000 + + await_cross_record_stream_backpressure(owner) + + deadline_at = started_at + timeout + remaining = deadline_at - System.monotonic_time(:millisecond) + assert remaining > 0 + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, remaining + 400 + + reader = + Task.Supervisor.async_nolink(:http_fetch_task_supervisor, fn -> + assert_raise RuntimeError, ~r/^stream read failed: :(request_timeout|timeout)$/, fn -> + HTTP.Response.read_all(response) + end + end) + + monitor_test_process(reader.pid) + await_cross_record_read_all(reader.pid) + send(holder, :release_cross_record_stream_chunk) + + assert %RuntimeError{message: message} = Task.await(reader, 5_000) + assert message in ["stream read failed: :request_timeout", "stream read failed: :timeout"] + + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + end + for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do test "fails when a closed ex_ssl HTTP/2 connection still needs request body writes for a #{completion} response" do test_pid = self() @@ -717,6 +1249,32 @@ defmodule HTTP.SocketClientHTTP2Test do :ok = apply(transport, :send, [socket, iodata]) end + defp await_test_gate(gate) do + receive do + ^gate -> :ok + after + 5_000 -> exit({:test_gate_timeout, gate}) + end + end + + defp monitor_test_process(pid) do + on_exit(fn -> + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + Process.monitor(pid) + end + + defp cleanup_owner(owner, controller) do + if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) + + if Process.alive?(controller) do + HTTP.AbortController.abort(controller) + else + if Process.alive?(owner), do: send(owner, :abort) + end + end + defp await_owner(controller) do case :sys.get_state(controller).request_id do owner when is_pid(owner) -> owner @@ -728,6 +1286,75 @@ defmodule HTTP.SocketClientHTTP2Test do await_owner_tls_data_and_close(owner, nil, System.monotonic_time(:millisecond) + 5_000) end + defp await_owner_tls_data(owner, expected_data) do + await_owner_tls_data(owner, expected_data, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_owner_tls_data(owner, expected_data, deadline_at) do + case Process.info(owner, :messages) do + {:messages, messages} -> + case Enum.find(messages, fn + {:ssl, %SSL.Socket{}, ^expected_data} -> true + _message -> false + end) do + {:ssl, %SSL.Socket{pid: tls_pid}, ^expected_data} -> {tls_pid, expected_data} + nil -> await_owner_tls_data_or_fail(owner, expected_data, deadline_at) + end + + nil -> + flunk("socket owner exited before receiving the first TLS record") + end + end + + defp await_owner_tls_data_or_fail(owner, expected_data, deadline_at) do + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("socket owner did not receive the first TLS record as one active-once delivery") + else + receive do + after + min(10, remaining) -> await_owner_tls_data(owner, expected_data, deadline_at) + end + end + end + + defp assert_owner_has_only_tls_data(owner, tls_pid, expected_count) do + {:messages, messages} = Process.info(owner, :messages) + + assert messages + |> Enum.count(&match?({:ssl, %SSL.Socket{pid: ^tls_pid}, _data}, &1)) == expected_count + end + + defp assert_tls_buffered_after_peer_close(tls_pid, minimum_size) do + assert_tls_buffered_after_peer_close( + tls_pid, + minimum_size, + System.monotonic_time(:millisecond) + 5_000 + ) + end + + defp assert_tls_buffered_after_peer_close(tls_pid, minimum_size, deadline_at) do + {_phase, state} = :sys.get_state(tls_pid) + + if state.closed and state.size >= minimum_size do + assert state.active == false + :ok + else + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("second TLS record was not retained in the TLS receive buffer after peer close") + else + receive do + after + min(10, remaining) -> + assert_tls_buffered_after_peer_close(tls_pid, minimum_size, deadline_at) + end + end + end + end + defp await_owner_loop(owner) do await_owner_loop(owner, System.monotonic_time(:millisecond) + 5_000) end @@ -782,6 +1409,146 @@ defmodule HTTP.SocketClientHTTP2Test do end end + defp cross_record_stream_drain_fixture(test_pid, timeout) do + body = "backpressured-final-record" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode( + :headers, + @end_headers, + 1, + HPACK.encode_headers([{":status", "200"}, {"x-protocol", "h2"}]) + ) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:cross_record_stream_server_ready, self()}) + + await_test_gate(:send_cross_record_stream_first) + send_all(socket, transport, first_record) + send(test_pid, :cross_record_stream_first_sent) + + await_test_gate(:send_cross_record_stream_second) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :cross_record_stream_second_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + timeout: timeout, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:cross_record_stream_server_ready, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + {server_pid, controller, promise, owner, first_record_binary, second_record} + end + + defp queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) do + true = :erlang.suspend_process(owner) + send(server_pid, :send_cross_record_stream_first) + assert_receive :cross_record_stream_first_sent, 5_000 + + {tls_pid, ^first_record} = await_owner_tls_data(owner, first_record) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_cross_record_stream_second) + assert_receive :cross_record_stream_second_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + owner_monitor = Process.monitor(owner) + tls_monitor = Process.monitor(tls_pid) + true = :erlang.resume_process(owner) + {tls_pid, owner_monitor, tls_monitor} + end + + defp hold_cross_record_stream_chunk(stream, test_pid) do + spawn(fn -> + send(stream, {:read_chunk, self(), :ack}) + + receive do + {:stream_chunk, ^stream, chunk, ack_ref} -> + send(test_pid, {:held_cross_record_stream_chunk, self(), stream, chunk, ack_ref}) + + receive do + :release_cross_record_stream_chunk -> + send(stream, {:stream_chunk_ack, ack_ref}) + after + 5_000 -> + send(test_pid, {:held_cross_record_stream_chunk_timeout, self()}) + end + after + 5_000 -> + send(test_pid, {:held_cross_record_stream_chunk_timeout, self()}) + end + end) + end + + defp await_cross_record_stream_backpressure(owner) do + await_cross_record_stream_backpressure(owner, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_cross_record_stream_backpressure(owner, deadline_at) do + if Process.info(owner, :current_function) == {:current_function, {HTTP.Stream, :chunk, 3}} and + Process.info(owner, :status) == {:status, :waiting} do + :ok + else + await_cross_record_condition( + owner, + deadline_at, + "socket owner did not block on stream backpressure", + fn -> await_cross_record_stream_backpressure(owner, deadline_at) end + ) + end + end + + defp await_cross_record_read_all(reader) do + await_cross_record_read_all(reader, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_cross_record_read_all(reader, deadline_at) do + if Process.info(reader, :current_function) == + {:current_function, {HTTP.Response, :collect_stream, 2}} and + Process.info(reader, :status) == {:status, :waiting} do + :ok + else + await_cross_record_condition( + reader, + deadline_at, + "response reader did not wait for stream error", + fn -> await_cross_record_read_all(reader, deadline_at) end + ) + end + end + + defp await_cross_record_condition(_pid, deadline_at, message, fun) do + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk(message) + else + receive do + after + min(10, remaining) -> fun.() + end + end + end + defp chunk_binary(binary, size) when byte_size(binary) <= size, do: [binary] defp chunk_binary(binary, size) do From 25f8a9229d259151970a8b82fb5b48d62d63eb0c Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 17:26:20 +0800 Subject: [PATCH 13/17] test(tls): cover unsupported options and ALPN profile conflicts --- .../test/http/ssl_transport_test.exs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/apps/http_fetch/test/http/ssl_transport_test.exs b/apps/http_fetch/test/http/ssl_transport_test.exs index ebc2a52..cb954f6 100644 --- a/apps/http_fetch/test/http/ssl_transport_test.exs +++ b/apps/http_fetch/test/http/ssl_transport_test.exs @@ -24,6 +24,31 @@ defmodule HTTP.SSLTransportTest do HTTP.FetchOptions.new(http_version: :http3) end + test "rejects unsupported ex_ssl socket options through HTTP.fetch" do + for socket_opts <- [[nodelay: true], [send_timeout_close: false]] do + assert {:error, {:options, _reason}} = + "https://127.0.0.1:1/secure" + |> HTTP.fetch(tls_backend: :ex_ssl, socket_opts: socket_opts) + |> HTTP.Promise.await() + end + end + + test "propagates ex_ssl profile and ALPN conflicts through HTTP.fetch" do + profile = %SSL.ClientHello.WireProfile{extensions: [{:alpn, ["http/1.1"]}]} + + assert {:error, {:options, {:alpn_advertised_protocols, :profile_conflict}}} = + "https://127.0.0.1:1/secure" + |> HTTP.fetch( + tls_backend: :ex_ssl, + http_version: :http2, + ssl: [ + alpn_advertised_protocols: ["h2"], + ex_ssl: [profile: profile] + ] + ) + |> HTTP.Promise.await() + end + test "rejects self-signed certificates by default" do url = start_https_server!(fn socket -> send_response(socket, "secure") end) From e102e5289bf0bcb5b6a6d8dd3a45fb3e238d9815 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 17:26:20 +0800 Subject: [PATCH 14/17] test(http2): preserve early-response baseline regressions --- apps/http_core/test/http/http2_test.exs | 202 +++++++++- .../test/http/socket_client_http2_test.exs | 378 +++++++++++++++++- 2 files changed, 552 insertions(+), 28 deletions(-) diff --git a/apps/http_core/test/http/http2_test.exs b/apps/http_core/test/http/http2_test.exs index 8aa63a5..777593a 100644 --- a/apps/http_core/test/http/http2_test.exs +++ b/apps/http_core/test/http/http2_test.exs @@ -13,7 +13,7 @@ defmodule HTTP.HTTP2Test do @max_window_size 2_147_483_647 describe "outbound_control_only?/1" do - test "only permits acknowledgements and window updates after the upload is sent" do + test "classifies only acknowledgements and window updates" do controls = [ Frame.encode(:settings, @ack, 0, ""), Frame.encode(:ping, @ack, 0, "12345678"), @@ -22,7 +22,7 @@ defmodule HTTP.HTTP2Test do conn = %HTTP.HTTP2{outbound: controls} assert HTTP.HTTP2.outbound_control_only?(conn) - refute HTTP.HTTP2.outbound_control_only?(%{conn | pending_body: "unsent"}) + assert HTTP.HTTP2.outbound_control_only?(%{conn | pending_body: "unsent"}) for required <- [ Frame.encode(:data, @end_stream, 1, "upload"), @@ -203,21 +203,209 @@ defmodule HTTP.HTTP2Test do end test "combines HEADERS and CONTINUATION before decoding" do - conn = HTTP.HTTP2.new(:get) + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) header_block = HPACK.encode_headers([{":status", "204"}, {"x-test", "split"}]) size = div(IO.iodata_length(header_block), 2) header_block = IO.iodata_to_binary(header_block) <> = header_block + assert {:ok, conn, []} = + HTTP.HTTP2.stream(conn, Frame.encode(:headers, @end_stream, 1, first)) + + refute HTTP.HTTP2.complete_response?(conn) + refute HTTP.HTTP2.request_stopped?(conn) + assert conn.pending_body == "xxxxx" + + assert {:ok, conn, [{:headers, 204, headers}, :done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:continuation, @end_headers, 1, second)) + + assert HTTP.Headers.get(headers, "x-test") == "split" + assert HTTP.HTTP2.complete_response?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + end + + test "does not complete a bodyless response until END_STREAM arrives" do + assert {:ok, conn, [{:headers, 204, _headers}]} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([{":status", "204"}]) + ) + + refute HTTP.HTTP2.complete_response?(conn) + assert {:error, :closed} = HTTP.HTTP2.close(conn) + + assert {:ok, conn, [:done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:data, @end_stream, 1, "")) + + assert HTTP.HTTP2.complete_response?(conn) + end + + test "stops an upload explicitly without discarding non-upload frames" do + conn = %HTTP.HTTP2{ + pending_body: "unsent", + outbound: [ + Frame.encode(:data, @end_stream, 1, "upload"), + Frame.encode(:settings, @ack, 0, ""), + Frame.encode(:settings, 0, 0, "") + ] + } + + conn = HTTP.HTTP2.stop_request(conn) + {conn, outbound} = HTTP.HTTP2.take_outbound(conn) + assert [] = conn.outbound + + assert {:ok, %Frame{type: :settings, flags: 0, payload: ""}, outbound} = + outbound |> IO.iodata_to_binary() |> Frame.decode() + + assert {:ok, %Frame{type: :settings, flags: @ack, payload: ""}, ""} = + Frame.decode(outbound) + + refute HTTP.HTTP2.outbound_control_only?(%{ + conn + | outbound: [Frame.encode(:settings, 0, 0, "")] + }) + + refute HTTP.HTTP2.complete_response?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + + assert {:ok, conn, []} = + HTTP.HTTP2.stream( + conn, + Frame.encode(:window_update, 0, 1, <<0::1, 1::31>>) + ) + + assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) + end + + test "stops a pending upload after an early final response and preserves control frames" do + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) + + final_response = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + frames = [ - Frame.encode(:headers, @end_stream, 1, first), - Frame.encode(:continuation, @end_headers, 1, second) + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + final_response ] - assert {:ok, _conn, [{:headers, 204, headers}, :done]} = + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = HTTP.HTTP2.stream(conn, IO.iodata_to_binary(frames)) - assert HTTP.Headers.get(headers, "x-test") == "split" + assert HTTP.HTTP2.complete_response?(conn) + + {conn, outbound} = HTTP.HTTP2.take_outbound(conn) + + assert {:ok, %Frame{type: :settings, flags: flags, payload: ""}, ""} = + outbound |> IO.iodata_to_binary() |> Frame.decode() + + assert (flags &&& @ack) == @ack + assert HTTP.HTTP2.outbound_control_only?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + + assert {:ok, conn, []} = + HTTP.HTTP2.stream( + conn, + IO.iodata_to_binary([ + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>) + ]) + ) + + assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) + end + + test "does not restart an upload when WINDOW_UPDATE follows an early final response" do + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) + + final_response = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + frames = [ + final_response, + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>) + ] + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(conn, IO.iodata_to_binary(frames)) + + assert {_conn, []} = HTTP.HTTP2.take_outbound(conn) + assert HTTP.HTTP2.request_stopped?(conn) + end + + test "accepts NO_ERROR reset after a complete response without duplicate events" do + final_headers = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + reset = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), final_headers) + + assert HTTP.HTTP2.complete_response?(conn) + assert {:ok, ^conn, []} = HTTP.HTTP2.stream(conn, reset) + assert {:ok, ^conn, []} = HTTP.HTTP2.stream(conn, reset) + + assert {:ok, _conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), final_headers <> reset) + + assert {:error, {:stream_reset, :no_error}} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + Frame.encode(:rst_stream, 0, 1, <<0::32>>) + ) + end + + test "keeps an incomplete response reset as an error" do + headers = response_headers_frame([{":status", "200"}]) + reset = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + assert {:ok, conn, [{:headers, 200, _headers}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers) + + refute HTTP.HTTP2.complete_response?(conn) + assert {:error, {:stream_reset, :no_error}} = HTTP.HTTP2.stream(conn, reset) end test "decodes hpack static, dynamic, and huffman response headers" do diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index be0f882..0cac81e 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -532,7 +532,8 @@ defmodule HTTP.SocketClientHTTP2Test do end @tag :cross_record - test "fails when SETTINGS acknowledgement cannot flush a pending HTTP/2 request body" do + @tag :early_response + test "delivers a complete 413 response when the HTTP/2 request body is still pending" do test_pid = self() body = :binary.copy("p", @initial_window_size + 5) @@ -548,10 +549,12 @@ defmodule HTTP.SocketClientHTTP2Test do await_test_gate(:send_response_and_close) + response_body = "payload-too-large" + send_all(socket, transport, [ Frame.encode(:settings, 0, 0, ""), - Frame.encode(:headers, @end_headers, 1, response_headers("complete")), - Frame.encode(:data, @end_stream, 1, "complete") + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)), + Frame.encode(:data, @end_stream, 1, response_body) ]) :ok = :ssl.close(socket) @@ -587,12 +590,330 @@ defmodule HTTP.SocketClientHTTP2Test do owner_monitor = Process.monitor(owner) true = :erlang.resume_process(owner) - assert {:error, :closed} = - HTTP.Promise.await(promise) + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == "payload-too-large" + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :early_response + test "drains a cross-record 413 response without resuming its pending HTTP/2 upload" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + response_body = "cross-record-payload-too-large" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + + second_record = [ + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + Frame.encode(:data, @end_stream, 1, response_body) + ] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_early_response_first_record) + send_all(socket, transport, first_record) + send(test_pid, :early_response_first_record_sent) + + await_test_gate(:send_early_response_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :early_response_second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_early_response_first_record) + assert_receive :early_response_first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_early_response_second_record_and_close) + assert_receive :early_response_second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == response_body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :early_response + test "delivers a no-body 413 response through ex_ssl while the HTTP/2 upload is pending" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + await_test_gate(:send_no_body_early_response_and_close) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers ||| @end_stream, 1, response_headers(413, "")) + ]) + + :ok = :ssl.close(socket) + send(test_pid, :no_body_early_response_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + true = :erlang.suspend_process(owner) + send(server_pid, :send_no_body_early_response_and_close) + assert_receive :no_body_early_response_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + + assert response.status == 413 + assert HTTP.Response.read_all(response) == "" assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 end + @tag :early_response + test "delivers an early 413 response through OTP TLS while the peer stays open for SETTINGS ACK" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + response_body = "otp-payload-too-large" + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)), + Frame.encode(:data, @end_stream, 1, response_body) + ]) + + buffer = assert_settings_ack(socket, transport, buffer) + buffer = assert_window_update(socket, transport, buffer, 0, byte_size(response_body)) + buffer = assert_window_update(socket, transport, buffer, 1, byte_size(response_body)) + assert buffer == "" + send(test_pid, {:otp_early_response_settings_acknowledged, self()}) + await_test_gate(:close_otp_early_response) + end) + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + ssl: [verify: :verify_none] + ) + + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == response_body + assert_receive {:otp_early_response_settings_acknowledged, server_pid}, 5_000 + send(server_pid, :close_otp_early_response) + end + + @tag :early_response + test "delivers one complete response when a later buffered NO_ERROR reset follows END_STREAM" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + body = "no-error-reset-after-end-stream" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)), + Frame.encode(:data, @end_stream, 1, body) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_complete_response_record) + send_all(socket, transport, first_record) + send(test_pid, :complete_response_record_sent) + + await_test_gate(:send_no_error_reset_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :no_error_reset_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_complete_response_record) + assert_receive :complete_response_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_no_error_reset_and_close) + assert_receive :no_error_reset_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do + @tag :early_response + test "handles a #{completion} NO_ERROR reset in the same HTTP/2 response batch" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + body = "no-error-reset" + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)), + Frame.encode(:data, unquote(end_stream_flag), 1, body), + Frame.encode(:rst_stream, 0, 1, <<0::32>>) + ]) + + send(test_pid, {:no_error_response_sent, self()}) + await_test_gate(:close_no_error_response) + end) + + result = + url + |> HTTP.fetch( + method: :post, + body: request_body, + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + case unquote(completion) do + :complete -> + assert result.status == 200 + assert HTTP.Response.read_all(result) == body + + :incomplete -> + assert result == {:error, {:stream_reset, :no_error}} + end + + assert_receive {:no_error_response_sent, server_pid}, 5_000 + send(server_pid, :close_no_error_response) + end + end + @tag :cross_record test "drains a final streaming HTTP/2 frame buffered by ex_ssl after peer close" do test_pid = self() @@ -794,7 +1115,8 @@ defmodule HTTP.SocketClientHTTP2Test do end for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do - test "fails when a closed ex_ssl HTTP/2 connection still needs request body writes for a #{completion} response" do + @tag :early_response + test "handles a closed ex_ssl HTTP/2 connection with request body writes for a #{completion} response" do test_pid = self() body = :binary.copy("p", @initial_window_size + 5) @@ -808,18 +1130,20 @@ defmodule HTTP.SocketClientHTTP2Test do assert initial_body == binary_part(body, 0, @initial_window_size) send(test_pid, {:server_received_request, self()}) - receive do - :send_response_and_close -> - send_all(socket, transport, [ - Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), - Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), - Frame.encode(:headers, @end_headers, 1, response_headers("complete")), - Frame.encode(:data, unquote(end_stream_flag), 1, "complete") - ]) - - :ssl.close(socket) - send(test_pid, :server_closed) - end + await_test_gate(:send_response_and_close) + + frames = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + Frame.encode(:headers, @end_headers, 1, response_headers("complete")), + Frame.encode(:data, unquote(end_stream_flag), 1, "complete") + ] + + send_all(socket, transport, frames) + + :ssl.close(socket) + send(test_pid, :server_closed) end) controller = HTTP.AbortController.new() @@ -854,7 +1178,17 @@ defmodule HTTP.SocketClientHTTP2Test do owner_monitor = Process.monitor(owner) true = :erlang.resume_process(owner) - assert {:error, :closed} = HTTP.Promise.await(promise) + result = HTTP.Promise.await(promise) + + case unquote(completion) do + :complete -> + assert result.status == 200 + assert HTTP.Response.read_all(result) == "complete" + + :incomplete -> + assert result == {:error, :closed} + end + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 end end @@ -1164,9 +1498,11 @@ defmodule HTTP.SocketClientHTTP2Test do ]) end - defp response_headers(body) do + defp response_headers(body), do: response_headers(200, body) + + defp response_headers(status, body) do HPACK.encode_headers([ - {":status", "200"}, + {":status", Integer.to_string(status)}, {"content-length", Integer.to_string(byte_size(body))}, {"x-protocol", "h2"} ]) From 036b29391b389c628bd7e2c54da6df8f818260b2 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 17:27:50 +0800 Subject: [PATCH 15/17] docs: draft 0.12.0 changelog --- CHANGELOG.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f6eb4..aeb385b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [0.12.0] - Unreleased + +### Added +- Add source-candidate coverage for verified ex_ssl TLS 1.2 across Fetch, + HTTP/2, WebSocket, and EventSource, and for TLS 1.3 ticket resumption over + fresh HTTP/1.1 connections. +- Forward validated ex_ssl client identities, TCP socket options, and ordered + TLS 1.3 algorithm preferences. Pin client identities to the redirect origin. ### Changed - Keep OTP `:ssl` as the default TCP TLS backend while allowing verified TLS - 1.3 through the optional `:ex_ssl` backend (`~> 0.3.0`). HTTP/3 and + 1.3 through the optional `:ex_ssl` backend. HTTP/3 and WebTransport continue to use QUIC's independent TLS implementation. - Run individual app tests and E2E suites from the umbrella root after explicit test-environment preparation, so transitive runtime applications are compiled @@ -30,6 +37,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 DATA, including queued DATA released by WINDOW_UPDATE. Accept NO_ERROR resets only after response completion; require END_STREAM and a complete field block even for responses with no body. +- Enforce HTTP/2 response Content-Length and bounded frame/header parsing while + preserving valid cross-record and streaming completion after peer closure. ## [0.11.0] - 2026-07-04 From f4f71ee139ab0b2cc18308c639162cf38efba8ef Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 17:32:22 +0800 Subject: [PATCH 16/17] test(tls): hold HTTP/2 peer until large response is read --- scripts/ex_ssl_tls12_peer.py | 5 +++++ scripts/ex_ssl_tls12_test.exs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/scripts/ex_ssl_tls12_peer.py b/scripts/ex_ssl_tls12_peer.py index bb455a3..cd9a3eb 100644 --- a/scripts/ex_ssl_tls12_peer.py +++ b/scripts/ex_ssl_tls12_peer.py @@ -4,6 +4,7 @@ import base64 import hashlib import json +import select import socket import ssl import struct @@ -238,6 +239,10 @@ def main(): websocket(connection) else: sse(connection, index) + if args.mode == "h2": + if not select.select([sys.stdin], [], [], 10)[0] or sys.stdin.buffer.readline() != b"go\n": + raise ValueError("missing_h2_release") + event("released") if args.mode != "sse": try: connection.unwrap().close() diff --git a/scripts/ex_ssl_tls12_test.exs b/scripts/ex_ssl_tls12_test.exs index ce99248..8abdc58 100644 --- a/scripts/ex_ssl_tls12_test.exs +++ b/scripts/ex_ssl_tls12_test.exs @@ -48,6 +48,11 @@ defmodule CandidateTLS12Test do response = HTTP.Promise.await(promise, @timeout) assert response.status == 200 assert HTTP.Response.read_all(response) == :binary.copy("B", expected_bytes) + + if mode == "h2" do + true = Port.command(peer.handle, "go\n") + assert {:ok, _} = event(peer, "released") + end end) end end From 4ff4d87b952d5331e0ae7bf804cd84ad0d6064e7 Mon Sep 17 00:00:00 2001 From: Jonathan Gao Date: Tue, 22 Sep 2026 17:44:20 +0800 Subject: [PATCH 17/17] feat(tls): adopt published ex_ssl 0.4.0 --- CHANGELOG.md | 11 +-- README.md | 61 +++++++------- apps/http_core/mix.exs | 2 +- .../test/http/tls_transport_test.exs | 13 ++- .../test/http/socket_client_http2_test.exs | 2 +- .../test/http/ssl_transport_test.exs | 2 +- docs/ex-ssl-consumer-contract.md | 82 ++++++++++++------- docs/pr-14-validation.md | 4 + mix.lock | 2 +- scripts/ex_ssl_source_smoke.sh | 25 +++++- scripts/external_consumer_smoke.exs | 7 +- 11 files changed, 137 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aeb385b..04b7f22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,18 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.12.0] - Unreleased +## [0.12.0] - 2026-09-22 ### Added -- Add source-candidate coverage for verified ex_ssl TLS 1.2 across Fetch, - HTTP/2, WebSocket, and EventSource, and for TLS 1.3 ticket resumption over - fresh HTTP/1.1 connections. +- Add verified ex_ssl 0.4.0 TLS 1.2 support across Fetch, HTTP/2, WebSocket, + and EventSource, and opt-in TLS 1.3 ticket resumption over fresh HTTP/1.1 + connections. The default remains verified TLS 1.3 with tickets disabled. - Forward validated ex_ssl client identities, TCP socket options, and ordered TLS 1.3 algorithm preferences. Pin client identities to the redirect origin. ### Changed - Keep OTP `:ssl` as the default TCP TLS backend while allowing verified TLS - 1.3 through the optional `:ex_ssl` backend. HTTP/3 and + 1.3 or explicitly selected TLS 1.2 through the optional `:ex_ssl` backend. + HTTP/3 and WebTransport continue to use QUIC's independent TLS implementation. - Run individual app tests and E2E suites from the umbrella root after explicit test-environment preparation, so transitive runtime applications are compiled diff --git a/README.md b/README.md index 92d2445..04bc79c 100644 --- a/README.md +++ b/README.md @@ -155,15 +155,14 @@ configuration. The backend is captured when the request/client is created and retained through redirects and EventSource reconnects; runtime configuration changes affect new operations. Invalid selections fail explicitly. -`http_core` declares `ex_ssl ~> 0.3.0` as a transitive runtime dependency, -allowing 0.3.x patch updates. Consumers do not need to add it separately. -`ssl: [...]` supplies TLS settings to the selected backend. The `ex_ssl` 0.3.0 -backend uses the `SSL` module and requires peer verification and TLS 1.3. It uses -system CA certificates unless `cacerts` or `cacertfile` is supplied. DNS names -and IP addresses are verified against the peer certificate. TLS 1.2, -`verify: :verify_none`, client certificates, and arbitrary TCP socket options -are unsupported and return errors; connections never fall back to another -backend automatically. +`http_core` declares `ex_ssl ~> 0.4.0` as a transitive runtime dependency. +Consumers do not need to add it separately. `ssl: [...]` supplies TLS settings +to the selected backend. The `ex_ssl` backend uses its own `SSL` protocol engine +and requires peer verification. TLS 1.3 is the default; verified TLS 1.2 is +explicitly selectable. It uses system CA certificates unless `cacerts` or +`cacertfile` is supplied. DNS names and IP addresses are verified against the +peer certificate. `verify: :verify_none` and unsupported TLS or TCP options +return errors; connections never fall back to another backend automatically. A complete HTTP/2 response remains deliverable if the peer closes before the client can write its remaining WINDOW_UPDATE or acknowledgement frames. This @@ -189,39 +188,38 @@ Truncation, required writes before completion, abnormal closure, cancellation and timeout remain errors. The original deadline and streaming backpressure are preserved. -For `:ex_ssl`, `socket_opts` accepts `send_timeout` and -`send_timeout_close: true`. The unreleased source candidate also implements -`nodelay`, `keepalive`, `sndbuf`, `recbuf`, and local `ip`/`port`; the adapter -forwards only this allowlist and ex_ssl validates values. IPv6 literals infer the -family; an IPv6 local `ip` tuple selects IPv6 DNS resolution. Both option -containers must be keyword lists. These override matching entries in `ssl`. Custom +For `:ex_ssl`, `socket_opts` accepts `send_timeout`, +`send_timeout_close: true`, `nodelay`, `keepalive`, `sndbuf`, `recbuf`, and local +`ip`/`port`. The adapter forwards only this allowlist and ex_ssl validates values. +IPv6 literals infer the family; an IPv6 local `ip` tuple selects IPv6 DNS +resolution. Both option containers must be keyword lists. Socket options +override matching entries in `ssl`. Custom ClientHello profiles can be passed through `ssl: [ex_ssl: [profile: profile]]`; any ALPN list added by HTTP/2 selection must match the profile's ALPN list exactly. -For a source candidate that implements client authentication, ex_ssl credentials -stay within the initial request origin during automatic redirects. A scheme, +Configured ex_ssl client credentials stay within the initial request origin +during automatic redirects. A scheme, hostname or effective-port change returns `{:error, :client_identity_cross_origin_redirect}`. To authorize another origin, use `redirect: :manual` and explicitly make a new request with that identity. -The OTP backend retains its existing redirect behavior. Published ex_ssl 0.3.0 -does not yet support client identity options. +The OTP backend retains its existing redirect behavior. -The unreleased ex_ssl source candidate also supports verified TLS 1.2 for +ex_ssl 0.4.0 supports verified TLS 1.2 for HTTP/1.1, HTTP/2, WSS and EventSource. Select it with `ssl: [versions: [:"tlsv1.2"]]`; a mixed TLS 1.3/TLS 1.2 offer selects the peer's supported -version. Its independent OpenSSL source gate includes 262,144-byte HTTP/2 responses -with observed connection and stream WINDOW_UPDATE frames. This does not change -the published 0.3.0 contract or the OTP default. +version. The independent OpenSSL package gate includes 262,144-byte HTTP/2 +responses with observed connection and stream WINDOW_UPDATE frames. The OTP +default is unchanged. -TLS 1.3 session resumption in that candidate is explicit: +TLS 1.3 session resumption is explicit: `ssl: [versions: [:"tlsv1.3"], session_tickets: :auto]`. Tickets are disabled by default. Auto mode currently rejects client identities and mixed/TLS 1.2 version offers; early data and PSK-only exchange are unsupported. When a server declines a ticket, a full handshake continues on the same connection without -replaying request bytes. The source-only package test checks two fresh HTTP/1.1 connections +replaying request bytes. The package test checks two fresh HTTP/1.1 connections against an independent OpenSSL peer and requires server-observed session reuse. This is a bounded subset, not full OTP `:ssl` parity. -See the [ex_ssl compatibility contract](https://github.com/gsmlg-dev/ex_ssl/blob/v0.3.0/docs/COMPATIBILITY.md). +See the [ex_ssl compatibility contract](https://github.com/gsmlg-dev/ex_ssl/blob/v0.4.0/docs/COMPATIBILITY.md). The [consumer contract inventory](docs/ex-ssl-consumer-contract.md) maps the implemented subset and intentional restrictions to its tests. @@ -645,8 +643,11 @@ mix format --check-formatted MIT License -For unreleased ex_ssl algorithm and mTLS candidates, the source-only integration gate is +For cross-repository source checks, the source integration gate is `EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh`. -It validates new algorithms and mTLS against all five fresh package artifacts with a temporary -source override. Published ex_ssl 0.3.0 retains its documented algorithm subset; -see [the consumer contract](docs/ex-ssl-consumer-contract.md). +It validates algorithms, mTLS, TLS 1.2, and resumption against all five fresh +package artifacts with a temporary source override. Add +`EX_SSL_DEP_MODE=published` to resolve ex_ssl 0.4.0 from Hex while using the +source checkout only for test certificate fixtures. The external consumer smoke +also checks the published dependency; see +[the consumer contract](docs/ex-ssl-consumer-contract.md). diff --git a/apps/http_core/mix.exs b/apps/http_core/mix.exs index fb258c2..537bd73 100644 --- a/apps/http_core/mix.exs +++ b/apps/http_core/mix.exs @@ -37,7 +37,7 @@ defmodule HttpCore.MixProject do defp deps do [ - {:ex_ssl, "~> 0.3.0"}, + {:ex_ssl, "~> 0.4.0"}, {:quic, "~> 1.6", runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} ] diff --git a/apps/http_core/test/http/tls_transport_test.exs b/apps/http_core/test/http/tls_transport_test.exs index 754e22f..9f85106 100644 --- a/apps/http_core/test/http/tls_transport_test.exs +++ b/apps/http_core/test/http/tls_transport_test.exs @@ -98,15 +98,22 @@ defmodule HTTP.TLSTransportTest do test "ex_ssl rejects unsupported TLS and TCP options before connecting" do for ssl <- [ [verify: :verify_none], - [versions: [:"tlsv1.2"]], - [versions: [:"tlsv1.3", :"tlsv1.2"]], + [versions: [:"tlsv1.1"]], [certfile: @cert] ] do assert {:error, {:options, _}} = ExSSL.connect("localhost", 0, [ssl: ssl], 100) end assert {:error, {:options, {:nodelay, :unsupported_or_invalid}}} = - ExSSL.connect("localhost", 0, [socket_opts: [nodelay: true]], 100) + ExSSL.connect("localhost", 0, [socket_opts: [nodelay: :invalid]], 100) + + assert {:error, :econnrefused} = + ExSSL.connect("127.0.0.1", 0, [socket_opts: [nodelay: true]], 100) + + for versions <- [[:"tlsv1.2"], [:"tlsv1.3", :"tlsv1.2"]] do + assert {:error, :econnrefused} = + ExSSL.connect("127.0.0.1", 0, [ssl: [versions: versions]], 100) + end assert {:error, {:options, {:send_timeout_close, :unsupported_or_invalid}}} = ExSSL.connect("localhost", 0, [socket_opts: [send_timeout_close: false]], 100) diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index fe47ea3..3149ff9 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -1763,7 +1763,7 @@ defmodule HTTP.SocketClientHTTP2Test do end defp assert_tls_buffered_after_peer_close(tls_pid, expected_size) do - assert Application.spec(:ex_ssl, :vsn) == ~c"0.3.0", + assert Application.spec(:ex_ssl, :vsn) == ~c"0.4.0", "revalidate this private buffer probe before testing another ex_ssl version" assert_tls_buffered_after_peer_close( diff --git a/apps/http_fetch/test/http/ssl_transport_test.exs b/apps/http_fetch/test/http/ssl_transport_test.exs index cb954f6..a235a88 100644 --- a/apps/http_fetch/test/http/ssl_transport_test.exs +++ b/apps/http_fetch/test/http/ssl_transport_test.exs @@ -25,7 +25,7 @@ defmodule HTTP.SSLTransportTest do end test "rejects unsupported ex_ssl socket options through HTTP.fetch" do - for socket_opts <- [[nodelay: true], [send_timeout_close: false]] do + for socket_opts <- [[nodelay: :invalid], [send_timeout_close: false]] do assert {:error, {:options, _reason}} = "https://127.0.0.1:1/secure" |> HTTP.fetch(tls_backend: :ex_ssl, socket_opts: socket_opts) diff --git a/docs/ex-ssl-consumer-contract.md b/docs/ex-ssl-consumer-contract.md index 2202d9c..9804b00 100644 --- a/docs/ex-ssl-consumer-contract.md +++ b/docs/ex-ssl-consumer-contract.md @@ -1,9 +1,8 @@ # TCP TLS consumer contract -Audited against PR #14 head `690258ac38e50b0d1a968d9d5e510c560f45f5d4` -on 2026-09-22. OTP `:ssl` stays the default; ex_ssl is explicitly selected. -This inventory separates implemented consumer requirements from planned library -features. It does not claim full OTP compatibility. +Audited against PR #14 baseline `690258ac38e50b0d1a968d9d5e510c560f45f5d4` +and released ex_ssl 0.4.0 on 2026-09-22. OTP `:ssl` stays the default; ex_ssl +is explicitly selected. This inventory does not claim full OTP compatibility. | Requirement | Status | Production boundary | Executable coverage | | --- | --- | --- | --- | @@ -19,7 +18,8 @@ features. It does not claim full OTP compatibility. | Wrong CA/reference hostname and profile/ALPN conflicts | implemented | adapter forwards to verified ex_ssl options | `tls_transport_test.exs` (adapter-level evidence) | | No automatic backend fallback | implemented | fixed adapter, explicit option/handshake errors | `tls_backend_test.exs`, TLS-1.2-only peer negative in `tls_transport_test.exs` | | `socket_opts: [send_timeout: ..., send_timeout_close: true]` | implemented | ExSSL translates these two options; socket values override matching `ssl` values | `tls_transport_test.exs` | -| Arbitrary TCP options, verify-none, client certificates, TLS 1.2/mixed versions | deliberately unsupported in ex_ssl 0.3.0 | explicit redacted option errors | `tls_transport_test.exs` | +| Safe TCP options, client certificates, TLS 1.2/mixed versions, ordered TLS policy | implemented in ex_ssl 0.4.0 | allowlisted adapter options and independent ex_ssl engine | packaged-source 47-test gate, published-dependency smoke, and scoped transport tests | +| Arbitrary TCP options, verify-none, early data, TLS 1.2 resumption | unsupported | explicit option/protocol errors | negative option and authentication tests | | Introspection beyond negotiated ALPN, active-N, packet modes | not required by audited consumer | no production callsites | library roadmap; not a Phase 0 blocker | | HTTP/3 and WebTransport | separate QUIC implementation | explicit TCP backend rejected; shared default ignored | fetch SSL transport tests and WebTransport option/TLS config tests | @@ -35,28 +35,30 @@ is dropped to make a connection succeed. The cross-record fixture suspends the HTTP owner after it enters its receive loop, proves its sole first plaintext delivery, releases later peer output, and checks authenticated closure with retained plaintext before resuming. Its private -ex_ssl state probe is test-only, guarded to version 0.3.0, and checks exact byte +ex_ssl state probe is test-only, guarded to version 0.4.0, and checks exact byte count plus passive mode. Larger streaming responses drain incrementally through flow control before the final gated records; they do not require an oversized passive TLS buffer. -Current validation is tracked in the ex_ssl worktree's +Current validation is tracked in the ex_ssl repository's `docs/EX_SSL_HTTP_FETCH_PROGRESS.md`. Historical PR validation remains in [pr-14-validation.md](pr-14-validation.md). Commands must run from the umbrella root. Adapter-level security tests do not establish per-client-family coverage of every invalid option, and local OTP peers do not prove every server or runtime. -## Phase 1 source candidate +## Phase 1: algorithms in ex_ssl 0.4.0 -The unreleased ex_ssl candidate adds P-384 ECDHE/ECDSA, Ed25519 and -RSA-PSS-PSS SHA-256/384/512. Consumer package metadata still uses ex_ssl 0.3.0; -these new algorithms require the candidate source until a separate release. +ex_ssl 0.4.0 adds P-384 ECDHE/ECDSA, Ed25519 and RSA-PSS-PSS SHA-256/384/512. +The source smoke below records the pre-release cross-repository validation; +the current consumer dependency resolves the published 0.4.0 package. Run `EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh` from the umbrella root. This builds all five fresh package artifacts into a temporary consumer and explicitly overrides ex_ssl there; repository manifests, lockfiles and installed sources are unchanged. It is separate from the existing five-package released-dependency smoke, which has no ex_ssl override. +Set `EX_SSL_DEP_MODE=published` with the same fixture source directory to run +the 47-test feature gate against the Hex release rather than the source override. Seed 36 on OTP 28 / Elixir 1.18.5: 12 tests, zero failures (ten positive exchanges cover each new signature over HTTP/1.1+P-384 HRR and HTTP/2+direct P-384; five @@ -67,9 +69,9 @@ both HTTP mode and exact profile ALPN and waits for the SETTINGS acknowledgement No production workaround was added. Log: `/tmp/http-fetch-tls-plan-algorithms.log`. -## Phase 2 source candidate +## Phase 2: client identity in ex_ssl 0.4.0 -Candidate ex_ssl `fc1319d` supports one bounded initial-handshake client identity +ex_ssl 0.4.0 supports one bounded initial-handshake client identity through `ssl: [certfile: ..., keyfile: ...]` or the documented in-memory forms. It remains separate from the server's `cacerts`/`cacertfile` trust. Encrypted keys, hardware signing and multiple identities remain unsupported. The library's @@ -90,19 +92,18 @@ expired/wrong-purpose/incompatible credentials, pre-I/O key mismatch and bad server hostname. WSS verifies passive Upgrade, active-once frames and close; EventSource verifies the identity across same-origin reconnects. Redirect tests cover all three origin components, DNS casing, manual reuse and unchanged OTP. -These are source-candidate tests; released ex_ssl 0.3.0 still rejects client -identity options. No release manifest or installed dependency has been changed. +These tests were first run against source and are now rerun against the published +0.4.0 dependency without modifying installed dependency sources. -## Phase 3 source candidate +## Phase 3: options in ex_ssl 0.4.0 The adapter forwards the safe TCP allowlist: `nodelay`, `keepalive`, `sndbuf`, -`recbuf`, local `ip`/`port`, plus the existing send deadline options. The source -candidate validates values and supports mutable driver options; released ex_ssl -0.3.0 continues to reject newly unsupported keys. Keyword containers and duplicate +`recbuf`, local `ip`/`port`, plus the existing send deadline options. ex_ssl 0.4.0 +validates values and supports mutable driver options. Keyword containers and duplicate keys reject before fetch adds deadlines or ALPN, including improper lists. Raw active/packet controls, linger and arbitrary socket backends remain rejected. -The candidate accepts ordered TLS1.3 `ciphers`, `signature_algs`, +ex_ssl 0.4.0 accepts ordered TLS 1.3 `ciphers`, `signature_algs`, `signature_algs_cert` and `supported_groups` through `ssl`. Generated profiles preserve order; explicit profile conflicts fail before I/O. The certificate signature policy is separate from handshake CertificateVerify. No supplied @@ -114,19 +115,18 @@ selection, IPv6 local binding/DNS, raw option precedence/mutation, authenticatio failures and pre-I/O rejection of malformed/unsafe/conflicting options. The same shared adapter serves WSS and SSE. No TLS1.2 or default change is introduced here. -## Phase 4 source candidate +## Phase 4: TLS 1.2 in ex_ssl 0.4.0 -The source candidate adds verified TLS 1.2 ECDHE AES-GCM to the shared ex_ssl -adapter. The independent OpenSSL packaged-source gate covers HTTP/1.1 and HTTP/2 +ex_ssl 0.4.0 adds verified TLS 1.2 ECDHE AES-GCM to the shared adapter. The +independent OpenSSL packaged gate covers HTTP/1.1 and HTTP/2 with both TLS 1.2-only and mixed offers, mixed-offer TLS 1.3 selection, WSS Upgrade/frame/close, and EventSource reconnect with pinned backend and Last-Event-ID. All seven scenarios pass in the final 47-test source gate. The expanded HTTP/2 fixture sends 262,144 bytes, honors connection/stream flow control, and requires observed WINDOW_UPDATE frames; its seven-test gate -also passes. The released ex_ssl 0.3.0 remains TLS 1.3-only. TLS 1.2 session resumption and full -OTP option parity are not claimed. +also passes. TLS 1.2 session resumption and full OTP option parity are not claimed. -## Phase 5 source candidate +## Phase 5: opt-in TLS 1.3 resumption in ex_ssl 0.4.0 `ssl: [versions: [:"tlsv1.3"], session_tickets: :auto]` enables bounded TLS 1.3 ticket reuse for a fresh connection to the same authenticated context. The @@ -137,7 +137,33 @@ handshake processing on the same socket. `scripts/ex_ssl_resumption_test.exs` uses two packaged HTTP/1.1 fetches against one Python/OpenSSL context and checks the peer's `session_reused` value is false -then true. It is a source-candidate test and does not extend the released -dependency contract. Cache policy isolation and measured performance are +then true. Cache policy isolation and measured performance are recorded in the library readiness report. This consumer check proves only the HTTP/1.1 adapter path; HTTP/2/WSS/SSE resumption is not separately verified. + +## Published 0.4.0 validation (2026-09-22) + +The umbrella lock resolves Hex ex_ssl 0.4.0. The test-only cross-record buffer +probe was revalidated against its `closed`, `size`, and `active` fields and its +version guard advanced from 0.3.0 to 0.4.0. `MIX_ENV=test mix test +apps/http_fetch/test/http/socket_client_http2_test.exs --only cross_record` +passed 11 tests with 23 excluded. The full `MIX_ENV=test mix test` passed 185 +`http_core`, 176 `http_fetch` plus 20 doctests, 33 WebSocket, 24 WebTransport, +and 26 EventSource tests, all with zero failures. `MIX_ENV=test mix compile +--warnings-as-errors`, `mix format --check-formatted`, `mix credo` (116 files, +no issues), and `mix dialyzer` (four existing ignored warnings, zero unnecessary +skips) passed. + +The fresh five-package `bash scripts/external_consumer_smoke.sh` passed with a +transitive Hex ex_ssl 0.4.0 dependency and no source override. The broader +`EX_SSL_DEP_MODE=published EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash +scripts/ex_ssl_source_smoke.sh` resolved ex_ssl from Hex and passed 47 tests; +the source directory supplies only test fixture builders in that mode. Neither +smoke modifies installed dependency sources. On the first published run, the +unit suite had two failures from obsolete 0.3.0 negative expectations for +TLS 1.2 and `nodelay: true`; the first external smoke failed its obsolete +`~> 0.3.0` metadata assertion. Those assertions were updated to test supported +and still-rejected values before the passing reruns. An earlier source-mode +47-test run had one intermittent large HTTP/2 `:econnreset`; the independent +test peer now holds its close until the client reads the complete response. +The final source-mode and published-mode 47-test runs each had zero failures. diff --git a/docs/pr-14-validation.md b/docs/pr-14-validation.md index e2629d1..2a3db5d 100644 --- a/docs/pr-14-validation.md +++ b/docs/pr-14-validation.md @@ -1,5 +1,9 @@ # PR #14 validation +> Historical PR #14 validation at ex_ssl 0.3.0. For the released ex_ssl 0.4.0 +> dependency and current consumer limits, see +> [the TCP TLS consumer contract](ex-ssl-consumer-contract.md). + ## Phase 0 continuation (2026-09-22) Current PR head remains `690258ac38e50b0d1a968d9d5e510c560f45f5d4`. diff --git a/mix.lock b/mix.lock index 2684afd..3ec0606 100644 --- a/mix.lock +++ b/mix.lock @@ -6,7 +6,7 @@ "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, - "ex_ssl": {:hex, :ex_ssl, "0.3.0", "2ecfd0dc4ab1e86418f469aa360560eecc8c94538717e9c022130f6cc121e86c", [:mix], [], "hexpm", "629ba6272dad27593784383aa7fda9e1c60a28a366e33aa235918fec994adabd"}, + "ex_ssl": {:hex, :ex_ssl, "0.4.0", "43be385bd538e577e2279b420e8b860aaee03dc1f6136021b5aa7d46b1bc79a9", [:mix], [], "hexpm", "eeaa04e209ca644e01e3895c196809897b6524287a8e5912411bd6e744fdb61c"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, diff --git a/scripts/ex_ssl_source_smoke.sh b/scripts/ex_ssl_source_smoke.sh index 230246a..a633669 100755 --- a/scripts/ex_ssl_source_smoke.sh +++ b/scripts/ex_ssl_source_smoke.sh @@ -4,6 +4,12 @@ set -euo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) : "${EX_SSL_SOURCE_DIR:?Set EX_SSL_SOURCE_DIR to the candidate ex_ssl checkout}" export EX_SSL_SOURCE_DIR=$(cd "$EX_SSL_SOURCE_DIR" && pwd) +: "${EX_SSL_DEP_MODE:=source}" +if [[ "$EX_SSL_DEP_MODE" != source && "$EX_SSL_DEP_MODE" != published ]]; then + echo "EX_SSL_DEP_MODE must be source or published" >&2 + exit 2 +fi +export EX_SSL_DEP_MODE work_dir=$(mktemp -d) trap 'rm -rf "$work_dir"' EXIT export HTTP_FETCH_PACKAGE_DIR="$work_dir/packages" @@ -16,13 +22,20 @@ defmodule CandidateConsumer.MixProject do use Mix.Project def project do packages = System.fetch_env!("HTTP_FETCH_PACKAGE_DIR") + ex_ssl_dep = + if System.fetch_env!("EX_SSL_DEP_MODE") == "published" do + {:ex_ssl, "~> 0.4.0"} + else + {:ex_ssl, path: System.fetch_env!("EX_SSL_SOURCE_DIR"), override: true} + end + [app: :candidate_consumer, version: "0.0.0", deps: [ {:http_core, path: Path.join(packages, "http_core")}, {:http_fetch, path: Path.join(packages, "http_fetch")}, {:http_web_socket, path: Path.join(packages, "http_web_socket")}, {:http_event_source, path: Path.join(packages, "http_event_source")}, {:http_web_transport, path: Path.join(packages, "http_web_transport")}, - {:ex_ssl, path: System.fetch_env!("EX_SSL_SOURCE_DIR"), override: true} + ex_ssl_dep ]] end def application, do: [extra_applications: [:logger, :ssl, :public_key]] @@ -34,4 +47,12 @@ for test_file in "$repo_root"/scripts/ex_ssl_*_test.exs; do done cp "$repo_root/scripts/ex_ssl_tls12_peer.py" "$work_dir/consumer/test/ex_ssl_tls12_peer.py" printf 'ExUnit.start()\n' > "$work_dir/consumer/test/test_helper.exs" -(cd "$work_dir/consumer" && mix deps.get && mix compile --warnings-as-errors && mix test "$@" --seed 36) +( + cd "$work_dir/consumer" + mix deps.get + mix compile --warnings-as-errors + if [[ "$EX_SSL_DEP_MODE" == published ]]; then + mix run -e 'unless Application.spec(:ex_ssl, :vsn) == ~c"0.4.0", do: raise "expected published ex_ssl 0.4.0"' + fi + mix test "$@" --seed 36 +) diff --git a/scripts/external_consumer_smoke.exs b/scripts/external_consumer_smoke.exs index 3e21bb3..b4a525b 100644 --- a/scripts/external_consumer_smoke.exs +++ b/scripts/external_consumer_smoke.exs @@ -15,6 +15,9 @@ defmodule ExternalConsumerSmoke do assert Enum.any?(Application.started_applications(), fn {app, _, _} -> app == :ex_ssl end), ":ex_ssl was not started through http_core's package dependency" + assert Application.spec(:ex_ssl, :vsn) == ~c"0.4.0", + "published ex_ssl 0.4.0 must be loaded through http_core" + certfile = System.fetch_env!("HTTP_FETCH_CERTFILE") cacertfile = System.fetch_env!("HTTP_FETCH_CACERTFILE") keyfile = System.fetch_env!("HTTP_FETCH_KEYFILE") @@ -70,8 +73,8 @@ defmodule ExternalConsumerSmoke do core = metadata!(package_dir, "http_core") core_version = Map.fetch!(core, <<"version">>) - assert requirement!(core, <<"ex_ssl">>) == <<"~> 0.3.0">>, - "http_core package must require ex_ssl ~> 0.3.0" + assert requirement!(core, <<"ex_ssl">>) == <<"~> 0.4.0">>, + "http_core package must require ex_ssl ~> 0.4.0" for app <- ["http_fetch", "http_web_socket", "http_event_source", "http_web_transport"] do assert requirement!(metadata!(package_dir, app), <<"http_core">>) == "~> " <> core_version,