diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index 4af2ded..25760b9 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -1,7 +1,7 @@ [ # Dialyzer loses the Task.Supervisor.async_nolink/4 return shape in HTTP.fetch/2 - ~r/(apps\/http_fetch\/)?lib\/http\.ex:267.*invalid_contract/, - ~r/(apps\/http_fetch\/)?lib\/http\.ex:268.*no_return/, + ~r/(apps\/http_fetch\/)?lib\/http\.ex:270.*invalid_contract/, + ~r/(apps\/http_fetch\/)?lib\/http\.ex:271.*no_return/, # HTTP.Promise.then/3 opaque type issue with Task struct - Task.Supervisor returns opaque Task ~r/(apps\/http_fetch\/)?lib\/http\/promise\.ex:96.*contract_with_opaque/ diff --git a/.formatter.exs b/.formatter.exs index 11dbefe..98af6c6 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,5 +1,5 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs"], + inputs: ["{mix,.formatter}.exs", "scripts/*.exs"], subdirectories: ["apps/*"] ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 328557f..9ed056e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,13 @@ jobs: - name: Compile with warnings as errors run: mix compile --warnings-as-errors + - name: Compile test environment with warnings as errors + env: + MIX_ENV: test + run: | + mix deps.get + mix compile --warnings-as-errors + format: name: Format Check runs-on: ubuntu-latest @@ -214,3 +221,25 @@ jobs: - name: Build package working-directory: apps/${{ matrix.app }} run: mix hex.build --unpack -o /tmp/${{ matrix.app }}_pkg + + external-consumer: + name: External consumer smoke + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Elixir + uses: erlef/setup-beam@v1 + with: + elixir-version: ${{ env.ELIXIR_VERSION }} + otp-version: ${{ env.OTP_VERSION }} + + - name: Install dependencies + env: + MIX_ENV: prod + run: mix deps.get + + - name: Run external consumer smoke test + run: bash scripts/external_consumer_smoke.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ee3a8de..84b3560 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -62,8 +62,15 @@ jobs: ${{ runner.os }}-mix- - name: Install Elixir dependencies + env: + MIX_ENV: test run: mix deps.get + - name: Compile umbrella dependencies + env: + MIX_ENV: test + run: mix compile --warnings-as-errors + - name: Set up Go if: matrix.app == 'http_fetch' uses: actions/setup-go@v5 @@ -112,15 +119,14 @@ jobs: env: MIX_ENV: test E2E_BASE_URL: ${{ env.E2E_BASE_URL }} - run: mix test.e2e + run: mix test apps/http_fetch/e2e - name: Run app E2E test suite if present if: matrix.app != 'http_fetch' - working-directory: apps/${{ matrix.app }} run: | set -euo pipefail - if [ -d e2e ]; then - MIX_ENV=test mix test e2e/ + if [ -d "apps/${{ matrix.app }}/e2e" ]; then + MIX_ENV=test mix test "apps/${{ matrix.app }}/e2e" else echo "No e2e suite for ${{ matrix.app }}" fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7668146..813b0f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,8 +53,16 @@ jobs: ${{ runner.os }}-mix- - name: Install dependencies + env: + MIX_ENV: test run: mix deps.get + - name: Compile umbrella dependencies + env: + MIX_ENV: test + run: mix compile --warnings-as-errors + - name: Run tests - working-directory: apps/${{ matrix.app }} - run: mix test + env: + MIX_ENV: test + run: mix test apps/${{ matrix.app }}/test diff --git a/AGENTS.md b/AGENTS.md index 4c8141a..4d4a9ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,22 @@ mix docs # ExDoc HTML Run a single test file or line: `mix test apps/http_fetch/test/http/response_test.exs:42`. First-time Dialyzer setup: `mix dialyzer --plt` (2-3 min, cached in `apps/http_fetch/priv/plts/`). +### Testing an individual umbrella app + +The child applications share the umbrella's `_build`, `deps`, and lockfile, but +a child Mix project does not put runtime applications of an `in_umbrella` +dependency on its own code path. Run scoped tests through the root Mix project +after the root preparation step: + +```bash +MIX_ENV=test mix deps.get +MIX_ENV=test mix compile --warnings-as-errors +MIX_ENV=test mix test apps/http_fetch/test +``` + +Use the same root preparation and replace the path for any other app under +`apps/`. The E2E workflow uses the same root-scoped form for `apps/*/e2e`. + ## Project layout This is a Mix umbrella with independent child apps under `apps/`. Shared HTTP diff --git a/CHANGELOG.md b/CHANGELOG.md index 92bfaa7..04b7f22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,41 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [0.12.0] - 2026-09-22 + +### Added +- Add verified ex_ssl 0.4.0 TLS 1.2 support across Fetch, HTTP/2, WebSocket, + and EventSource, and opt-in TLS 1.3 ticket resumption over fresh HTTP/1.1 + connections. The default remains verified TLS 1.3 with tickets disabled. +- Forward validated ex_ssl client identities, TCP socket options, and ordered + TLS 1.3 algorithm preferences. Pin client identities to the redirect origin. + +### Changed +- Keep OTP `:ssl` as the default TCP TLS backend while allowing verified TLS + 1.3 or explicitly selected TLS 1.2 through the optional `:ex_ssl` backend. + HTTP/3 and + WebTransport continue to use QUIC's independent TLS implementation. +- Run individual app tests and E2E suites from the umbrella root after explicit + test-environment preparation, so transitive runtime applications are compiled + and on the code path even on cold checkouts. +- Validate all five built packages in an isolated external consumer, including + transitive dependencies and verified local TLS requests. + +### Fixed +- Deliver complete HTTP/2 responses when the peer closes immediately after an + END_STREAM DATA frame and a non-essential WINDOW_UPDATE or acknowledgement + returns `:closed`. Incomplete responses and required request writes before + response completion still fail normally. +- Preserve unread ex_ssl TLS records when an HTTP/2 control write fails after + normal peer closure. Drain them through the existing active-once receiver and + require a complete HTTP/2 response, without extending deadlines or accepting + truncated responses, error resets, or protocol errors. +- Preserve valid HTTP/2 early final responses while cancelling remaining upload + DATA, including queued DATA released by WINDOW_UPDATE. Accept NO_ERROR resets + only after response completion; require END_STREAM and a complete field block + even for responses with no body. +- Enforce HTTP/2 response Content-Length and bounded frame/header parsing while + preserving valid cross-record and streaming completion after peer closure. ## [0.11.0] - 2026-07-04 diff --git a/CLAUDE.md b/CLAUDE.md index 48db8f7..49850c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,6 +84,14 @@ mix docs MIX_ENV=prod mix compile ``` +For individual app tests, stay at the umbrella root: run `MIX_ENV=test mix +deps.get`, `MIX_ENV=test mix compile --warnings-as-errors`, then `mix test +apps//test`. Running Mix inside a child app does not traverse the runtime +dependencies of its `in_umbrella` dependencies. E2E uses the same root preparation +and `mix test apps//e2e`, or `mix test.e2e` for all E2E suites. +Run `bash scripts/external_consumer_smoke.sh` to build all five packages and +verify an isolated non-umbrella consumer, including transitive TLS dependencies. + ## Important Implementation Details ### Request Options Mapping diff --git a/README.md b/README.md index 961a97e..04bc79c 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,17 @@ A modern HTTP client library for Elixir that provides a fetch API similar to web browsers, built on Erlang's built-in socket modules. +For development, prepare the umbrella before running a scoped app test: +`MIX_ENV=test mix deps.get && MIX_ENV=test mix compile --warnings-as-errors` +followed by `MIX_ENV=test mix test apps/http_fetch/test`. Running the test from +the root keeps runtime applications of `in_umbrella` dependencies, including +`ex_ssl`, on the code path without adding duplicate child dependencies. + ## Features - **Browser-like API**: Familiar fetch interface with promises and async/await patterns - **Full HTTP support**: GET, POST, PUT, DELETE, PATCH, HEAD methods -- **Internal HTTP/1.1 transport**: Uses `:gen_tcp` for HTTP, `:ssl` for HTTPS, and Unix domain sockets +- **Internal HTTP/1.1 transport**: Uses `:gen_tcp` for HTTP, selectable TLS for HTTPS, and Unix domain sockets - **Unix Domain Sockets**: HTTP over Unix sockets for Docker daemon, systemd, and other local services - **Form data support**: HTTP.FormData for multipart/form-data and file uploads - **Streaming request bodies**: Fetch-style `duplex: "half"` uploads over HTTP/1.1 @@ -21,7 +27,7 @@ A modern HTTP client library for Elixir that provides a fetch API similar to web - **Promise-based**: Async operations with chaining support - **Request cancellation**: AbortController support for cancelling requests - **Automatic JSON parsing**: Built-in JSON response handling -- **Zero dependencies**: Uses only Erlang/OTP built-in modules +- **Selectable TLS**: OTP `:ssl` by default, with opt-in `:ex_ssl` for verified TLS 1.3 ## Browser Fetch API Compatibility @@ -119,6 +125,109 @@ response = IO.puts("Docker Version: #{docker_info["Version"]}") ``` +## TLS Backend Selection + +HTTPS fetch (HTTP/1.1 and HTTP/2), secure WebSocket, and HTTPS EventSource share +one TLS default. OTP `:ssl` remains the default when no configuration is set: + +```elixir +# config/config.exs or config/runtime.exs +config :http_core, tls_backend: :ex_ssl +``` + +A flat per-call option overrides that default: + +```elixir +HTTP.fetch("https://example.com", tls_backend: :ssl) + +HTTP.fetch("https://example.com", + tls_backend: :ex_ssl, + ssl: [cacertfile: "/path/to/ca.pem"] +) + +HTTP.WebSocket.new("wss://example.com/socket", [], tls_backend: :ex_ssl) +HTTP.EventSource.new("https://example.com/events", tls_backend: :ex_ssl) +``` + +`tls_backend` accepts `:ssl`, `:ex_ssl`, `"ssl"`, or `"ex_ssl"`. Maps also accept +`"tls_backend"` and `"tlsBackend"` keys. Omitted or `nil` values inherit the shared +configuration. The backend is captured when the request/client is created and +retained through redirects and EventSource reconnects; runtime configuration +changes affect new operations. Invalid selections fail explicitly. + +`http_core` declares `ex_ssl ~> 0.4.0` as a transitive runtime dependency. +Consumers do not need to add it separately. `ssl: [...]` supplies TLS settings +to the selected backend. The `ex_ssl` backend uses its own `SSL` protocol engine +and requires peer verification. TLS 1.3 is the default; verified TLS 1.2 is +explicitly selectable. It uses system CA certificates unless `cacerts` or +`cacertfile` is supplied. DNS names and IP addresses are verified against the +peer certificate. `verify: :verify_none` and unsupported TLS or TCP options +return errors; connections never fall back to another backend automatically. + +A complete HTTP/2 response remains deliverable if the peer closes before the +client can write its remaining WINDOW_UPDATE or acknowledgement frames. This +also covers responses buffered across multiple TLS records by `ex_ssl` after a +normal peer shutdown: the client drains the receive side before deciding whether +the response completed. Only `:closed` on optional control writes qualifies. +A complete early response (such as 413) stops the remaining upload, including +request DATA queued by WINDOW_UPDATE in the same batch. It also survives a +subsequent RST_STREAM(NO_ERROR), as required by +[RFC 9113 ยง8.1](https://www.rfc-editor.org/rfc/rfc9113.html#section-8.1). +Completion requires END_STREAM and the complete HEADERS/CONTINUATION field +block; an unfinished upload neither proves nor prevents response completion. +HTTP/2 also validates Content-Length against unpadded DATA bytes before +completion, rejects body overruns immediately, and reports mismatches as +`:content_length_mismatch`. Valid HEAD/304 representation lengths do not require +a body. Malformed/conflicting lengths, values longer than 20 decimal digits, +and values outside the unsigned 64-bit bound return `:invalid_content_length`. +Inbound frames are limited to the advertised 16,384-byte payload size and +compressed header blocks to 65,536 bytes, including CONTINUATION fragments. +Content-Length is forbidden on informational/204 responses and in trailers; +DATA or HEADERS after END_STREAM is rejected rather than completed again. +Truncation, required writes before completion, abnormal closure, cancellation +and timeout remain errors. The original deadline and streaming backpressure +are preserved. + +For `:ex_ssl`, `socket_opts` accepts `send_timeout`, +`send_timeout_close: true`, `nodelay`, `keepalive`, `sndbuf`, `recbuf`, and local +`ip`/`port`. The adapter forwards only this allowlist and ex_ssl validates values. +IPv6 literals infer the family; an IPv6 local `ip` tuple selects IPv6 DNS +resolution. Both option containers must be keyword lists. Socket options +override matching entries in `ssl`. Custom +ClientHello profiles can be passed through `ssl: [ex_ssl: [profile: profile]]`; +any ALPN list added by HTTP/2 selection must match the profile's ALPN list exactly. +Configured ex_ssl client credentials stay within the initial request origin +during automatic redirects. A scheme, +hostname or effective-port change returns +`{:error, :client_identity_cross_origin_redirect}`. To authorize another origin, +use `redirect: :manual` and explicitly make a new request with that identity. +The OTP backend retains its existing redirect behavior. + +ex_ssl 0.4.0 supports verified TLS 1.2 for +HTTP/1.1, HTTP/2, WSS and EventSource. Select it with `ssl: [versions: +[:"tlsv1.2"]]`; a mixed TLS 1.3/TLS 1.2 offer selects the peer's supported +version. The independent OpenSSL package gate includes 262,144-byte HTTP/2 +responses with observed connection and stream WINDOW_UPDATE frames. The OTP +default is unchanged. + +TLS 1.3 session resumption is explicit: +`ssl: [versions: [:"tlsv1.3"], session_tickets: :auto]`. Tickets are disabled +by default. Auto mode currently rejects client identities and mixed/TLS 1.2 +version offers; early data and PSK-only exchange are unsupported. When a server +declines a ticket, a full handshake continues on the same connection without +replaying request bytes. The package test checks two fresh HTTP/1.1 connections +against an independent OpenSSL peer and requires server-observed session reuse. +This is a bounded subset, not full OTP `:ssl` parity. + +See the [ex_ssl compatibility contract](https://github.com/gsmlg-dev/ex_ssl/blob/v0.4.0/docs/COMPATIBILITY.md). +The [consumer contract inventory](docs/ex-ssl-consumer-contract.md) maps the +implemented subset and intentional restrictions to its tests. + +Plain HTTP, WS, and Unix sockets retain their existing transports. HTTP/3 and +WebTransport use QUIC's separate TLS implementation and ignore the shared +setting. An explicit non-`nil` `tls_backend` on either QUIC API returns +`{:error, :tls_backend_not_supported_for_quic}` (through the promise for fetch). + ## Form Data With File Upload ```elixir @@ -371,7 +480,7 @@ request = %HTTP.Request{ ``` **Transport Options:** -- `transport_options`: Socket transport options such as `timeout`, `connect_timeout`, `ssl`, +- `transport_options`: Socket transport options such as `timeout`, `connect_timeout`, `tls_backend`, `ssl`, `socket_opts`, and `redirect` `redirect` defaults to `:follow` with the socket transport. Pass `redirect: :manual` @@ -456,10 +565,22 @@ open doc/index.html ### Running Tests +Run these commands from the umbrella root, including when testing one app. +The root dependency graph includes the runtime dependencies of every umbrella +app; invoking Mix inside a child app does not traverse its `in_umbrella` +dependencies in the same way. + ```bash +# Prepare dependencies, including on a cold checkout +MIX_ENV=test mix deps.get +MIX_ENV=test mix compile --warnings-as-errors + # Run all unit tests mix test +# Run one app (replace the app name as needed) +mix test apps/http_fetch/test + # Run specific test file mix test apps/http_fetch/test/http/response_test.exs @@ -486,6 +607,22 @@ MIX_ENV=test mix test.e2e ``` In CI, the `e2e.yml` workflow handles all of this automatically. +`mix test.e2e` keeps execution at the umbrella root. To run one suite, use +`MIX_ENV=test mix test apps/http_web_socket/e2e` (or another app's `e2e` +directory) after the same preparation as the unit tests. + +### Testing Packaged Consumers + +```bash +bash scripts/external_consumer_smoke.sh +``` + +This builds all five current Hex packages and installs their unpacked contents +into a temporary project outside the umbrella, with independent dependencies +and build output and no repository lockfile. Local paths resolve the unpublished +internal packages; `ex_ssl` is resolved only through `http_core`. The smoke +checks runtime application startup, verified local TLS 1.3 requests with both +TCP TLS backends, and the separate WebTransport QUIC boundary. ### Code Formatting @@ -505,3 +642,12 @@ mix format --check-formatted ## License MIT License + +For cross-repository source checks, the source integration gate is +`EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh`. +It validates algorithms, mTLS, TLS 1.2, and resumption against all five fresh +package artifacts with a temporary source override. Add +`EX_SSL_DEP_MODE=published` to resolve ex_ssl 0.4.0 from Hex while using the +source checkout only for test certificate fixtures. The external consumer smoke +also checks the published dependency; see +[the consumer contract](docs/ex-ssl-consumer-contract.md). diff --git a/apps/http_core/lib/http/http2.ex b/apps/http_core/lib/http/http2.ex index 644a77a..0eec59b 100644 --- a/apps/http_core/lib/http/http2.ex +++ b/apps/http_core/lib/http/http2.ex @@ -12,8 +12,10 @@ defmodule HTTP.HTTP2 do @client_stream_id 1 @initial_window_size 65_535 @initial_max_frame_size 16_384 + @max_header_block_size 65_536 @max_window_size 2_147_483_647 @max_max_frame_size 16_777_215 + @max_content_length "18446744073709551615" @flag_end_stream 0x1 @flag_ack 0x1 @@ -34,6 +36,9 @@ defmodule HTTP.HTTP2 do max_frame_size: @initial_max_frame_size, pending_body: "", outbound: [], + request_stopped?: false, + expected_content_length: nil, + received_content_length: 0, done?: false @type event :: {:headers, non_neg_integer(), Headers.t()} | {:body, binary()} | :done @@ -98,10 +103,37 @@ defmodule HTTP.HTTP2 do {%{conn | outbound: []}, Enum.reverse(outbound)} end + @spec complete_response?(t()) :: boolean() + def complete_response?(%__MODULE__{done?: done?}), do: done? + + @spec request_stopped?(t()) :: boolean() + def request_stopped?(%__MODULE__{request_stopped?: stopped?}), do: stopped? + + @spec stop_request(t()) :: t() + def stop_request(%__MODULE__{} = conn) do + %{ + conn + | pending_body: "", + outbound: Enum.reject(conn.outbound, &request_data_frame?/1), + request_stopped?: true + } + end + + @spec outbound_control_only?(t()) :: boolean() + def outbound_control_only?(%__MODULE__{outbound: outbound}) do + Enum.all?(outbound, &control_frame?/1) + end + defp append_buffer(%__MODULE__{buffer: buffer} = conn, data) do %{conn | buffer: buffer <> data} end + # We do not advertise a larger receive frame size. Peer SETTINGS only change + # the size of frames we send, not this inbound bound. + defp parse(%__MODULE__{buffer: <>}, _events) + when length > @initial_max_frame_size, + do: {:error, :frame_size_error} + defp parse(%__MODULE__{} = conn, events) do case Frame.decode(conn.buffer) do :more -> @@ -148,6 +180,9 @@ defmodule HTTP.HTTP2 do defp handle_frame(_conn, %Frame{type: :settings}), do: {:error, :invalid_settings_stream} + defp handle_frame(%__MODULE__{done?: true}, %Frame{type: :headers, stream_id: @client_stream_id}), + do: {:error, :stream_closed} + defp handle_frame(conn, %Frame{type: :headers, stream_id: @client_stream_id} = frame) do with {:ok, fragment} <- headers_fragment(frame) do end_stream? = Frame.flag?(frame.flags, @flag_end_stream) @@ -158,6 +193,7 @@ defmodule HTTP.HTTP2 do continuation = %{ stream_id: frame.stream_id, fragments: [fragment], + size: byte_size(fragment), end_stream?: end_stream? } @@ -172,43 +208,66 @@ defmodule HTTP.HTTP2 do %__MODULE__{continuation: %{stream_id: stream_id} = continuation} = conn, %Frame{type: :continuation, stream_id: stream_id} = frame ) do - fragments = [frame.payload | continuation.fragments] + size = continuation.size + byte_size(frame.payload) - if Frame.flag?(frame.flags, @flag_end_headers) do - header_block = fragments |> Enum.reverse() |> IO.iodata_to_binary() - - conn - |> Map.put(:continuation, nil) - |> decode_response_headers(header_block, continuation.end_stream?) + if size > @max_header_block_size do + {:error, :header_block_too_large} else - {:ok, %{conn | continuation: %{continuation | fragments: fragments}}, []} + fragments = + if frame.payload == "", + do: continuation.fragments, + else: [frame.payload | continuation.fragments] + + if Frame.flag?(frame.flags, @flag_end_headers) do + header_block = fragments |> Enum.reverse() |> IO.iodata_to_binary() + + conn + |> Map.put(:continuation, nil) + |> decode_response_headers(header_block, continuation.end_stream?) + else + {:ok, %{conn | continuation: %{continuation | fragments: fragments, size: size}}, []} + end end end defp handle_frame(_conn, %Frame{type: :continuation}), do: {:error, :unexpected_continuation} + defp handle_frame(%__MODULE__{done?: true}, %Frame{type: :data, stream_id: @client_stream_id}), + do: {:error, :stream_closed} + defp handle_frame(%__MODULE__{status: nil}, %Frame{type: :data, stream_id: @client_stream_id}) do {:error, :data_before_response_headers} end defp handle_frame(conn, %Frame{type: :data, stream_id: @client_stream_id} = frame) do with {:ok, data, flow_controlled_size} <- data_payload(frame), - {:ok, conn} <- consume_receive_window(conn, flow_controlled_size) do + {:ok, conn} <- consume_receive_window(conn, flow_controlled_size), + :ok <- validate_response_body(conn, data), + {:ok, conn} <- count_response_body(conn, data) do end_stream? = Frame.flag?(frame.flags, @flag_end_stream) forbidden? = response_body_forbidden?(conn) - conn = if end_stream?, do: %{conn | done?: true}, else: conn - events = - [] - |> maybe_body_event(data, forbidden?) - |> maybe_done_event(end_stream?) + with {:ok, conn} <- maybe_complete_response(conn, end_stream?) do + events = + [] + |> maybe_body_event(data, forbidden?) + |> maybe_done_event(end_stream?) - {:ok, conn, events} + {:ok, conn, events} + end end end defp handle_frame(_conn, %Frame{type: :data}), do: {:error, :invalid_data_stream} + defp handle_frame(%__MODULE__{done?: true} = conn, %Frame{ + type: :rst_stream, + stream_id: @client_stream_id, + payload: <<0::32>> + }) do + {:ok, conn, []} + end + defp handle_frame(_conn, %Frame{ type: :rst_stream, stream_id: @client_stream_id, @@ -277,8 +336,9 @@ defmodule HTTP.HTTP2 do when is_integer(status) do with true <- end_stream? || {:error, :invalid_response_trailers}, {:ok, hpack, headers} <- HPACK.decode(conn.hpack, header_block), - :ok <- validate_response_trailers(headers) do - {:ok, %{conn | hpack: hpack, done?: true}, [:done]} + :ok <- validate_response_trailers(headers), + {:ok, conn} <- complete_response(conn) do + {:ok, %{conn | hpack: hpack}, [:done]} end end @@ -288,14 +348,19 @@ defmodule HTTP.HTTP2 do conn = %{conn | hpack: hpack} if status in 100..199 do - {:ok, conn, []} + with :ok <- validate_informational_response_headers(regular_headers) do + {:ok, conn, []} + end else headers = Headers.new(regular_headers) - done? = end_stream? or HTTP.HTTP1.body_forbidden?(conn.method, status) - conn = %{conn | status: status, done?: done?} - events = [{:headers, status, headers}] |> maybe_done_event(done?) + conn = %{conn | status: status} - {:ok, conn, events} + with {:ok, conn} <- set_expected_content_length(conn, regular_headers), + {:ok, conn} <- maybe_complete_response(conn, end_stream?) do + events = [{:headers, status, headers}] |> maybe_done_event(end_stream?) + + {:ok, conn, events} + end end end end @@ -315,7 +380,9 @@ defmodule HTTP.HTTP2 do end defp validate_response_trailers(headers) do - if Enum.any?(headers, fn {name, _value} -> String.starts_with?(name, ":") end) do + if Enum.any?(headers, fn {name, _value} -> + String.starts_with?(name, ":") or name == "content-length" + end) do {:error, :invalid_response_trailers} else :ok @@ -464,6 +531,103 @@ defmodule HTTP.HTTP2 do defp maybe_done_event(events, false), do: events defp maybe_done_event(events, true), do: events ++ [:done] + defp set_expected_content_length(conn, headers) do + headers + |> response_content_lengths() + |> parse_content_lengths() + |> case do + {:ok, nil} -> + {:ok, conn} + + {:ok, length} -> + cond do + conn.status in 100..199 or conn.status == 204 -> + {:error, :invalid_content_length} + + response_body_forbidden?(conn) -> + {:ok, conn} + + true -> + {:ok, %{conn | expected_content_length: length}} + end + + {:error, _reason} = error -> + error + end + end + + defp validate_informational_response_headers(headers) do + case headers |> response_content_lengths() |> parse_content_lengths() do + {:ok, nil} -> :ok + _ -> {:error, :invalid_content_length} + end + end + + defp response_content_lengths(headers) do + headers + |> Enum.filter(fn {name, _value} -> name == "content-length" end) + |> Enum.map(fn {_name, value} -> value end) + end + + defp parse_content_lengths([]), do: {:ok, nil} + + defp parse_content_lengths(values) do + if Enum.any?(values, &invalid_content_length_value?/1) do + {:error, :invalid_content_length} + else + lengths = Enum.map(values, &String.to_integer/1) + + case Enum.uniq(lengths) do + [length] -> {:ok, length} + _ -> {:error, :invalid_content_length} + end + end + end + + defp invalid_content_length_value?(value) do + byte_size(value) not in 1..20 or + (byte_size(value) == 20 and value > @max_content_length) or + not Enum.all?(:binary.bin_to_list(value), &(&1 in ?0..?9)) + end + + defp validate_response_body(conn, data) do + if response_body_forbidden?(conn) and data != "" do + {:error, :invalid_response_body} + else + :ok + end + end + + defp count_response_body(conn, data) do + conn = Map.update!(conn, :received_content_length, &(&1 + byte_size(data))) + + if response_content_length_valid_or_pending?(conn) do + {:ok, conn} + else + {:error, :content_length_mismatch} + end + end + + defp maybe_complete_response(conn, false), do: {:ok, conn} + defp maybe_complete_response(conn, true), do: complete_response(conn) + + defp response_content_length_valid?(%__MODULE__{expected_content_length: nil}), do: true + + defp response_content_length_valid?(%__MODULE__{ + expected_content_length: expected, + received_content_length: received + }), + do: expected == received + + defp response_content_length_valid_or_pending?(%__MODULE__{expected_content_length: nil}), + do: true + + defp response_content_length_valid_or_pending?(%__MODULE__{ + expected_content_length: expected, + received_content_length: received + }), + do: received <= expected + defp response_body_forbidden?(%__MODULE__{status: nil}), do: false defp response_body_forbidden?(%__MODULE__{method: method, status: status}) do @@ -474,7 +638,24 @@ defmodule HTTP.HTTP2 do %{conn | outbound: [iodata | outbound]} end + defp control_frame?(iodata) do + case Frame.decode(IO.iodata_to_binary(iodata)) do + {:ok, %Frame{type: :window_update}, ""} -> true + {:ok, %Frame{type: :settings, flags: flags}, ""} -> Frame.flag?(flags, @flag_ack) + {:ok, %Frame{type: :ping, flags: flags}, ""} -> Frame.flag?(flags, @flag_ack) + _ -> false + end + end + + defp request_data_frame?(iodata) do + case Frame.decode(IO.iodata_to_binary(iodata)) do + {:ok, %Frame{type: :data, stream_id: @client_stream_id}, ""} -> true + _ -> false + end + end + defp flush_pending_body(%__MODULE__{pending_body: ""} = conn), do: conn + defp flush_pending_body(%__MODULE__{request_stopped?: true} = conn), do: conn defp flush_pending_body(%__MODULE__{} = conn) do writable = min(conn.connection_send_window, conn.stream_send_window) @@ -495,6 +676,17 @@ defmodule HTTP.HTTP2 do end end + defp complete_response(conn) do + if response_content_length_valid?(conn) do + {:ok, + conn + |> Map.put(:done?, true) + |> stop_request()} + else + {:error, :content_length_mismatch} + end + end + defp pseudo_headers(%Request{} = request) do [ {":method", Request.method_token(request.method)}, diff --git a/apps/http_core/lib/http/tls_backend.ex b/apps/http_core/lib/http/tls_backend.ex new file mode 100644 index 0000000..6ae8607 --- /dev/null +++ b/apps/http_core/lib/http/tls_backend.ex @@ -0,0 +1,19 @@ +defmodule HTTP.TLSBackend do + @moduledoc false + + @type t :: :ssl | :ex_ssl + + @spec resolve(term()) :: {:ok, t()} | {:error, :invalid_tls_backend} + def resolve(value \\ nil) + + def resolve(nil), do: normalize(Application.get_env(:http_core, :tls_backend, :ssl)) + def resolve(value), do: normalize(value) + + @spec transport(t()) :: HTTP.Transport.SSL | HTTP.Transport.ExSSL + def transport(:ssl), do: HTTP.Transport.SSL + def transport(:ex_ssl), do: HTTP.Transport.ExSSL + + defp normalize(value) when value in [:ssl, "ssl"], do: {:ok, :ssl} + defp normalize(value) when value in [:ex_ssl, "ex_ssl"], do: {:ok, :ex_ssl} + defp normalize(_value), do: {:error, :invalid_tls_backend} +end diff --git a/apps/http_core/lib/http/transport.ex b/apps/http_core/lib/http/transport.ex index 66db7b2..1084ca9 100644 --- a/apps/http_core/lib/http/transport.ex +++ b/apps/http_core/lib/http/transport.ex @@ -1,14 +1,16 @@ defmodule HTTP.Transport do @moduledoc false - @type socket :: port() | :ssl.sslsocket() + @type socket :: port() | :ssl.sslsocket() | SSL.Socket.t() @type message :: {:data, binary()} | :closed | {:error, term()} | :unknown @callback connect(String.t(), non_neg_integer(), keyword(), timeout()) :: {:ok, socket()} | {:error, term()} @callback controlling_process(socket(), pid()) :: :ok | {:error, term()} @callback send(socket(), iodata()) :: :ok | {:error, term()} + @callback recv(socket(), non_neg_integer(), timeout()) :: {:ok, binary()} | {:error, term()} + @callback negotiated_protocol(socket()) :: {:ok, binary() | nil} | {:error, term()} @callback setopts(socket(), keyword()) :: :ok | {:error, term()} - @callback close(socket()) :: :ok + @callback close(socket()) :: :ok | {:error, term()} @callback normalize_message(term(), socket()) :: message() end diff --git a/apps/http_core/lib/http/transport/ex_ssl.ex b/apps/http_core/lib/http/transport/ex_ssl.ex new file mode 100644 index 0000000..0304ad2 --- /dev/null +++ b/apps/http_core/lib/http/transport/ex_ssl.ex @@ -0,0 +1,98 @@ +defmodule HTTP.Transport.ExSSL do + @moduledoc false + + @behaviour HTTP.Transport + + @impl true + def connect(host, port, opts, timeout) do + with {:ok, ssl_opts} <- tls_options(opts) do + SSL.connect(connect_host(host), port, ssl_opts, timeout) + end + end + + @impl true + def controlling_process(socket, pid), do: SSL.controlling_process(socket, pid) + + @impl true + def send(socket, iodata), do: SSL.send(socket, iodata) + + @impl true + def recv(socket, length, timeout), do: SSL.recv(socket, length, timeout) + + @impl true + def setopts(socket, opts), do: SSL.setopts(socket, opts) + + @impl true + def close(socket), do: SSL.close(socket) + + @impl true + def negotiated_protocol(socket) do + case SSL.negotiated_protocol(socket) do + {:error, :protocol_not_negotiated} -> {:ok, nil} + result -> result + end + end + + @impl true + def normalize_message({:ssl, socket, data}, socket), do: {:data, data} + def normalize_message({:ssl_closed, socket}, socket), do: :closed + def normalize_message({:ssl_error, socket, reason}, socket), do: {:error, reason} + def normalize_message(_, _), do: :unknown + + defp tls_options(opts) do + ssl_opts = Keyword.get(opts, :ssl, []) + socket_opts = Keyword.get(opts, :socket_opts, []) + + with :ok <- validate_keyword(ssl_opts), + :ok <- validate_keyword(socket_opts), + :ok <- validate_socket_options(socket_opts) do + # SSL supplies system trust only when neither caller CA option is present, + # and infers DNS SNI or IP identity from the connection host. + defaults = [ + mode: :binary, + packet: :raw, + active: false, + verify: :verify_peer, + versions: [:"tlsv1.3"], + depth: 4, + customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)] + ] + + {:ok, defaults |> Keyword.merge(ssl_opts) |> Keyword.merge(socket_opts)} + end + end + + defp validate_keyword(opts) do + if Keyword.keyword?(opts) and + length(Keyword.keys(opts)) == length(Enum.uniq(Keyword.keys(opts))) do + :ok + else + {:error, {:options, :invalid_options}} + end + end + + defp validate_socket_options(opts) do + allowed = [ + :send_timeout, + :send_timeout_close, + :nodelay, + :keepalive, + :sndbuf, + :recbuf, + :ip, + :port + ] + + case Enum.find(opts, fn {key, _} -> key not in allowed end) do + nil -> :ok + {key, _} -> {:error, {:options, {key, :unsupported_or_invalid}}} + end + end + + defp connect_host(host) do + case :inet.parse_address(String.to_charlist(host)) do + {:ok, address} -> address + {:error, :einval} -> host + end + end +end diff --git a/apps/http_core/lib/http/transport/ssl.ex b/apps/http_core/lib/http/transport/ssl.ex index f11130e..872a3ba 100644 --- a/apps/http_core/lib/http/transport/ssl.ex +++ b/apps/http_core/lib/http/transport/ssl.ex @@ -23,12 +23,16 @@ defmodule HTTP.Transport.SSL do @impl true def send(socket, iodata), do: :ssl.send(socket, iodata) + @impl true + def recv(socket, length, timeout), do: :ssl.recv(socket, length, timeout) + @impl true def setopts(socket, opts), do: :ssl.setopts(socket, opts) @impl true def close(socket), do: :ssl.close(socket) + @impl true @spec negotiated_protocol(:ssl.sslsocket()) :: {:ok, binary() | nil} | {:error, :closed} def negotiated_protocol(socket) do case :ssl.negotiated_protocol(socket) do diff --git a/apps/http_core/lib/http/transport/tcp.ex b/apps/http_core/lib/http/transport/tcp.ex index b6eea78..54ac28b 100644 --- a/apps/http_core/lib/http/transport/tcp.ex +++ b/apps/http_core/lib/http/transport/tcp.ex @@ -21,6 +21,12 @@ defmodule HTTP.Transport.TCP do @impl true def send(socket, iodata), do: :gen_tcp.send(socket, iodata) + @impl true + def recv(socket, length, timeout), do: :gen_tcp.recv(socket, length, timeout) + + @impl true + def negotiated_protocol(_socket), do: {:ok, nil} + @impl true def setopts(socket, opts), do: :inet.setopts(socket, opts) diff --git a/apps/http_core/lib/http/transport/unix.ex b/apps/http_core/lib/http/transport/unix.ex index 358130e..d9178a9 100644 --- a/apps/http_core/lib/http/transport/unix.ex +++ b/apps/http_core/lib/http/transport/unix.ex @@ -21,6 +21,12 @@ defmodule HTTP.Transport.Unix do @impl true def send(socket, iodata), do: :gen_tcp.send(socket, iodata) + @impl true + def recv(socket, length, timeout), do: :gen_tcp.recv(socket, length, timeout) + + @impl true + def negotiated_protocol(_socket), do: {:ok, nil} + @impl true def setopts(socket, opts), do: :inet.setopts(socket, opts) diff --git a/apps/http_core/mix.exs b/apps/http_core/mix.exs index 485a432..537bd73 100644 --- a/apps/http_core/mix.exs +++ b/apps/http_core/mix.exs @@ -37,6 +37,7 @@ defmodule HttpCore.MixProject do defp deps do [ + {:ex_ssl, "~> 0.4.0"}, {:quic, "~> 1.6", runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} ] diff --git a/apps/http_core/test/http/http2_limits_test.exs b/apps/http_core/test/http/http2_limits_test.exs new file mode 100644 index 0000000..5179f04 --- /dev/null +++ b/apps/http_core/test/http/http2_limits_test.exs @@ -0,0 +1,65 @@ +defmodule HTTP.HTTP2LimitsTest do + use ExUnit.Case, async: true + + alias HTTP.HTTP2 + alias HTTP.HTTP2.{Frame, HPACK} + + test "rejects an oversized advertised frame before retaining its payload at every header split" do + header = <<16_385::24, 0, 0, 1::32>> + + for split <- 1..8 do + <> = header + assert {:ok, conn, []} = HTTP2.stream(HTTP2.new(:get), first) + assert {:error, :frame_size_error} = HTTP2.stream(conn, rest) + end + end + + test "bounds accumulated CONTINUATION fragments even without END_HEADERS" do + fragment = :binary.copy(<<0>>, 16_384) + assert {:ok, conn, []} = HTTP2.stream(HTTP2.new(:get), Frame.encode(:headers, 0, 1, fragment)) + + conn = + Enum.reduce(1..3, conn, fn _, conn -> + assert {:ok, conn, []} = HTTP2.stream(conn, Frame.encode(:continuation, 0, 1, fragment)) + conn + end) + + assert {:error, :header_block_too_large} = + HTTP2.stream(conn, Frame.encode(:continuation, 0, 1, <<0>>)) + end + + test "accepts a bounded header block split over legal frames and TCP chunks" do + value = :binary.copy("v", 20_000) + + block = + HPACK.encode_headers([{":status", "200"}, {"x-large", value}]) |> IO.iodata_to_binary() + + <> = block + wire = Frame.encode(:headers, 1, 1, first) <> Frame.encode(:continuation, 4, 1, rest) + + {conn, events} = + for <>, reduce: {HTTP2.new(:get), []} do + {conn, events} -> + assert {:ok, conn, next} = HTTP2.stream(conn, chunk) + {conn, events ++ next} + end + + assert HTTP2.complete_response?(conn) + assert [{:headers, 200, headers}, :done] = events + assert HTTP.Headers.get(headers, "x-large") == value + end + + test "rejects whitespace around HTTP/2 Content-Length" do + for value <- [" 2", "2 ", "\t2"] do + frame = + Frame.encode( + :headers, + 4, + 1, + HPACK.encode_headers([{":status", "200"}, {"content-length", value}]) + ) + + assert {:error, :invalid_content_length} = HTTP2.stream(HTTP2.new(:get), frame) + end + end +end diff --git a/apps/http_core/test/http/http2_test.exs b/apps/http_core/test/http/http2_test.exs index bc539c3..0eeb63b 100644 --- a/apps/http_core/test/http/http2_test.exs +++ b/apps/http_core/test/http/http2_test.exs @@ -9,9 +9,32 @@ defmodule HTTP.HTTP2Test do @end_stream 0x1 @ack 0x1 @end_headers 0x4 + @padded 0x8 @initial_window_size 65_535 @max_window_size 2_147_483_647 + describe "outbound_control_only?/1" do + test "classifies only acknowledgements and window updates" do + controls = [ + Frame.encode(:settings, @ack, 0, ""), + Frame.encode(:ping, @ack, 0, "12345678"), + Frame.encode(:window_update, 0, 1, <<0::1, 1::31>>) + ] + + conn = %HTTP.HTTP2{outbound: controls} + assert HTTP.HTTP2.outbound_control_only?(conn) + assert HTTP.HTTP2.outbound_control_only?(%{conn | pending_body: "unsent"}) + + for required <- [ + Frame.encode(:data, @end_stream, 1, "upload"), + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:ping, 0, 0, "12345678") + ] do + refute HTTP.HTTP2.outbound_control_only?(%{conn | outbound: [required | controls]}) + end + end + end + describe "serialize_request/1" do test "serializes the connection preface, settings, and request headers" do request = %HTTP.Request{ @@ -180,22 +203,383 @@ defmodule HTTP.HTTP2Test do assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) end + test "rejects a final DATA frame shorter or longer than Content-Length" do + for {declared_length, body} <- [{3, "ok"}, {2, "too"}] do + frames = [ + response_headers_frame([ + {":status", "200"}, + {"content-length", Integer.to_string(declared_length)} + ]), + Frame.encode(:data, @end_stream, 1, body) + ] + + assert {:error, :content_length_mismatch} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + end + + test "rejects DATA exceeding Content-Length before END_STREAM" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, 0, 1, "too") + ] + + assert {:error, :content_length_mismatch} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + + test "rejects malformed and conflicting Content-Length response headers" do + for headers <- [ + [{":status", "200"}, {"content-length", "two"}], + [{":status", "200"}, {"content-length", "2"}, {"content-length", "3"}], + [{":status", "200"}, {"content-length", String.duplicate("9", 21)}], + [{":status", "200"}, {"content-length", "18446744073709551616"}] + ] do + assert {:error, :invalid_content_length} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), response_headers_frame(headers)) + end + end + + test "accepts the largest unsigned 64-bit Content-Length" do + assert {:ok, %{expected_content_length: 18_446_744_073_709_551_615}, + [{:headers, 200, _headers}]} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([ + {":status", "200"}, + {"content-length", "18446744073709551615"} + ]) + ) + end + + test "rejects Content-Length on informational response headers" do + assert {:error, :invalid_content_length} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([{":status", "100"}, {"content-length", "0"}]) + ) + end + + test "counts DATA payload without padding toward Content-Length" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, @padded ||| @end_stream, 1, <<2, "ok", 0, 0>>) + ] + + assert {:ok, _conn, [{:headers, 200, _headers}, {:body, "ok"}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + + test "validates Content-Length when trailers complete a streamed response" do + headers = response_headers_frame([{":status", "200"}, {"content-length", "4"}]) + + trailers = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{"x-checksum", "ok"}]) + ) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "he"}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers <> Frame.encode(:data, 0, 1, "he")) + + assert {:error, :content_length_mismatch} = + HTTP.HTTP2.stream(conn, trailers) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "four"}]} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + headers <> Frame.encode(:data, 0, 1, "four") + ) + + assert {:ok, _conn, [:done]} = HTTP.HTTP2.stream(conn, trailers) + end + + test "rejects Content-Length in response trailers" do + headers = response_headers_frame([{":status", "200"}, {"content-length", "2"}]) + + trailers = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{"content-length", "2"}]) + ) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "ok"}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers <> Frame.encode(:data, 0, 1, "ok")) + + assert {:error, :invalid_response_trailers} = HTTP.HTTP2.stream(conn, trailers) + end + + test "completes a Content-Length response streamed across calls" do + headers = response_headers_frame([{":status", "200"}, {"content-length", "4"}]) + + assert {:ok, conn, [{:headers, 200, _headers}, {:body, "he"}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers <> Frame.encode(:data, 0, 1, "he")) + + assert {:ok, _conn, [{:body, "ll"}, :done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:data, @end_stream, 1, "ll")) + end + + test "allows HEAD and 304 representation lengths without response content" do + for {method, status} <- [{:head, "200"}, {:get, "304"}] do + frame = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", status}, {"content-length", "10"}]) + ) + + assert {:ok, _conn, [{:headers, _status, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(method), frame) + end + end + + test "rejects Content-Length on a 204 response" do + assert {:error, :invalid_content_length} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([{":status", "204"}, {"content-length", "0"}]) + ) + end + + test "rejects nonempty DATA on a body-forbidden response" do + headers = response_headers_frame([{":status", "204"}]) + + assert {:ok, conn, [{:headers, 204, _headers}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers) + + assert {:error, :invalid_response_body} = + HTTP.HTTP2.stream(conn, Frame.encode(:data, @end_stream, 1, "nope")) + end + + test "rejects DATA and HEADERS after response completion" do + complete = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), complete) + + for frame <- [ + Frame.encode(:data, @end_stream, 1, ""), + response_headers_frame([{":status", "200"}]) + ] do + assert {:error, :stream_closed} = HTTP.HTTP2.stream(conn, frame) + end + end + test "combines HEADERS and CONTINUATION before decoding" do - conn = HTTP.HTTP2.new(:get) + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) header_block = HPACK.encode_headers([{":status", "204"}, {"x-test", "split"}]) size = div(IO.iodata_length(header_block), 2) header_block = IO.iodata_to_binary(header_block) <> = header_block + assert {:ok, conn, []} = + HTTP.HTTP2.stream(conn, Frame.encode(:headers, @end_stream, 1, first)) + + refute HTTP.HTTP2.complete_response?(conn) + refute HTTP.HTTP2.request_stopped?(conn) + assert conn.pending_body == "xxxxx" + + assert {:ok, conn, [{:headers, 204, headers}, :done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:continuation, @end_headers, 1, second)) + + assert HTTP.Headers.get(headers, "x-test") == "split" + assert HTTP.HTTP2.complete_response?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + end + + test "does not complete a bodyless response until END_STREAM arrives" do + assert {:ok, conn, [{:headers, 204, _headers}]} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + response_headers_frame([{":status", "204"}]) + ) + + refute HTTP.HTTP2.complete_response?(conn) + assert {:error, :closed} = HTTP.HTTP2.close(conn) + + assert {:ok, conn, [:done]} = + HTTP.HTTP2.stream(conn, Frame.encode(:data, @end_stream, 1, "")) + + assert HTTP.HTTP2.complete_response?(conn) + end + + test "stops an upload explicitly without discarding non-upload frames" do + conn = %HTTP.HTTP2{ + pending_body: "unsent", + outbound: [ + Frame.encode(:data, @end_stream, 1, "upload"), + Frame.encode(:settings, @ack, 0, ""), + Frame.encode(:settings, 0, 0, "") + ] + } + + conn = HTTP.HTTP2.stop_request(conn) + {conn, outbound} = HTTP.HTTP2.take_outbound(conn) + assert [] = conn.outbound + + assert {:ok, %Frame{type: :settings, flags: 0, payload: ""}, outbound} = + outbound |> IO.iodata_to_binary() |> Frame.decode() + + assert {:ok, %Frame{type: :settings, flags: @ack, payload: ""}, ""} = + Frame.decode(outbound) + + refute HTTP.HTTP2.outbound_control_only?(%{ + conn + | outbound: [Frame.encode(:settings, 0, 0, "")] + }) + + refute HTTP.HTTP2.complete_response?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + + assert {:ok, conn, []} = + HTTP.HTTP2.stream( + conn, + Frame.encode(:window_update, 0, 1, <<0::1, 1::31>>) + ) + + assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) + end + + test "stops a pending upload after an early final response and preserves control frames" do + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) + + final_response = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + frames = [ - Frame.encode(:headers, @end_stream, 1, first), - Frame.encode(:continuation, @end_headers, 1, second) + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + final_response ] - assert {:ok, _conn, [{:headers, 204, headers}, :done]} = + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = HTTP.HTTP2.stream(conn, IO.iodata_to_binary(frames)) - assert HTTP.Headers.get(headers, "x-test") == "split" + assert HTTP.HTTP2.complete_response?(conn) + + {conn, outbound} = HTTP.HTTP2.take_outbound(conn) + + assert {:ok, %Frame{type: :settings, flags: flags, payload: ""}, ""} = + outbound |> IO.iodata_to_binary() |> Frame.decode() + + assert (flags &&& @ack) == @ack + assert HTTP.HTTP2.outbound_control_only?(conn) + assert HTTP.HTTP2.request_stopped?(conn) + + assert {:ok, conn, []} = + HTTP.HTTP2.stream( + conn, + IO.iodata_to_binary([ + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>) + ]) + ) + + assert {^conn, []} = HTTP.HTTP2.take_outbound(conn) + end + + test "does not restart an upload when WINDOW_UPDATE follows an early final response" do + body = :binary.copy("x", @initial_window_size + 5) + + request = %HTTP.Request{ + method: :post, + url: URI.parse("https://example.com/widgets"), + body: body + } + + {conn, _wire} = HTTP.HTTP2.prepare_request(HTTP.HTTP2.new(:post), request) + + final_response = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + frames = [ + final_response, + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>) + ] + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(conn, IO.iodata_to_binary(frames)) + + assert {_conn, []} = HTTP.HTTP2.take_outbound(conn) + assert HTTP.HTTP2.request_stopped?(conn) + end + + test "accepts NO_ERROR reset after a complete response without duplicate events" do + final_headers = + Frame.encode( + :headers, + @end_headers ||| @end_stream, + 1, + HPACK.encode_headers([{":status", "200"}]) + ) + + reset = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + assert {:ok, conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), final_headers) + + assert HTTP.HTTP2.complete_response?(conn) + assert {:ok, ^conn, []} = HTTP.HTTP2.stream(conn, reset) + assert {:ok, ^conn, []} = HTTP.HTTP2.stream(conn, reset) + + assert {:ok, _conn, [{:headers, 200, _headers}, :done]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), final_headers <> reset) + + assert {:error, {:stream_reset, :no_error}} = + HTTP.HTTP2.stream( + HTTP.HTTP2.new(:get), + Frame.encode(:rst_stream, 0, 1, <<0::32>>) + ) + end + + test "keeps an incomplete response reset as an error" do + headers = response_headers_frame([{":status", "200"}]) + reset = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + assert {:ok, conn, [{:headers, 200, _headers}]} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), headers) + + refute HTTP.HTTP2.complete_response?(conn) + assert {:error, {:stream_reset, :no_error}} = HTTP.HTTP2.stream(conn, reset) end test "decodes hpack static, dynamic, and huffman response headers" do @@ -244,6 +628,28 @@ defmodule HTTP.HTTP2Test do HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) end + test "does not discard a reset following END_STREAM in the same batch" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, @end_stream, 1, "ok"), + Frame.encode(:rst_stream, 0, 1, <<0x8::32>>) + ] + + assert {:error, {:stream_reset, :cancel}} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + + test "does not discard a protocol error following END_STREAM in the same batch" do + frames = [ + response_headers_frame([{":status", "200"}, {"content-length", "2"}]), + Frame.encode(:data, @end_stream, 1, "ok"), + Frame.encode(:window_update, 0, 0, <<0::32>>) + ] + + assert {:error, :invalid_window_update_increment} = + HTTP.HTTP2.stream(HTTP.HTTP2.new(:get), IO.iodata_to_binary(frames)) + end + test "rejects data before final response headers" do assert {:error, :data_before_response_headers} = HTTP.HTTP2.stream( diff --git a/apps/http_core/test/http/tls_backend_test.exs b/apps/http_core/test/http/tls_backend_test.exs new file mode 100644 index 0000000..2404859 --- /dev/null +++ b/apps/http_core/test/http/tls_backend_test.exs @@ -0,0 +1,42 @@ +defmodule HTTP.TLSBackendTest do + use ExUnit.Case, async: false + + alias HTTP.TLSBackend + + setup do + previous = Application.fetch_env(:http_core, :tls_backend) + Application.delete_env(:http_core, :tls_backend) + + on_exit(fn -> + case previous do + {:ok, value} -> Application.put_env(:http_core, :tls_backend, value) + :error -> Application.delete_env(:http_core, :tls_backend) + end + end) + end + + test "defaults to OTP and resolves explicit atoms and strings" do + assert {:ok, :ssl} = TLSBackend.resolve() + assert {:ok, :ssl} = TLSBackend.resolve("ssl") + assert {:ok, :ex_ssl} = TLSBackend.resolve(:ex_ssl) + assert {:ok, :ex_ssl} = TLSBackend.resolve("ex_ssl") + end + + test "nil inherits the configured backend and explicit options override it" do + Application.put_env(:http_core, :tls_backend, :ex_ssl) + assert {:ok, :ex_ssl} = TLSBackend.resolve(nil) + assert {:ok, :ssl} = TLSBackend.resolve(:ssl) + Application.put_env(:http_core, :tls_backend, "ssl") + assert {:ok, :ssl} = TLSBackend.resolve() + end + + test "invalid configuration or selection fails instead of falling back" do + Application.put_env(:http_core, :tls_backend, :unknown) + assert {:error, :invalid_tls_backend} = TLSBackend.resolve() + assert {:ok, :ex_ssl} = TLSBackend.resolve(:ex_ssl) + + for value <- [:unknown, "SSL", false, 1, %{}] do + assert {:error, :invalid_tls_backend} = TLSBackend.resolve(value) + end + end +end diff --git a/apps/http_core/test/http/tls_transport_test.exs b/apps/http_core/test/http/tls_transport_test.exs new file mode 100644 index 0000000..9f85106 --- /dev/null +++ b/apps/http_core/test/http/tls_transport_test.exs @@ -0,0 +1,281 @@ +defmodule HTTP.TLSTransportTest do + use ExUnit.Case, async: true + + alias HTTP.Transport.ExSSL + + @fixtures Path.expand("../../../http_fetch/test/support/fixtures", __DIR__) + @ca Path.join(@fixtures, "localhost-ca.pem") + @cert Path.join(@fixtures, "localhost.pem") + @key Path.join(@fixtures, "localhost.key") + + for transport <- [HTTP.Transport.SSL, ExSSL] do + @transport transport + + describe "#{inspect(transport)}" do + test "verified passive traffic and absent ALPN" do + port = + peer(fn socket -> + assert {:ok, "ping"} = :ssl.recv(socket, 4, 5_000) + :ok = :ssl.send(socket, "pong") + end) + + assert {:ok, socket} = + @transport.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + + assert {:ok, nil} = @transport.negotiated_protocol(socket) + assert :ok = @transport.send(socket, ["pi", "ng"]) + assert {:ok, "pong"} = @transport.recv(socket, 4, 5_000) + assert {:error, :closed} = @transport.recv(socket, 0, 5_000) + assert :ok = @transport.close(socket) + end + + test "worker ownership transfer, ALPN and active-once delivery survive worker exit" do + port = + peer( + fn socket -> + assert {:ok, "ping"} = :ssl.recv(socket, 4, 5_000) + :ok = :ssl.send(socket, "pong") + end, + alpn_preferred_protocols: ["h2"] + ) + + parent = self() + + {worker, monitor} = + spawn_monitor(fn -> + {:ok, socket} = + @transport.connect( + "localhost", + port, + [ssl: [cacertfile: @ca, alpn_advertised_protocols: ["h2"]]], + 5_000 + ) + + :ok = @transport.controlling_process(socket, parent) + send(parent, {:connected, socket}) + end) + + assert_receive {:connected, socket}, 5_000 + assert_receive {:DOWN, ^monitor, :process, ^worker, :normal}, 5_000 + assert {:ok, "h2"} = @transport.negotiated_protocol(socket) + assert :ok = @transport.setopts(socket, active: :once) + assert :ok = @transport.send(socket, "ping") + assert_receive {:ssl, ^socket, "pong"} = message, 5_000 + assert {:data, "pong"} = @transport.normalize_message(message, socket) + assert :ok = @transport.setopts(socket, active: :once) + assert_receive {:ssl_closed, ^socket} = message, 5_000 + assert :closed = @transport.normalize_message(message, socket) + assert :unknown = @transport.normalize_message({:ssl, :other_socket, "pong"}, socket) + assert :ok = @transport.close(socket) + end + + test "rejects an untrusted peer" do + port = peer(fn _socket -> :ok end) + assert {:error, _} = @transport.connect("localhost", port, [], 5_000) + end + + test "rejects a mismatched reference hostname" do + port = peer(fn _socket -> :ok end) + + assert {:error, _} = + @transport.connect( + "localhost", + port, + [ssl: [cacertfile: @ca, server_name_indication: ~c"wrong.example"]], + 5_000 + ) + end + end + end + + test "ex_ssl verifies IP literals without treating the IP as DNS SNI" do + port = peer(fn socket -> :ssl.send(socket, "ip") end) + assert {:ok, socket} = ExSSL.connect("127.0.0.1", port, [ssl: [cacertfile: @ca]], 5_000) + assert {:ok, "ip"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl rejects unsupported TLS and TCP options before connecting" do + for ssl <- [ + [verify: :verify_none], + [versions: [:"tlsv1.1"]], + [certfile: @cert] + ] do + assert {:error, {:options, _}} = ExSSL.connect("localhost", 0, [ssl: ssl], 100) + end + + assert {:error, {:options, {:nodelay, :unsupported_or_invalid}}} = + ExSSL.connect("localhost", 0, [socket_opts: [nodelay: :invalid]], 100) + + assert {:error, :econnrefused} = + ExSSL.connect("127.0.0.1", 0, [socket_opts: [nodelay: true]], 100) + + for versions <- [[:"tlsv1.2"], [:"tlsv1.3", :"tlsv1.2"]] do + assert {:error, :econnrefused} = + ExSSL.connect("127.0.0.1", 0, [ssl: [versions: versions]], 100) + end + + assert {:error, {:options, {:send_timeout_close, :unsupported_or_invalid}}} = + ExSSL.connect("localhost", 0, [socket_opts: [send_timeout_close: false]], 100) + end + + test "ex_ssl rejects malformed and duplicate options" do + for opts <- [[ssl: nil], [socket_opts: [:binary]], [ssl: [depth: 1, depth: 2]]] do + assert {:error, {:options, :invalid_options}} = ExSSL.connect("localhost", 0, opts, 100) + end + end + + test "ex_ssl socket send options take precedence over matching TLS options" do + port = peer(fn socket -> :ssl.send(socket, "ok") end) + + assert {:ok, socket} = + ExSSL.connect( + "localhost", + port, + [ + ssl: [cacertfile: @ca, send_timeout_close: false], + socket_opts: [send_timeout: 1_000, send_timeout_close: true] + ], + 5_000 + ) + + assert {:ok, "ok"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl rejects TLS 1.2-only peers without backend fallback" do + port = peer(fn _socket -> :ok end, versions: [:"tlsv1.2"]) + assert {:error, _} = ExSSL.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + end + + test "ex_ssl keeps partial passive bytes across receive timeouts" do + parent = self() + + port = + peer(fn socket -> + :ok = :ssl.send(socket, "a") + send(parent, {:partial_sent, self()}) + + receive do + :finish -> :ssl.send(socket, "b") + after + 5_000 -> flunk("test did not finish the response") + end + end) + + assert {:ok, socket} = ExSSL.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + assert_receive {:partial_sent, server}, 5_000 + assert {:error, :timeout} = ExSSL.recv(socket, 2, 20) + send(server, :finish) + assert {:ok, "ab"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl closes a passive connection when its application owner exits" do + parent = self() + + port = + peer(fn socket -> + send(parent, {:peer_closed, :ssl.recv(socket, 0, 5_000)}) + end) + + {owner, monitor} = + spawn_monitor(fn -> + {:ok, socket} = ExSSL.connect("localhost", port, [ssl: [cacertfile: @ca]], 5_000) + send(parent, {:owned, socket}) + + receive do + :exit -> :ok + end + end) + + assert_receive {:owned, socket}, 5_000 + assert {:error, :not_owner} = ExSSL.controlling_process(socket, self()) + send(owner, :exit) + assert_receive {:DOWN, ^monitor, :process, ^owner, :normal}, 5_000 + assert_receive {:peer_closed, {:error, _}}, 5_000 + assert :ok = ExSSL.close(socket) + end + + test "ex_ssl propagates custom-profile ALPN conflicts without connecting" do + profile = %SSL.ClientHello.WireProfile{extensions: [{:alpn, ["http/1.1"]}]} + + assert {:error, {:options, {:alpn_advertised_protocols, :profile_conflict}}} = + ExSSL.connect( + "localhost", + 0, + [ + ssl: [ + cacertfile: @ca, + alpn_advertised_protocols: ["h2"], + ex_ssl: [profile: profile] + ] + ], + 100 + ) + end + + test "ex_ssl accepts the default custom profile with HTTP ALPN" do + port = peer(fn socket -> :ssl.send(socket, "ok") end, alpn_preferred_protocols: ["http/1.1"]) + + assert {:ok, socket} = + ExSSL.connect( + "localhost", + port, + [ + ssl: [ + cacertfile: @ca, + alpn_advertised_protocols: ["http/1.1"], + ex_ssl: [profile: :default] + ] + ], + 5_000 + ) + + assert {:ok, "http/1.1"} = ExSSL.negotiated_protocol(socket) + assert {:ok, "ok"} = ExSSL.recv(socket, 2, 5_000) + assert :ok = ExSSL.close(socket) + end + + defp peer(handler, options \\ []) do + {:ok, listener} = + :ssl.listen( + 0, + [ + :binary, + packet: :raw, + active: false, + ip: {127, 0, 0, 1}, + certfile: @cert, + keyfile: @key + ] ++ Keyword.put_new(options, :versions, [:"tlsv1.3"]) + ) + + {:ok, {{127, 0, 0, 1}, port}} = :ssl.sockname(listener) + + pid = + spawn_link(fn -> + case :ssl.transport_accept(listener, 5_000) do + {:ok, tcp} -> + case :ssl.handshake(tcp, 5_000) do + {:ok, socket} -> + handler.(socket) + :ssl.close(socket) + + {:error, _} -> + :ok + end + + {:error, _} -> + :ok + end + end) + + on_exit(fn -> + :ssl.close(listener) + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + port + end +end diff --git a/apps/http_event_source/lib/http/event_source.ex b/apps/http_event_source/lib/http/event_source.ex index 8ba2237..26f009a 100644 --- a/apps/http_event_source/lib/http/event_source.ex +++ b/apps/http_event_source/lib/http/event_source.ex @@ -10,6 +10,10 @@ defmodule HTTP.EventSource do Custom server-sent event names are delivered through the message event's `type` field. + + For `https` connections, pass `tls_backend: :ssl | :ex_ssl` (or the equivalent + string in a map). When omitted, the shared `:http_core` TLS backend setting is + captured when the source is created and retained across reconnects. """ alias HTTP.EventSource.Connection diff --git a/apps/http_event_source/lib/http/event_source/connection.ex b/apps/http_event_source/lib/http/event_source/connection.ex index 81e20c3..3d43c2b 100644 --- a/apps/http_event_source/lib/http/event_source/connection.ex +++ b/apps/http_event_source/lib/http/event_source/connection.ex @@ -25,6 +25,7 @@ defmodule HTTP.EventSource.Connection do idle_timeout: :infinity, ssl: [], socket_opts: [], + tls_backend: :ssl, unix_socket: nil, max_line_size: 64 * 1024, transport: nil, @@ -74,6 +75,7 @@ defmodule HTTP.EventSource.Connection do idle_timeout: options.idle_timeout, ssl: options.ssl, socket_opts: options.socket_opts, + tls_backend: options.tls_backend, unix_socket: options.unix_socket, max_line_size: options.max_line_size, last_event_id: options.last_event_id, @@ -177,8 +179,11 @@ defmodule HTTP.EventSource.Connection do {:ok, HTTP.Transport.TCP, host, port || 80} end - defp select_transport(%{uri: %URI{scheme: "https", host: host, port: port}}) do - {:ok, HTTP.Transport.SSL, host, port || 443} + defp select_transport(%{ + uri: %URI{scheme: "https", host: host, port: port}, + tls_backend: backend + }) do + {:ok, HTTP.TLSBackend.transport(backend), host, port || 443} end defp select_transport(%{uri: %URI{scheme: scheme}}), do: {:error, {:unsupported_scheme, scheme}} @@ -350,6 +355,7 @@ defmodule HTTP.EventSource.Connection do defp rearm(%{transport: transport, socket: socket} = state) do case transport.setopts(socket, active: :once) do :ok -> {:noreply, state} + {:error, :closed} -> handle_transport_closed(state) {:error, reason} -> {:noreply, reconnect(state, reason)} end end diff --git a/apps/http_event_source/lib/http/event_source/options.ex b/apps/http_event_source/lib/http/event_source/options.ex index 57dc4f8..a5d1b83 100644 --- a/apps/http_event_source/lib/http/event_source/options.ex +++ b/apps/http_event_source/lib/http/event_source/options.ex @@ -24,6 +24,8 @@ defmodule HTTP.EventSource.Options do "socket_opts" => :socket_opts, "socketOpts" => :socket_opts, "ssl" => :ssl, + "tls_backend" => :tls_backend, + "tlsBackend" => :tls_backend, "unix_socket" => :unix_socket, "unixSocket" => :unix_socket, "with_credentials" => :with_credentials, @@ -42,6 +44,7 @@ defmodule HTTP.EventSource.Options do idle_timeout: :infinity, ssl: [], socket_opts: [], + tls_backend: :ssl, unix_socket: nil, max_line_size: @default_max_line_size, ref: nil @@ -59,6 +62,7 @@ defmodule HTTP.EventSource.Options do idle_timeout: timeout(), ssl: keyword(), socket_opts: keyword(), + tls_backend: HTTP.TLSBackend.t(), unix_socket: String.t() | nil, max_line_size: pos_integer(), ref: reference() @@ -82,6 +86,7 @@ defmodule HTTP.EventSource.Options do idle_timeout: Keyword.get(init, :idle_timeout, :infinity), ssl: Keyword.get(init, :ssl, []), socket_opts: Keyword.get(init, :socket_opts, []), + tls_backend: Keyword.fetch!(init, :tls_backend), unix_socket: Keyword.get(init, :unix_socket), max_line_size: Keyword.get(init, :max_line_size, @default_max_line_size), ref: Keyword.get(init, :ref, make_ref()) @@ -144,6 +149,7 @@ defmodule HTTP.EventSource.Options do {:ok, ssl} <- normalize_keyword(Keyword.get(init, :ssl, []), :invalid_ssl_options), {:ok, socket_opts} <- normalize_keyword(Keyword.get(init, :socket_opts, []), :invalid_socket_options), + {:ok, tls_backend} <- HTTP.TLSBackend.resolve(Keyword.get(init, :tls_backend)), {:ok, unix_socket} <- normalize_unix_socket(Keyword.get(init, :unix_socket)), {:ok, max_line_size} <- normalize_pos_integer( @@ -162,6 +168,7 @@ defmodule HTTP.EventSource.Options do |> Keyword.put(:idle_timeout, idle_timeout) |> Keyword.put(:ssl, ssl) |> Keyword.put(:socket_opts, socket_opts) + |> Keyword.put(:tls_backend, tls_backend) |> Keyword.put(:unix_socket, unix_socket) |> Keyword.put(:max_line_size, max_line_size)} end diff --git a/apps/http_event_source/test/http/event_source/options_test.exs b/apps/http_event_source/test/http/event_source/options_test.exs index 1ffa97c..83fdf8d 100644 --- a/apps/http_event_source/test/http/event_source/options_test.exs +++ b/apps/http_event_source/test/http/event_source/options_test.exs @@ -1,5 +1,5 @@ defmodule HTTP.EventSource.OptionsTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias HTTP.EventSource.Options @@ -55,6 +55,33 @@ defmodule HTTP.EventSource.OptionsTest do }) end + test "selects TLS backends from atom and string map options" do + assert {:ok, %{tls_backend: :ssl}} = + Options.new("https://example.com/events", tls_backend: :ssl) + + assert {:ok, %{tls_backend: :ex_ssl}} = + Options.new("https://example.com/events", %{"tlsBackend" => "ex_ssl"}) + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("https://example.com/events", %{"tls_backend" => "ssl"}) + end + + test "pins the configured TLS backend when options are constructed" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + + Application.put_env(:http_core, :tls_backend, :ex_ssl) + assert {:ok, options} = Options.new("https://example.com/events") + assert options.tls_backend == :ex_ssl + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("https://example.com/events", tls_backend: :ssl) + + Application.put_env(:http_core, :tls_backend, :ssl) + assert options.tls_backend == :ex_ssl + assert {:ok, %{tls_backend: :ssl}} = Options.new("https://example.com/events") + end + test "rejects invalid init options" do assert {:error, :invalid_owner} = Options.new("http://example.com/events", owner: :bad) @@ -63,5 +90,11 @@ defmodule HTTP.EventSource.OptionsTest do assert {:error, :invalid_reconnect_time} = Options.new("http://example.com/events", reconnect_time: -1) + + assert {:error, :invalid_tls_backend} = + Options.new("https://example.com/events", tls_backend: :unknown) end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(value), do: Application.put_env(:http_core, :tls_backend, value) end diff --git a/apps/http_event_source/test/http/event_source_test.exs b/apps/http_event_source/test/http/event_source_test.exs index e3bfb92..ec544a2 100644 --- a/apps/http_event_source/test/http/event_source_test.exs +++ b/apps/http_event_source/test/http/event_source_test.exs @@ -1,11 +1,15 @@ defmodule HTTP.EventSourceTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias HTTP.EventSource alias HTTP.EventSource.Event.Error alias HTTP.EventSource.Event.Message alias HTTP.EventSource.Event.Open + @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) + @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) + @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) + test "defines browser ready state constants" do assert EventSource.connecting() == 0 assert EventSource.open() == 1 @@ -104,7 +108,106 @@ defmodule HTTP.EventSourceTest do assert :ok = EventSource.close(source) end + test "delivers events over verified TLS 1.3 for each backend" do + for backend <- [:ssl, :ex_ssl] do + {:ok, _server, port} = + HTTPEventSource.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + body: "data: secure\n\n", + close: false + ) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + tls_backend: backend, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {EventSource, ^source, %Open{}}, 1_000 + assert_receive {EventSource, ^source, %Message{data: "secure"}}, 1_000 + assert :ok = EventSource.close(source) + assert_receive :event_source_server_closed, 1_000 + end + end + + test "retains the configured TLS backend across reconnects" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + Application.put_env(:http_core, :tls_backend, :ex_ssl) + + {:ok, server, port} = + HTTPEventSource.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + responses: [ + [body: "data: first\n\n", wait_for: :close_first_response], + [body: "data: second\n\n", close: false] + ] + ) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + ssl: [cacertfile: @cacertfile], + reconnect_time: 10 + ) + + assert_receive {:event_source_server_request, _first_request}, 1_000 + assert_receive {EventSource, ^source, %Message{data: "first"}}, 1_000 + Application.put_env(:http_core, :tls_backend, :invalid) + refute_receive {:event_source_server_request, _request}, 50 + send(server, :close_first_response) + assert_receive {EventSource, ^source, %Message{data: "second"}}, 1_000 + assert :ok = EventSource.close(source) + end + test "rejects invalid constructor input synchronously" do assert {:error, {:unsupported_scheme, "ftp"}} = EventSource.new("ftp://example.com/events") end + + test "finalizes events when TLS closes before a delayed consumer rearms" do + {:ok, server, port} = + HTTPEventSource.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + body: fn -> + receive do + :send_body -> "data: final\r\r" + after + 5_000 -> flunk("consumer did not release the server") + end + end + ) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile], + reconnect_time: 60_000 + ) + + assert_receive {EventSource, ^source, %Open{}}, 5_000 + %{socket: %{pid: tls_pid}} = :sys.get_state(source.pid) + monitor = Process.monitor(tls_pid) + :ok = :sys.suspend(source.pid) + + # Let TLS deliver data and its terminal notification while the consumer is + # paused. Rearming after it parses the data now necessarily returns :closed. + try do + send(server, :send_body) + assert_receive {:DOWN, ^monitor, :process, ^tls_pid, _}, 5_000 + after + :sys.resume(source.pid) + end + + assert_receive {EventSource, ^source, %Message{data: "final"}}, 5_000 + assert_receive {EventSource, ^source, %Error{reason: :eof}}, 5_000 + assert :ok = EventSource.close(source) + end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(value), do: Application.put_env(:http_core, :tls_backend, value) end diff --git a/apps/http_event_source/test/support/event_source_server.ex b/apps/http_event_source/test/support/event_source_server.ex index c82954c..7fda39f 100644 --- a/apps/http_event_source/test/support/event_source_server.ex +++ b/apps/http_event_source/test/support/event_source_server.ex @@ -4,66 +4,78 @@ defmodule HTTPEventSource.TestServer do def start_link(opts \\ []) do parent = self() responses = Keyword.get(opts, :responses, [opts]) + tls? = Keyword.get(opts, :tls, false) + transport = socket_module(tls?) + {:ok, listen_socket} = listen(tls?, opts) + {:ok, {{127, 0, 0, 1}, port}} = sockname(tls?, listen_socket) - {:ok, listen_socket} = - :gen_tcp.listen(0, [ - :binary, - packet: :raw, - active: false, - reuseaddr: true, - ip: {127, 0, 0, 1} - ]) - - {:ok, {{127, 0, 0, 1}, port}} = :inet.sockname(listen_socket) - - pid = - spawn_link(fn -> - accept_loop(listen_socket, parent, responses) - end) - + pid = spawn_link(fn -> accept_loop(tls?, transport, listen_socket, parent, responses) end) {:ok, pid, port} end - defp accept_loop(listen_socket, _parent, []) do - :gen_tcp.close(listen_socket) + defp listen(false, _opts) do + :gen_tcp.listen(0, [:binary, packet: :raw, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) end - defp accept_loop(listen_socket, parent, [response | rest]) do - case :gen_tcp.accept(listen_socket, 5_000) do + defp listen(true, opts) do + :ssl.listen(0, + mode: :binary, + packet: :raw, + active: false, + reuseaddr: true, + ip: {127, 0, 0, 1}, + versions: [:"tlsv1.3"], + certfile: Keyword.fetch!(opts, :certfile), + keyfile: Keyword.fetch!(opts, :keyfile) + ) + end + + defp accept_loop(_tls?, transport, listen_socket, _parent, []), + do: transport.close(listen_socket) + + defp accept_loop(tls?, transport, listen_socket, parent, [response | rest]) do + case accept(tls?, listen_socket) do {:ok, socket} -> - serve(socket, parent, response) - accept_loop(listen_socket, parent, rest) + serve(transport, socket, parent, response) + accept_loop(tls?, transport, listen_socket, parent, rest) {:error, reason} -> send(parent, {:event_source_server_error, reason}) - :gen_tcp.close(listen_socket) + transport.close(listen_socket) end end - defp serve(socket, parent, response) do - case recv_until(socket, "\r\n\r\n", <<>>) do + defp accept(false, listen_socket), do: :gen_tcp.accept(listen_socket, 5_000) + + defp accept(true, listen_socket) do + with {:ok, transport_socket} <- :ssl.transport_accept(listen_socket, 5_000), + do: :ssl.handshake(transport_socket) + end + + defp socket_module(false), do: :gen_tcp + defp socket_module(true), do: :ssl + defp sockname(false, listen_socket), do: :inet.sockname(listen_socket) + defp sockname(true, listen_socket), do: :ssl.sockname(listen_socket) + + defp serve(transport, socket, parent, response) do + case recv_until(transport, socket, "\r\n\r\n", <<>>) do {:ok, request} -> send(parent, {:event_source_server_request, request}) - :ok = :gen_tcp.send(socket, response_head(response)) - :ok = send_body(socket, Keyword.get(response, :body, "")) + :ok = transport.send(socket, response_head(response)) + :ok = send_body(transport, socket, Keyword.get(response, :body, "")) - if Keyword.get(response, :close, true) do - :gen_tcp.close(socket) - else - wait_for_close(socket, parent) - end + close_response(transport, socket, parent, response) {:error, reason} -> send(parent, {:event_source_server_error, reason}) - :gen_tcp.close(socket) + transport.close(socket) end end - defp recv_until(socket, marker, buffer) do + defp recv_until(transport, socket, marker, buffer) do if :binary.match(buffer, marker) == :nomatch do - with {:ok, data} <- :gen_tcp.recv(socket, 0, 1_000) do - recv_until(socket, marker, buffer <> data) - end + with {:ok, data} <- transport.recv(socket, 0, 1_000), + do: recv_until(transport, socket, marker, buffer <> data) else {:ok, buffer} end @@ -71,53 +83,56 @@ defmodule HTTPEventSource.TestServer do defp response_head(response) do status = Keyword.get(response, :status, 200) - reason = reason_phrase(status) headers = Keyword.get(response, :headers, []) content_type = Keyword.get(response, :content_type, "text/event-stream") - - content_type_header = - if content_type do - [{"Content-Type", content_type}] - else - [] - end + content_type_header = if content_type, do: [{"Content-Type", content_type}], else: [] [ "HTTP/1.1 ", Integer.to_string(status), " ", - reason, + reason_phrase(status), "\r\n", header_lines(content_type_header ++ headers), - "Connection: close\r\n", - "\r\n" + "Connection: close\r\n\r\n" ] end - defp header_lines(headers) do - Enum.map(headers, fn {name, value} -> [to_string(name), ": ", to_string(value), "\r\n"] end) - end + defp header_lines(headers), + do: + Enum.map(headers, fn {name, value} -> [to_string(name), ": ", to_string(value), "\r\n"] end) - defp send_body(_socket, ""), do: :ok + defp send_body(_transport, _socket, ""), do: :ok - defp send_body(socket, chunks) when is_list(chunks) do - Enum.each(chunks, fn chunk -> - :ok = :gen_tcp.send(socket, chunk) - end) - end + defp send_body(transport, socket, body) when is_function(body, 0), + do: send_body(transport, socket, body.()) - defp send_body(socket, body), do: :gen_tcp.send(socket, body) + defp send_body(transport, socket, chunks) when is_list(chunks), + do: Enum.each(chunks, &transport.send(socket, &1)) - defp wait_for_close(socket, parent) do - case :gen_tcp.recv(socket, 0, 2_000) do - {:error, :closed} -> - send(parent, :event_source_server_closed) + defp send_body(transport, socket, body), do: transport.send(socket, body) - {:error, reason} -> - send(parent, {:event_source_server_error, reason}) + defp close_response(transport, socket, parent, response) do + case Keyword.get(response, :wait_for) do + wait_for when is_atom(wait_for) and not is_nil(wait_for) -> + receive do + ^wait_for -> transport.close(socket) + end + + nil -> + if Keyword.get(response, :close, true) do + transport.close(socket) + else + wait_for_close(transport, socket, parent) + end + end + end - {:ok, _data} -> - wait_for_close(socket, parent) + defp wait_for_close(transport, socket, parent) do + case transport.recv(socket, 0, 2_000) do + {:error, :closed} -> send(parent, :event_source_server_closed) + {:error, reason} -> send(parent, {:event_source_server_error, reason}) + {:ok, _data} -> wait_for_close(transport, socket, parent) end end diff --git a/apps/http_event_source/test/support/fixtures/localhost-ca.pem b/apps/http_event_source/test/support/fixtures/localhost-ca.pem new file mode 100644 index 0000000..15227f1 --- /dev/null +++ b/apps/http_event_source/test/support/fixtures/localhost-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKzCCAhOgAwIBAgIUbWn02fvDN0zvFEGHFliwdXqASNIwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqaXND6GhQeV+tXwmYZYy +q+kFvpGWXCa49yG4/nJLfUYtpANqii88Y08J+6fAskHxoWYLidcnAvC5V51IcreX +g/N108fCvxTQ5jYfwIyj8oAEL4QZbnKklQrcLRFqN1Ef9f6cdNWL4pZZk7/ChbXP +ULHTNQF8FIoDUS4H5bjobU2lBER4H7Qa8oevkMGotgfjrzqWftTWt68dJiTsfoX1 +z+VH6t6t5vvT/L+0J7M9JAB++sckXSY9J+yIeG7Cu3sGtwTqIvuiiBH0latUYC9M +sJ0LS400sBonIZC4BdvVc9NCU8M1sx5yWV6eNRe7Cz3hrzQVcWMbAB73qXFMvpEO +OwIDAQABo2MwYTAdBgNVHQ4EFgQUUayz7IdQZHRIjmttS9hkKzG01M4wHwYDVR0j +BBgwFoAUUayz7IdQZHRIjmttS9hkKzG01M4wDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggEBAIrCD3yHZ+F/P/s0FXZRFFNB +AIqYjEwYr2OBN/ShHhceoqM55FbThtYPxUlVJH+uWkxpvnExDOauCapW7Cxlz9FF +3NvLjWf8fs80Uno5Dy6mik4rL6OsThYX8Ko2u6gkSiQm9yuT1iKvigGjU4q1qyft +ZJH8an7/egIIHxZya5oBuGKBnX241s6n9Osdtww/Eu5IOlZZeORJ0mAA7gTHaZyI +VquDo6Ml66LMyuIQkbtr19UHAEHrSfhmrujI5GruA/ejnma2O5sE1r1hVa91TV+L +TuaVxPvkdS+CIBvfwy90zmDbPhLt0kU5FiY2x6FlR1MrvVBn1uUQ59kYJ7yNI9k= +-----END CERTIFICATE----- diff --git a/apps/http_event_source/test/support/fixtures/localhost.key b/apps/http_event_source/test/support/fixtures/localhost.key new file mode 100644 index 0000000..d05ba0d --- /dev/null +++ b/apps/http_event_source/test/support/fixtures/localhost.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtsMy7LvUMU+H3 +27785rkCaSrkj5ljfUUPDA8+LM1ReN4ovqRaKjN8Tz88AOOwsZkXHLEJ5Uv8RbpS +8ruITF9moSXhVNFvVkBtf+SIBF2M5sDwsV/q8VFwUcvncDyLOsZwqsal9j+e9LjY +SlQtzzapBWfZm8O540RNpmpsa6k0M7jfp/m8P/l3RIlYzEuWPfCBd1ZmQMUN0T6H ++iYBBfNj6ioDUOUP5VfwA8cVXgblYbFKPF6uTtv95Ep4CFin0u6S0xVRlX0PERU5 +kyNLkmXUibClAY93IEkX5KDe0wb0WU4QBC9BAC8CegTFKd57MUlH0FtABCakLDCz +uHKFOc5bAgMBAAECggEAF95J5Bcp2Y3Vaagq8LoMBobJPCt3mrJmQREZLyQc+usv +LE42a6WM+bIyMF6261scfI6WzZNTA9HZLvjoSGymV4YhnHt5ZGFn7SQ8FWz0Jh63 +uNvHIJQU6uhyqtVDnKZxCp6NOdVDHOboV+Cu/LSTAFsb0E2VcgAHLNyHl2qcD+4p +qVDfyz/2BcAciQUTyQmq8tVPyJV226iZAkTJjah8H+MOXo6XqieNJ2WTz3H57PCE +tmm5YLeNl6S47O2369VRFmhYZZBS2tiBEfXNwwFCc6Q44esmJG1UtLUp6LMenNon +FJTod0QuZMjqfomzYzvJ9sCi8cvrABDVs8HQiw8RIQKBgQDY4ReBy63jdUJkDYQC +00SnrGywE7YNLYu+ZkHz+J14Xn/ylKFK0sz1WPfGZZQWtLWEXCYw3f2Z8d42Rd0Z +l/E7H2KiRq2oPrqc/mwfWtwjL0sqOuLswAzdL8AKLbl6vou1UjrJennuvaOh2C2s +sKJutX7CoM8GvXvWweNGY9xvuQKBgQDNBV8IkKuE4B8TIF7x49lkOFpWVgf6E1bS +My22JFIk8C0Jnxh8e/6mujeEJ2jMO/kpYPhWkDhKp4CTm+S6Ga6CSFzBjnA0dfaB +8Htyb4R5bJn7wD9+Zj+tHSG6gwVxJl/u2YfDAZL+zlKgVx7RoTGkF8mskHVmgrFT +DIJyFs0wswKBgFMSjR2DdfzNOnv2jV1DrWWIby1Wr4IGsyNgKd0YmsCzedDiS5HM +gwNra6UL3ZiA6ZJkdaB8N5qTAanKQvF9uMILuI0uA3CRbouaDLJJ7E5x3Bm16pwC +yCqlEqsTbptshzkR4UCxcCkZbKcelggytFUxofdM/1+2jsvpAnRA5fvJAoGAXS+/ +zkjTbQXhmfPws8l4mhDzHqLj5Uq8/7W7ZTqFC70O+3yQyKQjTuz9JtgyzgHEcoZc +2hubOnOAAZeuEthxdU4muuNfJLkpXk5MDeuaLwapxr/PHEilUK4ZEolTA+cJW6sM +BhrFEYP+ElsG6wl1YrxdMk5Gzl1A9BqPgAPVJ/kCgYEArpwxPGurJDQ3JszIWDJU +CX6KhwAncK9LALQT6EPB3Y54Ou4cq1XMSWlmqUUmN821J/mbR4f66F+nFy/q29Br +giwArg0Ngkt+uaKTfo6IeR7+b2vAjzZ+z3cEbNZpuyCAbQwytnXT8t7ZUonAQ7dO +TmHo1zxsuNSt2EYos+EP+VA= +-----END PRIVATE KEY----- diff --git a/apps/http_event_source/test/support/fixtures/localhost.pem b/apps/http_event_source/test/support/fixtures/localhost.pem new file mode 100644 index 0000000..178c710 --- /dev/null +++ b/apps/http_event_source/test/support/fixtures/localhost.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHTCCAgWgAwIBAgIUEJVRR6DfNeTUcvIJYQL9oVN8xyEwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbDMuy71DFPh99u+/Oa5Amkq5I+ZY31F +DwwPPizNUXjeKL6kWiozfE8/PADjsLGZFxyxCeVL/EW6UvK7iExfZqEl4VTRb1ZA +bX/kiARdjObA8LFf6vFRcFHL53A8izrGcKrGpfY/nvS42EpULc82qQVn2ZvDueNE +TaZqbGupNDO436f5vD/5d0SJWMxLlj3wgXdWZkDFDdE+h/omAQXzY+oqA1DlD+VX +8APHFV4G5WGxSjxerk7b/eRKeAhYp9LuktMVUZV9DxEVOZMjS5Jl1ImwpQGPdyBJ +F+Sg3tMG9FlOEAQvQQAvAnoExSneezFJR9BbQAQmpCwws7hyhTnOWwIDAQABo14w +XDAaBgNVHREEEzARhwR/AAABgglsb2NhbGhvc3QwHQYDVR0OBBYEFN1Qq2IrwIwg +nlCE50Q3jpSthaODMB8GA1UdIwQYMBaAFFGss+yHUGR0SI5rbUvYZCsxtNTOMA0G +CSqGSIb3DQEBCwUAA4IBAQBwO6NAaG2iwdO0iAaatt9cyP7ZvtJM0I0se1Dd1AtU +3iBlrEFNAWwph3ycD9Wr5HKswN7DhK+cOeklAeeBaxgB8egyHuJHKJLmoZQip+FQ +lCoMgPOBXH5IepiczMN4mUUCJn2ZNf0j/QkO0CjjefiYBYKunMQDNMtmhiB7LVIL +RAFo6kawJ49F3l/rTSKMMTGjlsW2wSM3pexevTvG/8dagT9ULQfLOuuHd+2paqKU +EoLAuaagqefwZaNASEvKP1Tl9I1GvZmjVTK8qJDmMJE+/D5HwscrDZmBvSDJdLLf +oZPdnbpZv0TXYvlpC5c1pHRPeA5LGa54GhM4Akc8ieDO +-----END CERTIFICATE----- diff --git a/apps/http_fetch/lib/http.ex b/apps/http_fetch/lib/http.ex index 95e5a4a..102e389 100644 --- a/apps/http_fetch/lib/http.ex +++ b/apps/http_fetch/lib/http.ex @@ -109,6 +109,9 @@ defmodule HTTP do - `:signal`: An `HTTP.AbortController` PID. If provided, the request can be aborted via this controller. - `:http_version`: `:http1`, `:http2`, `:http3`, `:h2c`, or `:auto`. + - `:tls_backend`: `:ssl` (default) or `:ex_ssl` for HTTPS requests. It may also be + configured globally with `config :http_core, :tls_backend, :ex_ssl`; an explicit + request value takes precedence. HTTP/3 does not support this option. - `:timeout`, `:connect_timeout`, `:ssl`, and `:socket_opts`: Elixir-specific transport extensions used by the socket or QUIC transport. - `:unix_socket`: Path to a Unix Domain Socket file (e.g., "/var/run/docker.sock"). diff --git a/apps/http_fetch/lib/http/fetch_options.ex b/apps/http_fetch/lib/http/fetch_options.ex index 9907abc..b6d67a7 100644 --- a/apps/http_fetch/lib/http/fetch_options.ex +++ b/apps/http_fetch/lib/http/fetch_options.ex @@ -19,7 +19,9 @@ defmodule HTTP.FetchOptions do - `connect_timeout` - connection timeout in milliseconds - `http_version` - protocol selection, one of `:http1`, `:http2`, `:http3`, `:h2c`, or `:auto`; defaults to `:http1` - - `ssl` - TLS options passed to `:ssl` + - `tls_backend` - TLS implementation, `:ssl` or `:ex_ssl`; defaults to the + shared `:http_core, :tls_backend` configuration + - `ssl` - TLS options passed to the selected TLS backend - `socket_opts` - socket options passed to the underlying transport - `unix_socket` - Unix Domain Socket path """ @@ -40,6 +42,8 @@ defmodule HTTP.FetchOptions do "socket_opts" => :socket_opts, "socketOpts" => :socket_opts, "ssl" => :ssl, + "tls_backend" => :tls_backend, + "tlsBackend" => :tls_backend, "timeout" => :timeout, "unix_socket" => :unix_socket, "unixSocket" => :unix_socket @@ -54,6 +58,7 @@ defmodule HTTP.FetchOptions do unix_socket: nil, redirect: :follow, http_version: :http1, + tls_backend: nil, timeout: nil, connect_timeout: nil, ssl: nil, @@ -61,6 +66,7 @@ defmodule HTTP.FetchOptions do @type redirect :: :follow | :manual | :error @type http_version :: :http1 | :http2 | :http3 | :h2c | :auto + @type tls_backend :: term() @type t :: %__MODULE__{ method: atom(), @@ -72,6 +78,7 @@ defmodule HTTP.FetchOptions do unix_socket: String.t() | nil, redirect: redirect(), http_version: http_version(), + tls_backend: tls_backend(), timeout: integer() | nil, connect_timeout: integer() | nil, ssl: list() | nil, @@ -109,6 +116,7 @@ defmodule HTTP.FetchOptions do |> maybe_add(:socket_opts, options.socket_opts) |> maybe_add(:redirect, options.redirect) |> maybe_add(:http_version, options.http_version) + |> maybe_add(:tls_backend, options.tls_backend) end @doc """ @@ -170,6 +178,9 @@ defmodule HTTP.FetchOptions do {:http_version, http_version}, acc -> %{acc | http_version: http_version} + {:tls_backend, tls_backend}, acc -> + %{acc | tls_backend: tls_backend} + {:timeout, timeout}, acc -> %{acc | timeout: timeout} @@ -196,12 +207,15 @@ defmodule HTTP.FetchOptions do defp normalize_headers(_), do: HTTP.Headers.new() defp normalize_options(%__MODULE__{} = options) do + http_version = normalize_http_version(options.http_version) + %{ options | method: normalize_method(options.method), redirect: normalize_redirect(options.redirect), duplex: normalize_duplex(options.duplex), - http_version: normalize_http_version(options.http_version) + http_version: http_version, + tls_backend: normalize_tls_backend(options.tls_backend, http_version) } end @@ -273,6 +287,24 @@ defmodule HTTP.FetchOptions do "unsupported http_version: #{inspect(http_version)}; expected :http1, :http2, :http3, :h2c, or :auto" end + defp normalize_tls_backend(nil, :http3), do: nil + defp normalize_tls_backend(tls_backend, :http3), do: tls_backend + defp normalize_tls_backend(tls_backend, _http_version), do: resolve_tls_backend(tls_backend) + + defp resolve_tls_backend(tls_backend) do + case HTTP.TLSBackend.resolve(tls_backend) do + {:ok, backend} -> + backend + + {:error, :invalid_tls_backend} -> + raise ArgumentError, tls_backend_error_message(tls_backend) + end + end + + defp tls_backend_error_message(tls_backend) do + "unsupported tls_backend: #{inspect(tls_backend)}; expected :ssl or :ex_ssl" + end + defp maybe_add(list, _key, nil), do: list defp maybe_add(list, key, value), do: Keyword.put(list, key, value) end diff --git a/apps/http_fetch/lib/http/socket_client.ex b/apps/http_fetch/lib/http/socket_client.ex index 0d28370..8ffc6ab 100644 --- a/apps/http_fetch/lib/http/socket_client.ex +++ b/apps/http_fetch/lib/http/socket_client.ex @@ -10,6 +10,9 @@ defmodule HTTP.SocketClient do @spec request(Request.t(), pid() | nil, String.t() | nil) :: Response.t() | {:error, term()} def request(%Request{} = request, abort_controller_pid \\ nil, unix_socket_path \\ nil) do cond do + http_version(request) == :http3 and tls_backend(request) != nil -> + {:error, :tls_backend_not_supported_for_quic} + http_version(request) == :http3 and Request.streaming_body?(request) -> {:error, :streaming_request_body_unsupported_for_http3} @@ -17,7 +20,9 @@ defmodule HTTP.SocketClient do request_http3(request, abort_controller_pid, unix_socket_path) true -> - request_socket(request, abort_controller_pid, unix_socket_path) + with {:ok, request} <- pin_tls_backend(request) do + request_socket(request, abort_controller_pid, unix_socket_path) + end end end @@ -261,10 +266,14 @@ defmodule HTTP.SocketClient do case state.protocol_module.stream(state.protocol, data) do {:ok, protocol, events} -> state = %{state | protocol: protocol} + discard_closed_controls? = discard_closed_control_writes?(state, events) - case flush_protocol_writes(state) do - {:ok, state} -> handle_events(state, events) - {:error, reason} -> fail(state, reason) + case flush_protocol_writes(state, discard_closed_controls?) do + {:ok, state} -> + handle_events(state, events) + + {:error, reason} -> + fail(state, reason) end {:error, reason} -> @@ -366,6 +375,7 @@ defmodule HTTP.SocketClient do defp rearm(state) do case state.transport.setopts(state.socket, active: :once) do :ok -> owner_loop(state) + {:error, :closed} -> handle_closed(state) {:error, reason} -> fail(state, reason) end end @@ -387,6 +397,9 @@ defmodule HTTP.SocketClient do :done + {:error, :client_identity_cross_origin_redirect = reason} -> + fail(state, reason) + {:error, _reason} -> send_response(state.parent, state.ref, response) finish(state) @@ -486,8 +499,9 @@ defmodule HTTP.SocketClient do defp connected_protocol(_transport, _socket, %{mode: :http1}), do: {:ok, :http1} defp connected_protocol(_transport, _socket, %{mode: :h2c}), do: {:ok, :http2} - defp connected_protocol(HTTP.Transport.SSL, socket, %{mode: :force_h2}) do - with {:ok, protocol} <- HTTP.Transport.SSL.negotiated_protocol(socket) do + defp connected_protocol(transport, socket, %{mode: :force_h2}) + when transport in [HTTP.Transport.SSL, HTTP.Transport.ExSSL] do + with {:ok, protocol} <- transport.negotiated_protocol(socket) do case normalize_alpn_protocol(protocol) do "h2" -> {:ok, :http2} other -> {:error, {:http2_not_negotiated, other}} @@ -495,8 +509,9 @@ defmodule HTTP.SocketClient do end end - defp connected_protocol(HTTP.Transport.SSL, socket, %{mode: :auto_https}) do - with {:ok, protocol} <- HTTP.Transport.SSL.negotiated_protocol(socket) do + defp connected_protocol(transport, socket, %{mode: :auto_https}) + when transport in [HTTP.Transport.SSL, HTTP.Transport.ExSSL] do + with {:ok, protocol} <- transport.negotiated_protocol(socket) do case normalize_alpn_protocol(protocol) do "h2" -> {:ok, :http2} _other -> {:ok, :http1} @@ -510,15 +525,35 @@ defmodule HTTP.SocketClient do defp connect(transport, host, port, request, selection, timeout) do connect_timeout = min(connect_timeout(request), timeout) - interruptible_connect( - transport, - host, - port, - transport_opts(request, selection, timeout), - connect_timeout - ) + with :ok <- validate_transport_option_lists(transport, request) do + interruptible_connect( + transport, + host, + port, + transport_opts(request, selection, timeout), + connect_timeout + ) + end + end + + defp validate_transport_option_lists(HTTP.Transport.ExSSL, request) do + valid? = + Enum.all?([:ssl, :socket_opts], fn key -> + opts = Keyword.get(request.transport_options, key, []) + + Keyword.keyword?(opts) and + length(Keyword.keys(opts)) == length(Enum.uniq(Keyword.keys(opts))) + end) + + if valid? do + :ok + else + {:error, {:options, :invalid_options}} + end end + defp validate_transport_option_lists(_transport, _request), do: :ok + defp interruptible_connect(transport, host, port, opts, timeout) do parent = self() ref = make_ref() @@ -611,7 +646,9 @@ defmodule HTTP.SocketClient do {:ok, pid} -> receive do {:send_result, ^ref, result} -> - close_on_error(transport, socket, result) + # The caller owns cleanup. A failed control write can leave readable + # TLS data behind, so sending must not destroy the receive side. + result :abort -> transport.close(socket) @@ -635,13 +672,6 @@ defmodule HTTP.SocketClient do end end - defp close_on_error(_transport, _socket, :ok), do: :ok - - defp close_on_error(transport, socket, {:error, reason}) do - transport.close(socket) - {:error, reason} - end - defp send_prepared_request(transport, socket, {:buffer, iodata}, deadline_at) do send_request(transport, socket, iodata, remaining_timeout(deadline_at)) end @@ -709,24 +739,54 @@ defmodule HTTP.SocketClient do defp ack_stream_chunk(_stream, nil), do: :ok defp ack_stream_chunk(stream, ack_ref), do: send(stream, {:stream_chunk_ack, ack_ref}) - defp flush_protocol_writes(%{protocol_module: HTTP.HTTP2, protocol: protocol} = state) do + defp flush_protocol_writes( + %{protocol_module: HTTP.HTTP2, protocol: protocol} = state, + discard_closed_controls? + ) do {protocol, iodata} = HTTP.HTTP2.take_outbound(protocol) state = %{state | protocol: protocol} case IO.iodata_to_binary(iodata) do "" -> {:ok, state} - data -> flush_protocol_write(state, data) + data -> flush_protocol_write(state, data, discard_closed_controls?) end end - defp flush_protocol_writes(state), do: {:ok, state} + defp flush_protocol_writes(state, _discard_closed_controls?), do: {:ok, state} - defp flush_protocol_write(state, data) do + defp discard_closed_control_writes?( + %{protocol_module: HTTP.HTTP2, protocol: protocol, transport: transport}, + events + ) do + # Classify queued frames independently from any upload waiting for credit. + # A complete early response stops that upload in the protocol layer. + # In ex_ssl 0.3.0 an established socket's peer close_notify rejects writes + # with :closed while retaining + # unread plaintext; abnormal TCP closure returns :econnreset instead. + # This owner has not closed the socket locally. Rearming it drains that + # plaintext (or reports EOF); only the HTTP parser can complete a response. + HTTP.HTTP2.outbound_control_only?(protocol) and + (transport == HTTP.Transport.ExSSL or + (HTTP.HTTP2.complete_response?(protocol) and :done in events)) + end + + defp discard_closed_control_writes?(_state, _events), do: false + + defp flush_protocol_write(state, data, discard_closed_controls?) do timeout = remaining_timeout(state.deadline_at) case send_request(state.transport, state.socket, data, timeout) do - :ok -> {:ok, state} - {:error, reason} -> {:error, reason} + :ok -> + {:ok, state} + + {:error, :closed} when discard_closed_controls? -> + # Sending is over, but receiving is not necessarily over. In particular, + # buffered WINDOW_UPDATE frames must not restart the abandoned upload + # while we drain the response through the original receive/deadline loop. + {:ok, %{state | protocol: HTTP.HTTP2.stop_request(state.protocol)}} + + {:error, reason} -> + {:error, reason} end end @@ -750,9 +810,12 @@ defmodule HTTP.SocketClient do {:ok, HTTP.Transport.TCP, host, uri.port || 80} end - defp select_transport(%Request{url: %URI{scheme: "https", host: host} = uri}, _socket_path) + defp select_transport( + %Request{url: %URI{scheme: "https", host: host} = uri} = request, + _socket_path + ) when is_binary(host) do - {:ok, HTTP.Transport.SSL, host, uri.port || 443} + {:ok, HTTP.TLSBackend.transport(tls_backend(request)), host, uri.port || 443} end defp select_transport(%Request{url: %URI{scheme: scheme}}, _socket_path) do @@ -782,7 +845,8 @@ defmodule HTTP.SocketClient do end end - defp protocol_selection(%Request{url: %URI{scheme: "https"}} = request, HTTP.Transport.SSL) do + defp protocol_selection(%Request{url: %URI{scheme: "https"}} = request, transport) + when transport in [HTTP.Transport.SSL, HTTP.Transport.ExSSL] do case http_version(request) do :http1 -> {:ok, %{mode: :http1, alpn_protocols: []}} @@ -802,6 +866,18 @@ defmodule HTTP.SocketClient do Keyword.get(request.transport_options, :http_version, :http1) end + defp tls_backend(%Request{} = request), do: Keyword.get(request.transport_options, :tls_backend) + + defp pin_tls_backend(%Request{} = request) do + with {:ok, backend} <- HTTP.TLSBackend.resolve(tls_backend(request)) do + {:ok, + %{ + request + | transport_options: Keyword.put(request.transport_options, :tls_backend, backend) + }} + end + end + defp request_timeout(%Request{} = request), do: Keyword.get(request.transport_options, :timeout, HTTP.Config.default_request_timeout()) @@ -906,7 +982,8 @@ defmodule HTTP.SocketClient do defp redirect_request(request, response) do with location when is_binary(location) <- Headers.get(response.headers, "location"), - %URI{} = uri <- URI.merge(request.url, location) do + %URI{} = uri <- URI.merge(request.url, location), + :ok <- validate_client_identity_redirect(request, uri) do request = request |> rewrite_redirect_method(response.status) @@ -914,10 +991,29 @@ defmodule HTTP.SocketClient do {:ok, %{request | url: uri}} else + {:error, _} = error -> error _ -> {:error, :invalid_redirect} end end + defp validate_client_identity_redirect(request, uri) do + ssl_options = Keyword.get(request.transport_options, :ssl, []) + + if tls_backend(request) == :ex_ssl and + Enum.any?([:cert, :certfile, :key, :keyfile], &Keyword.has_key?(ssl_options, &1)) and + client_identity_origin(request.url) != client_identity_origin(uri) do + {:error, :client_identity_cross_origin_redirect} + else + :ok + end + end + + defp client_identity_origin(uri) do + scheme = String.downcase(uri.scheme || "") + + {scheme, String.downcase(uri.host || ""), uri.port || HTTP.HTTP1.default_port(scheme)} + end + defp rewrite_redirect_method(%{method: :post} = request, status) when status in [301, 302], do: drop_redirect_body(request) diff --git a/apps/http_fetch/test/http/fetch_options_test.exs b/apps/http_fetch/test/http/fetch_options_test.exs index 7c55ae5..f4d1dd2 100644 --- a/apps/http_fetch/test/http/fetch_options_test.exs +++ b/apps/http_fetch/test/http/fetch_options_test.exs @@ -21,6 +21,7 @@ defmodule HTTP.FetchOptionsTest do "method" => "POST", "redirect" => "manual", "httpVersion" => "h2", + "tlsBackend" => "ex_ssl", "connectTimeout" => 2_000 }) @@ -28,6 +29,7 @@ defmodule HTTP.FetchOptionsTest do method: :post, redirect: :manual, http_version: :http2, + tls_backend: :ex_ssl, connect_timeout: 2_000 } = options @@ -91,6 +93,37 @@ defmodule HTTP.FetchOptionsTest do HTTP.FetchOptions.new(http_version: :spdy) end end + + test "normalizes TLS backend selection" do + assert %HTTP.FetchOptions{tls_backend: :ssl} = HTTP.FetchOptions.new([]) + assert %HTTP.FetchOptions{tls_backend: :ssl} = HTTP.FetchOptions.new(tls_backend: "ssl") + + assert %HTTP.FetchOptions{tls_backend: :ex_ssl} = + HTTP.FetchOptions.new(%{"tlsBackend" => "ex_ssl"}) + end + + test "rejects invalid TLS backend selection" do + assert_raise ArgumentError, ~r/unsupported tls_backend/, fn -> + HTTP.FetchOptions.new(tls_backend: :invalid) + end + + assert_raise ArgumentError, ~r/unsupported tls_backend/, fn -> + HTTP.FetchOptions.new(tls_backend: "SSL") + end + end + + test "does not resolve a nil TLS backend for HTTP/3" do + assert %HTTP.FetchOptions{http_version: :http3, tls_backend: nil} = + HTTP.FetchOptions.new(http_version: :http3) + end + + test "preserves explicit TLS backend markers for HTTP/3" do + assert %HTTP.FetchOptions{tls_backend: false} = + HTTP.FetchOptions.new(http_version: :http3, tls_backend: false) + + assert %HTTP.FetchOptions{tls_backend: :unknown} = + HTTP.FetchOptions.new(http_version: :http3, tls_backend: :unknown) + end end describe "to_transport_options/1" do @@ -101,6 +134,7 @@ defmodule HTTP.FetchOptionsTest do connect_timeout: 2_000, redirect: :manual, http_version: :http2, + tls_backend: :ex_ssl, ssl: [verify: :verify_none], socket_opts: [:inet6] ) @@ -110,6 +144,7 @@ defmodule HTTP.FetchOptionsTest do assert transport_options[:connect_timeout] == 2_000 assert transport_options[:redirect] == :manual assert transport_options[:http_version] == :http2 + assert transport_options[:tls_backend] == :ex_ssl assert transport_options[:ssl] == [verify: :verify_none] assert transport_options[:socket_opts] == [:inet6] end diff --git a/apps/http_fetch/test/http/socket_client_http2_test.exs b/apps/http_fetch/test/http/socket_client_http2_test.exs index a85a8ea..3149ff9 100644 --- a/apps/http_fetch/test/http/socket_client_http2_test.exs +++ b/apps/http_fetch/test/http/socket_client_http2_test.exs @@ -7,6 +7,7 @@ defmodule HTTP.SocketClientHTTP2Test do alias HTTP.HTTP2.HPACK @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) + @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) @ack 0x1 @@ -169,6 +170,1312 @@ defmodule HTTP.SocketClientHTTP2Test do |> HTTP.Promise.await() end + test "auto over https negotiates h2 through ex_ssl" do + url = + start_https_h2_server!([<<"h2">>, <<"http/1.1">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response(socket, transport, "ex-ssl-h2") + assert_settings_ack(socket, transport, buffer) + end) + + response = + url + |> HTTP.fetch( + http_version: :auto, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "ex-ssl-h2" + end + + test "forced HTTPS HTTP/2 succeeds through ex_ssl" do + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response(socket, transport, "forced-ex-ssl-h2") + assert_settings_ack(socket, transport, buffer) + end) + + response = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert HTTP.Response.read_all(response) == "forced-ex-ssl-h2" + end + + test "delivers a complete ex_ssl HTTP/2 response queued before a peer close" do + test_pid = self() + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + receive do + :send_response_and_close -> + send_h2_response(socket, transport, "queued-before-close") + :ok = :ssl.close(socket) + send(test_pid, :server_closed) + end + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_response_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "queued-before-close" + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "drains a second ex_ssl TLS record after control writes fail following peer close" do + test_pid = self() + body = "second-record-body" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "drains a split HTTP/2 frame from the ex_ssl receive buffer after peer close" do + test_pid = self() + body = "split-frame-body" + headers = Frame.encode(:headers, @end_headers, 1, response_headers(body)) + split_at = 5 + <> = headers + + first_record = [Frame.encode(:settings, 0, 0, ""), headers_start] + first_record_binary = IO.iodata_to_binary(first_record) + second_record = [headers_rest, Frame.encode(:data, @end_stream, 1, body)] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + for mode <- [:buffered_short, :buffered_long, :streamed_short] do + @tag :cross_record + @tag :content_length_gate + @tag length_mode: mode + test "rejects #{mode} Content-Length mismatch after cross-record peer close", %{ + length_mode: mode + } do + test_pid = self() + body = "invalid-length-body" + + declared_size = + case mode do + :buffered_short -> byte_size(body) + 1 + :buffered_long -> byte_size(body) - 1 + :streamed_short -> HTTP.Config.streaming_threshold() + 1 + end + + headers = + HPACK.encode_headers([ + {":status", "200"}, + {"content-length", Integer.to_string(declared_size)} + ]) + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, headers) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + case mode do + :streamed_short -> + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream_monitor = Process.monitor(response.stream) + + assert_raise RuntimeError, "stream read failed: :content_length_mismatch", fn -> + HTTP.Response.read_all(response) + end + + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + + _ -> + assert {:error, :content_length_mismatch} = HTTP.Promise.await(promise) + end + + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + end + + @tag :cross_record + test "rejects a truncated second ex_ssl TLS record after a control write fails" do + test_pid = self() + body = "truncated-second-record" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, 0, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, :closed} = HTTP.Promise.await(promise) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "does not turn a reset buffered after peer close into a successful response" do + test_pid = self() + body = "reset-after-drain" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + + second_record = [ + Frame.encode(:data, @end_stream, 1, body), + Frame.encode(:rst_stream, 0, 1, <<0x8::32>>) + ] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_first_record) + send_all(socket, transport, first_record) + send(test_pid, :first_record_sent) + + await_test_gate(:send_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_first_record) + assert_receive :first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + send(server_pid, :send_second_record_and_close) + assert_receive :second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert {:error, {:stream_reset, :cancel}} = HTTP.Promise.await(promise) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + @tag :early_response + test "delivers a complete 413 response when the HTTP/2 request body is still pending" do + test_pid = self() + body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_response_and_close) + + response_body = "payload-too-large" + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)), + Frame.encode(:data, @end_stream, 1, response_body) + ]) + + :ok = :ssl.close(socket) + send(test_pid, :server_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_response_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == "payload-too-large" + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :early_response + test "drains a cross-record 413 response without resuming its pending HTTP/2 upload" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + response_body = "cross-record-payload-too-large" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + + second_record = [ + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + Frame.encode(:data, @end_stream, 1, response_body) + ] + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_early_response_first_record) + send_all(socket, transport, first_record) + send(test_pid, :early_response_first_record_sent) + + await_test_gate(:send_early_response_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :early_response_second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_early_response_first_record) + assert_receive :early_response_first_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_early_response_second_record_and_close) + assert_receive :early_response_second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == response_body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :early_response + test "delivers a no-body 413 response through ex_ssl while the HTTP/2 upload is pending" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + await_test_gate(:send_no_body_early_response_and_close) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers ||| @end_stream, 1, response_headers(413, "")) + ]) + + :ok = :ssl.close(socket) + send(test_pid, :no_body_early_response_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_no_body_early_response_and_close) + assert_receive :no_body_early_response_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + + assert response.status == 413 + assert HTTP.Response.read_all(response) == "" + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :early_response + test "delivers an early 413 response through OTP TLS while the peer stays open for SETTINGS ACK" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + response_body = "otp-payload-too-large" + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(413, response_body)), + Frame.encode(:data, @end_stream, 1, response_body) + ]) + + buffer = assert_settings_ack(socket, transport, buffer) + buffer = assert_window_update(socket, transport, buffer, 0, byte_size(response_body)) + buffer = assert_window_update(socket, transport, buffer, 1, byte_size(response_body)) + assert buffer == "" + send(test_pid, {:otp_early_response_settings_acknowledged, self()}) + await_test_gate(:close_otp_early_response) + end) + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + ssl: [verify: :verify_none] + ) + + response = HTTP.Promise.await(promise) + assert response.status == 413 + assert HTTP.Response.read_all(response) == response_body + assert_receive {:otp_early_response_settings_acknowledged, server_pid}, 5_000 + send(server_pid, :close_otp_early_response) + end + + @tag :early_response + test "delivers one complete response when a later buffered NO_ERROR reset follows END_STREAM" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + body = "no-error-reset-after-end-stream" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)), + Frame.encode(:data, @end_stream, 1, body) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:rst_stream, 0, 1, <<0::32>>) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_complete_response_record) + send_all(socket, transport, first_record) + send(test_pid, :complete_response_record_sent) + + await_test_gate(:send_no_error_reset_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :no_error_reset_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: request_body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_complete_response_record) + assert_receive :complete_response_record_sent, 5_000 + + {tls_pid, ^first_record_binary} = await_owner_tls_data(owner, first_record_binary) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_no_error_reset_and_close) + assert_receive :no_error_reset_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do + @tag :early_response + test "handles a #{completion} NO_ERROR reset in the same HTTP/2 response batch" do + test_pid = self() + request_body = :binary.copy("p", @initial_window_size + 5) + body = "no-error-reset" + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(request_body, 0, @initial_window_size) + + send_all(socket, transport, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, response_headers(body)), + Frame.encode(:data, unquote(end_stream_flag), 1, body), + Frame.encode(:rst_stream, 0, 1, <<0::32>>) + ]) + + send(test_pid, {:no_error_response_sent, self()}) + await_test_gate(:close_no_error_response) + end) + + result = + url + |> HTTP.fetch( + method: :post, + body: request_body, + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + case unquote(completion) do + :complete -> + assert result.status == 200 + assert HTTP.Response.read_all(result) == body + + :incomplete -> + assert result == {:error, {:stream_reset, :no_error}} + end + + assert_receive {:no_error_response_sent, server_pid}, 5_000 + send(server_pid, :close_no_error_response) + end + end + + @tag :cross_record + test "drains a final streaming HTTP/2 frame buffered by ex_ssl after peer close" do + test_pid = self() + body = :binary.copy("q", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, body) + buffer = assert_settings_ack(socket, transport, buffer) + chunks = chunk_binary(body, 16_384) + preceding_chunks = Enum.drop(chunks, -3) + [first_final_chunk, second_final_chunk, third_final_chunk] = Enum.take(chunks, -3) + + Enum.reduce(preceding_chunks, buffer, fn chunk, current_buffer -> + send_all(socket, transport, Frame.encode(:data, 0, 1, chunk)) + + current_buffer = + assert_window_update(socket, transport, current_buffer, 0, byte_size(chunk)) + + assert_window_update(socket, transport, current_buffer, 1, byte_size(chunk)) + end) + + first_final_frame = Frame.encode(:data, 0, 1, first_final_chunk) + split_at = 8_000 + <> = first_final_frame + + first_record = [Frame.encode(:ping, 0, 0, "final123"), first_frame_start] + + second_record = [ + first_frame_rest, + Frame.encode(:data, 0, 1, second_final_chunk), + Frame.encode(:data, @end_stream, 1, third_final_chunk) + ] + + send( + test_pid, + {:server_ready_to_finish, self(), IO.iodata_to_binary(first_record), second_record} + ) + + await_test_gate(:send_final_first_record) + send_all(socket, transport, first_record) + send(test_pid, :final_first_record_sent) + + await_test_gate(:send_final_second_record_and_close) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :final_second_record_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + stream_monitor = monitor_test_process(response.stream) + reader = Task.async(fn -> HTTP.Response.read_all(response) end) + monitor_test_process(reader.pid) + + assert_receive {:server_ready_to_finish, server_pid, first_record, second_record}, 10_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_final_first_record) + assert_receive :final_first_record_sent, 5_000 + + {tls_pid, ^first_record} = await_owner_tls_data(owner, first_record) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_final_second_record_and_close) + assert_receive :final_second_record_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, :erlang.iolist_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + tls_monitor = Process.monitor(tls_pid) + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert Task.await(reader, 10_000) == body + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + @tag :cross_record + test "aborts an ex_ssl cross-record drain while stream backpressure holds the final body" do + {server_pid, controller, promise, owner, first_record, second_record} = + cross_record_stream_drain_fixture(self(), 5_000) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + {tls_pid, owner_monitor, tls_monitor} = + queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream = response.stream + stream_monitor = monitor_test_process(stream) + + holder = hold_cross_record_stream_chunk(stream, self()) + monitor_test_process(holder) + + assert_receive {:held_cross_record_stream_chunk, ^holder, ^stream, _chunk, _ack_ref}, + 5_000 + + await_cross_record_stream_backpressure(owner) + + :ok = HTTP.AbortController.abort(controller) + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + + reader = + Task.Supervisor.async_nolink(:http_fetch_task_supervisor, fn -> + assert_raise RuntimeError, "stream read failed: :aborted", fn -> + HTTP.Response.read_all(response) + end + end) + + monitor_test_process(reader.pid) + await_cross_record_read_all(reader.pid) + send(holder, :release_cross_record_stream_chunk) + + assert %RuntimeError{message: "stream read failed: :aborted"} = Task.await(reader, 5_000) + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + end + + @tag :cross_record + test "keeps the original deadline while an ex_ssl cross-record drain is backpressured" do + timeout = 2_000 + started_at = System.monotonic_time(:millisecond) + + {server_pid, controller, promise, owner, first_record, second_record} = + cross_record_stream_drain_fixture(self(), timeout) + + on_exit(fn -> + cleanup_owner(owner, controller) + end) + + # This explicit timer gate consumes half the request budget before entering + # the drain. A drain that reset the request timeout would outlive the + # original absolute deadline asserted below. + Process.send_after(self(), :begin_cross_record_deadline_drain, div(timeout, 2)) + assert_receive :begin_cross_record_deadline_drain, div(timeout, 2) + 250 + + {tls_pid, owner_monitor, tls_monitor} = + queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) + + response = HTTP.Promise.await(promise) + assert response.status == 200 + assert is_pid(response.stream) + stream = response.stream + stream_monitor = monitor_test_process(stream) + + holder = hold_cross_record_stream_chunk(stream, self()) + monitor_test_process(holder) + + assert_receive {:held_cross_record_stream_chunk, ^holder, ^stream, _chunk, _ack_ref}, + 5_000 + + await_cross_record_stream_backpressure(owner) + + deadline_at = started_at + timeout + remaining = deadline_at - System.monotonic_time(:millisecond) + assert remaining > 0 + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, remaining + 400 + + reader = + Task.Supervisor.async_nolink(:http_fetch_task_supervisor, fn -> + assert_raise RuntimeError, ~r/^stream read failed: :(request_timeout|timeout)$/, fn -> + HTTP.Response.read_all(response) + end + end) + + monitor_test_process(reader.pid) + await_cross_record_read_all(reader.pid) + send(holder, :release_cross_record_stream_chunk) + + assert %RuntimeError{message: message} = Task.await(reader, 5_000) + assert message in ["stream read failed: :request_timeout", "stream read failed: :timeout"] + + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, :normal}, 5_000 + end + + for {completion, end_stream_flag} <- [complete: @end_stream, incomplete: 0] do + @tag :early_response + test "handles a closed ex_ssl HTTP/2 connection with request body writes for a #{completion} response" do + test_pid = self() + body = :binary.copy("p", @initial_window_size + 5) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + + {initial_body, _buffer} = + recv_request_body_until(socket, transport, buffer, @initial_window_size) + + assert initial_body == binary_part(body, 0, @initial_window_size) + send(test_pid, {:server_received_request, self()}) + + await_test_gate(:send_response_and_close) + + frames = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:window_update, 0, 0, <<0::1, 5::31>>), + Frame.encode(:window_update, 0, 1, <<0::1, 5::31>>), + Frame.encode(:headers, @end_headers, 1, response_headers("complete")), + Frame.encode(:data, unquote(end_stream_flag), 1, "complete") + ] + + send_all(socket, transport, frames) + + :ssl.close(socket) + send(test_pid, :server_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + method: :post, + body: body, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:server_received_request, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + if Process.info(owner, :status) == {:status, :suspended}, + do: :erlang.resume_process(owner) + + if Process.alive?(controller), do: HTTP.AbortController.abort(controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_response_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + result = HTTP.Promise.await(promise) + + case unquote(completion) do + :complete -> + assert result.status == 200 + assert HTTP.Response.read_all(result) == "complete" + + :incomplete -> + assert result == {:error, :closed} + end + + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + end + + test "delivers a complete ex_ssl HTTP/2 response when the peer closes immediately" do + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send_h2_response(socket, transport, "closed-after-response") + end) + + response = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "closed-after-response" + end + + test "delivers a complete large ex_ssl HTTP/2 response without a final window update" do + test_pid = self() + body = :binary.copy("z", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, body) + buffer = assert_settings_ack(socket, transport, buffer) + chunks = chunk_binary(body, 16_384) + last_index = length(chunks) - 1 + + {final_chunk, preceding_chunks} = List.pop_at(chunks, last_index) + + Enum.reduce(preceding_chunks, buffer, fn chunk, buffer -> + send_all(socket, transport, Frame.encode(:data, 0, 1, chunk)) + buffer = assert_window_update(socket, transport, buffer, 0, byte_size(chunk)) + assert_window_update(socket, transport, buffer, 1, byte_size(chunk)) + end) + + send(test_pid, {:server_ready_to_finish, self()}) + + receive do + :send_final_chunk_and_close -> + send_all(socket, transport, Frame.encode(:data, @end_stream, 1, final_chunk)) + :ok = :ssl.close(socket) + send(test_pid, :server_closed) + end + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + response = + HTTP.Promise.await(promise) + + stream_monitor = Process.monitor(response.stream) + reader = Task.async(fn -> HTTP.Response.read_all(response) end) + assert_receive {:server_ready_to_finish, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + on_exit(fn -> + if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) + if Process.alive?(controller), do: HTTP.AbortController.abort(controller) + end) + + true = :erlang.suspend_process(owner) + send(server_pid, :send_final_chunk_and_close) + assert_receive :server_closed, 5_000 + tls_pid = await_owner_tls_data_and_close(owner) + tls_monitor = Process.monitor(tls_pid) + assert_receive {:DOWN, ^tls_monitor, :process, ^tls_pid, _reason}, 5_000 + owner_monitor = Process.monitor(owner) + true = :erlang.resume_process(owner) + + assert response.status == 200 + assert Task.await(reader, 5_000) == body + assert_receive {:DOWN, ^stream_monitor, :process, _stream, :normal}, 5_000 + assert_receive {:DOWN, ^owner_monitor, :process, ^owner, :normal}, 5_000 + end + + test "rejects an ex_ssl HTTP/2 response that closes without END_STREAM" do + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, "truncated") + send_all(socket, transport, Frame.encode(:data, 0, 1, "truncated")) + end) + + assert {:error, :closed} = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + end + + test "auto HTTPS falls back to HTTP/1.1 through ex_ssl without ALPN" do + url = + start_https_h2_server!([], fn socket, _transport -> + assert {:ok, request} = recv_http1_headers(socket, <<>>) + assert request =~ "GET /test HTTP/1.1\r\n" + + :ok = + :ssl.send( + socket, + "HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nfallback" + ) + end) + + response = + url + |> HTTP.fetch( + http_version: :auto, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert HTTP.Response.read_all(response) == "fallback" + end + + test "streams large forced HTTPS HTTP/2 responses through ex_ssl" do + body = :binary.copy("h", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, buffer} = recv_client_h2_request(socket, transport) + send_h2_response_headers(socket, transport, body) + buffer = assert_settings_ack(socket, transport, buffer) + + chunks = chunk_binary(body, 16_384) + last_index = length(chunks) - 1 + + Enum.reduce(Enum.with_index(chunks), buffer, fn {chunk, index}, buffer -> + flags = if index == last_index, do: @end_stream, else: 0 + send_all(socket, transport, Frame.encode(:data, flags, 1, chunk)) + + buffer = assert_window_update(socket, transport, buffer, 0, byte_size(chunk)) + assert_window_update(socket, transport, buffer, 1, byte_size(chunk)) + end) + end) + + response = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert is_pid(response.stream) + assert HTTP.Response.read_all(response) == body + end + + test "forced HTTPS HTTP/2 through ex_ssl fails without h2 ALPN" do + url = + start_https_h2_server!([], fn socket, _transport -> + :timer.sleep(100) + :ssl.close(socket) + end) + + assert {:error, {:http2_not_negotiated, nil}} = + url + |> HTTP.fetch( + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + end + defp start_h2c_server!(handler) do {:ok, listen_socket} = :gen_tcp.listen(0, [ @@ -228,7 +1535,7 @@ defmodule HTTP.SocketClientHTTP2Test do case :ssl.handshake(transport_socket) do {:ok, socket} -> handler.(socket, :ssl) - :ssl.close(socket) + :ok = :ssl.close(socket) {:error, _reason} -> :ok @@ -263,6 +1570,17 @@ defmodule HTTP.SocketClientHTTP2Test do {headers, buffer} end + defp recv_http1_headers(socket, acc) do + if String.contains?(acc, "\r\n\r\n") do + {:ok, acc} + else + case :ssl.recv(socket, 0, 5_000) do + {:ok, data} -> recv_http1_headers(socket, acc <> data) + {:error, reason} -> {:error, reason} + end + end + end + defp send_h2_response(socket, transport, body) do headers = response_headers(body) @@ -280,9 +1598,11 @@ defmodule HTTP.SocketClientHTTP2Test do ]) end - defp response_headers(body) do + defp response_headers(body), do: response_headers(200, body) + + defp response_headers(status, body) do HPACK.encode_headers([ - {":status", "200"}, + {":status", Integer.to_string(status)}, {"content-length", Integer.to_string(byte_size(body))}, {"x-protocol", "h2"} ]) @@ -365,6 +1685,310 @@ defmodule HTTP.SocketClientHTTP2Test do :ok = apply(transport, :send, [socket, iodata]) end + defp await_test_gate(gate) do + receive do + ^gate -> :ok + after + 5_000 -> exit({:test_gate_timeout, gate}) + end + end + + defp monitor_test_process(pid) do + on_exit(fn -> + if Process.alive?(pid), do: Process.exit(pid, :kill) + end) + + Process.monitor(pid) + end + + defp cleanup_owner(owner, controller) do + if Process.info(owner, :status) == {:status, :suspended}, do: :erlang.resume_process(owner) + + if Process.alive?(controller) do + HTTP.AbortController.abort(controller) + else + if Process.alive?(owner), do: send(owner, :abort) + end + end + + defp await_owner(controller) do + case :sys.get_state(controller).request_id do + owner when is_pid(owner) -> owner + nil -> flunk("socket owner was not registered before the request reached the server") + end + end + + defp await_owner_tls_data_and_close(owner) do + await_owner_tls_data_and_close(owner, nil, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_owner_tls_data(owner, expected_data) do + await_owner_tls_data(owner, expected_data, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_owner_tls_data(owner, expected_data, deadline_at) do + case Process.info(owner, :messages) do + {:messages, messages} -> + case Enum.find(messages, fn + {:ssl, %SSL.Socket{}, ^expected_data} -> true + _message -> false + end) do + {:ssl, %SSL.Socket{pid: tls_pid}, ^expected_data} -> {tls_pid, expected_data} + nil -> await_owner_tls_data_or_fail(owner, expected_data, deadline_at) + end + + nil -> + flunk("socket owner exited before receiving the first TLS record") + end + end + + defp await_owner_tls_data_or_fail(owner, expected_data, deadline_at) do + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("socket owner did not receive the first TLS record as one active-once delivery") + else + receive do + after + min(10, remaining) -> await_owner_tls_data(owner, expected_data, deadline_at) + end + end + end + + defp assert_owner_has_only_tls_data(owner, tls_pid, expected_count) do + {:messages, messages} = Process.info(owner, :messages) + + assert messages + |> Enum.count(&match?({:ssl, %SSL.Socket{pid: ^tls_pid}, _data}, &1)) == expected_count + end + + defp assert_tls_buffered_after_peer_close(tls_pid, expected_size) do + assert Application.spec(:ex_ssl, :vsn) == ~c"0.4.0", + "revalidate this private buffer probe before testing another ex_ssl version" + + assert_tls_buffered_after_peer_close( + tls_pid, + expected_size, + System.monotonic_time(:millisecond) + 5_000 + ) + end + + defp assert_tls_buffered_after_peer_close(tls_pid, expected_size, deadline_at) do + {_phase, state} = :sys.get_state(tls_pid) + + if state.closed do + assert state.size == expected_size + assert state.active == false + :ok + else + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("second TLS record was not retained in the TLS receive buffer after peer close") + else + receive do + after + min(10, remaining) -> + assert_tls_buffered_after_peer_close(tls_pid, expected_size, deadline_at) + end + end + end + end + + defp await_owner_loop(owner) do + await_owner_loop(owner, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_owner_loop(owner, deadline_at) do + if Process.info(owner, :current_function) == + {:current_function, {HTTP.SocketClient, :owner_loop, 1}} and + Process.info(owner, :status) == {:status, :waiting} do + :ok + else + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("socket owner did not return to its receive loop before the TLS close gate") + else + receive do + after + min(10, remaining) -> await_owner_loop(owner, deadline_at) + end + end + end + end + + defp await_owner_tls_data_and_close(owner, tls_pid, deadline_at) do + case Process.info(owner, :messages) do + {:messages, messages} -> + tls_pid = + tls_pid || + Enum.find_value(messages, fn + {:ssl, %SSL.Socket{pid: pid}, _data} -> pid + _message -> nil + end) + + if is_pid(tls_pid) and + Enum.any?(messages, &match?({:ssl_closed, %SSL.Socket{pid: ^tls_pid}}, &1)) do + tls_pid + else + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk("socket owner did not receive the queued TLS response and close") + else + receive do + after + min(10, remaining) -> await_owner_tls_data_and_close(owner, tls_pid, deadline_at) + end + end + end + + nil -> + flunk("socket owner exited before receiving the queued TLS response and close") + end + end + + defp cross_record_stream_drain_fixture(test_pid, timeout) do + body = "backpressured-final-record" + + first_record = [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode( + :headers, + @end_headers, + 1, + HPACK.encode_headers([{":status", "200"}, {"x-protocol", "h2"}]) + ) + ] + + first_record_binary = IO.iodata_to_binary(first_record) + second_record = Frame.encode(:data, @end_stream, 1, body) + + url = + start_https_h2_server!([<<"h2">>], fn socket, transport -> + {_request_headers, _buffer} = recv_client_h2_request(socket, transport) + send(test_pid, {:cross_record_stream_server_ready, self()}) + + await_test_gate(:send_cross_record_stream_first) + send_all(socket, transport, first_record) + send(test_pid, :cross_record_stream_first_sent) + + await_test_gate(:send_cross_record_stream_second) + send_all(socket, transport, second_record) + :ok = :ssl.close(socket) + send(test_pid, :cross_record_stream_second_closed) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + http_version: :http2, + signal: controller, + timeout: timeout, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {:cross_record_stream_server_ready, server_pid}, 5_000 + owner = await_owner(controller) + await_owner_loop(owner) + + {server_pid, controller, promise, owner, first_record_binary, second_record} + end + + defp queue_cross_record_stream_drain(server_pid, owner, first_record, second_record) do + true = :erlang.suspend_process(owner) + send(server_pid, :send_cross_record_stream_first) + assert_receive :cross_record_stream_first_sent, 5_000 + + {tls_pid, ^first_record} = await_owner_tls_data(owner, first_record) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + send(server_pid, :send_cross_record_stream_second) + assert_receive :cross_record_stream_second_closed, 5_000 + assert_tls_buffered_after_peer_close(tls_pid, byte_size(second_record)) + assert_owner_has_only_tls_data(owner, tls_pid, 1) + + owner_monitor = Process.monitor(owner) + tls_monitor = Process.monitor(tls_pid) + true = :erlang.resume_process(owner) + {tls_pid, owner_monitor, tls_monitor} + end + + defp hold_cross_record_stream_chunk(stream, test_pid) do + spawn(fn -> + send(stream, {:read_chunk, self(), :ack}) + + receive do + {:stream_chunk, ^stream, chunk, ack_ref} -> + send(test_pid, {:held_cross_record_stream_chunk, self(), stream, chunk, ack_ref}) + + receive do + :release_cross_record_stream_chunk -> + send(stream, {:stream_chunk_ack, ack_ref}) + after + 5_000 -> + send(test_pid, {:held_cross_record_stream_chunk_timeout, self()}) + end + after + 5_000 -> + send(test_pid, {:held_cross_record_stream_chunk_timeout, self()}) + end + end) + end + + defp await_cross_record_stream_backpressure(owner) do + await_cross_record_stream_backpressure(owner, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_cross_record_stream_backpressure(owner, deadline_at) do + if Process.info(owner, :current_function) == {:current_function, {HTTP.Stream, :chunk, 3}} and + Process.info(owner, :status) == {:status, :waiting} do + :ok + else + await_cross_record_condition( + owner, + deadline_at, + "socket owner did not block on stream backpressure", + fn -> await_cross_record_stream_backpressure(owner, deadline_at) end + ) + end + end + + defp await_cross_record_read_all(reader) do + await_cross_record_read_all(reader, System.monotonic_time(:millisecond) + 5_000) + end + + defp await_cross_record_read_all(reader, deadline_at) do + if Process.info(reader, :current_function) == + {:current_function, {HTTP.Response, :collect_stream, 2}} and + Process.info(reader, :status) == {:status, :waiting} do + :ok + else + await_cross_record_condition( + reader, + deadline_at, + "response reader did not wait for stream error", + fn -> await_cross_record_read_all(reader, deadline_at) end + ) + end + end + + defp await_cross_record_condition(_pid, deadline_at, message, fun) do + remaining = deadline_at - System.monotonic_time(:millisecond) + + if remaining <= 0 do + flunk(message) + else + receive do + after + min(10, remaining) -> fun.() + end + end + end + defp chunk_binary(binary, size) when byte_size(binary) <= size, do: [binary] defp chunk_binary(binary, size) do diff --git a/apps/http_fetch/test/http/ssl_transport_test.exs b/apps/http_fetch/test/http/ssl_transport_test.exs index 2e59fe7..a235a88 100644 --- a/apps/http_fetch/test/http/ssl_transport_test.exs +++ b/apps/http_fetch/test/http/ssl_transport_test.exs @@ -1,11 +1,54 @@ defmodule HTTP.SSLTransportTest do - use ExUnit.Case, async: true + use ExUnit.Case @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) describe "https transport" do + test "rejects explicit TLS backends for HTTP/3 through the promise" do + for tls_backend <- [false, :unknown, "ssl", "ex_ssl"] do + assert {:error, :tls_backend_not_supported_for_quic} = + "https://127.0.0.1:1/secure" + |> HTTP.fetch(http_version: :http3, tls_backend: tls_backend) + |> HTTP.Promise.await() + end + end + + test "does not resolve shared TLS configuration for HTTP/3" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + Application.put_env(:http_core, :tls_backend, :invalid) + + assert %HTTP.FetchOptions{http_version: :http3, tls_backend: nil} = + HTTP.FetchOptions.new(http_version: :http3) + end + + test "rejects unsupported ex_ssl socket options through HTTP.fetch" do + for socket_opts <- [[nodelay: :invalid], [send_timeout_close: false]] do + assert {:error, {:options, _reason}} = + "https://127.0.0.1:1/secure" + |> HTTP.fetch(tls_backend: :ex_ssl, socket_opts: socket_opts) + |> HTTP.Promise.await() + end + end + + test "propagates ex_ssl profile and ALPN conflicts through HTTP.fetch" do + profile = %SSL.ClientHello.WireProfile{extensions: [{:alpn, ["http/1.1"]}]} + + assert {:error, {:options, {:alpn_advertised_protocols, :profile_conflict}}} = + "https://127.0.0.1:1/secure" + |> HTTP.fetch( + tls_backend: :ex_ssl, + http_version: :http2, + ssl: [ + alpn_advertised_protocols: ["h2"], + ex_ssl: [profile: profile] + ] + ) + |> HTTP.Promise.await() + end + test "rejects self-signed certificates by default" do url = start_https_server!(fn socket -> send_response(socket, "secure") end) @@ -48,6 +91,205 @@ defmodule HTTP.SSLTransportTest do assert HTTP.Response.read_all(response) == "trusted" end + test "fetches a verified response through ex_ssl" do + url = + start_https_server!(fn socket -> + assert {:ok, request} = recv_headers(socket, <<>>) + assert request =~ "GET /secure HTTP/1.1\r\n" + send_response(socket, "ex-ssl") + end) + + response = + url + |> HTTP.fetch(tls_backend: :ex_ssl, ssl: [cacertfile: @cacertfile]) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "ex-ssl" + end + + test "streams chunked responses through ex_ssl" do + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + "5\r\nchunk\r\n", + "2\r\ned\r\n", + "0\r\n\r\n" + ]) + end) + + response = + url + |> HTTP.fetch(tls_backend: :ex_ssl, ssl: [cacertfile: @cacertfile]) + |> HTTP.Promise.await() + + assert is_pid(response.stream) + assert HTTP.Response.read_all(response) == "chunked" + end + + test "streams close-delimited responses through ex_ssl" do + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + :ok = :ssl.send(socket, "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nclose-delimited") + end) + + controller = HTTP.AbortController.new() + + response = + url + |> HTTP.fetch( + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert is_pid(response.stream) + + owner = :sys.get_state(controller).request_id + assert {:ssl_closed, %SSL.Socket{pid: socket_pid}} = await_owner_close(owner) + + socket_monitor = Process.monitor(socket_pid) + assert_receive {:DOWN, ^socket_monitor, :process, ^socket_pid, _reason}, 2_000 + + assert HTTP.Response.read_all(response) == "close-delimited" + end + + test "uploads large buffered bodies through ex_ssl" do + body = :binary.copy("u", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_server!(fn socket -> + assert %{body: ^body, headers: %{"content-length" => content_length}} = + recv_request(socket) + + assert content_length == Integer.to_string(byte_size(body)) + send_response(socket, "uploaded") + end) + + response = + url + |> HTTP.fetch( + method: :post, + body: body, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + |> HTTP.Promise.await() + + assert HTTP.Response.read_all(response) == "uploaded" + end + + test "cancels an in-flight ex_ssl request" do + test_pid = self() + + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send(test_pid, :request_received) + send(test_pid, {:server_recv_after_abort, :ssl.recv(socket, 0, 5_000)}) + end) + + controller = HTTP.AbortController.new() + + promise = + HTTP.fetch(url, + signal: controller, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive :request_received, 2_000 + :ok = HTTP.AbortController.abort(controller) + + assert {:error, :aborted} = HTTP.Promise.await(promise) + assert_receive {:server_recv_after_abort, {:error, :closed}}, 2_000 + end + + test "times out an ex_ssl request waiting for response headers" do + test_pid = self() + + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send(test_pid, :request_received) + send(test_pid, {:server_recv_after_timeout, :ssl.recv(socket, 0, 5_000)}) + end) + + promise = + HTTP.fetch(url, + timeout: 500, + tls_backend: :ex_ssl, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive :request_received, 2_000 + assert {:error, :request_timeout} = HTTP.Promise.await(promise, 2_000) + assert_receive {:server_recv_after_timeout, {:error, :closed}}, 2_000 + end + + test "uses TLS backend transport options on direct requests" do + url = + start_https_server!(fn socket -> + assert {:ok, request} = recv_headers(socket, <<>>) + assert request =~ "GET /secure HTTP/1.1\r\n" + send_response(socket, "direct") + end) + + response = + HTTP.SocketClient.request(%HTTP.Request{ + url: URI.parse(url), + transport_options: [tls_backend: "ex_ssl", ssl: [cacertfile: @cacertfile]] + }) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "direct" + end + + test "pins the configured backend through HTTPS redirects" do + test_pid = self() + + destination = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send_response(socket, "redirected") + end) + + source = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send(test_pid, {:redirect_request_received, self()}) + + receive do + :send_redirect -> + :ok = + :ssl.send(socket, [ + "HTTP/1.1 302 Found\r\n", + "Location: ", + destination, + "\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ]) + end + end) + + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + Application.put_env(:http_core, :tls_backend, :ex_ssl) + + promise = HTTP.fetch(source, ssl: [cacertfile: @cacertfile]) + assert_receive {:redirect_request_received, redirect_server} + Application.put_env(:http_core, :tls_backend, :invalid) + send(redirect_server, :send_redirect) + + assert %HTTP.Response{status: 200} = response = HTTP.Promise.await(promise) + assert HTTP.Response.read_all(response) == "redirected" + end + test "streams large responses over ssl" do body = String.duplicate("s", HTTP.Config.streaming_threshold() + 1) @@ -67,6 +309,25 @@ defmodule HTTP.SSLTransportTest do assert response.body == response.stream assert HTTP.Response.read_all(response) == body end + + test "streams large verified responses through ex_ssl" do + body = String.duplicate("s", HTTP.Config.streaming_threshold() + 1) + + url = + start_https_server!(fn socket -> + assert {:ok, _request} = recv_headers(socket, <<>>) + send_response(socket, body) + end) + + response = + url + |> HTTP.fetch(tls_backend: "ex_ssl", ssl: [cacertfile: @cacertfile]) + |> HTTP.Promise.await() + + assert response.status == 200 + assert is_pid(response.stream) + assert HTTP.Response.read_all(response) == body + end end defp start_https_server!(handler) when is_function(handler, 1) do @@ -118,6 +379,70 @@ defmodule HTTP.SSLTransportTest do end end + defp await_owner_close(owner) do + deadline = System.monotonic_time(:millisecond) + 2_000 + wait_for_owner_close(owner, deadline) + end + + defp wait_for_owner_close(owner, deadline) do + case Process.info(owner, :messages) do + {:messages, messages} -> + case Enum.find(messages, &match?({:ssl_closed, %SSL.Socket{}}, &1)) do + nil -> + if System.monotonic_time(:millisecond) < deadline do + Process.sleep(10) + wait_for_owner_close(owner, deadline) + else + flunk("fetch owner did not queue an ExSSL close message") + end + + message -> + message + end + + nil -> + flunk("fetch owner exited before its queued ExSSL close could be observed") + end + end + + defp recv_request(socket) do + {head, rest} = recv_header_block(socket, <<>>) + [_request_line | header_lines] = String.split(head, "\r\n") + + headers = + header_lines + |> Enum.map(fn line -> + [name, value] = String.split(line, ":", parts: 2) + {String.downcase(name), String.trim(value)} + end) + |> Map.new() + + content_length = headers |> Map.fetch!("content-length") |> String.to_integer() + %{headers: headers, body: recv_body(socket, rest, content_length)} + end + + defp recv_header_block(socket, acc) do + case :binary.match(acc, "\r\n\r\n") do + {index, 4} -> + head = binary_part(acc, 0, index) + rest = binary_part(acc, index + 4, byte_size(acc) - index - 4) + {head, rest} + + :nomatch -> + {:ok, data} = :ssl.recv(socket, 0, 5_000) + recv_header_block(socket, acc <> data) + end + end + + defp recv_body(_socket, data, content_length) when byte_size(data) >= content_length do + binary_part(data, 0, content_length) + end + + defp recv_body(socket, data, content_length) do + {:ok, more} = :ssl.recv(socket, content_length - byte_size(data), 5_000) + recv_body(socket, data <> more, content_length) + end + defp send_response(socket, body) do :ok = :ssl.send(socket, [ @@ -132,4 +457,7 @@ defmodule HTTP.SSLTransportTest do :ssl.close(socket) end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(backend), do: Application.put_env(:http_core, :tls_backend, backend) end diff --git a/apps/http_web_socket/lib/http/web_socket.ex b/apps/http_web_socket/lib/http/web_socket.ex index 79864de..c683a1a 100644 --- a/apps/http_web_socket/lib/http/web_socket.ex +++ b/apps/http_web_socket/lib/http/web_socket.ex @@ -11,6 +11,10 @@ defmodule HTTP.WebSocket do Plain Elixir binaries are sent as text frames. Use `array_buffer/1` or `HTTP.Blob` for binary frames. + + For `wss` connections, pass `tls_backend: :ssl | :ex_ssl` (or the equivalent + string in a map). When omitted, the shared `:http_core` TLS backend setting is + captured when the socket is created. """ alias HTTP.WebSocket.ArrayBuffer diff --git a/apps/http_web_socket/lib/http/web_socket/connection.ex b/apps/http_web_socket/lib/http/web_socket/connection.ex index 620c59e..092d88a 100644 --- a/apps/http_web_socket/lib/http/web_socket/connection.ex +++ b/apps/http_web_socket/lib/http/web_socket/connection.ex @@ -30,6 +30,7 @@ defmodule HTTP.WebSocket.Connection do connect_timeout: 30_000, ssl: [], socket_opts: [], + tls_backend: :ssl, max_send_queue: 16 * 1024 * 1024, transport: nil, socket: nil, @@ -75,6 +76,7 @@ defmodule HTTP.WebSocket.Connection do connect_timeout: options.connect_timeout, ssl: options.ssl, socket_opts: options.socket_opts, + tls_backend: options.tls_backend, max_send_queue: options.max_send_queue, binary_type: options.binary_type, parser: Frame.new_parser(max_message_size: options.max_message_size) @@ -94,13 +96,15 @@ defmodule HTTP.WebSocket.Connection do Telemetry.connect_stop(state.uri, state.protocol, duration) emit(state, %Open{target: state.target}) - if extra != <<>> do - send(self(), {:websocket_data, extra}) - end + state = %{state | ready_state: @open, connect_started_at: started_at} - case rearm(state) do - :ok -> {:noreply, %{state | ready_state: @open, connect_started_at: started_at}} - {:error, reason} -> fail_connection(state, reason, started_at) + if extra == <<>> do + case rearm(state) do + :ok -> {:noreply, state} + {:error, reason} -> fail_connection(state, reason, started_at) + end + else + handle_socket_data(extra, state) end {:error, reason} -> @@ -146,8 +150,6 @@ defmodule HTTP.WebSocket.Connection do end @impl true - def handle_info({:websocket_data, data}, state), do: handle_socket_data(data, state) - def handle_info(:close_timeout, state) do {:stop, :normal, finish_close(state, state.close_code || 1006, state.close_reason, false)} end @@ -167,7 +169,7 @@ defmodule HTTP.WebSocket.Connection do defp connect_and_upgrade(state) do key = state |> Map.get(:uri) |> generate_key() - with {:ok, transport, host, port} <- select_transport(state.uri), + with {:ok, transport, host, port} <- select_transport(state), {:ok, request} <- Handshake.build_request(state.uri, state.protocols, state.headers, key), {:ok, socket} <- connect(transport, host, port, state), @@ -190,15 +192,15 @@ defmodule HTTP.WebSocket.Connection do defp generate_key(_uri), do: :crypto.strong_rand_bytes(16) |> Base.encode64() - defp select_transport(%URI{scheme: "ws", host: host, port: port}) do + defp select_transport(%{uri: %URI{scheme: "ws", host: host, port: port}}) do {:ok, HTTP.Transport.TCP, host, port || 80} end - defp select_transport(%URI{scheme: "wss", host: host, port: port}) do - {:ok, HTTP.Transport.SSL, host, port || 443} + defp select_transport(%{uri: %URI{scheme: "wss", host: host, port: port}, tls_backend: backend}) do + {:ok, HTTP.TLSBackend.transport(backend), host, port || 443} end - defp select_transport(%URI{scheme: scheme}), do: {:error, {:unsupported_scheme, scheme}} + defp select_transport(%{uri: %URI{scheme: scheme}}), do: {:error, {:unsupported_scheme, scheme}} defp connect(transport, host, port, state) do transport.connect( @@ -224,8 +226,7 @@ defmodule HTTP.WebSocket.Connection do end end - defp recv(HTTP.Transport.TCP, socket, timeout), do: :gen_tcp.recv(socket, 0, timeout) - defp recv(HTTP.Transport.SSL, socket, timeout), do: :ssl.recv(socket, 0, timeout) + defp recv(transport, socket, timeout), do: transport.recv(socket, 0, timeout) defp fail_connection(state, reason, started_at) do duration = System.monotonic_time(:microsecond) - started_at diff --git a/apps/http_web_socket/lib/http/web_socket/options.ex b/apps/http_web_socket/lib/http/web_socket/options.ex index 28281ed..3496346 100644 --- a/apps/http_web_socket/lib/http/web_socket/options.ex +++ b/apps/http_web_socket/lib/http/web_socket/options.ex @@ -6,6 +6,11 @@ defmodule HTTP.WebSocket.Options do @default_max_message_size 16 * 1024 * 1024 @default_max_send_queue 16 * 1024 * 1024 + @string_keys %{ + "tls_backend" => :tls_backend, + "tlsBackend" => :tls_backend + } + defstruct uri: nil, url: nil, protocols: [], @@ -16,6 +21,7 @@ defmodule HTTP.WebSocket.Options do connect_timeout: @default_connect_timeout, ssl: [], socket_opts: [], + tls_backend: :ssl, max_message_size: @default_max_message_size, max_send_queue: @default_max_send_queue, ref: nil @@ -31,6 +37,7 @@ defmodule HTTP.WebSocket.Options do connect_timeout: timeout(), ssl: keyword(), socket_opts: keyword(), + tls_backend: HTTP.TLSBackend.t(), max_message_size: pos_integer(), max_send_queue: pos_integer(), ref: reference() @@ -54,6 +61,7 @@ defmodule HTTP.WebSocket.Options do connect_timeout: Keyword.get(init, :connect_timeout, @default_connect_timeout), ssl: Keyword.get(init, :ssl, []), socket_opts: Keyword.get(init, :socket_opts, []), + tls_backend: Keyword.fetch!(init, :tls_backend), max_message_size: Keyword.get(init, :max_message_size, @default_max_message_size), max_send_queue: Keyword.get(init, :max_send_queue, @default_max_send_queue), ref: Keyword.get(init, :ref, make_ref()) @@ -129,7 +137,11 @@ defmodule HTTP.WebSocket.Options do end end - defp normalize_init(init) when is_map(init), do: init |> Map.to_list() |> normalize_init() + defp normalize_init(init) when is_map(init) do + init + |> Enum.map(fn {key, value} -> {normalize_key(key), value} end) + |> normalize_init() + end defp normalize_init(init) when is_list(init) do with {:ok, headers} <- normalize_headers(Keyword.get(init, :headers, [])), @@ -137,19 +149,24 @@ defmodule HTTP.WebSocket.Options do {:ok, owner} <- normalize_owner(Keyword.get(init, :owner, self())), {:ok, ssl} <- normalize_keyword(Keyword.get(init, :ssl, []), :invalid_ssl_options), {:ok, socket_opts} <- - normalize_keyword(Keyword.get(init, :socket_opts, []), :invalid_socket_options) do + normalize_keyword(Keyword.get(init, :socket_opts, []), :invalid_socket_options), + {:ok, tls_backend} <- HTTP.TLSBackend.resolve(Keyword.get(init, :tls_backend)) do {:ok, init |> Keyword.put(:headers, headers) |> Keyword.put(:binary_type, binary_type) |> Keyword.put(:owner, owner) |> Keyword.put(:ssl, ssl) - |> Keyword.put(:socket_opts, socket_opts)} + |> Keyword.put(:socket_opts, socket_opts) + |> Keyword.put(:tls_backend, tls_backend)} end end defp normalize_init(_init), do: {:error, :invalid_options} + defp normalize_key(key) when is_binary(key), do: Map.get(@string_keys, key, key) + defp normalize_key(key), do: key + defp normalize_headers(%HTTP.Headers{headers: headers}), do: normalize_headers(headers) defp normalize_headers(headers) when is_map(headers) do diff --git a/apps/http_web_socket/test/http/web_socket/options_test.exs b/apps/http_web_socket/test/http/web_socket/options_test.exs index dd936be..3029feb 100644 --- a/apps/http_web_socket/test/http/web_socket/options_test.exs +++ b/apps/http_web_socket/test/http/web_socket/options_test.exs @@ -1,5 +1,5 @@ defmodule HTTP.WebSocket.OptionsTest do - use ExUnit.Case, async: true + use ExUnit.Case, async: false alias HTTP.WebSocket.Options @@ -56,4 +56,39 @@ defmodule HTTP.WebSocket.OptionsTest do assert options.max_message_size == 32 assert options.max_send_queue == 64 end + + test "selects TLS backends from atom and string map options" do + assert {:ok, %{tls_backend: :ssl}} = + Options.new("wss://example.com/socket", [], tls_backend: :ssl) + + assert {:ok, %{tls_backend: :ex_ssl}} = + Options.new("wss://example.com/socket", [], %{"tlsBackend" => "ex_ssl"}) + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("wss://example.com/socket", [], %{"tls_backend" => "ssl"}) + end + + test "pins the configured TLS backend when options are constructed" do + previous = Application.get_env(:http_core, :tls_backend) + on_exit(fn -> restore_tls_backend(previous) end) + + Application.put_env(:http_core, :tls_backend, :ex_ssl) + assert {:ok, options} = Options.new("wss://example.com/socket") + assert options.tls_backend == :ex_ssl + + assert {:ok, %{tls_backend: :ssl}} = + Options.new("wss://example.com/socket", [], tls_backend: :ssl) + + Application.put_env(:http_core, :tls_backend, :ssl) + assert options.tls_backend == :ex_ssl + assert {:ok, %{tls_backend: :ssl}} = Options.new("wss://example.com/socket") + end + + test "rejects invalid TLS backend selections" do + assert {:error, :invalid_tls_backend} = + Options.new("wss://example.com/socket", [], tls_backend: :unknown) + end + + defp restore_tls_backend(nil), do: Application.delete_env(:http_core, :tls_backend) + defp restore_tls_backend(value), do: Application.put_env(:http_core, :tls_backend, value) end diff --git a/apps/http_web_socket/test/http/web_socket_test.exs b/apps/http_web_socket/test/http/web_socket_test.exs index 924a96f..6cc5b10 100644 --- a/apps/http_web_socket/test/http/web_socket_test.exs +++ b/apps/http_web_socket/test/http/web_socket_test.exs @@ -8,6 +8,10 @@ defmodule HTTP.WebSocketTest do alias HTTP.WebSocket.Event.Message alias HTTP.WebSocket.Event.Open + @certfile Path.expand("../support/fixtures/localhost.pem", __DIR__) + @cacertfile Path.expand("../support/fixtures/localhost-ca.pem", __DIR__) + @keyfile Path.expand("../support/fixtures/localhost.key", __DIR__) + test "defines browser ready state constants" do assert WebSocket.connecting() == 0 assert WebSocket.open() == 1 @@ -81,6 +85,32 @@ defmodule HTTP.WebSocketTest do assert_receive {:websocket_server_received, :binary, <<4, 5, 6>>}, 1_000 end + test "upgrades, exchanges frames, and closes over verified TLS 1.3" do + for backend <- [:ssl, :ex_ssl] do + {:ok, _server, port} = + HTTPWebSocket.TestServer.start_link( + tls: true, + certfile: @certfile, + keyfile: @keyfile, + open_message: "welcome" + ) + + socket = + WebSocket.new("wss://127.0.0.1:#{port}/socket", [], + tls_backend: backend, + ssl: [cacertfile: @cacertfile] + ) + + assert_receive {WebSocket, ^socket, %Open{}}, 1_000 + assert_receive {WebSocket, ^socket, %Message{data: "welcome"}}, 1_000 + assert :ok = WebSocket.send(socket, "hello") + assert_receive {:websocket_server_received, :text, "hello"}, 1_000 + assert_receive {WebSocket, ^socket, %Message{data: "echo:hello"}}, 1_000 + assert :ok = WebSocket.close(socket, 1000, "done") + assert_receive {WebSocket, ^socket, %Close{code: 1000, was_clean: true}}, 1_000 + end + end + test "rejects invalid constructor input synchronously" do assert {:error, :fragment_not_allowed} = WebSocket.new("ws://example.com/socket#frag") end diff --git a/apps/http_web_socket/test/http/web_socket_tls_lifecycle_test.exs b/apps/http_web_socket/test/http/web_socket_tls_lifecycle_test.exs new file mode 100644 index 0000000..f32ed58 --- /dev/null +++ b/apps/http_web_socket/test/http/web_socket_tls_lifecycle_test.exs @@ -0,0 +1,74 @@ +defmodule HTTP.WebSocketTLSLifecycleTest do + use ExUnit.Case, async: false + + alias HTTP.WebSocket + alias HTTP.WebSocket.Event.Close + alias HTTP.WebSocket.Event.Message + alias HTTP.WebSocket.Event.Open + + @fixtures Path.expand("../support/fixtures", __DIR__) + + test "delivers upgrade-buffered frames after the TLS peer has closed" do + parent = self() + existing_connections = tls_connections() + + {:ok, server, port} = + HTTPWebSocket.TestServer.start_link( + tls: true, + certfile: Path.join(@fixtures, "localhost.pem"), + keyfile: Path.join(@fixtures, "localhost.key"), + # A complete text frame and close frame share the upgrade's TLS record. + upgrade_frames: <<0x81, 3, "bye", 0x88, 2, 1000::16>>, + close_after_upgrade: true + ) + + handler = {__MODULE__, make_ref()} + + :ok = + :telemetry.attach( + handler, + [:http_web_socket, :connect, :stop], + &__MODULE__.pause_after_upgrade/4, + {port, parent} + ) + + on_exit(fn -> :telemetry.detach(handler) end) + + socket = + WebSocket.new("wss://127.0.0.1:#{port}/socket", [], + tls_backend: :ex_ssl, + ssl: [cacertfile: Path.join(@fixtures, "localhost-ca.pem")] + ) + + assert_receive {:upgrade_received, consumer}, 5_000 + [tls_pid] = tls_connections() -- existing_connections + monitor = Process.monitor(tls_pid) + + try do + send(server, :close_tls) + assert_receive {:DOWN, ^monitor, :process, ^tls_pid, _}, 5_000 + after + send(consumer, :continue_upgrade) + end + + assert_receive {WebSocket, ^socket, %Open{}}, 5_000 + assert_receive {WebSocket, ^socket, %Message{data: "bye"}}, 5_000 + assert_receive {WebSocket, ^socket, %Close{code: 1000, was_clean: true}}, 5_000 + end + + def pause_after_upgrade(_event, _measurements, %{port: port}, {port, parent}) do + send(parent, {:upgrade_received, self()}) + + receive do + :continue_upgrade -> :ok + after + 5_000 -> :ok + end + end + + def pause_after_upgrade(_event, _measurements, _metadata, _config), do: :ok + + defp tls_connections do + for {_, pid, _, _} <- DynamicSupervisor.which_children(SSL.ConnectionSupervisor), do: pid + end +end diff --git a/apps/http_web_socket/test/support/fixtures/localhost-ca.pem b/apps/http_web_socket/test/support/fixtures/localhost-ca.pem new file mode 100644 index 0000000..15227f1 --- /dev/null +++ b/apps/http_web_socket/test/support/fixtures/localhost-ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDKzCCAhOgAwIBAgIUbWn02fvDN0zvFEGHFliwdXqASNIwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqaXND6GhQeV+tXwmYZYy +q+kFvpGWXCa49yG4/nJLfUYtpANqii88Y08J+6fAskHxoWYLidcnAvC5V51IcreX +g/N108fCvxTQ5jYfwIyj8oAEL4QZbnKklQrcLRFqN1Ef9f6cdNWL4pZZk7/ChbXP +ULHTNQF8FIoDUS4H5bjobU2lBER4H7Qa8oevkMGotgfjrzqWftTWt68dJiTsfoX1 +z+VH6t6t5vvT/L+0J7M9JAB++sckXSY9J+yIeG7Cu3sGtwTqIvuiiBH0latUYC9M +sJ0LS400sBonIZC4BdvVc9NCU8M1sx5yWV6eNRe7Cz3hrzQVcWMbAB73qXFMvpEO +OwIDAQABo2MwYTAdBgNVHQ4EFgQUUayz7IdQZHRIjmttS9hkKzG01M4wHwYDVR0j +BBgwFoAUUayz7IdQZHRIjmttS9hkKzG01M4wDwYDVR0TAQH/BAUwAwEB/zAOBgNV +HQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggEBAIrCD3yHZ+F/P/s0FXZRFFNB +AIqYjEwYr2OBN/ShHhceoqM55FbThtYPxUlVJH+uWkxpvnExDOauCapW7Cxlz9FF +3NvLjWf8fs80Uno5Dy6mik4rL6OsThYX8Ko2u6gkSiQm9yuT1iKvigGjU4q1qyft +ZJH8an7/egIIHxZya5oBuGKBnX241s6n9Osdtww/Eu5IOlZZeORJ0mAA7gTHaZyI +VquDo6Ml66LMyuIQkbtr19UHAEHrSfhmrujI5GruA/ejnma2O5sE1r1hVa91TV+L +TuaVxPvkdS+CIBvfwy90zmDbPhLt0kU5FiY2x6FlR1MrvVBn1uUQ59kYJ7yNI9k= +-----END CERTIFICATE----- diff --git a/apps/http_web_socket/test/support/fixtures/localhost.key b/apps/http_web_socket/test/support/fixtures/localhost.key new file mode 100644 index 0000000..d05ba0d --- /dev/null +++ b/apps/http_web_socket/test/support/fixtures/localhost.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtsMy7LvUMU+H3 +27785rkCaSrkj5ljfUUPDA8+LM1ReN4ovqRaKjN8Tz88AOOwsZkXHLEJ5Uv8RbpS +8ruITF9moSXhVNFvVkBtf+SIBF2M5sDwsV/q8VFwUcvncDyLOsZwqsal9j+e9LjY +SlQtzzapBWfZm8O540RNpmpsa6k0M7jfp/m8P/l3RIlYzEuWPfCBd1ZmQMUN0T6H ++iYBBfNj6ioDUOUP5VfwA8cVXgblYbFKPF6uTtv95Ep4CFin0u6S0xVRlX0PERU5 +kyNLkmXUibClAY93IEkX5KDe0wb0WU4QBC9BAC8CegTFKd57MUlH0FtABCakLDCz +uHKFOc5bAgMBAAECggEAF95J5Bcp2Y3Vaagq8LoMBobJPCt3mrJmQREZLyQc+usv +LE42a6WM+bIyMF6261scfI6WzZNTA9HZLvjoSGymV4YhnHt5ZGFn7SQ8FWz0Jh63 +uNvHIJQU6uhyqtVDnKZxCp6NOdVDHOboV+Cu/LSTAFsb0E2VcgAHLNyHl2qcD+4p +qVDfyz/2BcAciQUTyQmq8tVPyJV226iZAkTJjah8H+MOXo6XqieNJ2WTz3H57PCE +tmm5YLeNl6S47O2369VRFmhYZZBS2tiBEfXNwwFCc6Q44esmJG1UtLUp6LMenNon +FJTod0QuZMjqfomzYzvJ9sCi8cvrABDVs8HQiw8RIQKBgQDY4ReBy63jdUJkDYQC +00SnrGywE7YNLYu+ZkHz+J14Xn/ylKFK0sz1WPfGZZQWtLWEXCYw3f2Z8d42Rd0Z +l/E7H2KiRq2oPrqc/mwfWtwjL0sqOuLswAzdL8AKLbl6vou1UjrJennuvaOh2C2s +sKJutX7CoM8GvXvWweNGY9xvuQKBgQDNBV8IkKuE4B8TIF7x49lkOFpWVgf6E1bS +My22JFIk8C0Jnxh8e/6mujeEJ2jMO/kpYPhWkDhKp4CTm+S6Ga6CSFzBjnA0dfaB +8Htyb4R5bJn7wD9+Zj+tHSG6gwVxJl/u2YfDAZL+zlKgVx7RoTGkF8mskHVmgrFT +DIJyFs0wswKBgFMSjR2DdfzNOnv2jV1DrWWIby1Wr4IGsyNgKd0YmsCzedDiS5HM +gwNra6UL3ZiA6ZJkdaB8N5qTAanKQvF9uMILuI0uA3CRbouaDLJJ7E5x3Bm16pwC +yCqlEqsTbptshzkR4UCxcCkZbKcelggytFUxofdM/1+2jsvpAnRA5fvJAoGAXS+/ +zkjTbQXhmfPws8l4mhDzHqLj5Uq8/7W7ZTqFC70O+3yQyKQjTuz9JtgyzgHEcoZc +2hubOnOAAZeuEthxdU4muuNfJLkpXk5MDeuaLwapxr/PHEilUK4ZEolTA+cJW6sM +BhrFEYP+ElsG6wl1YrxdMk5Gzl1A9BqPgAPVJ/kCgYEArpwxPGurJDQ3JszIWDJU +CX6KhwAncK9LALQT6EPB3Y54Ou4cq1XMSWlmqUUmN821J/mbR4f66F+nFy/q29Br +giwArg0Ngkt+uaKTfo6IeR7+b2vAjzZ+z3cEbNZpuyCAbQwytnXT8t7ZUonAQ7dO +TmHo1zxsuNSt2EYos+EP+VA= +-----END PRIVATE KEY----- diff --git a/apps/http_web_socket/test/support/fixtures/localhost.pem b/apps/http_web_socket/test/support/fixtures/localhost.pem new file mode 100644 index 0000000..178c710 --- /dev/null +++ b/apps/http_web_socket/test/support/fixtures/localhost.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHTCCAgWgAwIBAgIUEJVRR6DfNeTUcvIJYQL9oVN8xyEwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSaHR0cF9mZXRjaCB0ZXN0IENBMB4XDTI2MDYxNzA4NDAw +OVoXDTM2MDYxNDA4NDAwOVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArbDMuy71DFPh99u+/Oa5Amkq5I+ZY31F +DwwPPizNUXjeKL6kWiozfE8/PADjsLGZFxyxCeVL/EW6UvK7iExfZqEl4VTRb1ZA +bX/kiARdjObA8LFf6vFRcFHL53A8izrGcKrGpfY/nvS42EpULc82qQVn2ZvDueNE +TaZqbGupNDO436f5vD/5d0SJWMxLlj3wgXdWZkDFDdE+h/omAQXzY+oqA1DlD+VX +8APHFV4G5WGxSjxerk7b/eRKeAhYp9LuktMVUZV9DxEVOZMjS5Jl1ImwpQGPdyBJ +F+Sg3tMG9FlOEAQvQQAvAnoExSneezFJR9BbQAQmpCwws7hyhTnOWwIDAQABo14w +XDAaBgNVHREEEzARhwR/AAABgglsb2NhbGhvc3QwHQYDVR0OBBYEFN1Qq2IrwIwg +nlCE50Q3jpSthaODMB8GA1UdIwQYMBaAFFGss+yHUGR0SI5rbUvYZCsxtNTOMA0G +CSqGSIb3DQEBCwUAA4IBAQBwO6NAaG2iwdO0iAaatt9cyP7ZvtJM0I0se1Dd1AtU +3iBlrEFNAWwph3ycD9Wr5HKswN7DhK+cOeklAeeBaxgB8egyHuJHKJLmoZQip+FQ +lCoMgPOBXH5IepiczMN4mUUCJn2ZNf0j/QkO0CjjefiYBYKunMQDNMtmhiB7LVIL +RAFo6kawJ49F3l/rTSKMMTGjlsW2wSM3pexevTvG/8dagT9ULQfLOuuHd+2paqKU +EoLAuaagqefwZaNASEvKP1Tl9I1GvZmjVTK8qJDmMJE+/D5HwscrDZmBvSDJdLLf +oZPdnbpZv0TXYvlpC5c1pHRPeA5LGa54GhM4Akc8ieDO +-----END CERTIFICATE----- diff --git a/apps/http_web_socket/test/support/web_socket_server.ex b/apps/http_web_socket/test/support/web_socket_server.ex index 7fd8f36..04a4a5b 100644 --- a/apps/http_web_socket/test/support/web_socket_server.ex +++ b/apps/http_web_socket/test/support/web_socket_server.ex @@ -7,110 +7,137 @@ defmodule HTTPWebSocket.TestServer do def start_link(opts \\ []) do parent = self() - - {:ok, listen_socket} = - :gen_tcp.listen(0, [ - :binary, - packet: :raw, - active: false, - reuseaddr: true, - ip: {127, 0, 0, 1} - ]) - - {:ok, {{127, 0, 0, 1}, port}} = :inet.sockname(listen_socket) + tls? = Keyword.get(opts, :tls, false) + transport = socket_module(tls?) + {:ok, listen_socket} = listen(tls?, opts) + {:ok, {{127, 0, 0, 1}, port}} = sockname(tls?, listen_socket) pid = spawn_link(fn -> - {:ok, socket} = :gen_tcp.accept(listen_socket) - :ok = :gen_tcp.close(listen_socket) - serve(socket, parent, opts) + with {:ok, socket} <- accept(tls?, listen_socket), + :ok <- transport.close(listen_socket) do + serve(socket, parent, opts, transport) + else + {:error, reason} -> send(parent, {:websocket_server_error, reason}) + end end) {:ok, pid, port} end - defp serve(socket, parent, opts) do - with {:ok, request} <- recv_until(socket, "\r\n\r\n", <<>>), + defp listen(false, _opts) do + :gen_tcp.listen(0, [:binary, packet: :raw, active: false, reuseaddr: true, ip: {127, 0, 0, 1}]) + end + + defp listen(true, opts) do + :ssl.listen(0, + mode: :binary, + packet: :raw, + active: false, + reuseaddr: true, + ip: {127, 0, 0, 1}, + versions: [:"tlsv1.3"], + certfile: Keyword.fetch!(opts, :certfile), + keyfile: Keyword.fetch!(opts, :keyfile) + ) + end + + defp accept(false, listen_socket), do: :gen_tcp.accept(listen_socket) + + defp accept(true, listen_socket) do + with {:ok, transport_socket} <- :ssl.transport_accept(listen_socket), + do: :ssl.handshake(transport_socket) + end + + defp socket_module(false), do: :gen_tcp + defp socket_module(true), do: :ssl + defp sockname(false, listen_socket), do: :inet.sockname(listen_socket) + defp sockname(true, listen_socket), do: :ssl.sockname(listen_socket) + + defp serve(socket, parent, opts, transport) do + with {:ok, request} <- recv_until(transport, socket, "\r\n\r\n", <<>>), {:ok, key} <- request_header(request, "sec-websocket-key") do - protocol = Keyword.get(opts, :protocol) - :ok = :gen_tcp.send(socket, handshake_response(key, protocol)) + :ok = + transport.send(socket, [ + handshake_response(key, Keyword.get(opts, :protocol)), + Keyword.get(opts, :upgrade_frames, <<>>) + ]) + send(parent, {:websocket_server_handshake, request}) - maybe_send_open_message(socket, Keyword.get(opts, :open_message)) + if Keyword.get(opts, :close_after_upgrade, false) do + receive do + :close_tls -> transport.close(socket) + after + 5_000 -> transport.close(socket) + end + else + maybe_send_open_message(transport, socket, Keyword.get(opts, :open_message)) - case maybe_send_close(socket, Keyword.get(opts, :close_after_open)) do - :closed -> :ok - :open -> loop(socket, parent, <<>>) + case maybe_send_close(transport, socket, Keyword.get(opts, :close_after_open)) do + :closed -> :ok + :open -> loop(transport, socket, parent, <<>>) + end end else {:error, reason} -> send(parent, {:websocket_server_error, reason}) - :gen_tcp.close(socket) + transport.close(socket) end end - defp loop(socket, parent, buffer) do + defp loop(transport, socket, parent, buffer) do case take_client_frame(buffer) do {:ok, opcode, payload, rest} -> - handle_frame(socket, parent, opcode, payload) - loop(socket, parent, rest) + handle_frame(transport, socket, parent, opcode, payload) + loop(transport, socket, parent, rest) :more -> - case :gen_tcp.recv(socket, 0, 1_000) do - {:ok, data} -> - loop(socket, parent, buffer <> data) - - {:error, :closed} -> - send(parent, :websocket_server_closed) - - {:error, reason} -> - send(parent, {:websocket_server_error, reason}) + case transport.recv(socket, 0, 1_000) do + {:ok, data} -> loop(transport, socket, parent, buffer <> data) + {:error, :closed} -> send(parent, :websocket_server_closed) + {:error, reason} -> send(parent, {:websocket_server_error, reason}) end end end - defp handle_frame(socket, parent, 0x1, payload) do + defp handle_frame(transport, socket, parent, 0x1, payload) do send(parent, {:websocket_server_received, :text, payload}) - :ok = send_server_frame(socket, 0x1, "echo:" <> payload) + :ok = send_server_frame(transport, socket, 0x1, "echo:" <> payload) end - defp handle_frame(_socket, parent, 0x2, payload) do - send(parent, {:websocket_server_received, :binary, payload}) - end + defp handle_frame(_transport, _socket, parent, 0x2, payload), + do: send(parent, {:websocket_server_received, :binary, payload}) - defp handle_frame(socket, parent, 0x8, payload) do + defp handle_frame(transport, socket, parent, 0x8, payload) do send(parent, {:websocket_server_received, :close, payload}) - :ok = send_server_frame(socket, 0x8, payload) - :gen_tcp.close(socket) + :ok = send_server_frame(transport, socket, 0x8, payload) + transport.close(socket) end - defp handle_frame(_socket, parent, opcode, payload) do - send(parent, {:websocket_server_received, opcode, payload}) - end + defp handle_frame(_transport, _socket, parent, opcode, payload), + do: send(parent, {:websocket_server_received, opcode, payload}) - defp maybe_send_open_message(_socket, nil), do: :ok + defp maybe_send_open_message(_transport, _socket, nil), do: :ok - defp maybe_send_open_message(socket, {:binary, payload}) do - send_server_frame(socket, 0x2, payload) - end + defp maybe_send_open_message(transport, socket, {:binary, payload}), + do: send_server_frame(transport, socket, 0x2, payload) - defp maybe_send_open_message(socket, message) when is_binary(message) do - send_server_frame(socket, 0x1, message) - end + defp maybe_send_open_message(transport, socket, message) when is_binary(message), + do: send_server_frame(transport, socket, 0x1, message) - defp maybe_send_close(_socket, nil), do: :open + defp maybe_send_close(_transport, _socket, nil), do: :open - defp maybe_send_close(socket, {code, reason}) do - :ok = send_server_frame(socket, 0x8, <>) - :gen_tcp.close(socket) + defp maybe_send_close(transport, socket, {code, reason}) do + :ok = send_server_frame(transport, socket, 0x8, <>) + transport.close(socket) :closed end - defp recv_until(socket, marker, buffer) do + defp recv_until(transport, socket, marker, buffer) do if :binary.match(buffer, marker) == :nomatch do - with {:ok, data} <- :gen_tcp.recv(socket, 0, 1_000) do - recv_until(socket, marker, buffer <> data) - end + with {:ok, data} <- transport.recv(socket, 0, 1_000), + do: recv_until(transport, socket, marker, buffer <> data) else {:ok, buffer} end @@ -121,11 +148,8 @@ defmodule HTTPWebSocket.TestServer do |> String.split("\r\n") |> Enum.find_value(fn line -> case String.split(line, ":", parts: 2) do - [name, value] -> - if String.downcase(name) == wanted_name, do: {:ok, String.trim(value)} - - _ -> - nil + [name, value] -> if String.downcase(name) == wanted_name, do: {:ok, String.trim(value)} + _ -> nil end end) |> case do @@ -134,33 +158,16 @@ defmodule HTTPWebSocket.TestServer do end end - defp handshake_response(key, nil) do - [ - "HTTP/1.1 101 Switching Protocols\r\n", - "Upgrade: websocket\r\n", - "Connection: Upgrade\r\n", - "Sec-WebSocket-Accept: ", - accept_key(key), - "\r\n\r\n" - ] - end - defp handshake_response(key, protocol) do [ - "HTTP/1.1 101 Switching Protocols\r\n", - "Upgrade: websocket\r\n", - "Connection: Upgrade\r\n", - "Sec-WebSocket-Accept: ", + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ", accept_key(key), - "\r\n", - "Sec-WebSocket-Protocol: ", - protocol, + if(protocol, do: ["\r\nSec-WebSocket-Protocol: ", protocol], else: []), "\r\n\r\n" ] end defp accept_key(key), do: :crypto.hash(:sha, key <> @guid) |> Base.encode64() - defp take_client_frame(buffer) when byte_size(buffer) < 6, do: :more defp take_client_frame(<>) do @@ -183,7 +190,6 @@ defmodule HTTPWebSocket.TestServer do defp take_length(126, _rest), do: :more defp take_length(127, <>), do: {:ok, length, rest} defp take_length(127, _rest), do: :more - defp take_masked_payload(rest, length) when byte_size(rest) < 4 + length, do: :more defp take_masked_payload(<>, length) do @@ -191,21 +197,17 @@ defmodule HTTPWebSocket.TestServer do {:ok, mask_key, payload, remaining} end - defp send_server_frame(socket, opcode, payload) do - :gen_tcp.send(socket, server_frame(opcode, payload)) - end + defp send_server_frame(transport, socket, opcode, payload), + do: transport.send(socket, server_frame(opcode, payload)) - defp server_frame(opcode, payload) when byte_size(payload) <= 125 do - <<0x80 ||| opcode, byte_size(payload), payload::binary>> - end + defp server_frame(opcode, payload) when byte_size(payload) <= 125, + do: <<0x80 ||| opcode, byte_size(payload), payload::binary>> defp unmask(payload, <>) do payload |> :binary.bin_to_list() |> Enum.with_index() - |> Enum.map(fn {byte, index} -> - bxor(byte, Enum.at([a, b, c, d], rem(index, 4))) - end) + |> Enum.map(fn {byte, index} -> bxor(byte, Enum.at([a, b, c, d], rem(index, 4))) end) |> :binary.list_to_bin() end end diff --git a/apps/http_web_transport/lib/http/web_transport.ex b/apps/http_web_transport/lib/http/web_transport.ex index ce3b9bd..45125a3 100644 --- a/apps/http_web_transport/lib/http/web_transport.ex +++ b/apps/http_web_transport/lib/http/web_transport.ex @@ -5,6 +5,10 @@ defmodule HTTP.WebTransport do This module implements the public API shape and lifecycle management for WebTransport sessions. The default backend speaks WebTransport extended CONNECT over HTTP/3 using the shared QUIC transport in `:http_core`. + + QUIC uses its own TLS implementation. The shared `:http_core, :tls_backend` + configuration does not apply here; an explicit non-nil `tls_backend` option + returns `{:error, :tls_backend_not_supported_for_quic}`. """ alias HTTP.WebTransport.CloseInfo diff --git a/apps/http_web_transport/lib/http/web_transport/options.ex b/apps/http_web_transport/lib/http/web_transport/options.ex index 44b4beb..3b73688 100644 --- a/apps/http_web_transport/lib/http/web_transport/options.ex +++ b/apps/http_web_transport/lib/http/web_transport/options.ex @@ -37,7 +37,9 @@ defmodule HTTP.WebTransport.Options do "server_certificate_hashes" => :server_certificate_hashes, "socketOpts" => :socket_opts, "socket_opts" => :socket_opts, - "ssl" => :ssl + "ssl" => :ssl, + "tlsBackend" => :tls_backend, + "tls_backend" => :tls_backend } defstruct uri: nil, @@ -149,7 +151,8 @@ defmodule HTTP.WebTransport.Options do end defp normalize_init(init) when is_list(init) do - with {:ok, headers} <- normalize_headers(Keyword.get(init, :headers, [])), + with :ok <- validate_tls_backend(Keyword.get(init, :tls_backend)), + {:ok, headers} <- normalize_headers(Keyword.get(init, :headers, [])), {:ok, owner} <- normalize_owner(Keyword.get(init, :owner, self())), {:ok, allow_pooling} <- normalize_boolean(Keyword.get(init, :allow_pooling, false), :invalid_allow_pooling), @@ -230,6 +233,9 @@ defmodule HTTP.WebTransport.Options do defp normalize_init(_init), do: {:error, :invalid_options} + defp validate_tls_backend(nil), do: :ok + defp validate_tls_backend(_backend), do: {:error, :tls_backend_not_supported_for_quic} + defp default_backend, do: HTTP.WebTransport.Transport.QUIC defp normalize_key(key) when is_binary(key), do: Map.get(@string_keys, key, key) diff --git a/apps/http_web_transport/test/http/web_transport/options_test.exs b/apps/http_web_transport/test/http/web_transport/options_test.exs index dfc87b5..b90cea5 100644 --- a/apps/http_web_transport/test/http/web_transport/options_test.exs +++ b/apps/http_web_transport/test/http/web_transport/options_test.exs @@ -78,4 +78,19 @@ defmodule HTTP.WebTransport.OptionsTest do server_certificate_hashes: [%{algorithm: "sha-256", value: <<0>>}] ) end + + test "rejects explicit TLS backends because WebTransport uses QUIC TLS" do + for backend <- [:ssl, :ex_ssl, "ssl", "ex_ssl", false] do + for options <- [ + [tls_backend: backend], + %{"tls_backend" => backend}, + %{"tlsBackend" => backend} + ] do + assert {:error, :tls_backend_not_supported_for_quic} = + Options.new("https://example.com/transport", options) + end + end + + assert {:ok, _} = Options.new("https://example.com/transport", tls_backend: nil) + end end diff --git a/apps/http_web_transport/test/http/web_transport/tls_config_test.exs b/apps/http_web_transport/test/http/web_transport/tls_config_test.exs new file mode 100644 index 0000000..94008c5 --- /dev/null +++ b/apps/http_web_transport/test/http/web_transport/tls_config_test.exs @@ -0,0 +1,24 @@ +defmodule HTTP.WebTransport.TLSConfigTest do + use ExUnit.Case, async: false + + alias HTTP.WebTransport.Options + alias HTTP.WebTransport.Transport.QUIC + + test "shared TCP TLS configuration does not affect QUIC options" do + previous = Application.fetch_env(:http_core, :tls_backend) + + on_exit(fn -> + case previous do + {:ok, value} -> Application.put_env(:http_core, :tls_backend, value) + :error -> Application.delete_env(:http_core, :tls_backend) + end + end) + + for backend <- [:ex_ssl, :invalid] do + Application.put_env(:http_core, :tls_backend, backend) + + assert {:ok, %{backend: QUIC}} = + Options.new("https://example.com/transport") + end + end +end diff --git a/docs/ex-ssl-consumer-contract.md b/docs/ex-ssl-consumer-contract.md new file mode 100644 index 0000000..9804b00 --- /dev/null +++ b/docs/ex-ssl-consumer-contract.md @@ -0,0 +1,169 @@ +# TCP TLS consumer contract + +Audited against PR #14 baseline `690258ac38e50b0d1a968d9d5e510c560f45f5d4` +and released ex_ssl 0.4.0 on 2026-09-22. OTP `:ssl` stays the default; ex_ssl +is explicitly selected. This inventory does not claim full OTP compatibility. + +| Requirement | Status | Production boundary | Executable coverage | +| --- | --- | --- | --- | +| Connect, ownership, send, passive recv, active-once, ALPN, close | implemented | `http_core` transport behaviour and SSL/ExSSL adapters | `apps/http_core/test/http/tls_transport_test.exs` | +| Shared default and explicit per-call backend | implemented | `HTTP.TLSBackend`, fetch/WSS/EventSource option normalization | `apps/http_core/test/http/tls_backend_test.exs` and each client's option tests | +| Verified HTTP/1.1 fixed/chunked/close-delimited responses, large uploads, cancellation/deadline | implemented | `HTTP.SocketClient` | `apps/http_fetch/test/http/ssl_transport_test.exs` | +| Verified ALPN HTTP/2, streaming/flow control, early response with pending upload | implemented | shared HTTP/2 parser and socket owner | `apps/http_core/test/http/http2_test.exs`, `apps/http_fetch/test/http/socket_client_http2_test.exs` | +| Authenticated close during optional control writes | implemented | ex_ssl-only `:closed` classification, existing receive loop and deadline | deterministic cross-record tests in `socket_client_http2_test.exs` | +| HTTP/2 Content-Length completion and input bounds | implemented in this continuation | counted unpadded DATA, u64 decimal length, 16 KiB frames, 64 KiB compressed header blocks | core HTTP/2 and limits tests; real TLS cross-record buffered/streamed mismatch regressions | +| Redirect backend pinning | implemented | backend resolved before redirect lifecycle | `ssl_transport_test.exs` | +| Passive WSS Upgrade followed by active-once frames | implemented | `HTTP.WebSocket.Connection`, shared transport recv | `apps/http_web_socket/test/http/web_socket_test.exs`, `web_socket_tls_lifecycle_test.exs` | +| EventSource reconnect with pinned backend | implemented | `HTTP.EventSource.Connection` | `apps/http_event_source/test/http/event_source_test.exs` | +| Wrong CA/reference hostname and profile/ALPN conflicts | implemented | adapter forwards to verified ex_ssl options | `tls_transport_test.exs` (adapter-level evidence) | +| No automatic backend fallback | implemented | fixed adapter, explicit option/handshake errors | `tls_backend_test.exs`, TLS-1.2-only peer negative in `tls_transport_test.exs` | +| `socket_opts: [send_timeout: ..., send_timeout_close: true]` | implemented | ExSSL translates these two options; socket values override matching `ssl` values | `tls_transport_test.exs` | +| Safe TCP options, client certificates, TLS 1.2/mixed versions, ordered TLS policy | implemented in ex_ssl 0.4.0 | allowlisted adapter options and independent ex_ssl engine | packaged-source 47-test gate, published-dependency smoke, and scoped transport tests | +| Arbitrary TCP options, verify-none, early data, TLS 1.2 resumption | unsupported | explicit option/protocol errors | negative option and authentication tests | +| Introspection beyond negotiated ALPN, active-N, packet modes | not required by audited consumer | no production callsites | library roadmap; not a Phase 0 blocker | +| HTTP/3 and WebTransport | separate QUIC implementation | explicit TCP backend rejected; shared default ignored | fetch SSL transport tests and WebTransport option/TLS config tests | + +Only `HTTP.Transport.SSL` calls OTP `:ssl` in production. TLS listen, accept, +handshake, and peer traffic in test support are reference-server operations. +QUIC calls in HTTP/3/WebTransport are outside this TCP TLS contract. + +Public `ssl:` and `socket_opts:` containers remain backend-specific. The OTP +adapter preserves its existing option behavior. The ex_ssl adapter validates +keyword shape and duplicates and rejects unsupported socket options. No option +is dropped to make a connection succeed. + +The cross-record fixture suspends the HTTP owner after it enters its receive +loop, proves its sole first plaintext delivery, releases later peer output, and +checks authenticated closure with retained plaintext before resuming. Its private +ex_ssl state probe is test-only, guarded to version 0.4.0, and checks exact byte +count plus passive mode. Larger streaming responses drain incrementally through +flow control before the final gated records; they do not require an oversized +passive TLS buffer. + +Current validation is tracked in the ex_ssl repository's +`docs/EX_SSL_HTTP_FETCH_PROGRESS.md`. Historical PR validation remains in +[pr-14-validation.md](pr-14-validation.md). Commands must run from the umbrella +root. Adapter-level security tests do not establish per-client-family coverage +of every invalid option, and local OTP peers do not prove every server or runtime. + +## Phase 1: algorithms in ex_ssl 0.4.0 + +ex_ssl 0.4.0 adds P-384 ECDHE/ECDSA, Ed25519 and RSA-PSS-PSS SHA-256/384/512. +The source smoke below records the pre-release cross-repository validation; +the current consumer dependency resolves the published 0.4.0 package. + +Run `EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash scripts/ex_ssl_source_smoke.sh` +from the umbrella root. This builds all five fresh package artifacts into a +temporary consumer and explicitly overrides ex_ssl there; repository manifests, +lockfiles and installed sources are unchanged. It is separate from the existing +five-package released-dependency smoke, which has no ex_ssl override. +Set `EX_SSL_DEP_MODE=published` with the same fixture source directory to run +the 47-test feature gate against the Hex release rather than the source override. + +Seed 36 on OTP 28 / Elixir 1.18.5: 12 tests, zero failures (ten positive exchanges +cover each new signature over HTTP/1.1+P-384 HRR and HTTP/2+direct P-384; five +hostname-negative scenarios in one test; one OTP-default assertion). Both +transport modes retain peer verification. New fixture setup initially omitted +`http_version: :http2`, so five HTTP/2 cases failed; the corrected fixture sets +both HTTP mode and exact profile ALPN and waits for the SETTINGS acknowledgement. +No production workaround was added. Log: `/tmp/http-fetch-tls-plan-algorithms.log`. + + +## Phase 2: client identity in ex_ssl 0.4.0 + +ex_ssl 0.4.0 supports one bounded initial-handshake client identity +through `ssl: [certfile: ..., keyfile: ...]` or the documented in-memory forms. +It remains separate from the server's `cacerts`/`cacertfile` trust. Encrypted keys, +hardware signing and multiple identities remain unsupported. The library's +compatibility matrix documents CertificateRequest selection limits. + +For `:ex_ssl` with any configured `cert`, `certfile`, `key` or `keyfile`, an +automatic redirect that changes scheme, case-insensitive hostname or effective +port returns `{:error, :client_identity_cross_origin_redirect}` before opening +the next connection. Same-origin redirects retain the identity. Use +`redirect: :manual` and issue a separate, deliberate request if another origin +is authorized to receive those credentials. OTP backend behavior is unchanged. +This policy also rejects a downgrade to plain HTTP. HTTP/3/WebTransport remain +on QUIC and do not use these credentials. + +The source smoke includes required RSA/EC/large-chain HTTP/1.1 and HTTP/2 +requests, exact server-observed client DER, optional auth, missing/wrong-CA/ +expired/wrong-purpose/incompatible credentials, pre-I/O key mismatch and bad +server hostname. WSS verifies passive Upgrade, active-once frames and close; +EventSource verifies the identity across same-origin reconnects. Redirect tests +cover all three origin components, DNS casing, manual reuse and unchanged OTP. +These tests were first run against source and are now rerun against the published +0.4.0 dependency without modifying installed dependency sources. + +## Phase 3: options in ex_ssl 0.4.0 + +The adapter forwards the safe TCP allowlist: `nodelay`, `keepalive`, `sndbuf`, +`recbuf`, local `ip`/`port`, plus the existing send deadline options. ex_ssl 0.4.0 +validates values and supports mutable driver options. Keyword containers and duplicate +keys reject before fetch adds deadlines or ALPN, including improper lists. +Raw active/packet controls, linger and arbitrary socket backends remain rejected. + +ex_ssl 0.4.0 accepts ordered TLS 1.3 `ciphers`, `signature_algs`, +`signature_algs_cert` and `supported_groups` through `ssl`. Generated profiles +preserve order; explicit profile conflicts fail before I/O. The certificate +signature policy is separate from handshake CertificateVerify. No supplied +`signature_algs_cert` preserves the earlier chain policy. See the library matrix +for supported names/maps and deliberate differences from OTP. + +`scripts/ex_ssl_options_test.exs` exercises real packaged HTTP policy and TCP +selection, IPv6 local binding/DNS, raw option precedence/mutation, authentication +failures and pre-I/O rejection of malformed/unsafe/conflicting options. The same +shared adapter serves WSS and SSE. No TLS1.2 or default change is introduced here. + +## Phase 4: TLS 1.2 in ex_ssl 0.4.0 + +ex_ssl 0.4.0 adds verified TLS 1.2 ECDHE AES-GCM to the shared adapter. The +independent OpenSSL packaged gate covers HTTP/1.1 and HTTP/2 +with both TLS 1.2-only and mixed offers, mixed-offer TLS 1.3 selection, WSS +Upgrade/frame/close, and EventSource reconnect with pinned backend and +Last-Event-ID. All seven scenarios pass in the final 47-test source gate. +The expanded HTTP/2 fixture sends 262,144 bytes, honors connection/stream +flow control, and requires observed WINDOW_UPDATE frames; its seven-test gate +also passes. TLS 1.2 session resumption and full OTP option parity are not claimed. + +## Phase 5: opt-in TLS 1.3 resumption in ex_ssl 0.4.0 + +`ssl: [versions: [:"tlsv1.3"], session_tickets: :auto]` enables bounded TLS 1.3 +ticket reuse for a fresh connection to the same authenticated context. The +default is `:disabled`; auto rejects TLS 1.2/mixed offers and configured client +identity. Early data, persistent tickets, PSK-only key exchange, and automatic +reconnect/replay are unsupported. An unaccepted ticket follows normal full +handshake processing on the same socket. + +`scripts/ex_ssl_resumption_test.exs` uses two packaged HTTP/1.1 fetches against +one Python/OpenSSL context and checks the peer's `session_reused` value is false +then true. Cache policy isolation and measured performance are +recorded in the library readiness report. This consumer check proves only the +HTTP/1.1 adapter path; HTTP/2/WSS/SSE resumption is not separately verified. + +## Published 0.4.0 validation (2026-09-22) + +The umbrella lock resolves Hex ex_ssl 0.4.0. The test-only cross-record buffer +probe was revalidated against its `closed`, `size`, and `active` fields and its +version guard advanced from 0.3.0 to 0.4.0. `MIX_ENV=test mix test +apps/http_fetch/test/http/socket_client_http2_test.exs --only cross_record` +passed 11 tests with 23 excluded. The full `MIX_ENV=test mix test` passed 185 +`http_core`, 176 `http_fetch` plus 20 doctests, 33 WebSocket, 24 WebTransport, +and 26 EventSource tests, all with zero failures. `MIX_ENV=test mix compile +--warnings-as-errors`, `mix format --check-formatted`, `mix credo` (116 files, +no issues), and `mix dialyzer` (four existing ignored warnings, zero unnecessary +skips) passed. + +The fresh five-package `bash scripts/external_consumer_smoke.sh` passed with a +transitive Hex ex_ssl 0.4.0 dependency and no source override. The broader +`EX_SSL_DEP_MODE=published EX_SSL_SOURCE_DIR=/absolute/path/to/ex_ssl bash +scripts/ex_ssl_source_smoke.sh` resolved ex_ssl from Hex and passed 47 tests; +the source directory supplies only test fixture builders in that mode. Neither +smoke modifies installed dependency sources. On the first published run, the +unit suite had two failures from obsolete 0.3.0 negative expectations for +TLS 1.2 and `nodelay: true`; the first external smoke failed its obsolete +`~> 0.3.0` metadata assertion. Those assertions were updated to test supported +and still-rejected values before the passing reruns. An earlier source-mode +47-test run had one intermittent large HTTP/2 `:econnreset`; the independent +test peer now holds its close until the client reads the complete response. +The final source-mode and published-mode 47-test runs each had zero failures. diff --git a/docs/pr-14-validation.md b/docs/pr-14-validation.md new file mode 100644 index 0000000..2a3db5d --- /dev/null +++ b/docs/pr-14-validation.md @@ -0,0 +1,577 @@ +# PR #14 validation + +> Historical PR #14 validation at ex_ssl 0.3.0. For the released ex_ssl 0.4.0 +> dependency and current consumer limits, see +> [the TCP TLS consumer contract](ex-ssl-consumer-contract.md). + +## Phase 0 continuation (2026-09-22) + +Current PR head remains `690258ac38e50b0d1a968d9d5e510c560f45f5d4`. +The new work is isolated on `codex/tls-backend-plan`. Baseline rerun passed +422 tests plus 20 doctests on Elixir 1.18.5 / OTP 28 with seed 22. +The cross-record closure and early-response fixes are preserved. + +The Phase 0 audit found a separate response-completion gap: Content-Length was +not checked against HTTP/2 DATA. Pure regressions initially failed in four +cases. The change counts unpadded DATA, validates all completion paths and +rejects malformed or conflicting lengths. Real TLS regressions use the existing +owner/peer barriers to cover short buffered, oversized buffered, and short +streamed bodies after authenticated close. The private ex_ssl state probe now +checks exact retained bytes and explicitly guards the tested 0.3.0 version. + +Final root-scoped suites at seed 25 passed **442 tests plus 20 doctests**, +zero failures, no exclusions. Core HTTP/2 plus bounds tests passed 42 tests, +and the HTTP/2 consumer suite passed 34 tests (seed 24). These include the +original deterministic closure/early-response tests. Formatting, dev/test +warnings-as-errors compilation and Credo passed. + +Current execution commands, outcomes and remaining phases are tracked in the +ex_ssl companion worktree at `docs/EX_SSL_HTTP_FETCH_PROGRESS.md`; historical +results below are not results for this continuation. + +## Early final responses (baseline a1312cc) + +The current round started at `a1312cc40c8d6aad2cb60e750bfba84f9b3ac1cf`, +which was also the remote PR HEAD; the working tree was clean. The previous +cross-record drain remains in place. This round corrects its overly broad +assumption that an unfinished request upload invalidates a completed response. +The earlier records below are historical; their blanket pending-upload and +NO_ERROR-reset failure rules are superseded by this section. + +### Protocol basis and root cause + +[RFC 9113 ยง8.1](https://www.rfc-editor.org/rfc/rfc9113.html#section-8.1) defines +response completion independently of request transmission. A server may finish +its response before receiving the entire request, then use RST_STREAM(NO_ERROR) +to stop the upload without invalidating that response. Completion requires +END_STREAM and completion of any associated HEADERS/CONTINUATION field block. +A final status, Content-Length, or body-forbidden response alone is insufficient. + +Three decisions had been conflated: + +1. `outbound_control_only?/1` classified the queued frames **and** required an + empty `pending_body`. This prevented normal-close draining and discarded + complete early responses even when only an ACK needed writing. +2. Parsing WINDOW_UPDATE could queue more request DATA before the final response + was recognized, and response completion did not remove that obsolete upload. +3. Every target-stream RST_STREAM was returned as an error, including NO_ERROR + after a complete response. Body-forbidden responses were also incorrectly + considered complete without END_STREAM. + +The implementation separates frame classification, stopped request transmission +and response completion. Valid completion stops the remaining upload and removes +queued request DATA while preserving control frames. A normal ex_ssl close +reported by an optional control write stops transmission without marking the +response complete; the original active-once/deadline loop drains and validates +what remains. Subsequent buffered WINDOW_UPDATE cannot restart that upload. +Actual request DATA write failures before this transition remain failures. +NO_ERROR reset is harmless only after verified response completion; CANCEL, +protocol/TLS errors, truncation, cancellation and deadline retain their errors. + +### Initial baseline evidence + +A fresh detached worktree at `.trees/r3-baseline` was prepared with independent +`deps` and `_build` using `MIX_ENV=test mix deps.get` and +`MIX_ENV=test mix compile --warnings-as-errors`. Only the regression test file +was copied in; production code remained at a1312cc. + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --only early_response --seed 0 +# Initial baseline: 25 discovered, 1 executed, 1 failure, 24 excluded. +``` + +The client POSTs 65,535 + 5 bytes. The server reads only the initial window, +then waits for the test to suspend the HTTP owner before sending a valid 413 +response with exact body `payload-too-large` and END_STREAM. The test confirms +normal TLS closure and the queued response before resuming the owner. +`HTTP.Promise.await` actually returns `{:error, :closed}`, so the expected 413 +response assertion fails. Log: `/tmp/http_fetch-pr14-r3/a1312cc-first-red.log`. + +### Corrected expectations and additional baseline evidence + +The old test named `fails when SETTINGS acknowledgement cannot flush a pending +HTTP/2 request body` is corrected to expect a 413 and exact response body after +the same deterministic peer-close barrier. The completed branch of the existing +WINDOW_UPDATE/upload test is likewise corrected to preserve the response and +discard unsent DATA. Its incomplete branch retains the actual DATA-write failure +expectation; NO_ERROR-reset rejection is separate, so a parser reset cannot hide +loss of required-write coverage. + +The queue-classification test now permits an ACK-only queue even when +`pending_body` is nonempty. Separate tests verify explicit upload stopping, +retention of non-upload frames, continued ordinary upload flow control, and no +false response completion. The old bodyless HEADERS/CONTINUATION fixture is made +protocol-valid by carrying END_STREAM on HEADERS and waiting for END_HEADERS on +CONTINUATION; the first fragment alone neither completes the response nor stops +the pending upload. + +The final core test file was copied into the unchanged a1312cc worktree and run +with these selections: + +```bash +MIX_ENV=test mix test apps/http_core/test/http/http2_test.exs:235 apps/http_core/test/http/http2_test.exs:288 apps/http_core/test/http/http2_test.exs:372 --seed 0 +# 25 discovered, 3 executed, 3 failures, 22 excluded. +``` + +These are actual behavior failures, not missing-new-API errors: premature `:done` +for 204 without END_STREAM; request DATA still queued after a complete response; +and `{:stream_reset, :no_error}` after a complete response. Log: +`/tmp/http_fetch-pr14-r3/a1312cc-core-final-red.log`. The fixed full core HTTP/2 +file passes all 25 tests. Same-batch and subsequent-call NO_ERROR resets are +covered, including repeated reset with no duplicate events, and reset before a +complete response remains an error. + +### Final integration regression evidence + +The final integration file was copied unchanged into the a1312cc worktree: + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --only early_response --seed 0 +# Baseline: 31 discovered, 9 executed, 6 failures, 22 excluded. +``` + +Five failures returned `{:error, :closed}` instead of the complete response: +413 with body, bodyless 413, cross-record 413, a completed response followed by +a buffered NO_ERROR reset, and WINDOW_UPDATE plus a completed response. The +sixth returned `{:error, {:stream_reset, :no_error}}` for a complete response +and reset parsed in the same batch. The three passing controls were OTP's early +response, incomplete-response NO_ERROR reset, and the actual required DATA-write +failure. Log: `/tmp/http_fetch-pr14-r3/a1312cc-final-integration-red.log`. + +All early-response fixtures use a 65,540-byte POST, with the peer reading the +initial 65,535-byte window. The simple 413 and bodyless variants provide no +additional upload credit. The cross-record variant adds WINDOW_UPDATE in its +buffered second batch to verify that draining cannot restart a stopped upload. +Message gates prove that the first plaintext batch is the paused owner's sole +TLS message before releasing the second write. A bounded probe then verifies +normal peer closure, inactive delivery, and the exact remaining plaintext buffer +size. Only these flags and sizes are observed; no TLS keys or state dumps are +printed. The owner resumes while the TLS receive buffer still exists. + +The separate-record NO_ERROR integration test permits the owner to finish after +the complete first batch and release the buffered reset during cleanup. Protocol +unit tests additionally parse the reset in a subsequent call and prove that it +produces no duplicate events. Same-batch integration exercises actual reset +parsing. NO_ERROR before completion, CANCEL, truncation, fragmented frames, +required writes, streaming backpressure, cancellation and deadline keep their +negative coverage. Owner and TLS process monitors verify release in the gated +close tests. Existing cross-record/frame-fragment and streaming tests are kept. + +The complete final integration file passed seeds 1, 2, 3, 4 and 5: **31 tests, +0 failures per run**, no skips or exclusions. The core HTTP/2 file passed seeds +101, 202, 303, 404 and 505: **25 tests, 0 failures per run**. These repetitions +check stability; the record/buffer barriers establish the timing itself. + +### Current local acceptance results + +Executed on Elixir **1.18.5**, Erlang/OTP **28** (ERTS 16.4.0.5), Go **1.26.6**. +All commands below ran from the umbrella root, except the explicitly noted Go +build and isolated baseline reproduction. Logs and exit codes are in +`/tmp/http_fetch-pr14-r3/final-*.log` and `final-results.json`. + +| Actual command | Final result | +| --- | --- | +| `mix deps.get`; `MIX_ENV=test mix deps.get` | Both passed; lockfile unchanged | +| `mix format --check-formatted` | Passed | +| `mix compile --warnings-as-errors` | Passed | +| `MIX_ENV=test mix compile --warnings-as-errors` | Passed | +| `mix test` | **422 tests + 20 doctests, 0 failures, 0 skipped** | +| `MIX_ENV=test mix test apps/http_core/test` | 168 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test` | 171 tests + 20 doctests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/test` | 33 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/test` | 26 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/test` | 24 tests, 0 failures | +| `mix test apps/http_fetch/test/http/socket_client_http2_test.exs --seed N` for N=1..5 | 31 tests/run, 0 failures | +| `mix test apps/http_core/test/http/http2_test.exs --seed N` for N=101,202,303,404,505 | 25 tests/run, 0 failures | +| `mix credo` | Passed, no issues | +| `mix dialyzer --format github` | Passed; 4 existing ignored diagnostics, 0 new diagnostics | +| `bash scripts/external_consumer_smoke.sh` | Passed: five packages built from current source, metadata checked, isolated dependency resolution/compilation/startup and eight local TLS exchanges | +| `MIX_ENV=test mix test apps/http_fetch/e2e` | 50 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/e2e` | 3 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/e2e` | 2 tests, 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/e2e` | 3 tests, 0 failures | +| `MIX_ENV=test mix test.e2e` | Same 58 E2E tests, 0 failures | + +The Go fixture was rebuilt with `go build -o ../test_server/server .` from +`apps/http_fetch/priv/test_server`, started before E2E, and stopped afterward. +`E2E_BASE_URL` pointed at its reported local port. `http_core` has no E2E suite; +its root-scoped unit suite covers that workflow matrix entry. All final selected +app and E2E runs had zero skipped/excluded tests. Baseline line/tag exclusions +above are deliberate selection, not skipped failing tests. + +Cold compilation was exercised in the independent a1312cc worktree; the final +source was also cold-compiled by the external consumer with its own `deps`, +`_build`, and newly resolved lockfile. The consumer obtains ex_ssl transitively +from the built http_core package, with no manually added ex_ssl dependency. +WebTransport retains its existing QUIC boundary check; it is not a TCP TLS test. + +No requested local checks remain unexecuted. These results are local, not proof +of remote CI on a future commit; new remote results belong to their exact SHA. +Known limits remain: tests probe private ex_ssl 0.3.0 buffer flags only for +synchronization; production uses public transport calls. OTP's previously +recorded immediate-close `:einval` is still an error, not a new exception. The +OTP early-response test keeps the peer open for control acknowledgements; the +ex_ssl regressions prove close-before-control-write behavior. No backend +fallback, POST retry, certificate-policy relaxation, new timeout, dependency +patch, or dependency upgrade was introduced. + +## Historical: cross-record drain at a1312cc + +Reviewed baseline and remote PR HEAD: `010b0b850745b43faab73093849d3fb6a1dc6650`. +The branch had no later commits when this follow-up started. The earlier fixes +below remain intact. Validation uses Elixir 1.18.5 / OTP 28, as required by CI. + +### Reproduction and cause + +The new test runs through `HTTP.fetch` with a trusted local TLS 1.3 server and +locked ex_ssl 0.3.0. It suspends the HTTP owner only after it reaches its receive +loop. The server sends SETTINGS/HEADERS and waits at a second message gate. +The test verifies that exactly this plaintext is the owner's sole TLS data +message, consuming `active: :once`, before releasing the second server write. +The server then sends DATA/END_STREAM and closes TLS without waiting for an ACK. +A bounded test-only probe checks only `closed`, `active`, and plaintext buffer +size in the TLS process: closed is true, active is false, and the second batch +is still buffered. The owner still has only the first data message. TLS process +exit and `ssl_closed` are deliberately **not** prerequisites for resuming it. + +The test was copied into `.trees/r2-baseline`, a detached checkout at exactly +`010b0b8` with independently downloaded dependencies and compiled test output. +Production files there were unchanged. This actual command failed: + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs:264 --seed 0 +# 18 tests discovered, 1 failure, 17 excluded by line selection +# HTTP.Promise.await returned {:error, :closed}, not the expected response. +``` + +With the fix, the same initial test and command passed: 18 discovered, 0 failures, +17 excluded. A separate control-classification regression, copied unchanged into +the same baseline and selected with `apps/http_core/test/http/http2_test.exs:15`, +also failed there (19 discovered, 1 failure, 18 excluded): an unsent `pending_body` +was previously classified as optional controls. The fixed full parser file passed +all 19 tests. Evidence: `/tmp/http_fetch-pr14-r2/010b0b8-cross-record-red.log` +and `cross-record-first-green.log`. Later coverage is listed separately below. + +The first batch creates a SETTINGS ACK but no `:done`. After peer close_notify, +`SSL.send` returns `:closed`; `send_request/4 -> close_on_error/3` then explicitly +closed the TLS socket before the caller could classify the error. That destroyed +the still-unconsumed second record. The previous complete-batch exception could +not help: the HTTP parser had not yet seen END_STREAM. + +Read-only inspection of the locked dependency confirms its contract: +`active: :once` consumes one delivery; peer close_notify rejects new writes but +retains plaintext until subsequent `setopts(active: :once)` or `recv` drains it. +A raw TCP close without close_notify is `:econnreset`; fatal TLS alerts remain +TLS errors. Explicit `SSL.close` terminates the buffered receive side. The +public API has no connection-state query, and production uses no private TLS +state. Test probes never print TLS keys or complete internal state. No dependency +source or lockfile was changed; this is an http_fetch integration bug. + +### Fix and safety boundaries + +Send results now reach the caller without implicit socket destruction. Initial +request/upload failures still return to the owner's cleanup; failed required +protocol writes still use `fail -> finish -> cleanup`. Cancellation, deadlines, +worker-start failure and send timeout retain their termination paths. + +Before removing outbound frames, the client checks the actual queue for only +WINDOW_UPDATE, SETTINGS ACK or PING ACK, **and** checks that `pending_body` is +empty. Thus both already-queued request DATA and uploads waiting for more window +credit prevent the exception. Only ex_ssl's established, exclusively owned +socket can use `:closed` to continue an incomplete response: this owner has not +locally closed it, and that backend distinguishes abnormal closure. Other +transports retain the previous completed-response/current-`:done` restriction. + +An optional-control `:closed` clears that attempted queue once and runs the +existing event handlers and active-once rearm. It does not report success. +Each further attempt requires newly received data; there is no retry loop or +new timer. Stream chunk acknowledgements retain backpressure and the original +absolute request deadline remains in force. EOF without END_STREAM fails through +`HTTP2.close/1`; resets and protocol errors still fail in the parser. No `:einval`, +`:econnreset`, TLS error, required write error, cancellation or timeout is ignored. + +### Final regression coverage and baseline comparison + +Eight new integration tests use the `:cross_record` tag. They cover: + +- Small buffered response with SETTINGS/HEADERS in the first delivery and + DATA/END_STREAM still inside ex_ssl after peer close. +- A HEADERS frame header split across deliveries while SETTINGS ACK is pending. +- The same receive-buffer sequence without END_STREAM: still `:closed`. +- DATA/END_STREAM followed by RST_STREAM in the later buffered batch: still + `{:stream_reset, :cancel}`, never a successful response. +- A complete response with only optional frames queued but an upload still + waiting for window credit: still fails. Its fixture now proves closure before + resuming the owner; an initial version exposed a fixture race in seed 202 and + was corrected rather than weakening the expected error. +- A 5,000,001-byte stream whose final three DATA frames have not been consumed + when the owner pauses. PING plus part of a DATA payload is the first delivery; + the rest and two further DATA frames remain in TLS buffers before resume. + Exact body equality and normal stream/TLS/owner exits are checked. +- Cancellation while the drained final chunk is held by a reader acknowledgement + barrier and the owner is demonstrably waiting in `HTTP.Stream.chunk/3`. +- The original deadline under the same backpressure. A timer gate consumes half + a 2-second request budget before entering drain; owner termination must occur + by the original monotonic deadline plus 400ms, not a restarted deadline. + +Server gates and state probes have timeouts. Failure cleanup resumes/aborts the +owner and releases test stream/reader processes. Successful and error scenarios +monitor the owner and TLS process; streaming cases also monitor the stream. +The existing complete-batch regressions and required-upload DATA failures with +and without END_STREAM remain in the file, alongside both TLS backends. + +After completing the tests, the final test file was copied back into the same +`010b0b8` checkout, still without any production changes: + +```bash +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --only cross_record --seed 1 +# Baseline: 25 discovered, 8 executed, 7 failures, 17 excluded. +# The truncated-response rejection passes; the other seven regressions fail. +``` + +The full log is `010b0b8-final-cross-record-red.log`. The fixed tree passes all +8 selected tests with seeds 1โ€“5 (40 executions). The **entire** HTTP/2 file was +also run with seeds 1โ€“5, with no filtering: 25 tests per run, 125 executions, +0 failures, 0 skipped or excluded. Each full result is in `http2-seed-N.log`. + +### Follow-up local validation results + +All commands below ran from the umbrella root unless stated. These results are +from the follow-up working tree, not the previous commit's CI. Unit and E2E +suites report 0 failures and 0 skipped tests. Scoped suites repeat root coverage. + +| Command | Actual result | +| --- | --- | +| `mix deps.get`; `MIX_ENV=test mix deps.get` | Passed; `mix.lock` unchanged | +| `mix format --check-formatted` | Passed | +| `mix compile --warnings-as-errors` | Passed | +| `MIX_ENV=test mix compile --warnings-as-errors` | Passed | +| `mix test` | 410 tests + 20 doctests; 0 failures | +| `MIX_ENV=test mix test apps/http_core/test` | 162 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test` | 165 tests + 20 doctests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/test` | 33 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/test` | 26 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/test` | 24 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs --seed N`, N=1..5 | Each: 25 tests, 0 failures | +| `mix credo` | No issues | +| `mix dialyzer --format github` | Passed; 4 existing intentional ignores, no new warnings | +| `bash scripts/external_consumer_smoke.sh` | Passed; five newly built packages, isolated dependency/build tree, eight trusted TLS exchanges and QUIC boundary checks | +| `go build -o ../test_server/server .` in `apps/http_fetch/priv/test_server` | Passed | +| `MIX_ENV=test mix test apps/http_fetch/e2e` | 50 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/e2e` | 2 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test.e2e` | 58 tests; 0 failures | + +For E2E, the built Go server was started, its reported port set in +`E2E_BASE_URL=http://127.0.0.1:`, and the server terminated afterward. +`http_core` has no E2E directory. The package smoke builds all five packages +with `mix hex.build` before starting a consumer that shares neither repository +`deps`/`_build` nor its lockfile and declares no extra ex_ssl dependency. +The detached baseline also started with independent empty build/dependency +folders; no user build files were deleted. + +No requested local check remains unexecuted. Evidence and the command/exit-code +manifest are under `/tmp/http_fetch-pr14-r2/` (`final-results.json`, `final-*.log`). +Remote checks for `010b0b8` are historical and are not follow-up validation. + +### Preserved limitations + +The pre-existing OTP immediate-close `:einval` limitation described below is +unchanged; no new error whitelist or TLS fallback was introduced. Existing +stream deadline expiry can report either `:request_timeout` (owner timer) or +`:timeout` (the chunk acknowledgement wait reaches the same absolute deadline). +The deadline regression accepts exactly these existing timeout errors, never +`:closed` or success, and does not normalize production error semantics. +The bounded test-only buffer probe intentionally targets the locked ex_ssl +0.3.0 state layout; production code uses only its public transport operations. + +## Earlier round (historical evidence at 010b0b8) + +Base reviewed and fetched: `f5c0fe60393bc8f0570b3f92e04fe45a0aa88ceb` +(`codex/replace-ssl-with-ex-ssl`). The remote PR had no later commits when work +started. Existing uncommitted fixes were preserved and completed. + +Environment: Elixir 1.18.5, Erlang/OTP 28 (ERTS 16.4.0.5), Linux; Go 1.26.6 +for the vendored E2E server (the workflow requires Go 1.22+). + +### Application build failure + +The failure is the child Mix dependency graph and code path, not a stale cache. +The child projects share the umbrella's `../../deps`, `../../_build`, and lock, +but an `in_umbrella` dependency is not recursively traversed in the same way +when running Mix from a child. `http_core` declares `ex_ssl`, while the four +client applications declare `http_core`; their child `mix deps` output lacks +`ex_ssl`. `http_core.app` still lists `ex_ssl` as a runtime application. + +In an isolated detached worktree at the reviewed HEAD, `http_core` child tests +pass but all four client child tests fail with: + +```text +** (Mix) Could not start application ex_ssl: +could not find application file: ex_ssl.app +``` + +Even after the root test-environment compile generates +`_build/test/lib/ex_ssl/ebin/ex_ssl.app`, child execution still fails. Thus +precompiling alone or clearing a cache does not fix the missing code path. +The original Test run 35490860631 and E2E run 35490860636 both show the same +failure, with undefined `SSL` module warnings during child compilation. + +Test and E2E workflows now explicitly prepare `MIX_ENV=test` at the root and +run `mix test apps//test` or `mix test apps//e2e` from the root. +The root `test.e2e` alias also stays at the root. No client duplicates the +`ex_ssl` dependency, disables runtime startup, or suppresses compiler warnings. + +### Dependency and transport contract + +`http_core` now requires `ex_ssl ~> 0.3.0`; the existing lock remains 0.3.0 and +no unrelated dependency was upgraded. The external smoke uses freshly built +package contents in a separate temporary consumer, without the umbrella lock, +deps or build output. Only unpublished internal package resolution uses local +paths; the consumer does not declare `ex_ssl` itself. + +OTP `:ssl` stays the default. `:ex_ssl` is explicitly selected or inherited +from `:http_core` configuration and remains captured across redirects and +EventSource reconnects. No backend fallback was added. The supported paths +are HTTP/1.1, HTTP/2, secure WebSocket and HTTPS EventSource. HTTP/3 and +WebTransport retain QUIC TLS and reject an explicit TCP TLS backend. + +The ex_ssl 0.3.0 integration still requires verified TLS 1.3. TLS 1.2, +`verify_none`, client certificates and unsupported socket options are rejected; +custom profile ALPN must match the requested ALPN list. + +### HTTP/2 response/close ordering + +The parser emits response events and queues receive-window updates for nonempty +DATA, including END_STREAM DATA. SocketClient previously flushed those writes +before delivering the events. After a normal TLS shutdown, the write returned +`:closed` and the client discarded the already-complete response. + +The change preserves that order. A failed write is non-fatal only when all four +conditions hold: HTTP/2 reports completion, this batch contains `:done`, all +queued frames are WINDOW_UPDATE / SETTINGS ACK / PING ACK, and the write error +is exactly `:closed`. Pending request DATA makes the failure fatal, including +when a coalesced WINDOW_UPDATE releases the final upload bytes. Reset and +protocol errors are returned by the parser before this exception is considered. +No `:einval`, `:econnreset`, certificate, cancellation or timeout errors are +ignored. Cleanup and stream delivery use the existing event handlers. + +The new real TLS 1.3/ex_ssl tests establish the following order: + +1. The local server waits on a message gate after receiving the request. +2. The test waits until the socket owner is waiting in `owner_loop/1`, then + suspends it; the separate TLS process keeps running. +3. The server sends complete response frames and successfully closes TLS. +4. The test observes both data and `ssl_closed` queued for the suspended owner + and monitors the TLS process exiting before resuming the owner. +5. The exact body is delivered, and the owner exits normally. For the streaming + case, a concurrent reader handles normal backpressure, the final DATA is + gated separately after earlier frames, and the stream also exits normally. + +The buffered case coalesces SETTINGS/HEADERS/DATA. The 5,000,001-byte streaming +case separates response headers and DATA; its server does not wait for the +final WINDOW_UPDATE before closing. Waiting predicates use bounded polling of +actual process/message state; elapsed sleeps do not establish the ordering. + +With these tests copied into the detached original-HEAD worktree, leaving both +production modules unchanged, the buffered test fails with `{:error, :closed}` +and the streaming test raises `stream read failed: :closed`. The final combined +line-selected baseline run executes both +regressions (17 discovered, 2 failures, 15 excluded by line selection). Both pass +with the fix. The final HTTP/2 file contains 17 tests, including +required upload-write failures with and without END_STREAM, truncation, and +both existing TLS backends. Two additional parser tests preserve errors when +RST_STREAM or an invalid WINDOW_UPDATE follows END_STREAM in the same batch. +The full 17-test HTTP/2 file was also repeated with seeds 1 through 5: all 85 +executions passed, with no excluded tests. Earlier HTTP/1.1, EventSource and +WebSocket close-order regression tests remain unchanged and pass in the full +suite. + +### Final local validation + +All commands below were actually executed from the root unless indicated. +Unit and E2E suites have zero failures and zero skipped tests. Per-app rows +repeat the corresponding root-suite coverage; they are not additional unique +tests. Line-selected red/green runs intentionally exclude other tests. + +| Command | Result | +| --- | --- | +| `mix deps.get`; `MIX_ENV=test mix deps.get` | Passed; lock unchanged | +| `mix format --check-formatted` | Passed, including the new smoke script | +| `mix compile --warnings-as-errors` | Passed | +| `MIX_ENV=test mix compile --warnings-as-errors` | Passed | +| `mix test` | 401 tests + 20 doctests; 0 failures, 0 skipped | +| `MIX_ENV=test mix test apps/http_core/test` | 161 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_fetch/test` | 157 tests + 20 doctests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/test` | 33 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/test` | 26 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/test` | 24 tests; 0 failures | +| `mix credo` | No issues | +| `mix dialyzer --format github` | Passed; 4 existing intentional ignores, 0 new warnings | +| `MIX_ENV=prod mix hex.build --unpack -o ` in each app | All five packages built, also exercised by smoke | +| `bash scripts/external_consumer_smoke.sh` | Passed; five current packages, eight verified TLS client exchanges | +| `bash -n scripts/external_consumer_smoke.sh` | Passed | +| `go build -o ../test_server/server .` in `apps/http_fetch/priv/test_server` | Passed | +| `MIX_ENV=test mix test apps/http_fetch/e2e` | 50 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_socket/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_event_source/e2e` | 2 tests; 0 failures | +| `MIX_ENV=test mix test apps/http_web_transport/e2e` | 3 tests; 0 failures | +| `MIX_ENV=test mix test.e2e` with the Go server running | All 58 E2E tests; 0 failures | + +`http_core` has no E2E directory; the existing workflow conditional reports that +fact rather than skipping an existing suite. The root-scoped commands above +are the commands used by the modified Test and E2E workflows. CI retains the +development compile and also checks test-environment compilation. + +Cold-start validation used detached worktrees under `.trees/` with independent +`deps` and `_build`, without deleting the user's existing build output. The +original root `mix deps.get` followed by child `mix test` reproduced undefined +SSL-module warnings and missing `ex_ssl.app`; cold root test preparation and +root-scoped execution succeeded. The external consumer likewise starts with +an empty temporary dependency/build tree and no repository lockfile on every +invocation. Three consecutive complete consumer smoke runs passed after its +server synchronization was finalized, followed by the final full-validation run. + +The smoke checks actual `hex_metadata.config` requirements, including +`optional: false`, and executes HTTP/1.1, HTTP/2 with ALPN, WSS upgrade/message, +and HTTPS EventSource open/message with both the omitted/default backend and +explicit ex_ssl. WebTransport explicitly rejects a TCP TLS backend and retains +QUIC options under shared ex_ssl configuration. The existing real QUIC E2E +suite also passed; the consumer smoke itself does not create a QUIC session. + +### Remaining limitation and diagnostic failures + +The first external smoke server closed its HTTP/2 TLS connection immediately +without coordinating with the client. Repetition exposed an OTP `:einval` error. +A separate diagnostic loaded `HTTP.SocketClient` from the original f5c0fe6 +worktree (the code path was printed and checked): 100 immediate-close OTP TLS +1.3 requests yielded 69 successes, 26 `:closed`, and 5 `:einval` errors. Thus +this risk predates this fix. This diagnostic is not a substitute for the +synchronized ex_ssl regression tests. + +The general consumer smoke now waits for a SETTINGS ACK before server closure; +it tests installed-package operation. The dedicated close-order regression +still closes before the final client acknowledgement/window update. The +production fix deliberately does not treat `:einval` as a normal close; the +pre-existing OTP immediate-close limitation remains outside this narrow fix. +No dependency source was modified, and no ex_ssl upstream defect was needed +to explain either integration fix. + +Local evidence logs are under `/tmp/http_fetch-pr14-validation/`, with original +workflow logs `/tmp/http_fetch-pr14-old-{test,e2e}.log`, cold reproduction logs +`/tmp/http_fetch-pr14-original-workflow-{deps,child}.log`, and the OTP baseline +diagnostic `/tmp/http_fetch-pr14-baseline-h2-immediate-close.log`. These local +logs are not shipped in the packages. + +The final red/green commands were: + +```bash +# In the detached original-HEAD worktree, after copying only the final test file +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs:214 apps/http_fetch/test/http/socket_client_http2_test.exs:350 --trace +# Expected: 2 failures, 15 excluded (the two close-order regressions) + +# In the fixed working tree; also repeated with --seed 1 through --seed 5 +MIX_ENV=test mix test apps/http_fetch/test/http/socket_client_http2_test.exs +# 17 tests, 0 failures, 0 excluded +``` diff --git a/mix.exs b/mix.exs index 2319747..67fa1c5 100644 --- a/mix.exs +++ b/mix.exs @@ -54,9 +54,7 @@ defmodule HttpFetch.Umbrella.MixProject do end defp run_e2e_tests([]) do - args = Enum.flat_map(@e2e_apps, &["--app", &1]) ++ ["cmd", "mix", "test.e2e"] - - Mix.Task.run("do", args) + Mix.Task.run("test", Enum.map(@e2e_apps, &"apps/#{&1}/e2e")) end defp run_e2e_tests(args) do diff --git a/mix.lock b/mix.lock index b5fb157..3ec0606 100644 --- a/mix.lock +++ b/mix.lock @@ -6,6 +6,7 @@ "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, "ex_doc": {:hex, :ex_doc, "0.38.2", "504d25eef296b4dec3b8e33e810bc8b5344d565998cd83914ffe1b8503737c02", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "732f2d972e42c116a70802f9898c51b54916e542cc50968ac6980512ec90f42b"}, + "ex_ssl": {:hex, :ex_ssl, "0.4.0", "43be385bd538e577e2279b420e8b860aaee03dc1f6136021b5aa7d46b1bc79a9", [:mix], [], "hexpm", "eeaa04e209ca644e01e3895c196809897b6524287a8e5912411bd6e744fdb61c"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, diff --git a/scripts/ex_ssl_algorithms_test.exs b/scripts/ex_ssl_algorithms_test.exs new file mode 100644 index 0000000..4869f05 --- /dev/null +++ b/scripts/ex_ssl_algorithms_test.exs @@ -0,0 +1,211 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +Code.require_file( + Path.join(System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support/signature_fixtures.ex") +) + +defmodule CandidateAlgorithmsTest do + use ExUnit.Case, async: false + alias ExSSL.TestSupport.SignatureFixtures + alias HTTP.HTTP2.{Frame, HPACK} + alias SSL.ClientHello.WireProfile + + setup_all do + directory = + Path.join(System.tmp_dir!(), "http-algorithms-#{System.unique_integer([:positive])}") + + File.mkdir_p!(directory) + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: SignatureFixtures.create(directory)} + end + + for {scheme, name} <- [ + {0x0503, :ecdsa_secp384r1_sha384}, + {0x0807, :eddsa_ed25519}, + {0x0809, :rsa_pss_pss_sha256}, + {0x080A, :rsa_pss_pss_sha384}, + {0x080B, :rsa_pss_pss_sha512} + ], + protocol <- ["http/1.1", "h2"] do + test "verified #{name} HTTP response over #{protocol} with P384", context do + fixture = Map.fetch!(context.fixtures, unquote(scheme)) + body = "verified-#{unquote(name)}" + + with_peer( + fixture, + unquote(name), + unquote(protocol), + fn port -> + response = + HTTP.fetch("https://127.0.0.1:#{port}/algorithm", + http_version: if(unquote(protocol) == "h2", do: :http2, else: :http1), + tls_backend: :ex_ssl, + ssl: [ + cacerts: [fixture.der], + server_name_indication: ~c"exssl.test", + ex_ssl: [profile: profile(unquote(scheme), unquote(protocol))] + ] + ) + |> HTTP.Promise.await() + + assert response.status == 200 + assert HTTP.Response.read_all(response) == body + end, + body + ) + end + end + + test "wrong reference identity still fails for every expanded signature", context do + for {scheme, name} <- [ + {0x0503, :ecdsa_secp384r1_sha384}, + {0x0807, :eddsa_ed25519}, + {0x0809, :rsa_pss_pss_sha256}, + {0x080A, :rsa_pss_pss_sha384}, + {0x080B, :rsa_pss_pss_sha512} + ] do + fixture = Map.fetch!(context.fixtures, scheme) + + with_peer( + fixture, + name, + "http/1.1", + fn port -> + assert {:error, _} = + HTTP.fetch("https://127.0.0.1:#{port}/algorithm", + tls_backend: :ex_ssl, + ssl: [ + cacerts: [fixture.der], + server_name_indication: ~c"wrong.test", + ex_ssl: [profile: profile(scheme, "http/1.1")] + ] + ) + |> HTTP.Promise.await() + end, + :reject + ) + end + end + + test "default backend remains OTP" do + assert HTTP.FetchOptions.new([]).tls_backend == :ssl + end + + defp profile(scheme, protocol) do + # HTTP/1.1 forces HRR; HTTP/2 sends P384 in the first ClientHello. + initial = if protocol == "http/1.1", do: 0x001D, else: 0x0018 + + %WireProfile{ + name: :candidate_algorithm, + cipher_suites: [0x1301], + extensions: [ + {:server_name, :from_connection}, + {:supported_versions, [0x0304]}, + {:supported_groups, [initial, 0x0018] |> Enum.uniq()}, + {:signature_algorithms, [scheme]}, + {:key_share, [initial]}, + {:alpn, [protocol]} + ] + } + end + + defp with_peer(fixture, scheme, protocol, client, body) do + {:ok, listener} = + :ssl.listen(0, + certfile: String.to_charlist(fixture.certificate), + keyfile: String.to_charlist(fixture.key), + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true, + signature_algs: [scheme], + supported_groups: [:secp384r1], + alpn_preferred_protocols: [protocol] + ) + + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, socket} = :ssl.transport_accept(listener, 5_000) + + case :ssl.handshake(socket, 5_000) do + {:error, _} when body == :reject -> + :ok + + {:ok, socket} -> + try do + refute body == :reject + assert {:ok, ^protocol} = :ssl.negotiated_protocol(socket) + respond(socket, protocol, body) + after + :ssl.close(socket) + end + end + end) + + try do + client.(port) + assert :ok = Task.await(task, 10_000) + after + :ssl.close(listener) + Task.shutdown(task, :brutal_kill) + end + + assert {:error, :econnrefused} = + :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 1_000) + end + + defp respond(socket, "http/1.1", body) do + request = headers(socket, "") + assert String.starts_with?(request, "GET /algorithm HTTP/1.1\r\n") + + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: #{byte_size(body)}\r\n\r\n", + body + ]) + end + + defp respond(socket, "h2", body) do + preface = HTTP.HTTP2.connection_preface() + assert {:ok, ^preface} = :ssl.recv(socket, byte_size(preface), 5_000) + assert %Frame{type: :settings} = frame(socket) + assert %Frame{type: :headers, stream_id: 1, payload: block} = frame(socket) + assert {:ok, _, headers} = HPACK.decode(HPACK.new_decoder(), block) + assert {":path", "/algorithm"} in headers + + :ok = + :ssl.send(socket, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode( + :headers, + 4, + 1, + HPACK.encode_headers([ + {":status", "200"}, + {"content-length", Integer.to_string(byte_size(body))} + ]) + ), + Frame.encode(:data, 1, 1, body) + ]) + + assert %Frame{type: :settings, flags: 1, stream_id: 0, payload: ""} = frame(socket) + :ok + end + + defp frame(socket) do + assert {:ok, <> = header} = :ssl.recv(socket, 9, 5_000) + payload = if size == 0, do: "", else: elem(:ssl.recv(socket, size, 5_000), 1) + assert {:ok, frame, ""} = Frame.decode(header <> payload) + frame + end + + defp headers(socket, buffer) when byte_size(buffer) < 16_384 do + if String.contains?(buffer, "\r\n\r\n"), + do: buffer, + else: + ( + {:ok, bytes} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> bytes) + ) + end +end diff --git a/scripts/ex_ssl_mtls_redirects_test.exs b/scripts/ex_ssl_mtls_redirects_test.exs new file mode 100644 index 0000000..23d6164 --- /dev/null +++ b/scripts/ex_ssl_mtls_redirects_test.exs @@ -0,0 +1,167 @@ +# Candidate source checkout only; run through ex_ssl_source_smoke.sh. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file(Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support", fixture])) +end + +defmodule CandidateMTLSRedirectsTest do + use ExUnit.Case, async: false + alias ExSSL.TestSupport.ClientAuthFixtures + + setup_all do + directory = + Path.join(System.tmp_dir!(), "mtls-redirects-#{System.unique_integer([:positive])}") + + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "automatic ex_ssl redirects cannot move a client identity to another origin", %{ + fixtures: f + } do + for change <- [:port, :host, :scheme] do + {source, port} = listener(f) + {target, target_port} = listener(f) + + destination = + case change do + :port -> "https://127.0.0.1:#{target_port}/next" + :host -> "https://localhost:#{port}/next" + :scheme -> "http://127.0.0.1:#{port}/next" + end + + peer = serve(source, f.rsa.der, [redirect(destination)]) + assert {:error, :client_identity_cross_origin_redirect} = fetch(port, f) + assert :ok = Task.await(peer, 5_000) + assert {:error, :timeout} = :ssl.transport_accept(target, 0) + assert {:error, :timeout} = :ssl.transport_accept(source, 0) + :ssl.close(source) + :ssl.close(target) + end + end + + test "same-origin redirects retain the exact configured client identity", %{fixtures: f} do + {source, port} = listener(f) + peer = serve(source, f.rsa.der, [redirect("/next"), ok()]) + response = fetch(port, f) + assert response.status == 200 + assert response.redirected + assert HTTP.Response.read_all(response) == "mtls" + assert :ok = Task.await(peer, 5_000) + end + + test "DNS host casing does not change client identity origin", %{fixtures: f} do + {source, port} = listener(f) + peer = serve(source, f.rsa.der, [redirect("https://LOCALHOST:#{port}/next"), ok()]) + response = fetch_url("https://localhost:#{port}/start", f, []) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "mtls" + assert :ok = Task.await(peer, 5_000) + end + + test "manual redirect lets the caller deliberately reuse an identity in a new request", %{ + fixtures: f + } do + {source, port} = listener(f) + {target, target_port} = listener(f) + source_peer = serve(source, f.rsa.der, [redirect("https://127.0.0.1:#{target_port}/next")]) + response = fetch(port, f, redirect: :manual) + assert response.status == 302 + assert :ok = Task.await(source_peer, 5_000) + assert {:error, :timeout} = :ssl.transport_accept(target, 0) + target_peer = serve(target, f.rsa.der, [ok()]) + result = fetch(target_port, f) + assert result.status == 200 + assert HTTP.Response.read_all(result) == "mtls" + assert :ok = Task.await(target_peer, 5_000) + end + + test "OTP backend keeps its existing cross-origin client-identity behavior", %{fixtures: f} do + {source, port} = listener(f) + {target, target_port} = listener(f) + source_peer = serve(source, f.rsa.der, [redirect("https://127.0.0.1:#{target_port}/next")]) + target_peer = serve(target, f.rsa.der, [ok()]) + response = fetch(port, f, tls_backend: :ssl) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "mtls" + assert :ok = Task.await(source_peer, 5_000) + assert :ok = Task.await(target_peer, 5_000) + end + + defp listener(f) do + {:ok, socket} = + :ssl.listen(0, + certfile: f.server.certificate, + keyfile: f.server.key, + cacerts: [f.ca.der], + verify: :verify_peer, + fail_if_no_peer_cert: true, + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true + ) + + on_exit(fn -> :ssl.close(socket) end) + {:ok, {_, port}} = :ssl.sockname(socket) + {socket, port} + end + + defp serve(listener, expected, responses) do + task = + Task.async(fn -> + Enum.each(responses, fn response -> + {:ok, transport} = :ssl.transport_accept(listener, 5_000) + {:ok, socket} = :ssl.handshake(transport, 5_000) + + try do + assert {:ok, ^expected} = :ssl.peercert(socket) + headers(socket, <<>>) + assert :ok = :ssl.send(socket, response) + after + :ssl.close(socket) + end + end) + end) + + on_exit(fn -> if Process.alive?(task.pid), do: Process.exit(task.pid, :kill) end) + task + end + + defp headers(socket, buffer) do + if :binary.match(buffer, "\r\n\r\n") == :nomatch do + assert {:ok, data} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> data) + else + buffer + end + end + + defp fetch(port, f, extra \\ []) do + fetch_url("https://127.0.0.1:#{port}/start", f, extra) + end + + defp fetch_url(url, f, extra) do + HTTP.fetch( + url, + Keyword.merge( + [ + tls_backend: :ex_ssl, + timeout: 5_000, + ssl: [ + cacerts: [f.ca.der], + server_name_indication: ~c"exssl.test", + certfile: f.rsa.certificate, + keyfile: f.rsa.key + ] + ], + extra + ) + ) + |> HTTP.Promise.await(5_000) + end + + defp redirect(url), + do: "HTTP/1.1 302 Found\r\nLocation: #{url}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + + defp ok, do: "HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nmtls" +end diff --git a/scripts/ex_ssl_mtls_streams_test.exs b/scripts/ex_ssl_mtls_streams_test.exs new file mode 100644 index 0000000..5664daa --- /dev/null +++ b/scripts/ex_ssl_mtls_streams_test.exs @@ -0,0 +1,321 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file( + Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test", "support", fixture]) + ) +end + +defmodule CandidateMtlsStreamsTest do + use ExUnit.Case, async: false + import Bitwise + + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.EventSource + alias HTTP.EventSource.Event.Error + alias HTTP.EventSource.Event.Message, as: SSEMessage + alias HTTP.EventSource.Event.Open, as: SSEOpen + alias HTTP.WebSocket + alias HTTP.WebSocket.Event.Close + alias HTTP.WebSocket.Event.Message, as: WSMessage + alias HTTP.WebSocket.Event.Open, as: WSOpen + + @guid "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + @timeout 5_000 + + setup_all do + directory = + Path.join(System.tmp_dir!(), "http-mtls-streams-#{System.unique_integer([:positive])}") + + File.mkdir_p!(directory) + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "packaged WSS uses ex_ssl mTLS through passive upgrade and active-once echo close", %{ + fixtures: fixtures + } do + ref = make_ref() + %{listener: listener, port: port, task: task} = start_web_socket_peer(self(), ref, fixtures) + + socket = + WebSocket.new("wss://127.0.0.1:#{port}/socket", [], + tls_backend: :ex_ssl, + ssl: client_ssl_options(fixtures) + ) + + try do + assert_receive {:mtls_wss, ^ref, :authenticated, client_der}, @timeout + assert client_der == fixtures.rsa.der + assert_receive {:mtls_wss, ^ref, :upgrade}, @timeout + assert_receive {WebSocket, ^socket, %WSOpen{}}, @timeout + + assert :ok = WebSocket.send(socket, "mtls-echo") + assert_receive {:mtls_wss, ^ref, {:text, "mtls-echo"}}, @timeout + assert_receive {WebSocket, ^socket, %WSMessage{data: "echo:mtls-echo"}}, @timeout + + assert :ok = WebSocket.close(socket, 1000, "done") + assert_receive {:mtls_wss, ^ref, {:close, <<1000::16, "done">>}}, @timeout + + assert_receive {WebSocket, ^socket, %Close{code: 1000, reason: "done", was_clean: true}}, + @timeout + + assert :ok = Task.await(task, @timeout) + after + _ = WebSocket.close(socket) + stop_peer(listener, task) + end + end + + test "packaged EventSource reconnects to the same mTLS endpoint with its pinned backend", %{ + fixtures: fixtures + } do + ref = make_ref() + %{listener: listener, port: port, task: task} = start_event_source_peer(self(), ref, fixtures) + previous_backend = Application.get_env(:http_core, :tls_backend, :unset) + + on_exit(fn -> restore_backend(previous_backend) end) + + source = + EventSource.new("https://127.0.0.1:#{port}/events", + tls_backend: :ex_ssl, + reconnect_time: 10, + ssl: client_ssl_options(fixtures) + ) + + try do + assert_receive {:mtls_sse, ^ref, :authenticated, 1, client_der}, @timeout + assert client_der == fixtures.rsa.der + assert_receive {:mtls_sse, ^ref, {:request, 1, first_request}}, @timeout + assert String.starts_with?(first_request, "GET /events HTTP/1.1\r\n") + assert_receive {EventSource, ^source, %SSEOpen{}}, @timeout + + assert_receive {EventSource, ^source, %SSEMessage{data: "first", last_event_id: "41"}}, + @timeout + + assert_receive {:mtls_sse, ^ref, :first_ready}, @timeout + + Application.put_env(:http_core, :tls_backend, :invalid) + assert %{tls_backend: :ex_ssl} = :sys.get_state(source.pid) + send(task.pid, {:close_first, ref}) + assert_receive {:mtls_sse, ^ref, :first_closed}, @timeout + assert_receive {EventSource, ^source, %Error{reason: :eof}}, @timeout + + assert_receive {:mtls_sse, ^ref, :authenticated, 2, second_der}, @timeout + assert second_der == fixtures.rsa.der + assert_receive {:mtls_sse, ^ref, {:request, 2, second_request}}, @timeout + assert String.starts_with?(second_request, "GET /events HTTP/1.1\r\n") + assert second_request =~ "Last-Event-ID: 41\r\n" + assert_receive {EventSource, ^source, %SSEOpen{}}, @timeout + + assert_receive {EventSource, ^source, %SSEMessage{data: "second", last_event_id: "41"}}, + @timeout + + assert :ok = EventSource.close(source) + assert :ok = Task.await(task, @timeout) + after + _ = EventSource.close(source) + stop_peer(listener, task) + end + end + + defp start_web_socket_peer(parent, ref, fixtures) do + {:ok, listener} = listen(fixtures) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, socket} = accept_authenticated(listener, fixtures.rsa.der) + send(parent, {:mtls_wss, ref, :authenticated, fixtures.rsa.der}) + request = recv_headers(socket, <<>>) + key = websocket_key(request) + :ok = :ssl.send(socket, websocket_upgrade_response(key)) + send(parent, {:mtls_wss, ref, :upgrade}) + + {:text, "mtls-echo", buffer} = recv_client_frame(socket, <<>>) + send(parent, {:mtls_wss, ref, {:text, "mtls-echo"}}) + :ok = :ssl.send(socket, frame(:text, "echo:mtls-echo")) + + {:close, payload, <<>>} = recv_client_frame(socket, buffer) + send(parent, {:mtls_wss, ref, {:close, payload}}) + :ok = :ssl.send(socket, frame(:close, payload)) + :ssl.close(socket) + :ssl.close(listener) + end) + + %{listener: listener, port: port, task: task} + end + + defp start_event_source_peer(parent, ref, fixtures) do + {:ok, listener} = listen(fixtures) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + serve_event_source_connection( + listener, + parent, + ref, + fixtures.rsa.der, + 1, + "id: 41\ndata: first\n\n", + true + ) + + serve_event_source_connection( + listener, + parent, + ref, + fixtures.rsa.der, + 2, + "data: second\n\n", + false + ) + + :ssl.close(listener) + end) + + %{listener: listener, port: port, task: task} + end + + defp serve_event_source_connection(listener, parent, ref, expected_der, index, body, close?) do + {:ok, socket} = accept_authenticated(listener, expected_der) + send(parent, {:mtls_sse, ref, :authenticated, index, expected_der}) + request = recv_headers(socket, <<>>) + send(parent, {:mtls_sse, ref, {:request, index, request}}) + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: ", + if(close?, do: "close", else: "keep-alive"), + "\r\n\r\n", + body + ]) + + if close? do + send(parent, {:mtls_sse, ref, :first_ready}) + + receive do + {:close_first, ^ref} -> :ok + after + @timeout -> raise "first EventSource connection was not explicitly released" + end + + :ssl.close(socket) + send(parent, {:mtls_sse, ref, :first_closed}) + else + assert {:error, :closed} = :ssl.recv(socket, 0, @timeout) + end + end + + defp listen(fixtures) do + :ssl.listen(0, [ + :binary, + active: false, + mode: :binary, + packet: :raw, + reuseaddr: true, + ip: {127, 0, 0, 1}, + versions: [:"tlsv1.3"], + certfile: String.to_charlist(fixtures.server.certificate), + keyfile: String.to_charlist(fixtures.server.key), + cacerts: [fixtures.ca.der], + verify: :verify_peer, + fail_if_no_peer_cert: true + ]) + end + + defp accept_authenticated(listener, expected_der) do + with {:ok, transport} <- :ssl.transport_accept(listener, @timeout), + {:ok, socket} <- :ssl.handshake(transport, @timeout), + {:ok, ^expected_der} <- :ssl.peercert(socket) do + {:ok, socket} + end + end + + defp client_ssl_options(fixtures) do + [ + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + certfile: fixtures.rsa.certificate, + keyfile: fixtures.rsa.key + ] + end + + defp recv_headers(socket, buffer) when byte_size(buffer) < 16_384 do + if :binary.match(buffer, "\r\n\r\n") == :nomatch do + {:ok, bytes} = :ssl.recv(socket, 0, @timeout) + recv_headers(socket, buffer <> bytes) + else + buffer + end + end + + defp websocket_key(request) do + [_, key] = Regex.run(~r/sec-websocket-key:\s*([^\r\n]+)/i, request) + String.trim(key) + end + + defp websocket_upgrade_response(key) do + accept = :crypto.hash(:sha, key <> @guid) |> Base.encode64() + + [ + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ", + accept, + "\r\n\r\n" + ] + end + + defp recv_client_frame(socket, buffer) do + case parse_client_frame(buffer) do + {:ok, frame, rest} -> + {frame_type(frame), frame.payload, rest} + + :more -> + {:ok, bytes} = :ssl.recv(socket, 0, @timeout) + recv_client_frame(socket, buffer <> bytes) + end + end + + defp parse_client_frame(<>) do + opcode = first &&& 0x0F + masked? = (second &&& 0x80) != 0 + length_code = second &&& 0x7F + + with true <- masked?, + {:ok, length, rest} <- frame_length(length_code, rest), + <> <- rest do + {:ok, %{opcode: opcode, payload: unmask(encrypted, mask)}, remainder} + else + _ -> :more + end + end + + defp parse_client_frame(_), do: :more + defp frame_length(length, rest) when length <= 125, do: {:ok, length, rest} + defp frame_length(126, <>), do: {:ok, length, rest} + defp frame_length(127, <>), do: {:ok, length, rest} + defp frame_length(_, _), do: :more + defp frame_type(%{opcode: 0x1}), do: :text + defp frame_type(%{opcode: 0x8}), do: :close + + defp frame(:text, payload) when byte_size(payload) <= 125, + do: <<0x81, byte_size(payload), payload::binary>> + + defp frame(:close, payload) when byte_size(payload) <= 125, + do: <<0x88, byte_size(payload), payload::binary>> + + defp unmask(payload, mask) do + payload + |> :binary.bin_to_list() + |> Enum.with_index() + |> Enum.map_join(fn {byte, index} -> <> end) + end + + defp stop_peer(listener, task) do + _ = :ssl.close(listener) + if Process.alive?(task.pid), do: Task.shutdown(task, :brutal_kill) + end + + defp restore_backend(:unset), do: Application.delete_env(:http_core, :tls_backend) + defp restore_backend(value), do: Application.put_env(:http_core, :tls_backend, value) +end diff --git a/scripts/ex_ssl_mtls_test.exs b/scripts/ex_ssl_mtls_test.exs new file mode 100644 index 0000000..73e5052 --- /dev/null +++ b/scripts/ex_ssl_mtls_test.exs @@ -0,0 +1,253 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +source = System.fetch_env!("EX_SSL_SOURCE_DIR") +Code.require_file(Path.join(source, "test/support/signature_fixtures.ex")) +Code.require_file(Path.join(source, "test/support/client_auth_fixtures.ex")) + +defmodule CandidateMTLSTest do + use ExUnit.Case, async: false + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.HTTP2.{Frame, HPACK} + + @body "packaged-mtls" + @path "/mtls" + + setup_all do + directory = Path.join(System.tmp_dir!(), "http-mtls-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + for identity <- [:rsa, :ec, :large], protocol <- ["http/1.1", "h2"] do + test "required #{identity} identity returns verified #{protocol} HTTP response", context do + fixture = context.fixtures[unquote(identity)] + if unquote(identity) == :large, do: assert(byte_size(fixture.der) > 16_384) + exchange(context.fixtures, fixture, :required, unquote(protocol)) + end + end + + test "optional client auth without an identity returns an HTTP response", context do + exchange(context.fixtures, nil, :optional, "http/1.1") + end + + test "missing, wrong-CA, expired, and wrong-purpose identities cannot fetch", context do + for identity <- [ + nil, + context.fixtures.wrong, + context.fixtures.expired, + context.fixtures.server + ] do + with_peer( + context.fixtures, + :required, + "http/1.1", + fn port -> + assert {:error, _} = fetch(port, context.fixtures, identity, "http/1.1") + end, + :reject + ) + end + end + + test "mismatched client key fails before network I/O", context do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, ip: {127, 0, 0, 1}]) + {:ok, {_, port}} = :inet.sockname(listener) + + try do + assert {:error, _} = + fetch(port, context.fixtures, context.fixtures.rsa, "http/1.1", + keyfile: context.fixtures.ec.key + ) + + assert {:error, :timeout} = :gen_tcp.accept(listener, 200) + after + :gen_tcp.close(listener) + end + end + + test "incompatible requested signature does not authenticate", context do + with_peer( + %{context.fixtures | server: context.fixtures.ec_server}, + :required, + "http/1.1", + fn port -> + assert {:error, _} = fetch(port, context.fixtures, context.fixtures.rsa, "http/1.1") + end, + :reject, + signature_algs: [:ecdsa_secp256r1_sha256] + ) + end + + test "wrong server hostname fails even when client credentials are valid", context do + with_peer( + context.fixtures, + :required, + "http/1.1", + fn port -> + assert {:error, _} = + fetch(port, context.fixtures, context.fixtures.rsa, "http/1.1", + server_name_indication: ~c"wrong.test" + ) + end, + :reject + ) + end + + defp exchange(fixtures, identity, mode, protocol) do + expected = if identity, do: identity.der, else: nil + + response = + with_peer( + fixtures, + mode, + protocol, + fn port -> fetch(port, fixtures, identity, protocol) end, + expected + ) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + defp fetch(port, fixtures, identity, protocol, overrides \\ []) do + ssl = [ + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + verify: :verify_peer, + alpn_advertised_protocols: [protocol] + ] + + credentials = + if identity, do: [certfile: identity.certificate, keyfile: identity.key], else: [] + + ssl = Keyword.merge(ssl ++ credentials, overrides) + + HTTP.fetch("https://127.0.0.1:#{port}#{@path}", + http_version: if(protocol == "h2", do: :http2, else: :http1), + tls_backend: :ex_ssl, + ssl: ssl, + timeout: 5_000, + connect_timeout: 5_000 + ) + |> HTTP.Promise.await(10_000) + end + + defp with_peer(fixtures, mode, protocol, client, expected, extra \\ []) do + ssl_options = + [ + certfile: String.to_charlist(fixtures.server.certificate), + keyfile: String.to_charlist(fixtures.server.key), + cacerts: [fixtures.ca.der], + verify: :verify_peer, + fail_if_no_peer_cert: mode == :required, + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true, + alpn_preferred_protocols: [protocol] + ] + |> Keyword.merge(extra) + + {:ok, listener} = :ssl.listen(0, ssl_options) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, transport} = :ssl.transport_accept(listener, 5_000) + + case :ssl.handshake(transport, 5_000) do + {:ok, socket} -> + try do + if expected == :reject do + case :ssl.recv(socket, 0, 5_000) do + {:error, _} -> :ok + {:ok, bytes} -> flunk("rejected peer sent HTTP bytes: #{byte_size(bytes)}") + end + else + assert {:ok, ^protocol} = :ssl.negotiated_protocol(socket) + + case expected do + nil -> assert {:error, :no_peercert} = :ssl.peercert(socket) + der -> assert {:ok, ^der} = :ssl.peercert(socket) + end + + respond(socket, protocol) + end + after + :ssl.close(socket) + end + + {:error, _} when expected == :reject -> + :ok + end + end) + + result = + try do + result = client.(port) + assert :ok = Task.await(task, 10_000) + result + after + :ssl.close(listener) + Task.shutdown(task, :brutal_kill) + end + + assert {:error, :econnrefused} = + :gen_tcp.connect(~c"127.0.0.1", port, [:binary, active: false], 1_000) + + result + end + + defp respond(socket, "http/1.1") do + request = headers(socket, "") + assert String.starts_with?(request, "GET #{@path} HTTP/1.1\r\n") + + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: #{byte_size(@body)}\r\n\r\n", + @body + ]) + end + + defp respond(socket, "h2") do + preface = HTTP.HTTP2.connection_preface() + assert {:ok, ^preface} = :ssl.recv(socket, byte_size(preface), 5_000) + assert %Frame{type: :settings} = frame(socket) + assert %Frame{type: :headers, stream_id: 1, payload: block} = frame(socket) + assert {:ok, _, headers} = HPACK.decode(HPACK.new_decoder(), block) + assert {":path", @path} in headers + + :ok = + :ssl.send(socket, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode( + :headers, + 4, + 1, + HPACK.encode_headers([ + {":status", "200"}, + {"content-length", Integer.to_string(byte_size(@body))} + ]) + ), + Frame.encode(:data, 1, 1, @body) + ]) + + assert %Frame{type: :settings, flags: 1, stream_id: 0, payload: ""} = frame(socket) + :ok + end + + defp frame(socket) do + assert {:ok, <> = header} = :ssl.recv(socket, 9, 5_000) + payload = if size == 0, do: "", else: elem(:ssl.recv(socket, size, 5_000), 1) + assert {:ok, frame, ""} = Frame.decode(header <> payload) + frame + end + + defp headers(socket, buffer) when byte_size(buffer) < 16_384 do + if String.contains?(buffer, "\r\n\r\n"), + do: buffer, + else: + ( + {:ok, bytes} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> bytes) + ) + end +end diff --git a/scripts/ex_ssl_options_test.exs b/scripts/ex_ssl_options_test.exs new file mode 100644 index 0000000..aa4f793 --- /dev/null +++ b/scripts/ex_ssl_options_test.exs @@ -0,0 +1,340 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +source = System.fetch_env!("EX_SSL_SOURCE_DIR") +Code.require_file(Path.join(source, "test/support/signature_fixtures.ex")) +Code.require_file(Path.join(source, "test/support/client_auth_fixtures.ex")) + +defmodule CandidateOptionsTest do + use ExUnit.Case, async: false + + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.Transport.ExSSL + alias SSL.ClientHello.WireProfile + + @body "ordered-options" + + setup_all do + directory = Path.join(System.tmp_dir!(), "http-options-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "HTTP.fetch honors ordered cipher, group, CertificateVerify, and issuer policies", + context do + response = + with_peer( + context.fixtures, + fn port -> + fetch(port, context.fixtures, + ciphers: ["TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256"], + supported_groups: [:secp256r1, :x25519], + signature_algs: [:rsa_pss_rsae_sha256, :ecdsa_secp256r1_sha256], + signature_algs_cert: [:rsa_pkcs1_sha256] + ) + end, + @body, + ciphers: [ + %{key_exchange: :any, cipher: :aes_256_gcm, mac: :aead, prf: :sha384} + ], + supported_groups: [:secp256r1] + ) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + test "an incompatible CertificateVerify or issuer policy fails authentication", context do + for policy <- [ + [signature_algs: [:ecdsa_secp256r1_sha256]], + [signature_algs: [:rsa_pss_rsae_sha256], signature_algs_cert: [:ecdsa_secp256r1_sha256]] + ] do + with_peer( + context.fixtures, + fn port -> + assert {:error, _} = fetch(port, context.fixtures, policy) + end, + :reject + ) + end + end + + test "a conflicting explicit profile fails before TCP I/O" do + profile = %WireProfile{ + name: :candidate_option_conflict, + cipher_suites: [0x1301], + extensions: [ + {:supported_versions, [0x0304]}, + {:supported_groups, [0x001D]}, + {:signature_algorithms, [0x0804]}, + {:key_share, [0x001D]}, + {:alpn, ["http/1.1"]} + ] + } + + assert_before_io(fn port -> + assert {:error, {:options, _}} = + fetch(port, nil, + ex_ssl: [profile: profile], + ciphers: ["TLS_AES_256_GCM_SHA384"] + ) + end) + end + + test "adapter applies safe TCP options and socket_opts override matching ssl entries", + context do + with_peer( + context.fixtures, + fn port -> + ssl = tls_options(context.fixtures) ++ [nodelay: false, keepalive: false] + + socket_opts = [ + nodelay: true, + keepalive: true, + sndbuf: 16_384, + recbuf: 16_384, + ip: {127, 0, 0, 1}, + port: 0 + ] + + {:ok, socket} = + ExSSL.connect("127.0.0.1", port, [ssl: ssl, socket_opts: socket_opts], 5_000) + + try do + {:connected, state} = :sys.get_state(socket.pid) + tcp = state.tcp + assert {:ok, values} = :inet.getopts(tcp, [:nodelay, :keepalive, :sndbuf, :recbuf]) + assert Keyword.fetch!(values, :nodelay) + assert Keyword.fetch!(values, :keepalive) + assert Keyword.fetch!(values, :sndbuf) >= 16_384 + assert Keyword.fetch!(values, :recbuf) >= 16_384 + assert {:ok, {{127, 0, 0, 1}, local_port}} = :inet.sockname(tcp) + assert local_port > 0 + after + ExSSL.close(socket) + end + end, + :no_http + ) + end + + test "HTTP.fetch forwards the safe TCP allowlist", context do + response = + with_peer(context.fixtures, fn port -> + fetch(port, context.fixtures, [], + socket_opts: [ + nodelay: true, + keepalive: true, + sndbuf: 16_384, + recbuf: 16_384, + ip: {127, 0, 0, 1}, + port: 0 + ] + ) + end) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + test "mutable TCP options apply, but an invalid combined setopts changes nothing", context do + with_peer( + context.fixtures, + fn port -> + {:ok, socket} = + ExSSL.connect("127.0.0.1", port, [ssl: tls_options(context.fixtures)], 5_000) + + try do + {:connected, state} = :sys.get_state(socket.pid) + tcp = state.tcp + + assert :ok = + ExSSL.setopts(socket, + nodelay: true, + keepalive: true, + sndbuf: 16_384, + recbuf: 16_384 + ) + + assert {:ok, values} = :inet.getopts(tcp, [:nodelay, :keepalive, :sndbuf, :recbuf]) + assert Keyword.fetch!(values, :nodelay) + assert Keyword.fetch!(values, :keepalive) + assert Keyword.fetch!(values, :sndbuf) >= 16_384 + assert Keyword.fetch!(values, :recbuf) >= 16_384 + + assert {:error, {:options, _}} = + ExSSL.setopts(socket, nodelay: false, ip: {127, 0, 0, 1}) + + assert {:ok, [nodelay: true]} = :inet.getopts(tcp, [:nodelay]) + after + ExSSL.close(socket) + end + end, + :no_http + ) + end + + test "IPv6 local bind selects the DNS address family with certificate verification", context do + response = + with_peer( + context.fixtures, + fn port -> + HTTP.fetch("https://localhost:#{port}/options", + tls_backend: :ex_ssl, + ssl: tls_options(context.fixtures), + socket_opts: [ip: {0, 0, 0, 0, 0, 0, 0, 1}], + timeout: 5_000, + connect_timeout: 5_000 + ) + |> HTTP.Promise.await(10_000) + end, + @body, + ip: {0, 0, 0, 0, 0, 0, 0, 1} + ) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == @body + end + + test "malformed and unsafe socket options fail before network I/O" do + for socket_opts <- [ + [linger: {true, 0}], + [active: true], + [packet: 4], + [nodelay: :invalid], + [nodelay: true, nodelay: false], + [ip: {999, 0, 0, 1}], + [port: -1], + [send_timeout_close: false, linger: {true, 0}], + [{:nodelay, true} | :improper] + ] do + assert_before_io(fn port -> + assert {:error, {:options, _}} = + HTTP.fetch("https://127.0.0.1:#{port}/options", + tls_backend: :ex_ssl, + socket_opts: socket_opts, + timeout: 2_000, + connect_timeout: 2_000 + ) + |> HTTP.Promise.await(3_000) + end) + end + end + + test "malformed ssl options fail before HTTP/2 ALPN construction and network I/O" do + assert_before_io(fn port -> + assert {:error, {:options, _}} = + HTTP.fetch("https://127.0.0.1:#{port}/options", + http_version: :http2, + tls_backend: :ex_ssl, + ssl: [{:verify, :verify_peer} | :improper], + timeout: 2_000, + connect_timeout: 2_000 + ) + |> HTTP.Promise.await(3_000) + end) + end + + defp fetch(port, fixtures, ssl_extra, opts \\ []) do + ssl = if(fixtures, do: tls_options(fixtures), else: []) ++ ssl_extra + + HTTP.fetch( + "https://127.0.0.1:#{port}/options", + [ + tls_backend: :ex_ssl, + ssl: ssl, + timeout: 5_000, + connect_timeout: 5_000 + ] ++ opts + ) + |> HTTP.Promise.await(10_000) + end + + defp tls_options(fixtures) do + [ + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + verify: :verify_peer, + alpn_advertised_protocols: ["http/1.1"] + ] + end + + defp with_peer(fixtures, client, response \\ @body, extra \\ []) do + options = + [ + certfile: String.to_charlist(fixtures.server.certificate), + keyfile: String.to_charlist(fixtures.server.key), + versions: [:"tlsv1.3"], + active: false, + mode: :binary, + reuseaddr: true, + alpn_preferred_protocols: ["http/1.1"] + ] + |> Keyword.merge(extra) + + {:ok, listener} = :ssl.listen(0, options) + {:ok, {_, port}} = :ssl.sockname(listener) + + task = + Task.async(fn -> + {:ok, transport} = :ssl.transport_accept(listener, 5_000) + + case :ssl.handshake(transport, 5_000) do + {:ok, socket} -> + try do + case response do + :reject -> + assert {:error, _} = :ssl.recv(socket, 0, 5_000) + + :no_http -> + assert {:error, _} = :ssl.recv(socket, 0, 5_000) + :ok + + body -> + assert {:ok, "http/1.1"} = :ssl.negotiated_protocol(socket) + assert String.starts_with?(headers(socket, ""), "GET /options HTTP/1.1\r\n") + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: #{byte_size(body)}\r\n\r\n", + body + ]) + end + after + :ssl.close(socket) + end + + {:error, _} when response == :reject -> + :ok + end + end) + + try do + result = client.(port) + assert :ok = Task.await(task, 10_000) + result + after + :ssl.close(listener) + Task.shutdown(task, :brutal_kill) + end + end + + defp headers(socket, buffer) when byte_size(buffer) < 16_384 do + if String.contains?(buffer, "\r\n\r\n") do + buffer + else + {:ok, bytes} = :ssl.recv(socket, 0, 5_000) + headers(socket, buffer <> bytes) + end + end + + defp assert_before_io(client) do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, ip: {127, 0, 0, 1}]) + {:ok, {_, port}} = :inet.sockname(listener) + + try do + client.(port) + assert {:error, :timeout} = :gen_tcp.accept(listener, 200) + after + :gen_tcp.close(listener) + end + end +end diff --git a/scripts/ex_ssl_resumption_test.exs b/scripts/ex_ssl_resumption_test.exs new file mode 100644 index 0000000..e0b7d9e --- /dev/null +++ b/scripts/ex_ssl_resumption_test.exs @@ -0,0 +1,137 @@ +# Source-candidate package test; run through scripts/ex_ssl_source_smoke.sh. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file(Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support", fixture])) +end + +defmodule CandidateResumptionTest do + use ExUnit.Case, async: false + + alias ExSSL.TestSupport.ClientAuthFixtures + + @timeout 10_000 + @python Path.join(__DIR__, "ex_ssl_tls12_peer.py") + + setup_all do + directory = + Path.join(System.tmp_dir!(), "http-resumption-#{System.unique_integer([:positive])}") + + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + test "packaged HTTP/1.1 forwards TLS 1.3 auto tickets across fresh connections", %{ + fixtures: fixtures + } do + peer = start_peer(fixtures) + + try do + for {index, resumed} <- [{1, false}, {2, true}] do + promise = + HTTP.fetch("https://127.0.0.1:#{peer.port}/resumption", + tls_backend: :ex_ssl, + ssl: [ + versions: [:"tlsv1.3"], + verify: :verify_peer, + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + alpn_advertised_protocols: ["http/1.1"], + session_tickets: :auto + ], + timeout: @timeout, + connect_timeout: @timeout + ) + + assert {:ok, + %{ + "index" => ^index, + "version" => "TLSv1.3", + "alpn" => "http/1.1", + "session_reused" => ^resumed, + "client_der_b64" => :null + }} = event(peer, "handshake") + + assert {:ok, %{"bytes" => 6}} = event(peer, "exchange") + response = HTTP.Promise.await(promise, @timeout) + assert response.status == 200 + assert HTTP.Response.read_all(response) == "BBBBBB" + end + after + stop_peer(peer) + end + end + + defp start_peer(fixtures) do + executable = System.find_executable("python3") || raise "python3 unavailable" + assert File.regular?(@python) + + handle = + Port.open({:spawn_executable, executable}, [ + :binary, + :exit_status, + :stderr_to_stdout, + {:line, 65_536}, + args: [ + "-B", + "-u", + @python, + "--certfile", + fixtures.server.certificate, + "--keyfile", + fixtures.server.key, + "--cafile", + fixtures.ca.certificate, + "--mode", + "resumption" + ] + ]) + + {:os_pid, os_pid} = Port.info(handle, :os_pid) + peer = %{handle: handle, os_pid: os_pid, port: nil} + + case event(peer, "ready") do + {:ok, %{"port" => port}} -> + %{peer | port: port} + + error -> + stop_peer(peer) + raise "resumption peer startup failed: #{inspect(error)}" + end + end + + defp event(%{handle: handle}, expected) do + receive do + {^handle, {:data, {:eol, line}}} -> + case :json.decode(line) do + %{"kind" => ^expected} = value -> {:ok, value} + %{"kind" => "failure"} = value -> {:error, value} + other -> {:error, {:unexpected_peer_event, other}} + end + + {^handle, {:data, {:noeol, _}}} -> + {:error, :peer_line_too_long} + + {^handle, {:exit_status, status}} -> + {:error, {:peer_exit, status}} + after + @timeout -> {:error, :peer_timeout} + end + rescue + _ -> {:error, :invalid_peer_event} + end + + defp stop_peer(%{handle: handle, os_pid: os_pid}) do + case Port.info(handle, :os_pid) do + {:os_pid, ^os_pid} -> + _ = System.cmd("kill", ["-TERM", Integer.to_string(os_pid)], stderr_to_stdout: true) + + receive do + {^handle, {:exit_status, _}} -> :ok + after + 2_000 -> raise "resumption peer did not stop" + end + + nil -> + :ok + end + end +end diff --git a/scripts/ex_ssl_source_smoke.sh b/scripts/ex_ssl_source_smoke.sh new file mode 100755 index 0000000..a633669 --- /dev/null +++ b/scripts/ex_ssl_source_smoke.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Cross-repository candidate validation. Release smoke remains external_consumer_smoke.sh. +set -euo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +: "${EX_SSL_SOURCE_DIR:?Set EX_SSL_SOURCE_DIR to the candidate ex_ssl checkout}" +export EX_SSL_SOURCE_DIR=$(cd "$EX_SSL_SOURCE_DIR" && pwd) +: "${EX_SSL_DEP_MODE:=source}" +if [[ "$EX_SSL_DEP_MODE" != source && "$EX_SSL_DEP_MODE" != published ]]; then + echo "EX_SSL_DEP_MODE must be source or published" >&2 + exit 2 +fi +export EX_SSL_DEP_MODE +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT +export HTTP_FETCH_PACKAGE_DIR="$work_dir/packages" +mkdir -p "$HTTP_FETCH_PACKAGE_DIR" "$work_dir/consumer/test" +for app in http_core http_fetch http_web_socket http_event_source http_web_transport; do + (cd "$repo_root/apps/$app" && MIX_ENV=prod mix hex.build --unpack -o "$HTTP_FETCH_PACKAGE_DIR/$app") +done +cat > "$work_dir/consumer/mix.exs" <<'MIX' +defmodule CandidateConsumer.MixProject do + use Mix.Project + def project do + packages = System.fetch_env!("HTTP_FETCH_PACKAGE_DIR") + ex_ssl_dep = + if System.fetch_env!("EX_SSL_DEP_MODE") == "published" do + {:ex_ssl, "~> 0.4.0"} + else + {:ex_ssl, path: System.fetch_env!("EX_SSL_SOURCE_DIR"), override: true} + end + + [app: :candidate_consumer, version: "0.0.0", deps: [ + {:http_core, path: Path.join(packages, "http_core")}, + {:http_fetch, path: Path.join(packages, "http_fetch")}, + {:http_web_socket, path: Path.join(packages, "http_web_socket")}, + {:http_event_source, path: Path.join(packages, "http_event_source")}, + {:http_web_transport, path: Path.join(packages, "http_web_transport")}, + ex_ssl_dep + ]] + end + def application, do: [extra_applications: [:logger, :ssl, :public_key]] +end +MIX +cp "$repo_root/mix.lock" "$work_dir/consumer/mix.lock" +for test_file in "$repo_root"/scripts/ex_ssl_*_test.exs; do + cp "$test_file" "$work_dir/consumer/test/$(basename "$test_file")" +done +cp "$repo_root/scripts/ex_ssl_tls12_peer.py" "$work_dir/consumer/test/ex_ssl_tls12_peer.py" +printf 'ExUnit.start()\n' > "$work_dir/consumer/test/test_helper.exs" +( + cd "$work_dir/consumer" + mix deps.get + mix compile --warnings-as-errors + if [[ "$EX_SSL_DEP_MODE" == published ]]; then + mix run -e 'unless Application.spec(:ex_ssl, :vsn) == ~c"0.4.0", do: raise "expected published ex_ssl 0.4.0"' + fi + mix test "$@" --seed 36 +) diff --git a/scripts/ex_ssl_tls12_peer.py b/scripts/ex_ssl_tls12_peer.py new file mode 100644 index 0000000..cd9a3eb --- /dev/null +++ b/scripts/ex_ssl_tls12_peer.py @@ -0,0 +1,258 @@ +"""Bounded independent OpenSSL peer for packaged ex_ssl consumer tests.""" + +import argparse +import base64 +import hashlib +import json +import select +import socket +import ssl +import struct +import sys + + +def event(kind, **fields): + print(json.dumps({"kind": kind, **fields}), flush=True) + + +def exact(connection, length): + result = bytearray() + while len(result) < length: + part = connection.recv(length - len(result)) + if not part: + raise EOFError("truncated") + result.extend(part) + return bytes(result) + + +def headers(connection): + result = bytearray() + while b"\r\n\r\n" not in result: + if len(result) >= 16384: + raise ValueError("headers_too_large") + part = connection.recv(4096) + if not part: + raise EOFError("headers_truncated") + result.extend(part) + if len(result) > 16384: + raise ValueError("headers_too_large") + return bytes(result) + + +def frame(connection): + head = exact(connection, 9) + length = int.from_bytes(head[:3], "big") + if length > 16384: + raise ValueError("h2_frame_too_large") + return head[3], head[4], int.from_bytes(head[5:], "big") & 0x7FFFFFFF, exact(connection, length) + + +def send_frame(connection, kind, flags, stream, payload): + if len(payload) > 16384: + raise ValueError("h2_frame_too_large") + connection.sendall(len(payload).to_bytes(3, "big") + bytes([kind, flags]) + struct.pack("!I", stream) + payload) + + +def http1(connection, large, path=b"/tls12"): + request = headers(connection) + if not request.startswith(b"GET " + path + b" HTTP/1.1\r\n"): + raise ValueError("wrong_http1_request") + body = b"B" * (262144 if large else 6) + connection.sendall(b"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: " + str(len(body)).encode() + b"\r\n\r\n" + body) + event("exchange", bytes=len(body)) + + +def http2(connection, large): + if exact(connection, 24) != b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n": + raise ValueError("wrong_h2_preface") + reads = 0 + + def next_frame(): + nonlocal reads + if reads >= 256: + raise ValueError("too_many_h2_control_frames") + reads += 1 + return frame(connection) + + kind, _, stream, settings = next_frame() + if kind != 4 or stream != 0: + raise ValueError("missing_h2_settings") + if len(settings) % 6: + raise ValueError("invalid_h2_settings") + stream_window = 65535 + for offset in range(0, len(settings), 6): + setting, value = struct.unpack("!HI", settings[offset:offset + 6]) + if setting == 4: + if value > 0x7FFFFFFF: + raise ValueError("invalid_h2_initial_window") + stream_window = value + connection_window = 65535 + saw_headers = False + acknowledged = False + window_updates = 0 + + def control(kind, flags, stream, payload): + nonlocal acknowledged, connection_window, stream_window, window_updates + if kind == 4 and flags == 1 and stream == 0 and not payload: + acknowledged = True + elif kind == 8 and stream in (0, 1) and len(payload) == 4: + increment = struct.unpack("!I", payload)[0] & 0x7FFFFFFF + if increment == 0: + raise ValueError("invalid_h2_window_update") + if stream == 0: + connection_window += increment + else: + stream_window += increment + if connection_window > 0x7FFFFFFF or stream_window > 0x7FFFFFFF: + raise ValueError("h2_window_overflow") + window_updates += 1 + elif kind in (3, 7): + raise ValueError("h2_stream_or_connection_reset") + + for _ in range(8): + kind, flags, stream, payload = next_frame() + control(kind, flags, stream, payload) + if kind == 1 and stream == 1: + saw_headers = True + break + if not saw_headers: + raise ValueError("missing_h2_headers") + body = b"B" * (262144 if large else 6) + send_frame(connection, 4, 0, 0, b"") + # HPACK indexed :status 200 (8); literal content-length with indexed name (28). + digits = str(len(body)).encode() + block = b"\x88\x0f\x0d" + bytes([len(digits)]) + digits + send_frame(connection, 1, 4, 1, block) + offset = 0 + while offset < len(body): + available = min(16384, connection_window, stream_window, len(body) - offset) + if available == 0: + control(*next_frame()) + continue + chunk = body[offset:offset + available] + flags = 1 if offset + available == len(body) else 0 + send_frame(connection, 0, flags, 1, chunk) + offset += available + connection_window -= available + stream_window -= available + if not acknowledged: + while not acknowledged: + control(*next_frame()) + if not acknowledged: + raise ValueError("missing_h2_settings_ack") + event("exchange", bytes=len(body), window_updates=window_updates) + + +def websocket(connection): + request = headers(connection) + if not request.startswith(b"GET /socket HTTP/1.1\r\n"): + raise ValueError("wrong_wss_request") + fields = request.split(b"\r\n") + keys = [line.split(b":", 1)[1].strip() for line in fields if line.lower().startswith(b"sec-websocket-key:")] + if len(keys) != 1: + raise ValueError("missing_wss_key") + accept = base64.b64encode(hashlib.sha1(keys[0] + b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11").digest()) + connection.sendall(b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: " + accept + b"\r\n\r\n") + event("upgrade") + head = exact(connection, 2) + if head[0] != 0x81 or head[1] != 0x80 + 10: + raise ValueError("wrong_wss_frame") + mask = exact(connection, 4) + payload = exact(connection, 10) + decoded = bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) + if decoded != b"tls12-echo": + raise ValueError("wrong_wss_payload") + echoed = b"echo:" + decoded + connection.sendall(bytes([0x81, len(echoed)]) + echoed) + head = exact(connection, 2) + if head[0] != 0x88 or not (head[1] & 0x80): + raise ValueError("missing_wss_close") + size = head[1] & 0x7F + if size > 125: + raise ValueError("wss_close_too_large") + mask = exact(connection, 4) + payload = exact(connection, size) + decoded = bytes(value ^ mask[index % 4] for index, value in enumerate(payload)) + connection.sendall(bytes([0x88, len(decoded)]) + decoded) + event("exchange", bytes=10) + + +def sse(connection, index): + request = headers(connection) + if not request.startswith(b"GET /events HTTP/1.1\r\n"): + raise ValueError("wrong_sse_request") + last_id_ok = b"Last-Event-ID: 41\r\n" in request + event("request", index=index, last_id_ok=last_id_ok) + if index == 1: + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\nid: 41\ndata: first\n\n") + event("first_ready") + if sys.stdin.buffer.readline() != b"go\n": + raise ValueError("missing_release") + else: + connection.sendall(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: keep-alive\r\n\r\ndata: second\n\n") + event("second_ready") + sys.stdin.buffer.readline() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--certfile", required=True) + parser.add_argument("--keyfile", required=True) + parser.add_argument("--cafile", required=True) + parser.add_argument("--mode", choices=["http1", "h2", "wss", "sse", "resumption"], required=True) + parser.add_argument("--max-version", choices=["tls12", "tls13"], default="tls12") + parser.add_argument("--large", action="store_true") + args = parser.parse_args() + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_3 if args.mode == "resumption" else ssl.TLSVersion.TLSv1_2 + context.maximum_version = ssl.TLSVersion.TLSv1_2 if args.max_version == "tls12" and args.mode != "resumption" else ssl.TLSVersion.TLSv1_3 + context.options |= ssl.OP_NO_COMPRESSION | ssl.OP_NO_RENEGOTIATION + context.load_cert_chain(args.certfile, args.keyfile) + context.load_verify_locations(cafile=args.cafile) + context.verify_mode = ssl.CERT_NONE if args.mode == "resumption" else ssl.CERT_REQUIRED + context.set_ciphers("ECDHE-RSA-AES128-GCM-SHA256") + context.set_ecdh_curve("prime256v1") + context.set_alpn_protocols(["h2" if args.mode == "h2" else "http/1.1"]) + + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(4) + listener.settimeout(10) + event("ready", port=listener.getsockname()[1], openssl=ssl.OPENSSL_VERSION) + count = 2 if args.mode in ("sse", "resumption") else 1 + for index in range(1, count + 1): + raw, _ = listener.accept() + raw.settimeout(10) + try: + with context.wrap_socket(raw, server_side=True) as connection: + der = connection.getpeercert(binary_form=True) + event("handshake", index=index, version=connection.version(), cipher=connection.cipher()[0], alpn=connection.selected_alpn_protocol(), client_der_b64=base64.b64encode(der).decode() if der else None, session_reused=connection.session_reused) + if args.mode == "http1": + http1(connection, args.large) + elif args.mode == "resumption": + http1(connection, False, b"/resumption") + elif args.mode == "h2": + http2(connection, args.large) + elif args.mode == "wss": + websocket(connection) + else: + sse(connection, index) + if args.mode == "h2": + if not select.select([sys.stdin], [], [], 10)[0] or sys.stdin.buffer.readline() != b"go\n": + raise ValueError("missing_h2_release") + event("released") + if args.mode != "sse": + try: + connection.unwrap().close() + except (ssl.SSLError, OSError, EOFError): + pass + except (ssl.SSLError, OSError, EOFError, ValueError) as error: + event("failure", kind_name=type(error).__name__) + raw.close() + listener.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/ex_ssl_tls12_test.exs b/scripts/ex_ssl_tls12_test.exs new file mode 100644 index 0000000..8abdc58 --- /dev/null +++ b/scripts/ex_ssl_tls12_test.exs @@ -0,0 +1,274 @@ +# Run only through scripts/ex_ssl_source_smoke.sh with an explicit source checkout. +for fixture <- ["signature_fixtures.ex", "client_auth_fixtures.ex"] do + Code.require_file(Path.join([System.fetch_env!("EX_SSL_SOURCE_DIR"), "test/support", fixture])) +end + +defmodule CandidateTLS12Test do + use ExUnit.Case, async: false + + alias ExSSL.TestSupport.ClientAuthFixtures + alias HTTP.EventSource + alias HTTP.WebSocket + + @timeout 10_000 + @python Path.join(__DIR__, "ex_ssl_tls12_peer.py") + + setup_all do + directory = Path.join(System.tmp_dir!(), "http-tls12-#{System.unique_integer([:positive])}") + on_exit(fn -> File.rm_rf!(directory) end) + {:ok, fixtures: ClientAuthFixtures.create(directory)} + end + + for protocol <- [:http1, :http2], versions <- [[:"tlsv1.2"], [:"tlsv1.3", :"tlsv1.2"]] do + test "packaged #{protocol} fetch uses required mTLS over #{inspect(versions)} and returns large response", + %{ + fixtures: fixtures + } do + mode = if unquote(protocol) == :http2, do: "h2", else: "http1" + + with_peer(fixtures, mode, ["--large"], fn peer -> + promise = + HTTP.fetch("https://127.0.0.1:#{peer.port}/tls12", + tls_backend: :ex_ssl, + http_version: unquote(protocol), + ssl: + client_ssl( + fixtures, + unquote(versions), + if(mode == "h2", do: "h2", else: "http/1.1") + ), + timeout: @timeout, + connect_timeout: @timeout + ) + + expected_bytes = 262_144 + assert_handshake(peer, fixtures, "TLSv1.2", if(mode == "h2", do: "h2", else: "http/1.1")) + assert {:ok, %{"bytes" => ^expected_bytes} = exchange} = event(peer, "exchange") + if mode == "h2", do: assert(exchange["window_updates"] > 0) + response = HTTP.Promise.await(promise, @timeout) + assert response.status == 200 + assert HTTP.Response.read_all(response) == :binary.copy("B", expected_bytes) + + if mode == "h2" do + true = Port.command(peer.handle, "go\n") + assert {:ok, _} = event(peer, "released") + end + end) + end + end + + test "mixed offer selects TLS 1.3 on a capable peer without retry", %{fixtures: fixtures} do + with_peer(fixtures, "http1", ["--max-version", "tls13"], fn peer -> + response = + HTTP.fetch("https://127.0.0.1:#{peer.port}/tls12", + tls_backend: :ex_ssl, + ssl: client_ssl(fixtures, [:"tlsv1.3", :"tlsv1.2"], "http/1.1"), + timeout: @timeout, + connect_timeout: @timeout + ) + |> HTTP.Promise.await(@timeout) + + assert response.status == 200 + assert HTTP.Response.read_all(response) == "BBBBBB" + assert_handshake(peer, fixtures, "TLSv1.3", "http/1.1") + assert {:ok, %{"bytes" => 6}} = event(peer, "exchange") + end) + end + + test "packaged WSS upgrades through TLS 1.2 mTLS and echoes an active-once frame", %{ + fixtures: fixtures + } do + with_peer(fixtures, "wss", [], fn peer -> + socket = + WebSocket.new("wss://127.0.0.1:#{peer.port}/socket", [], + tls_backend: :ex_ssl, + ssl: client_ssl(fixtures, [:"tlsv1.2"], "http/1.1") + ) + + try do + assert_handshake(peer, fixtures, "TLSv1.2", "http/1.1") + assert {:ok, _} = event(peer, "upgrade") + assert_receive {WebSocket, ^socket, %HTTP.WebSocket.Event.Open{}}, @timeout + assert :ok = WebSocket.send(socket, "tls12-echo") + + assert_receive {WebSocket, ^socket, + %HTTP.WebSocket.Event.Message{data: "echo:tls12-echo"}}, + @timeout + + assert :ok = WebSocket.close(socket, 1000, "done") + + assert_receive {WebSocket, ^socket, + %HTTP.WebSocket.Event.Close{code: 1000, was_clean: true}}, + @timeout + + assert {:ok, %{"bytes" => 10}} = event(peer, "exchange") + after + _ = WebSocket.close(socket) + end + end) + end + + test "packaged SSE reconnects to same TLS 1.2 mTLS origin with backend and Last-Event-ID pinned", + %{ + fixtures: fixtures + } do + previous = Application.get_env(:http_core, :tls_backend, :unset) + on_exit(fn -> restore_backend(previous) end) + + with_peer(fixtures, "sse", [], fn peer -> + source = + EventSource.new("https://127.0.0.1:#{peer.port}/events", + tls_backend: :ex_ssl, + reconnect_time: 10, + ssl: client_ssl(fixtures, [:"tlsv1.2"], "http/1.1") + ) + + try do + assert_handshake(peer, fixtures, "TLSv1.2", "http/1.1", 1) + assert {:ok, %{"index" => 1, "last_id_ok" => false}} = event(peer, "request") + assert_receive {EventSource, ^source, %HTTP.EventSource.Event.Open{}}, @timeout + + assert_receive {EventSource, ^source, + %HTTP.EventSource.Event.Message{data: "first", last_event_id: "41"}}, + @timeout + + assert {:ok, _} = event(peer, "first_ready") + Application.put_env(:http_core, :tls_backend, :invalid) + assert %{tls_backend: :ex_ssl} = :sys.get_state(source.pid) + true = Port.command(peer.handle, "go\n") + + assert_handshake(peer, fixtures, "TLSv1.2", "http/1.1", 2) + assert {:ok, %{"index" => 2, "last_id_ok" => true}} = event(peer, "request") + assert_receive {EventSource, ^source, %HTTP.EventSource.Event.Open{}}, @timeout + + assert_receive {EventSource, ^source, + %HTTP.EventSource.Event.Message{data: "second", last_event_id: "41"}}, + @timeout + + assert {:ok, _} = event(peer, "second_ready") + after + _ = EventSource.close(source) + end + end) + end + + defp client_ssl(fixtures, versions, protocol) do + [ + versions: versions, + verify: :verify_peer, + alpn_advertised_protocols: [protocol], + cacerts: [fixtures.ca.der], + server_name_indication: ~c"exssl.test", + certfile: fixtures.rsa.certificate, + keyfile: fixtures.rsa.key + ] + end + + defp with_peer(fixtures, mode, extra, function) do + peer = start_peer(fixtures, mode, extra) + + try do + function.(peer) + after + stop_peer(peer) + end + + assert {:error, :econnrefused} = + :gen_tcp.connect(~c"127.0.0.1", peer.port, [:binary, active: false], 1_000) + end + + defp start_peer(fixtures, mode, extra) do + assert File.regular?(@python) + executable = System.find_executable("python3") || raise "python3 unavailable" + + args = + [ + "-B", + "-u", + @python, + "--certfile", + fixtures.server.certificate, + "--keyfile", + fixtures.server.key, + "--cafile", + fixtures.ca.certificate, + "--mode", + mode + ] ++ extra + + handle = + Port.open({:spawn_executable, executable}, [ + :binary, + :exit_status, + :stderr_to_stdout, + {:line, 65_536}, + args: args + ]) + + {:os_pid, os_pid} = Port.info(handle, :os_pid) + peer = %{handle: handle, os_pid: os_pid, port: nil} + + case event(peer, "ready") do + {:ok, %{"port" => port}} -> + %{peer | port: port} + + error -> + stop_peer(peer) + raise "TLS 1.2 peer startup failed: #{inspect(error)}" + end + end + + defp assert_handshake(peer, fixtures, version, alpn, index \\ 1) do + assert {:ok, + %{ + "index" => ^index, + "version" => ^version, + "alpn" => ^alpn, + "client_der_b64" => client_der, + "cipher" => cipher + }} = event(peer, "handshake") + + assert Base.decode64!(client_der) == fixtures.rsa.der + if version == "TLSv1.2", do: assert(cipher == "ECDHE-RSA-AES128-GCM-SHA256") + end + + defp event(%{handle: handle}, expected) do + receive do + {^handle, {:data, {:eol, line}}} -> + case :json.decode(line) do + %{"kind" => ^expected} = value -> {:ok, value} + %{"kind" => "failure"} = value -> {:error, value} + other -> {:error, {:unexpected_peer_event, other}} + end + + {^handle, {:data, {:noeol, _}}} -> + {:error, :peer_line_too_long} + + {^handle, {:exit_status, status}} -> + {:error, {:peer_exit, status}} + after + @timeout -> {:error, :peer_timeout} + end + rescue + _ -> {:error, :invalid_peer_event} + end + + defp stop_peer(%{handle: handle, os_pid: os_pid}) do + case Port.info(handle, :os_pid) do + {:os_pid, ^os_pid} -> + _ = System.cmd("kill", ["-TERM", Integer.to_string(os_pid)], stderr_to_stdout: true) + + receive do + {^handle, {:exit_status, _}} -> :ok + after + 2_000 -> raise "TLS 1.2 peer did not stop" + end + + nil -> + :ok + end + end + + defp restore_backend(:unset), do: Application.delete_env(:http_core, :tls_backend) + defp restore_backend(value), do: Application.put_env(:http_core, :tls_backend, value) +end diff --git a/scripts/external_consumer_smoke.exs b/scripts/external_consumer_smoke.exs new file mode 100644 index 0000000..b4a525b --- /dev/null +++ b/scripts/external_consumer_smoke.exs @@ -0,0 +1,311 @@ +defmodule ExternalConsumerSmoke do + alias HTTP.HTTP2.Frame + alias HTTP.HTTP2.HPACK + + @http1_response "HTTP/1.1 200 OK\r\nContent-Length: 11\r\nConnection: close\r\n\r\nconsumer-ok" + @end_stream 0x1 + @end_headers 0x4 + @guid "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + def run do + for app <- [:http_fetch, :http_web_socket, :http_event_source, :http_web_transport] do + {:ok, _} = Application.ensure_all_started(app) + end + + assert Enum.any?(Application.started_applications(), fn {app, _, _} -> app == :ex_ssl end), + ":ex_ssl was not started through http_core's package dependency" + + assert Application.spec(:ex_ssl, :vsn) == ~c"0.4.0", + "published ex_ssl 0.4.0 must be loaded through http_core" + + certfile = System.fetch_env!("HTTP_FETCH_CERTFILE") + cacertfile = System.fetch_env!("HTTP_FETCH_CACERTFILE") + keyfile = System.fetch_env!("HTTP_FETCH_KEYFILE") + + assert_package_metadata!() + + {:ok, :ssl} = HTTP.TLSBackend.resolve() + fetch_http1!(certfile, keyfile, cacertfile) + fetch_h2!(certfile, keyfile, cacertfile) + web_socket!(certfile, keyfile, cacertfile) + event_source!(certfile, keyfile, cacertfile) + + {:error, :tls_backend_not_supported_for_quic} = + HTTP.WebTransport.new("https://localhost:443/", tls_backend: :ex_ssl) + + Application.put_env(:http_core, :tls_backend, :ex_ssl) + + {:ok, %{backend: HTTP.WebTransport.Transport.QUIC}} = + HTTP.WebTransport.Options.new("https://localhost:443/") + + Application.delete_env(:http_core, :tls_backend) + + IO.puts("external consumer smoke passed") + end + + defp fetch_http1!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, []) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {:ok, _} = recv_headers(socket) + :ok = :ssl.send(socket, @http1_response) + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [ssl: [cacertfile: cacertfile]], + ex_ssl: [tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + response = HTTP.fetch("https://localhost:#{port}/", options) |> HTTP.Promise.await() + + assert response.status == 200 and HTTP.Response.read_all(response) == "consumer-ok", + "HTTP/1.1 #{label} failed" + end + end + + defp assert_package_metadata! do + package_dir = System.fetch_env!("HTTP_FETCH_PACKAGE_DIR") + core = metadata!(package_dir, "http_core") + core_version = Map.fetch!(core, <<"version">>) + + assert requirement!(core, <<"ex_ssl">>) == <<"~> 0.4.0">>, + "http_core package must require ex_ssl ~> 0.4.0" + + for app <- ["http_fetch", "http_web_socket", "http_event_source", "http_web_transport"] do + assert requirement!(metadata!(package_dir, app), <<"http_core">>) == "~> " <> core_version, + "#{app} package must require the built http_core version" + end + end + + defp metadata!(package_dir, app) do + {:ok, entries} = + :file.consult(String.to_charlist(Path.join([package_dir, app, "hex_metadata.config"]))) + + Map.new(entries) + end + + defp requirement!(metadata, name) do + metadata + |> Map.fetch!(<<"requirements">>) + |> Enum.map(&Map.new/1) + |> Enum.find_value(fn requirement -> + if Map.fetch!(requirement, <<"name">>) == name do + assert Map.fetch!(requirement, <<"optional">>) == false, + "#{name} package dependency must not be optional" + + Map.fetch!(requirement, <<"requirement">>) + end + end) + |> case do + nil -> raise "package is missing #{name} requirement" + requirement -> requirement + end + end + + defp fetch_h2!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, alpn_preferred_protocols: [<<"h2">>]) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {preface, buffer} = recv_exact(socket, byte_size(HTTP.HTTP2.connection_preface()), <<>>) + assert preface == HTTP.HTTP2.connection_preface(), "missing HTTP/2 preface" + {_settings, buffer} = recv_frame(socket, buffer) + {%Frame{type: :headers, stream_id: 1}, _buffer} = recv_frame(socket, buffer) + headers = HPACK.encode_headers([{":status", "200"}, {"content-length", "2"}]) + + :ok = + :ssl.send(socket, [ + Frame.encode(:settings, 0, 0, ""), + Frame.encode(:headers, @end_headers, 1, headers), + Frame.encode(:data, @end_stream, 1, "h2") + ]) + + {%Frame{type: :settings, flags: 1, stream_id: 0, payload: ""}, _buffer} = + recv_frame(socket, "") + + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [http_version: :http2, ssl: [cacertfile: cacertfile]], + ex_ssl: [http_version: :http2, tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + response = HTTP.fetch("https://localhost:#{port}/", options) |> HTTP.Promise.await() + + assert response.status == 200 and HTTP.Response.read_all(response) == "h2", + "HTTP/2 #{label} failed" + end + end + + defp web_socket!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, []) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {:ok, request} = recv_headers(socket) + [_, key] = Regex.run(~r/sec-websocket-key:\s*([^\r\n]+)/i, request) + accept = :crypto.hash(:sha, String.trim(key) <> @guid) |> Base.encode64() + + :ok = + :ssl.send(socket, [ + "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ", + accept, + "\r\n\r\n", + <<0x81, 11, "consumer-ws">> + ]) + + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [ssl: [cacertfile: cacertfile]], + ex_ssl: [tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + socket = HTTP.WebSocket.new("wss://localhost:#{port}/", [], options) + assert match?(%HTTP.WebSocket{}, socket), "WebSocket #{label} did not start" + await_web_socket_open(socket) + await_web_socket_message(socket, "consumer-ws") + end + end + + defp event_source!(certfile, keyfile, cacertfile) do + {listen, port} = listen!(certfile, keyfile, []) + + spawn_link(fn -> + for _ <- 1..2 do + {:ok, socket} = accept!(listen) + {:ok, _} = recv_headers(socket) + + :ok = + :ssl.send( + socket, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\ndata: consumer-sse\n\n" + ) + + :ssl.close(socket) + end + + :ssl.close(listen) + end) + + for {label, options} <- [ + default: [ssl: [cacertfile: cacertfile]], + ex_ssl: [tls_backend: :ex_ssl, ssl: [cacertfile: cacertfile]] + ] do + source = HTTP.EventSource.new("https://localhost:#{port}/", options) + assert match?(%HTTP.EventSource{}, source), "EventSource #{label} did not start" + await_event_source_open(source) + await_event_source_message(source, "consumer-sse") + :ok = HTTP.EventSource.close(source) + end + end + + defp listen!(certfile, keyfile, extra) do + {:ok, listen} = + :ssl.listen( + 0, + [ + :binary, + active: false, + ip: {127, 0, 0, 1}, + reuseaddr: true, + versions: [:"tlsv1.3"], + certfile: certfile, + keyfile: keyfile + ] ++ extra + ) + + {:ok, {{127, 0, 0, 1}, port}} = :ssl.sockname(listen) + {listen, port} + end + + defp accept!(listen) do + with {:ok, transport} <- :ssl.transport_accept(listen, 5_000), + do: :ssl.handshake(transport, 5_000) + end + + defp recv_headers(socket, buffer \\ "") do + if String.contains?(buffer, "\r\n\r\n"), + do: {:ok, buffer}, + else: + with({:ok, data} <- :ssl.recv(socket, 0, 5_000), do: recv_headers(socket, buffer <> data)) + end + + defp recv_exact(_socket, 0, buffer), do: {"", buffer} + + defp recv_exact(_socket, size, buffer) when byte_size(buffer) >= size do + <> = buffer + {data, rest} + end + + defp recv_exact(socket, size, buffer) do + {:ok, data} = :ssl.recv(socket, 0, 5_000) + recv_exact(socket, size, buffer <> data) + end + + defp recv_frame(socket, buffer) do + case Frame.decode(buffer) do + {:ok, frame, rest} -> + {frame, rest} + + :more -> + {:ok, data} = :ssl.recv(socket, 0, 5_000) + recv_frame(socket, buffer <> data) + end + end + + defp await_web_socket_open(socket) do + receive do + {HTTP.WebSocket, ^socket, %HTTP.WebSocket.Event.Open{}} -> :ok + _ -> await_web_socket_open(socket) + after + 5_000 -> raise "WebSocket did not open" + end + end + + defp await_web_socket_message(socket, expected) do + receive do + {HTTP.WebSocket, ^socket, %HTTP.WebSocket.Event.Message{data: ^expected}} -> :ok + _ -> await_web_socket_message(socket, expected) + after + 5_000 -> raise "WebSocket did not deliver #{expected}" + end + end + + defp await_event_source_open(source) do + receive do + {HTTP.EventSource, ^source, %HTTP.EventSource.Event.Open{}} -> :ok + _ -> await_event_source_open(source) + after + 5_000 -> raise "EventSource did not open" + end + end + + defp await_event_source_message(source, expected) do + receive do + {HTTP.EventSource, ^source, %HTTP.EventSource.Event.Message{data: ^expected}} -> :ok + _ -> await_event_source_message(source, expected) + after + 5_000 -> raise "EventSource did not deliver #{expected}" + end + end + + defp assert(value, message) + defp assert(true, _message), do: :ok + defp assert(false, message), do: raise(message) +end + +ExternalConsumerSmoke.run() diff --git a/scripts/external_consumer_smoke.sh b/scripts/external_consumer_smoke.sh new file mode 100644 index 0000000..d45ecbc --- /dev/null +++ b/scripts/external_consumer_smoke.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT + +package_dir="$work_dir/packages" +consumer_dir="$work_dir/consumer" +mkdir -p "$package_dir" + +apps=(http_core http_fetch http_web_socket http_event_source http_web_transport) + +for app in "${apps[@]}"; do + ( + cd "$repo_root/apps/$app" + MIX_ENV=prod mix hex.build --unpack -o "$package_dir/$app" + ) +done + +mix new "$consumer_dir" --sup >/dev/null + +cat >"$consumer_dir/mix.exs" <<'EOF' +defmodule ExternalConsumer.MixProject do + use Mix.Project + + def project do + [app: :external_consumer, version: "0.1.0", deps: deps()] + end + + def application do + [extra_applications: [:logger, :public_key, :ssl]] + end + + defp deps do + package_dir = System.fetch_env!("HTTP_FETCH_PACKAGE_DIR") + + [ + {:http_core, path: Path.join(package_dir, "http_core")}, + {:http_fetch, path: Path.join(package_dir, "http_fetch")}, + {:http_web_socket, path: Path.join(package_dir, "http_web_socket")}, + {:http_event_source, path: Path.join(package_dir, "http_event_source")}, + {:http_web_transport, path: Path.join(package_dir, "http_web_transport")} + ] + end +end +EOF + +cp "$repo_root/scripts/external_consumer_smoke.exs" "$consumer_dir/smoke.exs" + +( + cd "$consumer_dir" + export HTTP_FETCH_PACKAGE_DIR="$package_dir" + export HTTP_FETCH_CERTFILE="$repo_root/apps/http_fetch/test/support/fixtures/localhost.pem" + export HTTP_FETCH_CACERTFILE="$repo_root/apps/http_fetch/test/support/fixtures/localhost-ca.pem" + export HTTP_FETCH_KEYFILE="$repo_root/apps/http_fetch/test/support/fixtures/localhost.key" + MIX_ENV=prod mix deps.get + MIX_ENV=prod mix deps.tree --only runtime + MIX_ENV=prod mix compile --warnings-as-errors + MIX_ENV=prod mix run smoke.exs +)