diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6b1a20..022f9d8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,8 +19,12 @@ jobs: with: command: check licenses - test: + libcronet: needs: lint-audit + uses: ./.github/workflows/libcronet.yml + + test: + needs: [lint-audit, libcronet] name: cargo test (unit + integration) runs-on: ubuntu-latest steps: @@ -28,16 +32,24 @@ jobs: - name: Rust Cache uses: swatinem/rust-cache@v2 with: + save-if: ${{ github.event_name != 'pull_request' }} key: test-ubuntu - name: Check formatting run: cargo fmt --check - name: Lint with clippy run: cargo clippy --all-targets -- -D warnings + - name: Fetch libcronet + uses: actions/download-artifact@v8 + with: + name: libcronet-x86_64-unknown-linux-gnu + path: cronet + - name: Point the loader at it + run: echo "CRONET_LIB=$PWD/cronet/libcronet.so" >> "$GITHUB_ENV" - name: Run tests run: cargo test --all-targets linux-gnu: - needs: [lint-audit, test] + needs: [lint-audit, test, libcronet] name: Build Linux (GNU) runs-on: ubuntu-latest strategy: @@ -61,6 +73,7 @@ jobs: - name: Rust Cache uses: swatinem/rust-cache@v2 with: + save-if: ${{ github.event_name != 'pull_request' }} key: ${{ matrix.target_triple }} - name: Install base dependencies run: | @@ -71,6 +84,27 @@ jobs: run: sudo apt-get install -y ${{ matrix.apt_packages }} - name: Add Rust target run: rustup target add ${{ matrix.target_triple }} + - name: Fetch libcronet + uses: actions/download-artifact@v8 + with: + name: libcronet-${{ matrix.target_triple }} + path: cronet + - name: Link with lld + if: ${{ matrix.target == 'arm64' }} + env: ${{ matrix.custom_env }} + run: | + host=$(rustc -vV | sed -n 's/^host: //p') + bin="$(rustc --print sysroot)/lib/rustlib/$host/bin" + prefix=${CC%-gcc} + mkdir -p "$HOME/lld" + printf '#!/bin/sh\nexec "%s/rust-lld" -flavor gnu "$@"\n' "$bin" > "$HOME/lld/lld-wrapper" + chmod +x "$HOME/lld/lld-wrapper" + ln -sf lld-wrapper "$HOME/lld/ld" + ln -sf lld-wrapper "$HOME/lld/$prefix-ld" + echo "$HOME/lld" >> "$GITHUB_PATH" + echo "RUSTFLAGS=-C link-arg=-B$HOME/lld" >> "$GITHUB_ENV" + - name: Point the build at it + run: echo "CRONET_LIB_DIR=$PWD/cronet" >> "$GITHUB_ENV" - name: Build env: ${{ matrix.custom_env }} run: cargo build --release --target ${{ matrix.target_triple }} @@ -79,6 +113,7 @@ jobs: mkdir -p dist cp target/${{ matrix.target_triple }}/release/client dist/ cp target/${{ matrix.target_triple }}/release/server dist/ + cp cronet/LICENSE dist/CHROMIUM-LICENSE - name: Upload uses: actions/upload-artifact@v6 with: @@ -86,7 +121,7 @@ jobs: path: dist/ linux-musl: - needs: [lint-audit, test] + needs: [lint-audit, test, libcronet] name: Build Linux (musl) runs-on: ubuntu-latest strategy: @@ -114,6 +149,7 @@ jobs: - name: Rust Cache uses: swatinem/rust-cache@v2 with: + save-if: ${{ github.event_name != 'pull_request' }} key: ${{ matrix.target_triple }} - name: Install base dependencies run: | @@ -126,6 +162,27 @@ jobs: echo "/opt/${{ matrix.package }}/bin/" >> $GITHUB_PATH - name: Add Rust target run: rustup target add ${{ matrix.target_triple }} + - name: Fetch libcronet + uses: actions/download-artifact@v8 + with: + name: libcronet-${{ matrix.target_triple }} + path: cronet + - name: Link with lld + if: ${{ matrix.target == 'arm64' }} + env: ${{ matrix.custom_env }} + run: | + host=$(rustc -vV | sed -n 's/^host: //p') + bin="$(rustc --print sysroot)/lib/rustlib/$host/bin" + prefix=${CC%-gcc} + mkdir -p "$HOME/lld" + printf '#!/bin/sh\nexec "%s/rust-lld" -flavor gnu "$@"\n' "$bin" > "$HOME/lld/lld-wrapper" + chmod +x "$HOME/lld/lld-wrapper" + ln -sf lld-wrapper "$HOME/lld/ld" + ln -sf lld-wrapper "$HOME/lld/$prefix-ld" + echo "$HOME/lld" >> "$GITHUB_PATH" + echo "RUSTFLAGS=-C link-arg=-B$HOME/lld" >> "$GITHUB_ENV" + - name: Point the build at it + run: echo "CRONET_LIB_DIR=$PWD/cronet" >> "$GITHUB_ENV" - name: Build env: ${{ matrix.custom_env }} run: cargo build --release --target ${{ matrix.target_triple }} @@ -134,6 +191,7 @@ jobs: mkdir -p dist cp target/${{ matrix.target_triple }}/release/client dist/ cp target/${{ matrix.target_triple }}/release/server dist/ + cp cronet/LICENSE dist/CHROMIUM-LICENSE - name: Upload uses: actions/upload-artifact@v6 with: @@ -141,7 +199,7 @@ jobs: path: dist/ windows: - needs: [lint-audit, test] + needs: [lint-audit, test, libcronet] name: Build Windows runs-on: windows-latest strategy: @@ -156,10 +214,19 @@ jobs: - name: Rust Cache uses: swatinem/rust-cache@v2 with: + save-if: ${{ github.event_name != 'pull_request' }} key: ${{ matrix.target_triple }} - name: Install dependencies run: choco install cmake strawberryperl pkgconfiglite llvm nasm -y shell: cmd + - name: Fetch libcronet + uses: actions/download-artifact@v8 + with: + name: libcronet-${{ matrix.target_triple }} + path: cronet + - name: Point the build at it + shell: bash + run: echo "CRONET_LIB_DIR=$(cygpath -w "$PWD/cronet")" >> "$GITHUB_ENV" - name: Build env: RUSTFLAGS: "-C target-feature=+crt-static" @@ -172,6 +239,7 @@ jobs: New-Item -ItemType Directory -Force -Path dist Copy-Item "target\${{ matrix.target_triple }}\release\client.exe" -Destination dist\ Copy-Item "target\${{ matrix.target_triple }}\release\server.exe" -Destination dist\ + Copy-Item "cronet\LICENSE" -Destination dist\CHROMIUM-LICENSE - name: Upload uses: actions/upload-artifact@v6 with: @@ -179,7 +247,7 @@ jobs: path: dist/ darwin: - needs: [lint-audit, test] + needs: [lint-audit, test, libcronet] name: Build Darwin runs-on: macos-latest strategy: @@ -194,11 +262,19 @@ jobs: - name: Rust Cache uses: swatinem/rust-cache@v2 with: + save-if: ${{ github.event_name != 'pull_request' }} key: ${{ matrix.target_triple }} - name: Install dependencies run: brew install llvm - name: Add Rust target run: rustup target add ${{ matrix.target_triple }} + - name: Fetch libcronet + uses: actions/download-artifact@v8 + with: + name: libcronet-${{ matrix.target_triple }} + path: cronet + - name: Point the build at it + run: echo "CRONET_LIB_DIR=$PWD/cronet" >> "$GITHUB_ENV" - name: Build run: cargo build --release --target ${{ matrix.target_triple }} - name: Archive @@ -206,6 +282,7 @@ jobs: mkdir -p dist cp target/${{ matrix.target_triple }}/release/client dist/ cp target/${{ matrix.target_triple }}/release/server dist/ + cp cronet/LICENSE dist/CHROMIUM-LICENSE - name: Upload uses: actions/upload-artifact@v6 with: @@ -213,7 +290,7 @@ jobs: path: dist/ android: - needs: [lint-audit, test] + needs: [lint-audit, test, libcronet] name: Build Android runs-on: ubuntu-latest strategy: @@ -228,11 +305,25 @@ jobs: - name: Rust Cache uses: swatinem/rust-cache@v2 with: + save-if: ${{ github.event_name != 'pull_request' }} key: ${{ matrix.target_triple }} - name: Install Android Tools run: cargo install cargo-ndk - name: Add Rust target run: rustup target add ${{ matrix.target_triple }} + - name: Fetch libcronet + uses: actions/download-artifact@v8 + with: + name: libcronet-${{ matrix.target_triple }} + path: cronet + - name: Select the NDK + run: | + ndk="${ANDROID_NDK_LATEST_HOME:-${ANDROID_NDK_HOME:-}}" + if [ -n "$ndk" ]; then + echo "ANDROID_NDK_HOME=$ndk" >> "$GITHUB_ENV" + fi + - name: Point the build at it + run: echo "CRONET_LIB_DIR=$PWD/cronet" >> "$GITHUB_ENV" - name: Build run: cargo ndk --target ${{ matrix.target_triple }} --platform 28 build --release - name: Archive @@ -262,6 +353,7 @@ jobs: uses: actions/download-artifact@v8 with: path: artifacts + pattern: httproxy-* - name: Final Packaging run: | mkdir final_dist @@ -272,7 +364,7 @@ jobs: cd final_dist sha256sum *.zip > sha256sums.txt - name: Create GitHub Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: | final_dist/*.zip diff --git a/.github/workflows/libcronet.yml b/.github/workflows/libcronet.yml new file mode 100644 index 0000000..804665d --- /dev/null +++ b/.github/workflows/libcronet.yml @@ -0,0 +1,76 @@ +name: libcronet + +on: + workflow_call: + workflow_dispatch: + +jobs: + build: + name: ${{ matrix.triple }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - {triple: x86_64-unknown-linux-gnu, os: ubuntu-22.04} + - {triple: aarch64-unknown-linux-gnu, os: ubuntu-22.04} + - {triple: x86_64-unknown-linux-musl, os: ubuntu-22.04} + - {triple: aarch64-unknown-linux-musl, os: ubuntu-22.04} + - {triple: x86_64-linux-android, os: ubuntu-22.04} + - {triple: aarch64-linux-android, os: ubuntu-22.04} + - {triple: x86_64-apple-darwin, os: macos-latest} + - {triple: aarch64-apple-darwin, os: macos-latest} + - {triple: x86_64-pc-windows-msvc, os: windows-2022} + - {triple: aarch64-pc-windows-msvc, os: windows-2022} + steps: + - uses: actions/checkout@v6 + - name: Restore the package + id: package + uses: actions/cache@v5 + with: + path: cronet/${{ matrix.triple }} + key: cronet-package-${{ matrix.triple }}-${{ hashFiles('naiveproxy.lock', 'scripts/build-libcronet.sh') }}-v1 + - name: Cache the toolchain + if: steps.package.outputs.cache-hit != 'true' + uses: actions/cache@v5 + with: + path: | + .cronet-build/naiveproxy/src/third_party/llvm-build + .cronet-build/naiveproxy/src/gn + .cronet-build/naiveproxy/src/out/sysroot-build + key: cronet-${{ runner.os }}-${{ hashFiles('naiveproxy.lock') }}-v1 + - name: Cache the Android NDK + if: contains(matrix.triple, 'android') && steps.package.outputs.cache-hit != 'true' + uses: actions/cache@v5 + with: + path: .cronet-build/naiveproxy/src/third_party/android_toolchain + key: cronet-ndk-${{ hashFiles('naiveproxy.lock') }}-v1 + - name: Install ninja + if: steps.package.outputs.cache-hit != 'true' + shell: bash + run: | + case "$RUNNER_OS" in + Linux) + sudo apt-get update + sudo apt-get install -y ninja-build ccache zstd xz-utils + ;; + macOS) + brew install ninja ccache + ;; + Windows) + choco install ninja -y --no-progress + if ! command -v xz >/dev/null 2>&1; then + curl -fsSL -o xz.zip https://tukaani.org/xz/xz-5.6.4-windows.zip + unzip -q xz.zip -d xz + echo "$PWD/xz/bin_x86-64" >> "$GITHUB_PATH" + fi + ;; + esac + - name: Build + if: steps.package.outputs.cache-hit != 'true' + shell: bash + run: sh scripts/build-libcronet.sh ${{ matrix.triple }} + - uses: actions/upload-artifact@v6 + with: + name: libcronet-${{ matrix.triple }} + path: cronet/${{ matrix.triple }} diff --git a/.gitignore b/.gitignore index 1de5659..0b8a6f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -target \ No newline at end of file +/target +/cronet +/.cronet-build diff --git a/CONFIGURATION.md b/CONFIGURATION.md index c15bcaf..e458ef7 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -88,7 +88,7 @@ padding_threshold = 500 | --------------------- | ------- | -------- | ----------- | ----------- | | `listen` | string | yes | — | Local listen address, e.g. `"127.0.0.1:8080"`. | | `remote` | string | yes | — | Server URL including the hidden path. | -| `address` | string | no | host of `remote` | Connection address override. TLS SNI and `Host` still use the `remote` domain. | +| `address` | string | no | system DNS | Connection address override. TLS SNI and `Host` still use the `remote` domain. | | `public_key` | string | yes | — | Server's ML-KEM-768 encapsulation key; required for the tunnel handshake. | | `proxy_auth` | table | no | — | Local proxy authentication (§2.2). | | `max_connections` | integer | no | `1024` | Concurrent local connections; excess are refused. | diff --git a/Cargo.lock b/Cargo.lock index e8bbdea..b7c1047 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "aho-corasick" version = "1.1.5" @@ -17,21 +11,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "android_system_properties" version = "0.1.6" @@ -271,24 +250,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bindgen" -version = "0.72.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex 1.3.0", - "syn 2.0.119", -] - [[package]] name = "bit-vec" version = "0.9.1" @@ -313,52 +274,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "brotli" -version = "8.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "btls" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f" -dependencies = [ - "bitflags", - "btls-sys", - "foreign-types", - "libc", - "openssl-macros", -] - -[[package]] -name = "btls-sys" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1b8638a2e1c38a5ae4efa90ae57e643baec35a30d03fc5b399b893adc4954b" -dependencies = [ - "bindgen", - "cmake", - "fs_extra", - "fslock", -] - [[package]] name = "bumpalo" version = "3.20.3" @@ -380,16 +295,7 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex 2.0.1", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", + "shlex", ] [[package]] @@ -426,17 +332,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "clang-sys" -version = "1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.6.7" @@ -525,15 +420,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32fast" -version = "1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64" -dependencies = [ - "cfg-if", -] - [[package]] name = "crossbeam-channel" version = "0.5.17" @@ -828,17 +714,6 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" -[[package]] -name = "flate2" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" -dependencies = [ - "crc32fast", - "miniz_oxide", - "zlib-rs", -] - [[package]] name = "fnv" version = "1.0.7" @@ -851,33 +726,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" -dependencies = [ - "foreign-types-macros", - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-macros" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5190182e6915eb873ddbc16e23b711b6eb1f9c00a0d0a3a91b5f6228475225" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.10", -] - -[[package]] -name = "foreign-types-shared" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -893,16 +741,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "fslock" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "fst" version = "0.4.7" @@ -1024,12 +862,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "glob" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" - [[package]] name = "group" version = "0.14.0" @@ -1123,25 +955,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http2" -version = "0.5.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d3114be2f413b2e491e686b93a28cda30c355cffc8d091a57f8be4b1342896" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "smallvec", - "tokio", - "tokio-util", -] - [[package]] name = "httparse" version = "1.10.1" @@ -1171,7 +984,6 @@ dependencies = [ "futures", "h2", "http", - "http-body-util", "httparse", "ip_network", "ip_network_table", @@ -1186,7 +998,6 @@ dependencies = [ "rustls", "serde", "serde_json", - "singleflight-async", "tokio", "tokio-rustls", "tokio-socks", @@ -1200,8 +1011,6 @@ dependencies = [ "url", "uuid", "webpki-roots", - "wreq", - "wreq-util", "x25519-dalek", "zeroize", ] @@ -1413,27 +1222,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e537132deb99c0eb4b752f0346b6a836200eaaa3516dd7e5514b63930a09e5d" -[[package]] -name = "ipnet" -version = "2.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" - [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.19" @@ -1508,16 +1302,6 @@ version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" -[[package]] -name = "libloading" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" -dependencies = [ - "cfg-if", - "windows-link", -] - [[package]] name = "libm" version = "0.2.16" @@ -1545,15 +1329,6 @@ version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" -[[package]] -name = "lru" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9f71e3591c7ec44d3e563e87bf14ddc2bada8519dca046e5b0e560cd004e2d" -dependencies = [ - "hashbrown 0.17.1", -] - [[package]] name = "lru-slab" version = "0.1.3" @@ -1593,16 +1368,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mio" version = "1.2.4" @@ -1719,17 +1484,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "p256" version = "0.14.0" @@ -2005,18 +1759,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - [[package]] name = "regex-automata" version = "0.4.18" @@ -2262,12 +2004,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "shlex" version = "2.0.1" @@ -2294,22 +2030,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - -[[package]] -name = "singleflight-async" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffae0d841b8012a86beec66a3f9c57b7b331a10366c764cd40bd6faebe3ad77c" -dependencies = [ - "parking_lot", - "tokio", -] - [[package]] name = "slab" version = "0.4.12" @@ -2534,16 +2254,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "tokio-btls" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e1fd638ec35427faf3b8f412e0fdd6fae76591d79dba40f38fa667d22bc44dd" -dependencies = [ - "btls", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.7.2" @@ -2775,32 +2485,6 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "typed-builder" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" -dependencies = [ - "typed-builder-macro", -] - -[[package]] -name = "typed-builder-macro" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "typenum" version = "1.20.1" @@ -2875,15 +2559,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "want" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2944,15 +2619,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-root-certs" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "webpki-roots" version = "1.0.12" @@ -2962,28 +2628,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.62.2" @@ -3143,80 +2787,6 @@ dependencies = [ "primefield", ] -[[package]] -name = "wreq" -version = "0.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b11a67378558357ed083260fd0bbced0481e4d32e2fb722be9c52b42ee08d4" -dependencies = [ - "btls", - "btls-sys", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "http2", - "httparse", - "ipnet", - "libc", - "lru", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tokio-btls", - "tower", - "url", - "webpki-root-certs", - "wreq-proto", - "wreq-rt", -] - -[[package]] -name = "wreq-proto" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4d829f0427a5064829976fe790641ca21abb1d83d844812baf022dc0bdf2db" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "http2", - "httparse", - "pin-project-lite", - "smallvec", - "tokio", - "tokio-util", - "want", -] - -[[package]] -name = "wreq-rt" -version = "0.2.2-rc.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e9bce67a3fa3dd3f1503f066d86661c9caf399a763d3bd184da7afaf886c8b" -dependencies = [ - "pin-project-lite", - "tokio", - "wreq-proto", -] - -[[package]] -name = "wreq-util" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "636ad931b75aa030cc320899507ce8d5e44f1d9b3b44a81d2e1a225b0d4aad3b" -dependencies = [ - "brotli", - "flate2", - "typed-builder", - "wreq", - "zstd", -] - [[package]] name = "writeable" version = "0.6.4" @@ -3346,42 +2916,8 @@ dependencies = [ "syn 3.0.10", ] -[[package]] -name = "zlib-rs" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" - [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" - -[[package]] -name = "zstd" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "8.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.1.2+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/Cargo.toml b/Cargo.toml index cdeee6b..62421dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,6 @@ fst = "0.4.7" futures = "0.3.34" h2 = "0.4.20" http = "1.5.0" -http-body-util = "0.1.5" httparse = "1.10.1" ip_network = "0.4.1" ip_network_table = "0.2.0" @@ -54,8 +53,16 @@ rcgen = {version = "0.14.10", default-features = false, features = [ rustls = "0.23.46" serde = {version = "1.0.229", features = ["derive"]} serde_json = "1.0.154" -singleflight-async = "0.2.0" -tokio = {version = "1.54.1", features = ["rt-multi-thread", "test-util"]} +tokio = {version = "1.54.1", features = [ + "rt-multi-thread", + "macros", + "net", + "io-util", + "time", + "sync", + "fs", + "test-util", +]} tokio-rustls = "0.26.6" tokio-socks = "0.5.3" tokio-stream = {version = "0.1.19", features = ["net"]} @@ -72,8 +79,6 @@ tracing-subscriber = {version = "0.3.23", features = [ url = "2.5.8" uuid = "1.29.2" webpki-roots = "1.0.12" -wreq = "0.16.3" -wreq-util = "0.2.1" x25519-dalek = {version = "3.0.0", features = ["static_secrets", "getrandom", "zeroize"]} zeroize = "1.9.1" @@ -91,5 +96,3 @@ rustls = "0.23.46" serde_json = "1.0.154" tokio = {version = "1.54.1", features = ["full"]} tracing-subscriber = {version = "0.3.23", features = ["env-filter"]} -wreq = "0.16.3" -wreq-util = "0.2.1" diff --git a/README.md b/README.md index 3740843..9217ffe 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # httproxy -A high-performance HTTP proxy in Rust that emulates Chrome's TLS and HTTP/2 -fingerprints, so tunnel traffic is indistinguishable from ordinary browser +A high-performance HTTP proxy in Rust that carries tunnel traffic over +Chromium's own network stack, so it is indistinguishable from ordinary browser traffic. ## Features -- **Fingerprint emulation** — BoringSSL configured with Chrome's TLS and HTTP/2 - parameters. +- **Native Chromium fingerprint** — every upstream request is issued by + Chromium's network stack (Cronet): the browser's own TLS, HTTP/2 and QUIC + implementation, header order and padding, with no emulation layer. - **Flexible topology** — works through CDNs, web proxies and middleboxes. - **Traffic obfuscation** — configurable padding and inter-frame jitter. - **Mandatory authentication and encryption** — tunnel tokens authorise tunnel @@ -19,12 +20,30 @@ traffic. ## Getting Started -Requires a stable Rust toolchain. +Requires a stable Rust toolchain; the client also needs Cronet. ```bash cargo build --release # → target/release/{client,server} ``` +### Cronet + +`build.rs` links Cronet statically when `CRONET_LIB_DIR` points at a package +(`libcronet.a`, or `cronet.lib` on Windows); otherwise the client resolves +`libcronet` at startup through `CRONET_LIB`, the directory next to the binary, +or the system library path. + +Packages come from Chromium's `cronet_static` target, built by +`scripts/build-libcronet.sh ` from the tree pinned in +`naiveproxy.lock`: glibc and musl against Debian and OpenWrt sysroots, Android +against the NDK, macOS and Windows. CI builds every target in +`.github/workflows/libcronet.yml` and hands the packages to the build jobs. + +```bash +sh scripts/build-libcronet.sh x86_64-unknown-linux-gnu +CRONET_LIB_DIR=cronet/x86_64-unknown-linux-gnu cargo build --release +``` + Both binaries read a `config.toml`; every key is documented in **[CONFIGURATION.md](CONFIGURATION.md)**. diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..c82ef45 --- /dev/null +++ b/build.rs @@ -0,0 +1,139 @@ +use std::env; +use std::path::{Path, PathBuf}; + +fn main() { + link_android_builtins(); + println!("cargo:rerun-if-env-changed=CRONET_LIB_DIR"); + println!("cargo:rustc-check-cfg=cfg(cronet_static)"); + + let Some(directory) = env::var_os("CRONET_LIB_DIR").map(PathBuf::from) else { + return; + }; + if !directory.is_dir() { + panic!("CRONET_LIB_DIR is not a directory: {}", directory.display()); + } + + let Some(archive) = find(&directory, &["libcronet.a", "cronet.lib", "libcronet.lib"]) else { + panic!( + "CRONET_LIB_DIR={} holds no static libcronet.a/cronet.lib; \ + unset it to load libcronet at runtime instead", + directory.display() + ); + }; + + if archive.metadata().map(|meta| meta.len()).unwrap_or(0) < 4 * 1024 * 1024 { + println!( + "cargo:warning={} is far smaller than a static Cronet archive; \ + an import library for the shared Cronet cannot be linked in", + archive.display() + ); + } + + println!("cargo:rustc-link-search=native={}", directory.display()); + println!("cargo:rerun-if-changed={}", archive.display()); + let name = archive + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("libcronet.a"); + println!("cargo:rustc-link-lib=static:+verbatim={name}"); + println!("cargo:rustc-cfg=cronet_static"); + emit_linkage(&directory); +} + +fn find(directory: &Path, names: &[&str]) -> Option { + names + .iter() + .map(|name| directory.join(name)) + .find(|path| path.is_file()) +} + +fn emit_linkage(directory: &Path) { + let link_file = directory.join("cronet.link"); + let mut libraries = false; + if let Ok(text) = std::fs::read_to_string(&link_file) { + println!("cargo:rerun-if-changed={}", link_file.display()); + for line in text.lines() { + let line = line.trim(); + if let Some(name) = line.strip_prefix("lib=") { + println!("cargo:rustc-link-lib={name}"); + libraries = true; + } else if let Some(name) = line.strip_prefix("framework=") { + println!("cargo:rustc-link-lib=framework={name}"); + libraries = true; + } else if let Some(flag) = line.strip_prefix("arg=") { + println!("cargo:rustc-link-arg={flag}"); + } + } + } + if !libraries { + for name in default_libraries() { + println!("cargo:rustc-link-lib={name}"); + } + } +} + +fn link_android_builtins() { + if env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("android") { + return; + } + let arch = match env::var("CARGO_CFG_TARGET_ARCH").as_deref() { + Ok("aarch64") => "aarch64", + Ok("x86_64") => "x86_64", + _ => return, + }; + let Some(ndk) = ["ANDROID_NDK_HOME", "ANDROID_NDK_ROOT", "ANDROID_NDK"] + .into_iter() + .filter_map(env::var_os) + .map(PathBuf::from) + .find(|path| path.is_dir()) + else { + println!("cargo:warning=Android NDK not found; compiler-rt builtins stay unlinked"); + return; + }; + match builtins(&ndk, arch) { + Some(archive) => println!("cargo:rustc-link-arg={}", archive.display()), + None => println!( + "cargo:warning=no libclang_rt.builtins-{arch}-android.a under {}", + ndk.display() + ), + } +} + +fn builtins(ndk: &Path, arch: &str) -> Option { + let toolchains = ndk.join("toolchains").join("llvm").join("prebuilt"); + let mut newest: Option<(u64, PathBuf)> = None; + for host in std::fs::read_dir(toolchains).ok()?.flatten() { + let versions = host.path().join("lib").join("clang"); + for version in std::fs::read_dir(versions).ok()?.flatten() { + let rank = version + .file_name() + .to_str() + .and_then(|name| name.split('.').next()) + .and_then(|name| name.parse().ok()) + .unwrap_or(0); + let archive = version + .path() + .join("lib") + .join("linux") + .join(format!("libclang_rt.builtins-{arch}-android.a")); + if archive.is_file() && newest.as_ref().is_none_or(|(best, _)| rank > *best) { + newest = Some((rank, archive)); + } + } + } + Some(newest?.1) +} + +fn default_libraries() -> &'static [&'static str] { + if cfg!(windows) { + &[ + "ws2_32", "crypt32", "bcrypt", "advapi32", "user32", "ole32", "oleaut32", "shell32", + "shlwapi", "iphlpapi", "secur32", "dbghelp", "winmm", "userenv", "version", "psapi", + "wintrust", "rpcrt4", + ] + } else if cfg!(target_os = "linux") { + &["dl", "pthread", "m"] + } else { + &[] + } +} diff --git a/naiveproxy.lock b/naiveproxy.lock new file mode 100644 index 0000000..193aeb1 --- /dev/null +++ b/naiveproxy.lock @@ -0,0 +1,2 @@ +repository https://github.com/SagerNet/naiveproxy +commit 7c64cae24cd7bae18d9c603fdddc73b623c04462 diff --git a/scripts/build-libcronet.sh b/scripts/build-libcronet.sh new file mode 100644 index 0000000..07ef118 --- /dev/null +++ b/scripts/build-libcronet.sh @@ -0,0 +1,299 @@ +#!/bin/sh +set -eu + +root=$(cd "$(dirname "$0")/.." && pwd) +lock="$root/naiveproxy.lock" +work=${CRONET_BUILD_DIR:-$root/.cronet-build} + +triple=${1:?usage: build-libcronet.sh [destination]} +destination=${2:-$root/cronet} + +if [ ! -f "$lock" ]; then + echo "naiveproxy.lock is missing" >&2 + exit 1 +fi + +repository=$(awk '$1 == "repository" { print $2; exit }' "$lock") +commit=$(awk '$1 == "commit" { print $2; exit }' "$lock") +if [ -z "$repository" ] || [ -z "$commit" ]; then + echo "naiveproxy.lock needs a repository and a commit" >&2 + exit 1 +fi + +openwrt= +mac= +elf=0 +shared=0 +windows_static=0 +output=static + +case "$triple" in +x86_64-unknown-linux-gnu) + target='target_os="linux" target_cpu="x64"' + elf=1 + shared=1 + ;; +aarch64-unknown-linux-gnu) + target='target_os="linux" target_cpu="arm64"' + elf=1 + shared=1 + ;; +x86_64-unknown-linux-musl) + target='target_os="openwrt" target_cpu="x64" build_static=true use_allocator_shim=false use_partition_alloc=false' + openwrt='arch=x86_64 release=24.10.0 gcc_ver=13.3.0 target=x86 subtarget=64' + elf=1 + ;; +aarch64-unknown-linux-musl) + target='target_os="openwrt" target_cpu="arm64" build_static=true use_allocator_shim=false use_partition_alloc=false' + openwrt='arch=aarch64_generic release=24.10.0 gcc_ver=13.3.0 target=layerscape subtarget=armv8_64b' + elf=1 + ;; +x86_64-linux-android) + target='target_os="android" target_cpu="x64" is_desktop_android=true default_min_sdk_version=24 is_high_end_android=true' + elf=1 + ;; +aarch64-linux-android) + target='target_os="android" target_cpu="arm64" is_desktop_android=true default_min_sdk_version=24 is_high_end_android=true' + elf=1 + ;; +x86_64-apple-darwin) + target='target_os="mac" target_cpu="x64"' + mac=1 + ;; +aarch64-apple-darwin) + target='target_os="mac" target_cpu="arm64"' + mac=1 + ;; +x86_64-pc-windows-msvc) + target='target_os="win" target_cpu="x64"' + output=dll + windows_static=${CRONET_WINDOWS_STATIC:-1} + ;; +aarch64-pc-windows-msvc) + target='target_os="win" target_cpu="arm64"' + output=dll + windows_static=${CRONET_WINDOWS_STATIC:-1} + ;; +*) + echo "no Cronet build for $triple" >&2 + exit 1 + ;; +esac + +args='is_official_build=true +is_chrome_branded=true +exclude_unwind_tables=true +enable_resource_allowlist_generation=false +symbol_level=0 +is_clang=true +use_sysroot=false +fatal_linker_warnings=false +treat_warnings_as_errors=false +use_clang_modules=false +is_cronet_build=true +use_udev=false +use_aura=false +use_ozone=false +use_gio=false +use_glib=false +use_platform_icu_alternatives=true +is_perfetto_embedder=true +disable_file_support=true +enable_websockets=false +use_kerberos=false +disable_zstd_filter=false +enable_mdns=false +enable_reporting=false +include_transport_security_state_preload_list=false +enable_device_bound_sessions=false +enable_bracketed_proxy_uris=true +enable_quic_proxy_support=true +enable_disk_cache_sql_backend=false +use_nss_certs=false +enable_backup_ref_ptr_support=false +enable_dangling_raw_ptr_checks=false +use_thin_lto=false +is_cfi=false +use_cfi_icall=false +chrome_pgo_phase=2' + +if [ -n "$mac" ]; then + args="$args +mac_allow_system_xcode_for_official_builds_for_testing=true +enable_dsyms=false" +fi + +ninja_targets=cronet_static +if [ "$shared" = 1 ]; then + ninja_targets="cronet_static cronet" +fi +if [ "$output" = dll ]; then + ninja_targets=cronet + if [ "$windows_static" = 1 ]; then + ninja_targets="cronet cronet_static" + fi +fi + +if [ "${CRONET_DRY_RUN:-0}" = 1 ]; then + printf 'triple %s\nargs %s %s\nninja %s\n' "$triple" "$args" "$target" "$ninja_targets" + exit 0 +fi + +src=$work/naiveproxy/src + +fetch() { + if [ -d "$work/naiveproxy/.git" ] && [ "$(git -C "$work/naiveproxy" rev-parse HEAD 2>/dev/null || true)" = "$commit" ]; then + return + fi + rm -rf "$work/naiveproxy" + mkdir -p "$work/naiveproxy" + git -C "$work/naiveproxy" init --quiet + git -C "$work/naiveproxy" fetch --quiet --depth 1 "$repository" "$commit" + git -C "$work/naiveproxy" checkout --quiet --detach "$commit" +} + +disable_crel() { + case "$triple" in + aarch64-unknown-linux-*|x86_64-unknown-linux-*) ;; + *) return 0 ;; + esac + file=$src/build/config/compiler/BUILD.gn + if grep -q 'current_cpu != "x64" && current_cpu != "arm64" && current_cpu != "s390x"' "$file"; then + return 0 + fi + if ! grep -q 'current_cpu != "arm" && current_cpu != "s390x"' "$file"; then + echo "cannot disable CREL for $triple: $file no longer matches" >&2 + return 1 + fi + sed -i 's/current_cpu != "arm" && current_cpu != "s390x"/current_cpu != "arm" \&\& current_cpu != "x64" \&\& current_cpu != "arm64" \&\& current_cpu != "s390x"/' "$file" +} + +value() { + sed -n "s/^[[:space:]]*$1[[:space:]]*=[[:space:]]*//p" "$2" | head -n 1 +} + +manifest() { + ninja_file=$1 + to=$2 + : > "$to" + for lib in $(value libs "$ninja_file"); do + case "$lib" in + *.lds) ;; + -l:*) printf 'arg=%s\n' "$lib" >> "$to" ;; + -l*) printf 'lib=%s\n' "${lib#-l}" >> "$to" ;; + *) printf 'arg=%s\n' "$lib" >> "$to" ;; + esac + done + set -- $(value frameworks "$ninja_file") + while [ $# -gt 0 ]; do + if [ "$1" = "-framework" ] && [ $# -ge 2 ]; then + printf 'framework=%s\n' "$2" >> "$to" + shift 2 + else + shift + fi + done + for flag in $(value ldflags "$ninja_file"); do + case "$flag" in + -Wl,-wrap,*) printf 'arg=%s\n' "$flag" >> "$to" ;; + esac + done +} + +prelink() { + archive=$1 + to=$2 + bin=$src/third_party/llvm-build/Release+Asserts/bin + scratch=$work/prelink/$triple + mkdir -p "$scratch" + "$bin/llvm-nm" --defined-only --extern-only --format=just-symbols --quiet "$archive" | + grep -E '^(Cronet_|bidirectional_stream_|__wrap_)[A-Za-z0-9_.]*$|^(aligned_alloc|calloc|cfree|free|malloc|malloc_size|malloc_usable_size|memalign|posix_memalign|pvalloc|realloc|valloc|__libc_calloc|__libc_cfree|__libc_free|__libc_malloc|__libc_memalign|__libc_pvalloc|__libc_realloc|__libc_valloc|__posix_memalign)$' | + sort -u > "$scratch/globals.txt" || true + if ! grep -q '^Cronet_' "$scratch/globals.txt"; then + echo "$archive defines none of Cronet's C API" >&2 + exit 1 + fi + set -- "$bin/ld.lld" -r --force-group-allocation + while read -r symbol; do + set -- "$@" -u "$symbol" + done < "$scratch/globals.txt" + "$@" "$archive" -o "$scratch/libcronet.o" + "$bin/llvm-objcopy" --keep-global-symbols="$scratch/globals.txt" "$scratch/libcronet.o" + rm -f "$to" + "$bin/llvm-ar" rcsD "$to" "$scratch/libcronet.o" + rm -f "$scratch/libcronet.o" "$scratch/globals.txt" +} + +fetch +disable_crel + +sysroot=$(cd "$src" && EXTRA_FLAGS="$target" OPENWRT_FLAGS="$openwrt" sh -c '. ./get-sysroot.sh >/dev/null && printf %s "${WITH_SYSROOT:-}"') + +keyring=$src/build/linux/sysroot_scripts/keyring.gpg +if [ -n "$sysroot" ] && [ ! -d "$src/$sysroot/lib" ] && [ -f "$keyring" ] && command -v gpg >/dev/null 2>&1; then + rm -f "$keyring" + (cd "$src" && bash ./build/linux/sysroot_scripts/generate_keyring.sh) +fi + +( + cd "$src" + EXTRA_FLAGS="$target" OPENWRT_FLAGS="$openwrt" sh ./get-clang.sh +) + +if [ -n "$sysroot" ]; then + args="$args +target_sysroot=\"//$sysroot\"" +fi +if command -v ccache >/dev/null 2>&1; then + args="$args +cc_wrapper=\"ccache\"" +fi + +gn=$src/gn/out/gn +if [ -f "$src/gn/out/gn.exe" ]; then + gn=$src/gn/out/gn.exe +fi + +( + cd "$src" + DEPOT_TOOLS_WIN_TOOLCHAIN=0 "$gn" gen out/cronet --args="$args $target" + ninja -C out/cronet $ninja_targets +) + +out=$src/out/cronet +package=$destination/$triple +rm -rf "$package" +mkdir -p "$package" + +case "$output" in +dll) + cp "$out/cronet.dll" "$package/cronet.dll" + cp "$out/cronet.dll.lib" "$package/cronet.dll.lib" + if [ "$windows_static" = 1 ]; then + cp "$out/obj/components/cronet/cronet_static.lib" "$package/cronet.lib" + manifest "$out/obj/components/cronet/cronet_sample.ninja" "$package/cronet.link" + fi + ;; +*) + archive=$out/obj/components/cronet/libcronet_static.a + if [ "$elf" = 1 ] && [ "${CRONET_PRELINK:-1}" = 1 ]; then + prelink "$archive" "$package/libcronet.a" + else + cp "$archive" "$package/libcronet.a" + fi + if [ "$shared" = 1 ]; then + cp "$out/libcronet.so" "$package/libcronet.so" + fi + manifest "$out/obj/components/cronet/cronet_sample.ninja" "$package/cronet.link" + ;; +esac + +if [ "$elf" = 1 ] && grep -q '\.crel\.' "$package/libcronet.a"; then + echo "$package/libcronet.a still holds CREL relocations" >&2 + exit 1 +fi + +cp "$work/naiveproxy/CHROMIUM_VERSION" "$package/VERSION" +cp "$work/naiveproxy/LICENSE" "$package/LICENSE" + +echo "$(cd "$package" && pwd)" diff --git a/src/bin/client.rs b/src/bin/client.rs index 82b5967..c17122e 100644 --- a/src/bin/client.rs +++ b/src/bin/client.rs @@ -1,4 +1,3 @@ -use anyhow::Context; use clap::Parser; use std::fs; use std::sync::Arc; @@ -32,28 +31,7 @@ async fn main() -> anyhow::Result<()> { let addr: std::net::SocketAddr = config.client.listen.parse()?; let listener = TcpListener::bind(addr).await?; - let remote_url: url::Url = state.remote_str.parse()?; - let domain = remote_url.host_str().context("No domain in remote URL")?; - let final_addr = config - .client - .address - .clone() - .unwrap_or_else(|| domain.to_string()); - - let http_client = Arc::new( - wreq::Client::builder() - .tcp_nodelay(true) - .tcp_keepalive(Duration::from_secs(45)) - .tcp_keepalive_interval(Duration::from_secs(45)) - .pool_idle_timeout(Duration::from_secs(300)) - .pool_max_idle_per_host(6) - .emulation(httproxy::client::chrome_emulation()) - .no_proxy() - .dns_resolver(Arc::new(httproxy::client::state::ManualResolver { - target_addr: final_addr, - })) - .build()?, - ); + let http_client = httproxy::client::build_http_client(&config)?; info!(listen = %addr, "proxy listening"); diff --git a/src/client/actor/connection.rs b/src/client/actor/connection.rs index 8e45b18..c0bc514 100644 --- a/src/client/actor/connection.rs +++ b/src/client/actor/connection.rs @@ -10,15 +10,16 @@ use crate::client::constants::{ }; use crate::client::proxy; use crate::client::state::SharedState; +use crate::cronet::Client; pub struct ClientConnectionActor { socket: Option, - http_client: Arc, + http_client: Arc, state: Arc, } impl ClientConnectionActor { - pub fn new(socket: TcpStream, http_client: Arc, state: Arc) -> Self { + pub fn new(socket: TcpStream, http_client: Arc, state: Arc) -> Self { socket.set_nodelay(true).ok(); Self { socket: Some(socket), @@ -206,8 +207,8 @@ mod tests { addr } - fn test_client() -> Arc { - Arc::new(wreq::Client::builder().no_proxy().build().unwrap()) + fn test_client() -> Arc { + Client::new(None).unwrap() } #[tokio::test] diff --git a/src/client/actor/download_loop.rs b/src/client/actor/download_loop.rs index ffa5cb0..8559794 100644 --- a/src/client/actor/download_loop.rs +++ b/src/client/actor/download_loop.rs @@ -1,7 +1,6 @@ use anyhow::{Context, Result, anyhow}; use bytes::{Buf, Bytes, BytesMut}; use futures::StreamExt; -use http_body_util::BodyExt; use std::collections::VecDeque; use std::io::IoSlice; use std::sync::Arc; @@ -18,19 +17,20 @@ use crate::client::constants::{ PREFETCH_ROTATE_TIMEOUT, WRITE_BUF_CAPACITY, WRITE_FLUSH_TIMEOUT, WRITE_STALL_TIMEOUT, }; use crate::client::utils; +use crate::cronet::{Client, Response}; use crate::shaper::{self, DecodedFrame, EncodingType, FrameCipher}; enum Phase { Streaming { - response: Box, + response: Box, expected_seq: u64, prefetch_at: Option, prefetch_trigger: Option>, - prefetch_rx: Option>>, + prefetch_rx: Option>>, }, Rotating { expected_seq: u64, - prefetch_rx: Option>>, + prefetch_rx: Option>>, }, Done, } @@ -40,7 +40,7 @@ pub struct DownloadLoopActor { cipher: Arc, encoding: EncodingType, stream_id: Uuid, - http_client: Arc, + http_client: Arc, state: Arc, max_bytes: Option, phase: Phase, @@ -179,11 +179,11 @@ impl DecodeBuffers { impl DownloadLoopActor { pub fn new( - initial_response: wreq::Response, + initial_response: Response, write_half: OwnedWriteHalf, cipher: Arc, stream_id: Uuid, - http_client: Arc, + http_client: Arc, state: Arc, ) -> Self { let encoding = state.traffic_config.encoding_type; @@ -251,11 +251,11 @@ impl DownloadLoopActor { async fn do_streaming( &mut self, - response: wreq::Response, + response: Response, expected_seq: u64, prefetch_at: Option, prefetch_trigger: Option>, - prefetch_rx: Option>>, + prefetch_rx: Option>>, ) -> Result { let (bytes, next_seq) = self .drain_response(response, expected_seq, prefetch_at, prefetch_trigger) @@ -274,14 +274,14 @@ impl DownloadLoopActor { async fn drain_response( &mut self, - response: wreq::Response, + response: Response, mut expected_seq: u64, prefetch_at: Option, mut prefetch_trigger: Option>, ) -> Result<(u64, u64)> { let flush_deadline = tokio::time::sleep(WRITE_FLUSH_TIMEOUT); tokio::pin!(flush_deadline); - let mut data_stream = response.into_data_stream(); + let mut data_stream = response.into_body(); let mut bytes_received: u64 = 0; loop { @@ -334,7 +334,7 @@ impl DownloadLoopActor { async fn do_rotate( &mut self, expected_seq: u64, - prefetch_rx: Option>>, + prefetch_rx: Option>>, ) -> Result { let response = if let Some(rx) = prefetch_rx { tokio::pin!(rx); @@ -387,10 +387,10 @@ impl DownloadLoopActor { } async fn send_continue_request( - http_client: &wreq::Client, + http_client: &Client, state: &SharedState, stream_id: Uuid, -) -> Result { +) -> Result { let mut cookie = String::new(); utils::build_stream_cookie(&mut cookie, stream_id); let req = http_client @@ -405,13 +405,10 @@ async fn send_continue_request( } fn spawn_prefetch_continuation( - http_client: &Arc, + http_client: &Arc, state: &Arc, stream_id: Uuid, -) -> ( - oneshot::Sender<()>, - oneshot::Receiver>, -) { +) -> (oneshot::Sender<()>, oneshot::Receiver>) { let (trigger_tx, trigger_rx) = oneshot::channel(); let (result_tx, result_rx) = oneshot::channel(); let pre_client = Arc::clone(http_client); diff --git a/src/client/actor/upload_loop.rs b/src/client/actor/upload_loop.rs index 5b6d6d7..01932cf 100644 --- a/src/client/actor/upload_loop.rs +++ b/src/client/actor/upload_loop.rs @@ -13,6 +13,7 @@ use crate::client::constants::{ BATCH_BUF_INITIAL_CAPACITY, MAX_BATCH_BYTES, UPLOAD_REQUEST_TIMEOUT, }; use crate::client::utils; +use crate::cronet::Client; use crate::shaper::{self, ShaperStream}; enum Phase { @@ -22,7 +23,7 @@ enum Phase { } pub struct UploadLoopActor { - http_client: Arc, + http_client: Arc, state: Arc, stream_id: Uuid, shaped: ShaperStream, @@ -38,7 +39,7 @@ pub struct UploadLoopActor { impl UploadLoopActor { pub fn new( - http_client: Arc, + http_client: Arc, state: Arc, shaped: ShaperStream, stream_id: Uuid, @@ -206,7 +207,7 @@ impl UploadLoopActor { #[inline] async fn send_upload_post( - http_client: &wreq::Client, + http_client: &Client, state: &SharedState, body: Bytes, stream_id: Uuid, @@ -220,13 +221,10 @@ async fn send_upload_post( .header("Cache-Control", "no-store, no-transform") .header("Content-Type", "application/octet-stream") .header("Cookie", cookie); - let response = tokio::time::timeout( - UPLOAD_REQUEST_TIMEOUT, - req.body(wreq::Body::from(body)).send(), - ) - .await - .context("upload POST timed out")? - .context("http post failed")?; + let response = tokio::time::timeout(UPLOAD_REQUEST_TIMEOUT, req.body(body).send()) + .await + .context("upload POST timed out")? + .context("http post failed")?; let response = utils::check_response_status(response, "upstream rejected upload").await?; response.bytes().await.context("drain upload response")?; Ok(()) @@ -335,7 +333,7 @@ mod tests { (addr, stop_tx, handle) } - fn decode_all(received: &[u8]) -> Vec { + fn decode_all(received: &[u8]) -> Vec<(u64, Bytes)> { let cipher = test_cipher(); let mut json_scratch = Vec::new(); let mut src = BytesMut::from(received); @@ -349,15 +347,15 @@ mod tests { .unwrap() { match frame { - DecodedFrame::Owned { data, .. } => decoded.extend_from_slice(&data), + DecodedFrame::Owned { seq, data } => decoded.push((seq, data)), _ => panic!("unexpected borrowed frame"), } } decoded } - fn test_client() -> Arc { - Arc::new(wreq::Client::builder().no_proxy().build().unwrap()) + fn test_client() -> Arc { + Client::new(None).unwrap() } async fn tcp_pair() -> (tokio::net::tcp::OwnedReadHalf, TcpStream) { @@ -408,8 +406,16 @@ mod tests { let _ = stop_tx.send(()); let received = collector.await.unwrap(); - assert!(!received.is_empty(), "server must receive upload frames"); - assert_eq!(decode_all(&received), all); + let mut frames = decode_all(&received); + assert!(!frames.is_empty(), "server must receive upload frames"); + frames.sort_by_key(|(seq, _)| *seq); + let seqs: Vec = frames.iter().map(|(seq, _)| *seq).collect(); + assert_eq!(seqs, (0..frames.len() as u64).collect::>()); + let mut decoded = Vec::new(); + for (_, data) in frames { + decoded.extend_from_slice(&data); + } + assert_eq!(decoded, all); } #[tokio::test] diff --git a/src/client/connection.rs b/src/client/connection.rs index c4cd91e..451b8d6 100644 --- a/src/client/connection.rs +++ b/src/client/connection.rs @@ -8,11 +8,12 @@ use tracing::info; use crate::client::state::ClientPqFsm; use crate::client::{constants::WRITE_STALL_TIMEOUT, state::SharedState}; +use crate::cronet::Client; pub(crate) async fn open_tunnel( read_half: OwnedReadHalf, write_half: OwnedWriteHalf, - http_client: Arc, + http_client: Arc, state: Arc, payload: Bytes, target_host: &str, @@ -39,7 +40,7 @@ pub(crate) async fn open_tunnel( pub async fn handle_connection_actor( socket: TcpStream, - http_client: Arc, + http_client: Arc, state: Arc, ) -> Result<()> { use crate::client::actor::connection::ClientConnectionActor; diff --git a/src/client/handshake.rs b/src/client/handshake.rs index ecf407b..8976239 100644 --- a/src/client/handshake.rs +++ b/src/client/handshake.rs @@ -2,7 +2,6 @@ use anyhow::{Context, Result, anyhow}; use base64::Engine; use bytes::{Bytes, BytesMut}; use futures::StreamExt; -use http_body_util::BodyExt; use std::sync::Arc; use tokio::io::AsyncReadExt; use tracing::{Instrument, info}; @@ -11,6 +10,7 @@ use zeroize::Zeroizing; use crate::client::actor::download_loop::DownloadLoopActor; use crate::client::actor::upload_loop::UploadLoopActor; use crate::client::utils; +use crate::cronet::Client; use crate::crypto::{self, AesFrameCipher, TlsInspector}; use crate::shaper::{self, MAX_RAW_PAYLOAD, ShaperStream}; @@ -34,7 +34,7 @@ impl std::fmt::Display for RehandshakeRequired { impl std::error::Error for RehandshakeRequired {} pub async fn try_pq_connect( - http_client: &Arc, + http_client: &Arc, state: &Arc, ticket: &PqSessionTicket, target_host: &str, @@ -86,7 +86,7 @@ pub async fn try_pq_connect( http_client .post(state.remote_str.as_str()) .header("Cookie", &session_cookie) - .body(wreq::Body::from(early_data)) + .body(early_data) .send(), ) .await @@ -143,7 +143,7 @@ pub async fn try_pq_connect( } pub async fn perform_pq_handshake( - http_client: &wreq::Client, + http_client: &Client, state: &SharedState, server_pk: &crypto::EncapsulationKey, ) -> Result { @@ -197,7 +197,7 @@ pub async fn perform_pq_handshake( DOWNLOAD_CONNECT_TIMEOUT, http_client .post(state.remote_str.as_str()) - .body(wreq::Body::from(handshake_body)) + .body(Bytes::from(handshake_body)) .send(), ) .await @@ -207,7 +207,7 @@ pub async fn perform_pq_handshake( let response = utils::check_response_status(response, "server rejected handshake").await?; let mut body_buf = BytesMut::with_capacity(DECODE_BUF_CAPACITY); - let mut stream = response.into_data_stream(); + let mut stream = response.into_body(); let server_hello_data = loop { match stream.next().await { Some(Ok(chunk)) => { @@ -220,7 +220,7 @@ pub async fn perform_pq_handshake( break data; } } - Some(Err(e)) => return Err(e.into()), + Some(Err(e)) => return Err(e), None => return Err(anyhow!("ServerHello not received")), } }; @@ -265,7 +265,7 @@ pub async fn perform_pq_handshake( } pub async fn full_handshake( - http_client: &Arc, + http_client: &Arc, state: &Arc, server_pk: &crypto::EncapsulationKey, target_host: &str, diff --git a/src/client/mod.rs b/src/client/mod.rs index 5527c7e..688c61e 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -9,6 +9,7 @@ pub mod utils; use crate::client::constants::{MAX_IN_FLIGHT_BYTES, MAX_LOCAL_CONNECTIONS, UPLOAD_CONCURRENCY}; use crate::config::ClientTopConfig; +use crate::cronet::Client; use crate::crypto; use crate::shaper::ResolvedShaperConfig; @@ -17,10 +18,8 @@ use base64::Engine; use std::sync::Arc; use tokio::sync::Mutex; -pub fn chrome_emulation() -> wreq_util::Emulation { - wreq_util::Emulation::builder() - .profile(wreq_util::Profile::Chrome149) - .build() +pub fn build_http_client(cfg: &ClientTopConfig) -> Result> { + Client::new(cfg.client.address.as_deref()).context("failed to start cronet engine") } pub fn build_state(cfg: &ClientTopConfig) -> Result> { diff --git a/src/client/redirect.rs b/src/client/redirect.rs index 20cc86e..e4c3757 100644 --- a/src/client/redirect.rs +++ b/src/client/redirect.rs @@ -9,6 +9,7 @@ use tracing::{Instrument, debug, info, warn}; use crate::client::connection::open_tunnel; use crate::client::state::SharedState; use crate::config::RedirectSection; +use crate::cronet::Client; const ACCEPT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(100); @@ -57,7 +58,7 @@ impl RedirectRule { pub async fn spawn( rule: RedirectRule, - http_client: Arc, + http_client: Arc, state: Arc, conn_sem: Arc, ) -> Result { @@ -127,7 +128,7 @@ pub async fn spawn( async fn serve( socket: TcpStream, target: &str, - http_client: Arc, + http_client: Arc, state: Arc, ) -> Result<()> { socket.set_nodelay(true).ok(); diff --git a/src/client/state.rs b/src/client/state.rs index 5b693b8..5d37387 100644 --- a/src/client/state.rs +++ b/src/client/state.rs @@ -1,5 +1,4 @@ use bytes::Bytes; -use std::net::SocketAddr; use std::sync::Arc; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::sync::Mutex; @@ -9,34 +8,11 @@ use zeroize::Zeroizing; use crate::bypass::BypassRules; use crate::client::constants::MASTER_RESUME_WINDOW_SECS; use crate::client::handshake::{self, PqSessionTicket}; +use crate::cronet::Client; use crate::shaper::{ResolvedShaperConfig, TrafficConfig}; pub type InitialMasterEntry = (String, Zeroizing<[u8; 32]>, u64); -pub struct ManualResolver { - pub target_addr: String, -} - -impl wreq::dns::Resolve for ManualResolver { - fn resolve(&self, _name: wreq::dns::Name) -> wreq::dns::Resolving { - let target = self.target_addr.clone(); - Box::pin(async move { - let mut lookup_str = String::with_capacity(target.len() + 2); - lookup_str.push_str(&target); - lookup_str.push_str(":0"); - let addrs = tokio::net::lookup_host(lookup_str) - .await? - .map(|mut s| { - s.set_port(0); - s - }) - .collect::>(); - Ok(Box::new(addrs.into_iter()) - as Box + Send + 'static>) - }) - } -} - pub struct SharedState { pub remote_str: String, pub auth_token: String, @@ -103,7 +79,7 @@ impl ClientPqFsm { pub async fn run( mut self, - http_client: Arc, + http_client: Arc, state: Arc, ) -> anyhow::Result<()> { loop { @@ -170,7 +146,7 @@ impl ClientPqFsm { #[allow(clippy::result_large_err)] async fn try_resume( resuming: Resuming, - http_client: &Arc, + http_client: &Arc, state: &Arc, ) -> std::result::Result<(), (Resuming, anyhow::Error)> { let mut read_half = resuming.read_half; diff --git a/src/client/utils.rs b/src/client/utils.rs index 456ed6c..985a955 100644 --- a/src/client/utils.rs +++ b/src/client/utils.rs @@ -8,6 +8,7 @@ use tracing::warn; use uuid::Uuid; use crate::client::constants::{MIN_PADDING, PADDING_POOL}; +use crate::cronet::Response; use crate::shaper::{self, FrameCipher}; #[inline] @@ -140,10 +141,7 @@ pub fn is_silent_error(root: &(dyn std::error::Error + 'static)) -> bool { .contains("connection closed during header parsing") } -pub(crate) async fn check_response_status( - response: wreq::Response, - context: &str, -) -> Result { +pub(crate) async fn check_response_status(response: Response, context: &str) -> Result { if !response.status().is_success() { let status = response.status(); let _ = response.bytes().await; diff --git a/src/cronet/mod.rs b/src/cronet/mod.rs new file mode 100644 index 0000000..968953d --- /dev/null +++ b/src/cronet/mod.rs @@ -0,0 +1,403 @@ +mod runtime; +mod sys; + +use anyhow::{Context as _, Result, anyhow, bail}; +use bytes::{Bytes, BytesMut}; +use futures::{Stream, StreamExt}; +use http::StatusCode; +use std::ffi::{CStr, CString}; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::task::{Context, Poll}; +use tokio::sync::mpsc::{self, UnboundedReceiver}; + +use runtime::{Event, Executor, Spec, next_key}; +use sys::Ptr; + +fn engine_version(engine: Ptr) -> Result { + let raw = sys::api().engine_version(engine); + if raw.is_null() { + bail!("cronet: engine reported no version"); + } + Ok(unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned()) +} + +fn build_user_agent(version: &str) -> Option { + let major = version.split('.').next()?; + if major.is_empty() || !major.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + CString::new(format!( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{major}.0.0.0 Safari/537.36" + )) + .ok() +} + +pub struct Client { + exec: Arc, +} + +impl Client { + pub fn new(resolve: Option<&str>) -> Result> { + sys::load().map_err(|error| anyhow!(error))?; + let engine = Arc::new(Engine::start(resolve)?); + let callback = Arc::new(Callback::new()?); + let exec = Executor::new(Arc::clone(&engine), Arc::clone(&callback))?; + Ok(Arc::new(Self { exec })) + } + + pub fn post(&self, url: &str) -> RequestBuilder { + self.request("POST", url) + } + + pub fn patch(&self, url: &str) -> RequestBuilder { + self.request("PATCH", url) + } + + fn request(&self, method: &'static str, url: &str) -> RequestBuilder { + RequestBuilder { + exec: Arc::clone(&self.exec), + spec: Spec { + url: url.to_owned(), + method, + headers: Vec::new(), + body: None, + }, + } + } +} + +pub(crate) struct Engine(Ptr, AtomicBool); + +impl Engine { + fn start(resolve: Option<&str>) -> Result { + let options = resolve.map(experimental_options).transpose()?; + let api = sys::api(); + let engine = api.engine_create(); + if engine.is_null() { + bail!("cronet: engine create failed"); + } + let version = engine_version(engine).inspect_err(|_| api.engine_destroy(engine))?; + let Some(user_agent) = build_user_agent(&version) else { + api.engine_destroy(engine); + bail!("cronet: cannot derive a user agent from version {version}"); + }; + let params = api.engine_params_create(); + if params.is_null() { + api.engine_destroy(engine); + bail!("cronet: engine params create failed"); + } + api.engine_params_quic(params, true); + api.engine_params_http2(params, true); + api.engine_params_brotli(params, true); + api.engine_params_cache_mode(params, sys::HTTP_CACHE_DISABLED); + api.engine_params_user_agent(params, user_agent.as_ptr()); + if let Some(options) = options { + api.engine_params_experimental(params, options.as_ptr()); + } + let result = api.engine_start(engine, params); + api.engine_params_destroy(params); + if result != sys::RESULT_SUCCESS { + api.engine_destroy(engine); + bail!("cronet: engine start failed ({result})"); + } + Ok(Self(engine, AtomicBool::new(false))) + } + + pub(crate) fn shutdown(&self) { + if !self.1.swap(true, Ordering::AcqRel) { + let _ = sys::api().engine_shutdown(self.0); + } + } +} + +fn experimental_options(address: &str) -> Result { + let address = if address.contains(':') && !address.starts_with('[') { + format!("[{address}]") + } else { + address.to_owned() + }; + let options = format!(r#"{{"HostResolverRules":{{"host_resolver_rules":"MAP * {address}"}}}}"#); + CString::new(options).map_err(|_| anyhow!("cronet: invalid resolver address")) +} + +impl Drop for Engine { + fn drop(&mut self) { + self.shutdown(); + sys::api().engine_destroy(self.0); + } +} + +struct Callback(Ptr); + +impl Callback { + fn new() -> Result { + let callback = sys::api().callback_create( + Some(runtime::on_redirect), + Some(runtime::on_response_started), + Some(runtime::on_read_completed), + Some(runtime::on_succeeded), + Some(runtime::on_failed), + Some(runtime::on_canceled), + ); + if callback.is_null() { + bail!("cronet: callback create failed"); + } + Ok(Self(callback)) + } +} + +impl Drop for Callback { + fn drop(&mut self) { + sys::api().callback_destroy(self.0); + } +} + +pub struct RequestBuilder { + exec: Arc, + spec: Spec, +} + +impl RequestBuilder { + pub fn header(mut self, name: &str, value: impl AsRef) -> Self { + self.spec + .headers + .push((name.to_owned(), value.as_ref().to_owned())); + self + } + + pub fn body(mut self, body: Bytes) -> Self { + self.spec.body = Some(body); + self + } + + pub async fn send(self) -> Result { + url::Url::parse(&self.spec.url).context("invalid request url")?; + + let key = next_key(); + let (events, inbox) = mpsc::unbounded_channel(); + self.exec.open(key, events, self.spec); + + let mut guard = CloseGuard { + exec: Arc::clone(&self.exec), + key, + armed: true, + }; + let mut inbox = inbox; + loop { + match inbox.recv().await { + Some(Event::Status(status)) => { + guard.disarm(); + return Ok(Response { + status: StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY), + body: Body { + exec: self.exec, + key, + inbox, + }, + }); + } + Some(Event::Fail(reason)) => bail!("{reason}"), + Some(_) => {} + None => bail!("cronet: request aborted"), + } + } + } +} + +struct CloseGuard { + exec: Arc, + key: usize, + armed: bool, +} + +impl CloseGuard { + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for CloseGuard { + fn drop(&mut self) { + if self.armed { + self.exec.close(self.key); + } + } +} + +pub struct Response { + status: StatusCode, + body: Body, +} + +impl Response { + pub fn status(&self) -> StatusCode { + self.status + } + + pub fn into_body(self) -> Body { + self.body + } + + pub async fn bytes(mut self) -> Result { + let mut collected = BytesMut::new(); + while let Some(chunk) = self.body.next().await { + collected.extend_from_slice(&chunk?); + } + Ok(collected.freeze()) + } +} + +pub struct Body { + exec: Arc, + key: usize, + inbox: UnboundedReceiver, +} + +impl Stream for Body { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.inbox.poll_recv(cx) { + Poll::Ready(Some(Event::Data(chunk))) => { + this.exec.resume(this.key); + return Poll::Ready(Some(Ok(chunk))); + } + Poll::Ready(Some(Event::Fail(reason))) => { + return Poll::Ready(Some(Err(anyhow!(reason)))); + } + Poll::Ready(Some(Event::Status(_))) => {} + Poll::Ready(Some(Event::End)) | Poll::Ready(None) => return Poll::Ready(None), + Poll::Pending => return Poll::Pending, + } + } + } +} + +impl Drop for Body { + fn drop(&mut self) { + self.exec.close(self.key); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::SocketAddr; + use std::sync::Weak; + use std::time::Duration; + use tokio::time::{sleep, timeout}; + + async fn upstream() -> SocketAddr { + let router = axum::Router::new().route( + "/", + axum::routing::post(|body: Bytes| async move { body.len().to_string() }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + address + } + + async fn settle(watch: &Weak, live: usize) { + timeout(Duration::from_secs(10), async { + while watch.strong_count() > live { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the runtime still owns requests"); + } + + #[test] + fn derives_the_user_agent_from_the_cronet_version() { + assert_eq!( + build_user_agent("154.0.8037.49").unwrap().to_str().unwrap(), + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36" + ); + assert_eq!( + build_user_agent("120.0.6099.109") + .unwrap() + .to_str() + .unwrap(), + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ); + assert!(build_user_agent("").is_none()); + assert!(build_user_agent("Cronet").is_none()); + } + + #[tokio::test] + async fn sends_the_browser_user_agent() { + let router = axum::Router::new().route( + "/", + axum::routing::post(|headers: http::HeaderMap| async move { + headers + .get(http::header::USER_AGENT) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_owned() + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + + let client = Client::new(None).unwrap(); + let url = format!("http://{address}/"); + let body = client + .post(&url) + .body(Bytes::from_static(b"payload")) + .send() + .await + .unwrap() + .bytes() + .await + .unwrap(); + let expected = build_user_agent(&engine_version(client.exec.engine()).unwrap()).unwrap(); + assert_eq!(&body[..], expected.to_bytes()); + } + + #[tokio::test] + async fn releases_requests_uploads_and_engine() { + let client = Client::new(None).unwrap(); + let watch = client.exec.watch(); + let live = watch.strong_count(); + let address = upstream().await; + let url = format!("http://{address}/"); + + for _ in 0..8 { + let response = client + .post(&url) + .body(Bytes::from_static(b"payload")) + .send() + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(&response.bytes().await.unwrap()[..], b"7"); + settle(&watch, live).await; + } + + for _ in 0..4 { + let response = client + .post(&url) + .body(Bytes::from_static(b"payload")) + .send() + .await + .unwrap(); + drop(response); + settle(&watch, live).await; + } + + drop(client); + assert_eq!( + watch.strong_count(), + 0, + "cronet runtime outlived the client" + ); + } +} diff --git a/src/cronet/runtime.rs b/src/cronet/runtime.rs new file mode 100644 index 0000000..9d9e73b --- /dev/null +++ b/src/cronet/runtime.rs @@ -0,0 +1,652 @@ +use std::collections::HashMap; +use std::ffi::{CStr, CString}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::thread::JoinHandle; + +use anyhow::{Result, bail}; +use bytes::Bytes; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; + +use super::sys::{self, Ptr}; +use super::{Callback, Engine}; + +const READ_CHUNK: usize = 32 * 1024; + +static NEXT_KEY: AtomicUsize = AtomicUsize::new(1); + +pub(crate) fn next_key() -> usize { + NEXT_KEY.fetch_add(1, Ordering::Relaxed) +} + +pub(crate) enum Event { + Status(u16), + Data(Bytes), + End, + Fail(String), +} + +pub(crate) struct Spec { + pub(crate) url: String, + pub(crate) method: &'static str, + pub(crate) headers: Vec<(String, String)>, + pub(crate) body: Option, +} + +enum Command { + Open { + key: usize, + events: UnboundedSender, + spec: Spec, + }, + Resume(usize), + Close(usize), + Run(Ptr), + Collect, + Stop, +} + +struct Slot { + request: Ptr, + events: UnboundedSender, + upload: Option<(Ptr, Ptr)>, + reading: bool, + closed: bool, +} + +impl Slot { + fn fill(&mut self) { + if self.reading || self.closed { + return; + } + let api = sys::api(); + let buffer = api.buffer_create(); + if buffer.is_null() { + return; + } + api.buffer_alloc(buffer, READ_CHUNK as u64); + if api.buffer_size(buffer) < READ_CHUNK as u64 { + api.buffer_destroy(buffer); + return; + } + if api.request_read(self.request, buffer) == sys::RESULT_SUCCESS { + self.reading = true; + } else { + api.buffer_destroy(buffer); + } + } +} + +struct Upload { + shared: Arc, + body: Bytes, + position: usize, + closed: AtomicBool, +} + +struct Chunk { + buffer: Ptr, + length: usize, +} + +impl AsRef<[u8]> for Chunk { + fn as_ref(&self) -> &[u8] { + let data = sys::api().buffer_data(self.buffer).cast::(); + unsafe { std::slice::from_raw_parts(data, self.length) } + } +} + +impl Drop for Chunk { + fn drop(&mut self) { + sys::api().buffer_destroy(self.buffer); + } +} + +pub(crate) struct Shared { + commands: UnboundedSender, + slots: Mutex>, + garbage: Mutex>, +} + +pub(crate) struct Executor { + shared: Arc, + object: Ptr, + engine: Arc, + callback: Arc, + thread: Mutex>>, +} + +impl Executor { + pub(crate) fn new(engine: Arc, callback: Arc) -> Result> { + let api = sys::api(); + let (commands, inbox) = unbounded_channel(); + let shared = Arc::new(Shared { + commands, + slots: Mutex::new(HashMap::new()), + garbage: Mutex::new(Vec::new()), + }); + let object = api.executor_create(Some(execute)); + if object.is_null() { + bail!("cronet: executor create failed"); + } + api.executor_set_context(object, Arc::as_ptr(&shared).cast_mut().cast()); + api.callback_set_context(callback.0, Arc::as_ptr(&shared).cast_mut().cast()); + + let worker = Worker { + engine: engine.0, + callback: callback.0, + executor: object, + shared: Arc::clone(&shared), + }; + let thread = match std::thread::Builder::new() + .name("cronet".to_owned()) + .spawn(move || worker.run(inbox)) + { + Ok(thread) => thread, + Err(error) => { + api.executor_destroy(object); + return Err(error.into()); + } + }; + Ok(Arc::new(Self { + shared, + object, + engine, + callback, + thread: Mutex::new(Some(thread)), + })) + } + + pub(crate) fn open(&self, key: usize, events: UnboundedSender, spec: Spec) { + let _ = self + .shared + .commands + .send(Command::Open { key, events, spec }); + } + + pub(crate) fn resume(&self, key: usize) { + let _ = self.shared.commands.send(Command::Resume(key)); + } + + pub(crate) fn close(&self, key: usize) { + let _ = self.shared.commands.send(Command::Close(key)); + } + + #[cfg(test)] + pub(crate) fn engine(&self) -> Ptr { + self.engine.0 + } + + pub(crate) fn shutdown(&self) { + if let Some(thread) = self + .thread + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + { + let _ = self.shared.commands.send(Command::Stop); + let _ = thread.join(); + } + } + + #[cfg(test)] + pub(crate) fn watch(&self) -> std::sync::Weak { + Arc::downgrade(&self.shared) + } +} + +impl Drop for Executor { + fn drop(&mut self) { + self.shutdown(); + let api = sys::api(); + self.engine.shutdown(); + for (_, slot) in self + .shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain() + { + api.request_destroy(slot.request); + if let Some((provider, data)) = slot.upload { + reclaim(&self.shared, provider, data); + } + } + let mut garbage = self + .shared + .garbage + .lock() + .unwrap_or_else(PoisonError::into_inner); + for (provider, upload) in garbage.drain(..) { + api.upload_destroy(provider); + unsafe { drop(Box::from_raw(upload.raw().cast::())) }; + } + drop(garbage); + api.callback_set_context(self.callback.0, std::ptr::null_mut()); + api.executor_set_context(self.object, std::ptr::null_mut()); + api.executor_destroy(self.object); + } +} + +struct Worker { + engine: Ptr, + callback: Ptr, + executor: Ptr, + shared: Arc, +} + +impl Worker { + fn run(self, mut inbox: UnboundedReceiver) { + let mut stopping = false; + loop { + if stopping + && self + .shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner) + .is_empty() + { + break; + } + let Some(command) = inbox.blocking_recv() else { + break; + }; + match command { + Command::Run(runnable) => { + let api = sys::api(); + api.runnable_run(runnable); + api.runnable_destroy(runnable); + } + Command::Open { key, events, spec } => self.open(key, events, spec), + Command::Resume(key) => self.resume(key), + Command::Close(key) => self.close(key), + Command::Collect => self.collect(), + Command::Stop => { + stopping = true; + self.cancel_all(); + } + } + self.collect(); + } + } + + fn cancel_all(&self) { + let api = sys::api(); + let mut slots = self + .shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner); + for slot in slots.values_mut() { + if !slot.closed { + slot.closed = true; + api.request_cancel(slot.request); + } + } + } + + fn open(&self, key: usize, events: UnboundedSender, spec: Spec) { + let api = sys::api(); + let Ok(url) = CString::new(spec.url) else { + let _ = events.send(Event::Fail("cronet: invalid url".to_owned())); + return; + }; + let method = CString::new(spec.method).unwrap_or_default(); + + let params = api.params_create(); + if params.is_null() { + let _ = events.send(Event::Fail("cronet: params create failed".to_owned())); + return; + } + + api.params_method(params, method.as_ptr()); + api.params_no_cache(params, true); + for (name, value) in &spec.headers { + let (Ok(name), Ok(value)) = (CString::new(name.as_str()), CString::new(value.as_str())) + else { + continue; + }; + let header = api.header_create(); + if header.is_null() { + continue; + } + api.header_name(header, name.as_ptr()); + api.header_value(header, value.as_ptr()); + api.params_header_add(params, header); + api.header_destroy(header); + } + + let mut upload = None; + if let Some(body) = spec.body.filter(|body| !body.is_empty()) { + let owned = Box::into_raw(Box::new(Upload { + shared: Arc::clone(&self.shared), + body, + position: 0, + closed: AtomicBool::new(false), + })); + let provider = api.upload_create( + Some(upload_length), + Some(upload_read), + Some(upload_rewind), + Some(upload_close), + ); + if provider.is_null() { + unsafe { drop(Box::from_raw(owned)) }; + api.params_destroy(params); + let _ = events.send(Event::Fail( + "cronet: upload provider create failed".to_owned(), + )); + return; + } + let data = Ptr::new(owned.cast()); + api.upload_set_context(provider, owned.cast()); + api.params_upload(params, provider); + api.params_upload_executor(params, self.executor); + upload = Some((provider, data)); + } + + let request = api.request_create(); + if request.is_null() { + api.params_destroy(params); + if let Some((provider, data)) = upload { + reclaim(&self.shared, provider, data); + } + let _ = events.send(Event::Fail("cronet: request create failed".to_owned())); + return; + } + + api.request_set_context(request, key as *mut _); + self.shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert( + key, + Slot { + request, + events, + upload, + reading: false, + closed: false, + }, + ); + + let result = api.request_init( + request, + self.engine, + url.as_ptr(), + params, + self.callback, + self.executor, + ); + api.params_destroy(params); + + if result != sys::RESULT_SUCCESS { + let upload = finish( + &self.shared, + key, + Event::Fail(format!("cronet: init failed ({result})")), + ); + if let Some((provider, data)) = upload { + reclaim(&self.shared, provider, data); + } + return; + } + + let result = api.request_start(request); + if result != sys::RESULT_SUCCESS { + let upload = finish( + &self.shared, + key, + Event::Fail(format!("cronet: start failed ({result})")), + ); + if let Some((provider, data)) = upload { + reclaim(&self.shared, provider, data); + } + } + } + + fn resume(&self, key: usize) { + if let Some(slot) = self + .shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get_mut(&key) + { + slot.fill(); + } + } + + fn close(&self, key: usize) { + if let Some(slot) = self + .shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get_mut(&key) + && !slot.closed + { + slot.closed = true; + sys::api().request_cancel(slot.request); + } + } + + fn collect(&self) { + let api = sys::api(); + let mut garbage = self + .shared + .garbage + .lock() + .unwrap_or_else(PoisonError::into_inner); + for (provider, upload) in garbage.drain(..) { + api.upload_destroy(provider); + unsafe { drop(Box::from_raw(upload.raw().cast::())) }; + } + } +} + +fn reclaim(shared: &Shared, provider: Ptr, data: Ptr) { + let upload = unsafe { &*data.raw().cast::() }; + if upload.closed.swap(true, Ordering::AcqRel) { + return; + } + shared + .garbage + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push((provider, data)); +} + +fn finish(shared: &Shared, key: usize, event: Event) -> Option<(Ptr, Ptr)> { + let slot = shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&key)?; + sys::api().request_destroy(slot.request); + let _ = slot.events.send(event); + let _ = shared.commands.send(Command::Collect); + slot.upload +} + +fn shared_of(callback: Ptr) -> Option<&'static Shared> { + let raw = sys::api().callback_get_context(callback); + (!raw.is_null()).then(|| unsafe { &*raw.cast::() }) +} + +fn key_of(request: Ptr) -> usize { + sys::api().request_get_context(request) as usize +} + +pub(crate) unsafe extern "C" fn on_redirect( + _callback: Ptr, + request: Ptr, + _info: Ptr, + _location: sys::CronetString, +) { + sys::api().request_follow_redirect(request); +} + +pub(crate) unsafe extern "C" fn on_response_started(callback: Ptr, request: Ptr, info: Ptr) { + let Some(shared) = shared_of(callback) else { + return; + }; + let status = sys::api().response_status(info); + let key = key_of(request); + let mut slots = shared.slots.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(slot) = slots.get_mut(&key) { + slot.fill(); + let _ = slot + .events + .send(Event::Status(status.clamp(0, u16::MAX as i32) as u16)); + } +} + +pub(crate) unsafe extern "C" fn on_read_completed( + callback: Ptr, + request: Ptr, + _info: Ptr, + buffer: Ptr, + bytes: u64, +) { + let api = sys::api(); + let chunk = if bytes > 0 { + let length = (bytes as usize).min(api.buffer_size(buffer) as usize); + Some(Bytes::from_owner(Chunk { buffer, length })) + } else { + api.buffer_destroy(buffer); + None + }; + + let Some(shared) = shared_of(callback) else { + return; + }; + let key = key_of(request); + let mut slots = shared.slots.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(slot) = slots.get_mut(&key) { + slot.reading = false; + if let Some(chunk) = chunk { + let _ = slot.events.send(Event::Data(chunk)); + } + } +} + +pub(crate) unsafe extern "C" fn on_succeeded(callback: Ptr, request: Ptr, _info: Ptr) { + if let Some(shared) = shared_of(callback) + && let Some((provider, data)) = finish(shared, key_of(request), Event::End) + { + reclaim(shared, provider, data); + } +} + +pub(crate) unsafe extern "C" fn on_canceled(callback: Ptr, request: Ptr, _info: Ptr) { + if let Some(shared) = shared_of(callback) + && let Some((provider, data)) = finish(shared, key_of(request), Event::End) + { + reclaim(shared, provider, data); + } +} + +pub(crate) unsafe extern "C" fn on_failed(callback: Ptr, request: Ptr, _info: Ptr, error: Ptr) { + let Some(shared) = shared_of(callback) else { + return; + }; + let message = if error.is_null() { + None + } else { + let raw = sys::api().error_message(error); + (!raw.is_null()).then(|| { + unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned() + }) + }; + let message = message.unwrap_or_else(|| "cronet: request failed".to_owned()); + if let Some((provider, data)) = finish(shared, key_of(request), Event::Fail(message)) { + reclaim(shared, provider, data); + } +} + +unsafe extern "C" fn execute(object: Ptr, runnable: Ptr) { + let api = sys::api(); + let context = api.executor_get_context(object); + if context.is_null() { + api.runnable_destroy(runnable); + return; + } + let shared = unsafe { &*context.cast::() }; + if shared.commands.send(Command::Run(runnable)).is_err() { + api.runnable_destroy(runnable); + } +} + +fn upload_of(provider: Ptr) -> Option<&'static mut Upload> { + let raw = sys::api().upload_get_context(provider); + (!raw.is_null()).then(|| unsafe { &mut *raw.cast::() }) +} + +unsafe extern "C" fn upload_length(provider: Ptr) -> i64 { + upload_of(provider).map_or(0, |upload| upload.body.len() as i64) +} + +unsafe extern "C" fn upload_read(provider: Ptr, sink: Ptr, buffer: Ptr) { + let api = sys::api(); + let Some(upload) = upload_of(provider) else { + api.sink_read_error(sink, c"cronet: upload lost".as_ptr()); + return; + }; + let capacity = api.buffer_size(buffer) as usize; + let pending = &upload.body[upload.position..]; + let length = capacity.min(pending.len()); + if length > 0 { + unsafe { + std::ptr::copy_nonoverlapping( + pending.as_ptr(), + api.buffer_data(buffer).cast::(), + length, + ); + } + upload.position += length; + } + api.sink_read_ok(sink, length as u64, false); +} + +unsafe extern "C" fn upload_rewind(provider: Ptr, sink: Ptr) { + if let Some(upload) = upload_of(provider) { + upload.position = 0; + } + sys::api().sink_rewind_ok(sink); +} + +unsafe extern "C" fn upload_close(provider: Ptr) { + let raw = sys::api().upload_get_context(provider); + if raw.is_null() { + return; + } + let upload = unsafe { &*raw.cast::() }; + if upload.closed.swap(true, Ordering::AcqRel) { + return; + } + let shared = &upload.shared; + let data = Ptr::new(raw); + for slot in shared + .slots + .lock() + .unwrap_or_else(PoisonError::into_inner) + .values_mut() + { + if slot.upload.is_some_and(|(_, slot_data)| slot_data == data) { + slot.upload = None; + break; + } + } + upload + .shared + .garbage + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push((provider, data)); + let _ = shared.commands.send(Command::Collect); +} diff --git a/src/cronet/sys.rs b/src/cronet/sys.rs new file mode 100644 index 0000000..f4d9316 --- /dev/null +++ b/src/cronet/sys.rs @@ -0,0 +1,294 @@ +#[cfg(not(cronet_static))] +use std::ffi::{CString, OsString}; +use std::ffi::{c_char, c_void}; +use std::sync::OnceLock; + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct Ptr(usize); + +impl Ptr { + pub(crate) fn new(raw: *mut c_void) -> Self { + Self(raw as usize) + } + + pub(crate) fn raw(self) -> *mut c_void { + self.0 as *mut c_void + } + + pub(crate) fn is_null(self) -> bool { + self.0 == 0 + } +} + +pub(crate) const RESULT_SUCCESS: i32 = 0; +pub(crate) const HTTP_CACHE_DISABLED: i32 = 0; + +pub(crate) type CronetString = *const c_char; + +pub(crate) type OnRedirectReceived = unsafe extern "C" fn(Ptr, Ptr, Ptr, CronetString); +pub(crate) type OnResponseStarted = unsafe extern "C" fn(Ptr, Ptr, Ptr); +pub(crate) type OnReadCompleted = unsafe extern "C" fn(Ptr, Ptr, Ptr, Ptr, u64); +pub(crate) type OnSucceeded = unsafe extern "C" fn(Ptr, Ptr, Ptr); +pub(crate) type OnFailed = unsafe extern "C" fn(Ptr, Ptr, Ptr, Ptr); +pub(crate) type OnCanceled = unsafe extern "C" fn(Ptr, Ptr, Ptr); +pub(crate) type OnExecute = unsafe extern "C" fn(Ptr, Ptr); +pub(crate) type UploadLength = unsafe extern "C" fn(Ptr) -> i64; +pub(crate) type UploadRead = unsafe extern "C" fn(Ptr, Ptr, Ptr); +pub(crate) type UploadRewind = unsafe extern "C" fn(Ptr, Ptr); +pub(crate) type UploadClose = unsafe extern "C" fn(Ptr); + +macro_rules! api { + ($($field:ident: $symbol:ident($($arg:ident: $arg_ty:ty),* $(,)?) $(-> $ret:ty)?;)*) => { + #[cfg(cronet_static)] + #[allow(non_snake_case)] + mod symbols { + use super::*; + + unsafe extern "C" { + $(pub(super) fn $symbol($($arg: $arg_ty),*) $(-> $ret)?;)* + } + } + + pub(crate) struct Api { + $($field: unsafe extern "C" fn($($arg_ty),*) $(-> $ret)?,)* + } + + impl Api { + #[cfg(cronet_static)] + fn bind() -> Result { + Ok(Self { + $($field: symbols::$symbol + as unsafe extern "C" fn($($arg_ty),*) $(-> $ret)?,)* + }) + } + + #[cfg(not(cronet_static))] + fn bind() -> Result { + let library = Library::open()?; + Ok(Self { $($field: unsafe { library.symbol(stringify!($symbol))? },)* }) + } + + $( + #[inline] + pub(crate) fn $field(&self $(, $arg: $arg_ty)*) $(-> $ret)? { + unsafe { (self.$field)($($arg),*) } + } + )* + } + }; +} + +api! { + engine_create: Cronet_Engine_Create() -> Ptr; + engine_destroy: Cronet_Engine_Destroy(engine: Ptr); + engine_start: Cronet_Engine_StartWithParams(engine: Ptr, params: Ptr) -> i32; + engine_shutdown: Cronet_Engine_Shutdown(engine: Ptr) -> i32; + engine_version: Cronet_Engine_GetVersionString(engine: Ptr) -> CronetString; + engine_params_create: Cronet_EngineParams_Create() -> Ptr; + engine_params_destroy: Cronet_EngineParams_Destroy(params: Ptr); + engine_params_user_agent: Cronet_EngineParams_user_agent_set(params: Ptr, agent: CronetString); + engine_params_quic: Cronet_EngineParams_enable_quic_set(params: Ptr, enable: bool); + engine_params_http2: Cronet_EngineParams_enable_http2_set(params: Ptr, enable: bool); + engine_params_brotli: Cronet_EngineParams_enable_brotli_set(params: Ptr, enable: bool); + engine_params_cache_mode: Cronet_EngineParams_http_cache_mode_set(params: Ptr, mode: i32); + engine_params_experimental: Cronet_EngineParams_experimental_options_set( + params: Ptr, + options: CronetString, + ); + request_create: Cronet_UrlRequest_Create() -> Ptr; + request_destroy: Cronet_UrlRequest_Destroy(request: Ptr); + request_set_context: Cronet_UrlRequest_SetClientContext(request: Ptr, context: *mut c_void); + request_get_context: Cronet_UrlRequest_GetClientContext(request: Ptr) -> *mut c_void; + request_init: Cronet_UrlRequest_InitWithParams( + request: Ptr, + engine: Ptr, + url: CronetString, + params: Ptr, + callback: Ptr, + executor: Ptr, + ) -> i32; + request_start: Cronet_UrlRequest_Start(request: Ptr) -> i32; + request_follow_redirect: Cronet_UrlRequest_FollowRedirect(request: Ptr) -> i32; + request_read: Cronet_UrlRequest_Read(request: Ptr, buffer: Ptr) -> i32; + request_cancel: Cronet_UrlRequest_Cancel(request: Ptr); + params_create: Cronet_UrlRequestParams_Create() -> Ptr; + params_destroy: Cronet_UrlRequestParams_Destroy(params: Ptr); + params_method: Cronet_UrlRequestParams_http_method_set(params: Ptr, method: CronetString); + params_header_add: Cronet_UrlRequestParams_request_headers_add(params: Ptr, header: Ptr); + params_no_cache: Cronet_UrlRequestParams_disable_cache_set(params: Ptr, disable: bool); + params_upload: Cronet_UrlRequestParams_upload_data_provider_set(params: Ptr, provider: Ptr); + params_upload_executor: Cronet_UrlRequestParams_upload_data_provider_executor_set( + params: Ptr, + executor: Ptr, + ); + callback_create: Cronet_UrlRequestCallback_CreateWith( + redirect: Option, + started: Option, + read: Option, + succeeded: Option, + failed: Option, + canceled: Option, + ) -> Ptr; + callback_destroy: Cronet_UrlRequestCallback_Destroy(callback: Ptr); + callback_set_context: Cronet_UrlRequestCallback_SetClientContext( + callback: Ptr, + context: *mut c_void, + ); + callback_get_context: Cronet_UrlRequestCallback_GetClientContext(callback: Ptr) -> *mut c_void; + response_status: Cronet_UrlResponseInfo_http_status_code_get(info: Ptr) -> i32; + error_message: Cronet_Error_message_get(error: Ptr) -> CronetString; + buffer_create: Cronet_Buffer_Create() -> Ptr; + buffer_destroy: Cronet_Buffer_Destroy(buffer: Ptr); + buffer_alloc: Cronet_Buffer_InitWithAlloc(buffer: Ptr, size: u64); + buffer_data: Cronet_Buffer_GetData(buffer: Ptr) -> *mut c_void; + buffer_size: Cronet_Buffer_GetSize(buffer: Ptr) -> u64; + upload_create: Cronet_UploadDataProvider_CreateWith( + length: Option, + read: Option, + rewind: Option, + close: Option, + ) -> Ptr; + upload_destroy: Cronet_UploadDataProvider_Destroy(provider: Ptr); + upload_set_context: Cronet_UploadDataProvider_SetClientContext( + provider: Ptr, + context: *mut c_void, + ); + upload_get_context: Cronet_UploadDataProvider_GetClientContext(provider: Ptr) -> *mut c_void; + sink_read_ok: Cronet_UploadDataSink_OnReadSucceeded(sink: Ptr, bytes: u64, last: bool); + sink_read_error: Cronet_UploadDataSink_OnReadError(sink: Ptr, message: CronetString); + sink_rewind_ok: Cronet_UploadDataSink_OnRewindSucceeded(sink: Ptr); + executor_create: Cronet_Executor_CreateWith(execute: Option) -> Ptr; + executor_destroy: Cronet_Executor_Destroy(executor: Ptr); + executor_set_context: Cronet_Executor_SetClientContext(executor: Ptr, context: *mut c_void); + executor_get_context: Cronet_Executor_GetClientContext(executor: Ptr) -> *mut c_void; + runnable_run: Cronet_Runnable_Run(runnable: Ptr); + runnable_destroy: Cronet_Runnable_Destroy(runnable: Ptr); + header_create: Cronet_HttpHeader_Create() -> Ptr; + header_destroy: Cronet_HttpHeader_Destroy(header: Ptr); + header_name: Cronet_HttpHeader_name_set(header: Ptr, name: CronetString); + header_value: Cronet_HttpHeader_value_set(header: Ptr, value: CronetString); +} + +static API: OnceLock> = OnceLock::new(); + +pub(crate) fn load() -> Result<(), String> { + match API.get_or_init(Api::bind) { + Ok(_) => Ok(()), + Err(error) => Err(error.clone()), + } +} + +pub(crate) fn api() -> &'static Api { + match API.get_or_init(Api::bind) { + Ok(api) => api, + Err(error) => panic!("libcronet unavailable: {error}"), + } +} + +#[cfg(not(cronet_static))] +struct Library { + handle: *mut c_void, +} + +#[cfg(not(cronet_static))] +impl Library { + fn open() -> Result { + let mut tried = Vec::new(); + for candidate in candidates() { + let Some(handle) = platform::open(candidate.as_os_str()) else { + tried.push(candidate.to_string_lossy().into_owned()); + continue; + }; + return Ok(Self { handle }); + } + Err(format!("libcronet not found, tried: {}", tried.join(", "))) + } + + unsafe fn symbol(&self, name: &str) -> Result { + let name = CString::new(name).map_err(|_| "invalid symbol name".to_owned())?; + let raw = platform::symbol(self.handle, &name) + .ok_or_else(|| format!("missing symbol {}", name.to_string_lossy()))?; + Ok(unsafe { std::mem::transmute_copy(&raw) }) + } +} + +#[cfg(not(cronet_static))] +const LIBRARY_NAMES: &[&str] = if cfg!(windows) { + &["libcronet.dll", "cronet.dll"] +} else if cfg!(target_os = "macos") { + &["libcronet.dylib"] +} else { + &["libcronet.so", "libcronet.so.1"] +}; + +#[cfg(not(cronet_static))] +fn candidates() -> Vec { + let mut paths = Vec::new(); + if let Some(explicit) = std::env::var_os("CRONET_LIB") { + paths.push(explicit); + } + if let Ok(executable) = std::env::current_exe() + && let Some(directory) = executable.parent() + { + paths.extend( + LIBRARY_NAMES + .iter() + .map(|name| directory.join(name).into_os_string()), + ); + } + paths.extend(LIBRARY_NAMES.iter().map(OsString::from)); + paths +} + +#[cfg(all(windows, not(cronet_static)))] +mod platform { + use std::ffi::{CStr, OsStr, c_char, c_void}; + use std::os::windows::ffi::OsStrExt; + + #[link(name = "kernel32")] + unsafe extern "system" { + fn LoadLibraryW(name: *const u16) -> *mut c_void; + fn GetProcAddress(module: *mut c_void, name: *const c_char) -> *mut c_void; + } + + pub(super) fn open(path: &OsStr) -> Option<*mut c_void> { + let wide: Vec = path.encode_wide().chain([0]).collect(); + let handle = unsafe { LoadLibraryW(wide.as_ptr()) }; + (!handle.is_null()).then_some(handle) + } + + pub(super) fn symbol(handle: *mut c_void, name: &CStr) -> Option<*mut c_void> { + let address = unsafe { GetProcAddress(handle, name.as_ptr()) }; + (!address.is_null()).then_some(address) + } +} + +#[cfg(all(unix, not(cronet_static)))] +mod platform { + use std::ffi::{CStr, OsStr, c_char, c_int, c_void}; + use std::os::unix::ffi::OsStrExt; + + const RTLD_NOW: c_int = 2; + #[cfg(target_os = "macos")] + const RTLD_LOCAL: c_int = 4; + #[cfg(not(target_os = "macos"))] + const RTLD_LOCAL: c_int = 0; + + #[cfg_attr(all(target_os = "linux", target_env = "gnu"), link(name = "dl"))] + unsafe extern "C" { + fn dlopen(path: *const c_char, flags: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, name: *const c_char) -> *mut c_void; + } + + pub(super) fn open(path: &OsStr) -> Option<*mut c_void> { + let path = std::ffi::CString::new(path.as_bytes()).ok()?; + let handle = unsafe { dlopen(path.as_ptr(), RTLD_NOW | RTLD_LOCAL) }; + (!handle.is_null()).then_some(handle) + } + + pub(super) fn symbol(handle: *mut c_void, name: &CStr) -> Option<*mut c_void> { + let address = unsafe { dlsym(handle, name.as_ptr()) }; + (!address.is_null()).then_some(address) + } +} diff --git a/src/dns/client.rs b/src/dns/client.rs index efff07c..01da926 100644 --- a/src/dns/client.rs +++ b/src/dns/client.rs @@ -9,19 +9,19 @@ use domain::{ }; use moka::future::Cache; use rand::seq::SliceRandom; -use singleflight_async::SingleFlight; use std::{ + collections::HashMap, net::{IpAddr, SocketAddr}, str::FromStr, sync::{ - Arc, + Arc, Mutex, PoisonError, Weak, atomic::{AtomicBool, Ordering}, }, time::Duration, }; use tokio::{ net::TcpStream, - sync::Semaphore, + sync::{OwnedSemaphorePermit, Semaphore, watch}, task::JoinSet, time::{Instant, timeout}, }; @@ -34,6 +34,7 @@ use crate::warp::WarpClient; type CacheKey = (String, u16, Option); type SharedDnsResult = Result<(Vec, Duration), Arc>; +type InFlight = HashMap>>; #[derive(Clone)] struct CacheEntry { @@ -46,11 +47,41 @@ struct CacheEntry { pub struct DnsClient { config: DnsConfig, cache: Cache, - single_flight: SingleFlight, + in_flight: Mutex, semaphore: Arc, transport: Transport, } +struct InFlightGuard { + client: Weak, + key: CacheKey, +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + if let Some(client) = self.client.upgrade() { + client + .in_flight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&self.key); + } + } +} + +async fn wait_for_lookup( + mut receiver: watch::Receiver>, +) -> SharedDnsResult { + loop { + if let Some(result) = receiver.borrow().clone() { + return result; + } + if receiver.changed().await.is_err() { + return Err(Arc::new(anyhow!("DNS lookup task ended without a result"))); + } + } +} + impl DnsClient { pub async fn new(config: &DnsConfig) -> Result { let transport = transport::build_transport(config, None)?; @@ -66,7 +97,7 @@ impl DnsClient { config.options.max_ttl + config.options.swr_ttl, )) .build(), - single_flight: SingleFlight::new(), + in_flight: Mutex::new(HashMap::new()), semaphore: Arc::new(Semaphore::new(config.options.max_concurrent_queries)), transport, } @@ -102,12 +133,7 @@ impl DnsClient { ); tokio::spawn(async move { debug!("background refresh triggered: {} {:?}", dc, rtype); - let _ = sc - .single_flight - .work(kc.clone(), || async { - sc.query_upstream_and_cache(&dc, rtype, ecs, kc).await - }) - .await; + let _ = sc.lookup_uncached(dc, rtype, ecs, kc).await; flag.store(false, Ordering::Release); }); } @@ -116,21 +142,77 @@ impl DnsClient { } } - self.single_flight - .work(key.clone(), || async { - self.query_upstream_and_cache(domain, rtype, ecs, key).await - }) + self.lookup_uncached(domain.to_owned(), rtype, ecs, key) .await .map(|(ips, _)| ips) .map_err(|e| anyhow!("DNS resolution error: {}", e)) } + async fn lookup_uncached( + self: &Arc, + domain: String, + rtype: Rtype, + ecs: Option, + key: CacheKey, + ) -> SharedDnsResult { + let existing = self + .in_flight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&key) + .cloned(); + if let Some(receiver) = existing { + return wait_for_lookup(receiver).await; + } + + let permit = self + .semaphore + .clone() + .acquire_owned() + .await + .map_err(|error| Arc::new(anyhow!(error)))?; + + let (receiver, sender, leader) = { + let mut in_flight = self + .in_flight + .lock() + .unwrap_or_else(PoisonError::into_inner); + if let Some(receiver) = in_flight.get(&key) { + (receiver.clone(), None, false) + } else { + let (sender, receiver) = watch::channel(None); + in_flight.insert(key.clone(), receiver.clone()); + (receiver, Some(sender), true) + } + }; + + if leader { + let sender = sender.expect("leader owns the lookup result sender"); + let client = Arc::clone(self); + let cleanup = Arc::downgrade(self); + let cleanup_key = key.clone(); + tokio::spawn(async move { + let _cleanup = InFlightGuard { + client: cleanup, + key: cleanup_key, + }; + let result = client + .query_upstream_and_cache(&domain, rtype, ecs, key, permit) + .await; + let _ = sender.send(Some(result)); + }); + } + + wait_for_lookup(receiver).await + } + async fn query_upstream_and_cache( &self, domain: &str, rtype: Rtype, ecs: Option, key: CacheKey, + _permit: OwnedSemaphorePermit, ) -> SharedDnsResult { let (ips, ttl) = self .query_upstream(domain, rtype, ecs) @@ -152,8 +234,6 @@ impl DnsClient { rtype: Rtype, ecs: Option, ) -> Result<(Vec, Duration)> { - let _permit = self.semaphore.acquire().await?; - let mut query = self.build_query(domain, rtype, ecs, 0)?; let (resp, id) = self.transport.send(&mut query).await?; @@ -491,6 +571,65 @@ mod tests { DnsClient::new(&cfg).await.unwrap() } + #[tokio::test] + async fn canceled_lookup_does_not_leave_an_in_flight_entry() { + let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let config = DnsConfig { + upstream: upstream.local_addr().unwrap(), + tls_domain: None, + options: DnsOptions::default(), + }; + let client = Arc::new(DnsClient::new(&config).await.unwrap()); + let domain = "cancelled-random-query.example"; + let lookup = { + let client = Arc::clone(&client); + tokio::spawn(async move { client.lookup(domain, Rtype::A, None).await }) + }; + + let mut query = [0u8; 2048]; + let (length, peer) = + tokio::time::timeout(Duration::from_secs(2), upstream.recv_from(&mut query)) + .await + .unwrap() + .unwrap(); + lookup.abort(); + let _ = lookup.await; + + let id = u16::from_be_bytes([query[0], query[1]]); + let response = make_response_for( + id, + false, + Rcode::NOERROR, + 60, + &[Ipv4Addr::new(192, 0, 2, 1)], + domain, + Rtype::A, + ); + upstream.send_to(&response, peer).await.unwrap(); + + tokio::time::timeout(Duration::from_secs(2), async { + while !client + .in_flight + .lock() + .unwrap_or_else(PoisonError::into_inner) + .is_empty() + { + tokio::task::yield_now().await; + } + }) + .await + .expect("completed lookup must be removed from the in-flight map"); + assert!( + client + .cache + .get(&(domain.to_owned(), Rtype::A.to_int(), None)) + .await + .is_some(), + "detached query must complete and populate the DNS cache" + ); + assert!(length > 0); + } + #[tokio::test] async fn parse_response_accepts_valid_a_record() { let c = test_client().await; diff --git a/src/dns/transport.rs b/src/dns/transport.rs index 7d9440e..2f0dfcc 100644 --- a/src/dns/transport.rs +++ b/src/dns/transport.rs @@ -4,14 +4,14 @@ use std::{ future::Future, net::SocketAddr, pin::Pin, - sync::{Arc, OnceLock}, + sync::{Arc, Mutex as StdMutex, OnceLock, PoisonError}, task::{Context as TaskContext, Poll}, time::Duration, }; use tokio::{ io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf}, net::{TcpStream, UdpSocket}, - sync::{Mutex, mpsc, oneshot}, + sync::{mpsc, oneshot}, time::timeout, }; use tokio_rustls::{ @@ -30,17 +30,31 @@ const UDP_RECV_BUF: usize = 2048; const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); const TLS_TIMEOUT: Duration = Duration::from_secs(5); -type PendingMap = Arc>>>>>; +type PendingMap = Arc>>>>>; type BoxFuture = Pin + Send>>; type Connector = Arc BoxFuture> + Send + Sync>; -async fn assign_id_and_register( +struct PendingGuard { + pending: PendingMap, + id: u16, +} + +impl Drop for PendingGuard { + fn drop(&mut self) { + self.pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&self.id); + } +} + +fn assign_id_and_register( pending: &PendingMap, data: &mut [u8], tx: oneshot::Sender>>, ) -> u16 { - let mut map = pending.lock().await; + let mut map = pending.lock().unwrap_or_else(PoisonError::into_inner); let id = loop { let candidate: u16 = rand::random(); if !map.contains_key(&candidate) { @@ -144,7 +158,11 @@ impl StreamTransport { debug!("DNS stream transport connected"); } Err(e) => { - if let Some(tx) = actor_pending.lock().await.remove(&id) { + if let Some(tx) = actor_pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&id) + { let _ = tx.send(Err(anyhow!("connect failed: {}", e))); } continue; @@ -163,7 +181,11 @@ impl StreamTransport { if !matches!(write_result, Ok(Ok(()))) { warn!("DNS upstream write failed or timed out, dropping connection"); - for (_, tx) in actor_pending.lock().await.drain() { + for (_, tx) in actor_pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain() + { let _ = tx.send(Err(anyhow!("write failed, connection reset"))); } writer = None; @@ -182,7 +204,11 @@ impl StreamTransport { writer = None; reader_task = None; - for (_, tx) in actor_pending.lock().await.drain() { + for (_, tx) in actor_pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .drain() + { let _ = tx.send(Err(anyhow!("connection reset by remote"))); } } @@ -211,7 +237,11 @@ impl StreamTransport { } if buf.len() >= 2 { let id = u16::from_be_bytes([buf[0], buf[1]]); - if let Some(tx) = pending.lock().await.remove(&id) { + if let Some(tx) = pending + .lock() + .unwrap_or_else(PoisonError::into_inner) + .remove(&id) + { let _ = tx.send(Ok(buf)); } } @@ -221,20 +251,20 @@ impl StreamTransport { pub(super) async fn send(&self, data: &mut [u8]) -> Result<(Vec, u16)> { let (tx, rx) = oneshot::channel(); - let id = assign_id_and_register(&self.pending, data, tx).await; + let id = assign_id_and_register(&self.pending, data, tx); + let _pending_guard = PendingGuard { + pending: Arc::clone(&self.pending), + id, + }; if self.tx.send((data.to_vec(), id)).await.is_err() { - self.pending.lock().await.remove(&id); return Err(anyhow!("DNS transport actor closed")); } match timeout(Duration::from_secs(5), rx).await { Ok(Ok(res)) => Ok((res?, id)), Ok(Err(_)) => Err(anyhow!("DNS response channel closed")), - Err(_) => { - self.pending.lock().await.remove(&id); - Err(anyhow!("DNS query timeout")) - } + Err(_) => Err(anyhow!("DNS query timeout")), } } } diff --git a/src/lib.rs b/src/lib.rs index 1f25175..9ae4605 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub fn unix_now_secs() -> u64 { pub mod bypass; pub mod client; pub mod config; +pub mod cronet; pub mod crypto; pub mod dns; pub mod error; diff --git a/src/server/actor/tunnel.rs b/src/server/actor/tunnel.rs index 3f3bf53..823583a 100644 --- a/src/server/actor/tunnel.rs +++ b/src/server/actor/tunnel.rs @@ -13,8 +13,8 @@ use uuid::Uuid; use crate::now_secs; use crate::server::actor::upload::{UploadActor, UploadCmd}; use crate::server::constants::{ - ROTATION_STALENESS, SEGMENT_DONE_CHANNEL_CAPACITY, STREAM_IDLE_TIMEOUT_SECS, - UPLOAD_CMD_CHANNEL_CAPACITY, + MAX_PENDING_CONTINUES, ROTATION_STALENESS, SEGMENT_DONE_CHANNEL_CAPACITY, + STREAM_IDLE_TIMEOUT_SECS, UPLOAD_CMD_CHANNEL_CAPACITY, }; use crate::server::stream_registry::StreamRegistry; use crate::server::upstream::{UpstreamRead, UpstreamWrite}; @@ -365,9 +365,15 @@ impl TunnelActor { } } } - Phase::Active if self.segment_active => { + Phase::Active + if self.segment_active + && self.pending_continue.len() < MAX_PENDING_CONTINUES => + { self.pending_continue.push(reply); } + Phase::Active if self.segment_active => { + let _ = reply.send(None); + } _ => { let _ = reply.send(None); } @@ -570,6 +576,26 @@ mod tests { handle.await.unwrap(); } + #[test] + fn pending_continues_are_bounded() { + let (_cmd_tx, cmd_rx) = mpsc::channel::(16); + let mut actor = new_rotating_actor(cmd_rx, 1000); + actor.phase = Phase::Active; + actor.segment_active = true; + + let mut overflow = None; + for index in 0..=MAX_PENDING_CONTINUES { + let (reply_tx, reply_rx) = oneshot::channel(); + actor.dispatch_cmd(TunnelCmd::Continue { reply: reply_tx }); + if index == MAX_PENDING_CONTINUES { + overflow = Some(reply_rx); + } + } + + assert_eq!(actor.pending_continue.len(), MAX_PENDING_CONTINUES); + assert!(overflow.unwrap().blocking_recv().unwrap().is_none()); + } + #[tokio::test] async fn abandoned_continue_skips_segment_restart() { let (read_half, client_write, mut upstream_write) = tcp_pair().await; diff --git a/src/server/actor/upload.rs b/src/server/actor/upload.rs index 2e3b369..4dbbed3 100644 --- a/src/server/actor/upload.rs +++ b/src/server/actor/upload.rs @@ -200,8 +200,8 @@ impl UploadActor { } false } - UploadPhase::Draining { eos_waiters } => { - eos_waiters.push(ack); + UploadPhase::Draining { .. } => { + let _ = ack.send(Err(ServerError::bad_gateway("upload actor is draining"))); false } UploadPhase::Closed => { @@ -385,6 +385,27 @@ mod tests { handle.await.unwrap(); } + #[tokio::test] + async fn draining_eos_waiters_are_bounded() { + let (_rx, server_write) = tcp_pair().await; + let (_tx, rx) = mpsc::channel::(MAX_EOS_WAITERS + 2); + let mut actor = UploadActor::new(rx, server_write, 0); + actor.phase = UploadPhase::Draining { + eos_waiters: Vec::new(), + }; + + for _ in 0..=MAX_EOS_WAITERS { + let (ack_tx, mut ack_rx) = oneshot::channel(); + actor.handle_eos(0, ack_tx); + assert!(ack_rx.try_recv().unwrap().is_err()); + } + + let UploadPhase::Draining { eos_waiters } = &actor.phase else { + panic!("actor must remain draining"); + }; + assert!(eos_waiters.is_empty()); + } + #[tokio::test] async fn channel_close_acks_waiters_with_error() { let (_rx, server_write) = tcp_pair().await; diff --git a/src/server/constants.rs b/src/server/constants.rs index ff296b0..02469cb 100644 --- a/src/server/constants.rs +++ b/src/server/constants.rs @@ -6,6 +6,7 @@ pub const MAX_HANDSHAKE_BODY_SIZE: usize = 2 * MAX_FRAME_BUF_SIZE; pub const MAX_PENDING_BYTES: usize = 2 * 1024 * 1024; pub const MAX_PENDING_FRAMES: usize = 8 * 1024; pub const MAX_EOS_WAITERS: usize = 256; +pub const MAX_PENDING_CONTINUES: usize = 256; pub const MAX_REORDER_SECS: u64 = 10; pub const STREAM_IDLE_TIMEOUT_SECS: u64 = 120; diff --git a/src/warp/register.rs b/src/warp/register.rs index 053a04e..0da0e2c 100644 --- a/src/warp/register.rs +++ b/src/warp/register.rs @@ -4,11 +4,14 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result, bail}; use base64::Engine; use base64::engine::general_purpose::STANDARD as B64; +use bytes::Bytes; use p256::ecdsa::SigningKey; use p256::pkcs8::{DecodePrivateKey, EncodePrivateKey, EncodePublicKey}; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; +use crate::cronet::{Client, RequestBuilder}; + const API_URL: &str = "https://api.cloudflareclient.com"; const API_VERSION: &str = "v0a4471"; const CLIENT_VERSION: &str = "a-6.35-4471"; @@ -112,10 +115,7 @@ async fn save(path: &Path, creds: &WarpCredentials) -> Result<()> { } async fn register(opts: &RegisterOptions<'_>) -> Result { - let client = wreq::Client::builder() - .no_proxy() - .build() - .context("warp: build http client")?; + let client = Client::new(None).context("warp: build http client")?; let mut wg_key = [0u8; 32]; rand::Rng::fill_bytes(&mut rand::rng(), &mut wg_key); @@ -321,16 +321,15 @@ struct ApiErrorItem { message: String, } -fn json_body(value: &serde_json::Value) -> Result { - Ok(wreq::Body::from(serde_json::to_vec(value)?)) +fn json_body(value: &serde_json::Value) -> Result { + Ok(Bytes::from(serde_json::to_vec(value)?)) } -async fn send(req: wreq::RequestBuilder, stage: &str) -> Result { +async fn send(req: RequestBuilder, stage: &str) -> Result { let req = req .header("User-Agent", USER_AGENT) .header("CF-Client-Version", CLIENT_VERSION) - .header("Content-Type", "application/json; charset=UTF-8") - .header("Connection", "Keep-Alive"); + .header("Content-Type", "application/json; charset=UTF-8"); let response = tokio::time::timeout(API_TIMEOUT, req.send()) .await diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 635a020..40822ae 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -164,17 +164,7 @@ pub async fn spawn_client_with_redirects( let state = httproxy::client::build_state(&cfg).unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let proxy_addr = listener.local_addr().unwrap(); - let http_client = Arc::new( - wreq::Client::builder() - .tcp_nodelay(true) - .emulation(httproxy::client::chrome_emulation()) - .no_proxy() - .dns_resolver(Arc::new(httproxy::client::state::ManualResolver { - target_addr: server_addr.ip().to_string(), - })) - .build() - .unwrap(), - ); + let http_client = httproxy::client::build_http_client(&cfg).unwrap(); let sem = Arc::new(tokio::sync::Semaphore::new(state.max_connections)); let mut redirect_addrs = Vec::new();