diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3fc2ae6..28e88d6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,14 +6,21 @@ "version": "22" }, "ghcr.io/devcontainers/features/go:1": { - "version": "1.23" + "version": "1.25" }, "ghcr.io/devcontainers/features/php:1": { - "version": "8.3", + "version": "8.5", "installComposer": true + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12" + }, + "ghcr.io/devcontainers/features/rust:1": { + "version": "1.86", + "profile": "minimal" } }, - "postCreateCommand": "cd go && go mod tidy && cd ../php && composer install --no-interaction", + "postCreateCommand": "bash .devcontainer/setup.sh", "customizations": { "vscode": { "extensions": [ diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh new file mode 100755 index 0000000..5517de2 --- /dev/null +++ b/.devcontainer/setup.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install every dependency needed by the checked-in tests. Keep this script +# safe to rerun when a container is rebuilt or a dependency changes. +npm ci --ignore-scripts --no-audit --no-fund +python3 -m pip install --disable-pip-version-check -e 'python[dev]' +(cd php && composer install --no-interaction --prefer-dist) +(cd go && go mod download) +cargo fetch --locked --manifest-path rust/Cargo.toml +cargo fetch --locked --manifest-path ffi/Cargo.toml +cargo fetch --locked --manifest-path conformance/runners/run-rust/Cargo.toml diff --git a/.docker/php/Dockerfile b/.docker/php/Dockerfile new file mode 100644 index 0000000..6e48dc3 --- /dev/null +++ b/.docker/php/Dockerfile @@ -0,0 +1,8 @@ +FROM php:8.5-cli + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git libicu-dev unzip \ + && docker-php-ext-install intl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=composer:2 /usr/bin/composer /usr/bin/composer diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d4ed6b..09b045b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ jobs: with: node-version: "22" + - name: Install JavaScript dependencies + run: npm ci --ignore-scripts --no-audit --no-fund + - name: Run tests working-directory: javascript run: node test.js @@ -111,8 +114,8 @@ jobs: - uses: shivammathur/setup-php@b604ade2a87db23f8871b7182e69ec5e75effb45 # v2 with: - php-version: "8.3" - extensions: intl, mbstring + php-version: "8.5" + extensions: dom, intl, mbstring - name: Install dependencies working-directory: php @@ -129,6 +132,45 @@ jobs: php/src/ php/composer.json + python: + name: Python + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Python package and test dependencies + run: python -m pip install --disable-pip-version-check -e 'python[dev]' + + - name: Run tests + working-directory: python + run: python -m pytest + + - name: Verify generated signing vector + run: python tools/gen-test-vectors.py --check + + rust: + name: Rust + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 + with: + toolchain: "1.86" + + - name: Run tests + run: | + cargo test --locked --manifest-path rust/Cargo.toml + cargo test --locked --manifest-path ffi/Cargo.toml + conformance: name: Cross-language conformance runs-on: ubuntu-latest @@ -141,33 +183,33 @@ jobs: with: node-version: "22" + - name: Install JavaScript dependencies + run: npm ci --ignore-scripts --no-audit --no-fund + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version: "1.25" - uses: shivammathur/setup-php@b604ade2a87db23f8871b7182e69ec5e75effb45 # v2 with: - php-version: "8.3" - extensions: intl, mbstring + php-version: "8.5" + extensions: dom, intl, mbstring + + - name: Install PHP dependencies + working-directory: php + run: composer install --no-interaction --prefer-dist - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - # This action picks the toolchain from the ref name it is called by, so - # `@stable` used to be what selected stable Rust. Pinning to a commit - # removes that signal, and the toolchain must be named explicitly instead - # -- without the `with:` block below the step installs nothing usable. - # - # The pin is the head of the `stable` branch, which is a moving branch - # rather than a release tag: re-resolve it when bumping, and expect no - # semver tag to correspond. - - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable branch + # Pin the action implementation and name the compiler version explicitly. + - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 with: - toolchain: stable + toolchain: "1.86" - - name: Install Python binding dependencies - run: python3 -m pip install beautifulsoup4 + - name: Install Python binding and test dependencies + run: python3 -m pip install --disable-pip-version-check -e 'python[dev]' # REQUIRE_ALL_LANGUAGES=1 makes run-all.sh fail rather than skip when a # runner is missing, so every one of the five implementations is actually diff --git a/.gitignore b/.gitignore index dc9ee21..1e0663f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,16 +6,10 @@ Thumbs.db node_modules/ # Go -# go/go.sum is deliberately NOT ignored. It is a checksum file, not a version -# lock -- the library-doesn't-commit-its-lockfile convention that applies to -# php/composer.lock and rust/Cargo.lock below does not apply to it. Without a -# committed go.sum nothing verifies that the module contents CI builds are the -# same bytes anyone else gets, and `go mod verify` has nothing to check -# against. Commit go/go.sum alongside go/go.mod. +# go/go.sum is committed so CI can verify downloaded module contents. # PHP php/vendor/ -php/composer.lock # Python python/.venv/ @@ -29,11 +23,9 @@ __pycache__/ # Rust rust/target/ -rust/Cargo.lock # Conformance suite artifacts conformance/runners/run-rust/target/ -conformance/runners/run-rust/Cargo.lock # IDE .idea/ diff --git a/Makefile b/Makefile index 48fb8e0..a5c6e86 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,12 @@ # under `conformance/fixtures/`. `make conformance` exercises every # runnable language. -.PHONY: conformance conformance-update conformance-js conformance-go \ +.PHONY: test-docker conformance conformance-update conformance-js conformance-go \ conformance-php conformance-python conformance-rust help help: @echo "Targets:" + @echo " test-docker Test all five bindings in isolated containers." @echo " conformance Run every per-language conformance runner." @echo " conformance-update Regenerate fixture 'expected' fields from" @echo " the current Python+Rust output." @@ -19,6 +20,9 @@ help: conformance: ./conformance/run-all.sh +test-docker: + ./scripts/test-in-docker.sh + # Regenerate fixture expected fields. Run each available language with # --update; later runs overwrite earlier ones if they disagree, which # is what you want -- the last language to run is the source of truth. @@ -42,5 +46,5 @@ conformance-python: python3 conformance/runners/run-python.py conformance-rust: - cargo run --quiet --release \ + cargo run --quiet --release --locked \ --manifest-path conformance/runners/run-rust/Cargo.toml diff --git a/README.md b/README.md index 3dc7806..66d6930 100644 --- a/README.md +++ b/README.md @@ -1,203 +1,179 @@ # HTMLTrust Canonicalization -Canonical text normalization for the HTMLTrust content signing framework. Produces a stable, deterministic text representation so that the same content always hashes to the same value — regardless of which CMS, editor, or platform produced it. +HTMLTrust Canonicalization turns HTML and text into one stable byte sequence. +Use that sequence before hashing or signing content. The JavaScript, Go, PHP, +Python, and Rust bindings share the same fixtures and protocol rules. +The normative rules are maintained in the +[HTMLTrust IETF draft](https://github.com/HTMLTrust/htmltrust-spec/tree/main/ietf-draft). +The local [`spec.md`](spec.md) records the earlier text-only design for +historical reference. -All implementations follow the same [specification](spec.md) and pass the same verification test suite. +Status: `0.3.0` release candidate for `htmltrust-c14n-v1` +Previous protocol release: `v0.2.2` (`79b0d52fecd958f8fc7ade713fe0799ca1e79626`) +Readers: binding users and contributors -## Project status +## Test a fresh checkout -The current immutable release is **v0.2.2**, commit -`79b0d52fecd958f8fc7ade713fe0799ca1e79626`. It is the version used by the -HTMLTrust server, browser, CMS, and Hugo reference projects. The repository -contains the shared conformance fixtures plus five bindings. Changes to the -canonical output are protocol changes and must update every binding and the -conformance suite together. - -For reproducible builds, pin the release tag or full commit. For example: +Docker is the shortest path to a complete result. This command installs each +binding in its own container, runs its unit tests, then checks every shared +fixture: ```sh git clone https://github.com/HTMLTrust/htmltrust-canonicalization.git cd htmltrust-canonicalization -git checkout 79b0d52fecd958f8fc7ade713fe0799ca1e79626 -``` - -The JavaScript package can be installed from the same immutable commit: - -```sh -npm install https://github.com/HTMLTrust/htmltrust-canonicalization/archive/79b0d52fecd958f8fc7ade713fe0799ca1e79626.tar.gz +./scripts/test-in-docker.sh ``` -## Why Canonicalization? - -Content management systems silently transform text in ways that break naive hashing: - -- WordPress converts `"straight quotes"` to `"curly quotes"` -- Google Docs converts `--` to em dashes `—` -- Rich text editors swap `...` for the ellipsis character `…` -- Copy-paste introduces invisible Unicode characters (ZWSP, BOM, bidi marks) -- CJK editors interchange fullwidth and halfwidth forms - -Without canonicalization, the same authored content produces different hashes depending on which tool touched it last. This library normalizes all of these variations to a single canonical form. - -## Implementations +The script keeps dependency caches in Docker volumes scoped to the checkout's +absolute path. Concurrent worktrees do not share Cargo or language caches. Set +`HTMLTRUST_TEST_SESSION_ID` when concurrent test processes share one checkout. +Set `HTMLTRUST_CARGO_TARGET_MOUNT` to an absolute host directory when Cargo +artifacts must live outside Docker's volume store. -| Language | Path | Dependencies | Usage | -|---|---|---|---| -| **JavaScript** | [`javascript/`](javascript/) | None (browser + Node.js) | Browser extension, Hugo signing script | -| **Go** | [`go/`](go/) | `golang.org/x/text` (NFKC) | Hugo module | -| **PHP** | [`php/`](php/) | `ext-intl`, `ext-mbstring` | WordPress plugin | -| **Python** | [`python/`](python/) | `beautifulsoup4` | Tooling, tests | -| **Rust** | [`rust/`](rust/) | `scraper`, `unicode-normalization`, `url` | Conformance implementation | +## Install a binding -All implementations produce identical output for the same input. +Choose the binding that matches your application. Each binding declares its +runtime dependencies in its own manifest. -## Protocol Helpers - -The signing helpers use the legacy field name `domain`, but the value is a serialized Web origin such as `https://example.org` or `https://example.org:8443`, not a bare hostname. Helpers that build signature bindings reject host-only values. - -Hashes and signatures are encoded as canonical unpadded standard Base64. This is not base64url; conforming verification rejects padding, whitespace, `-`, and `_`. - -Canonical content includes signed semantic attribute records for `href`, `src`, `alt`, and `aria-label` across the JavaScript, Go, PHP, Python, and Rust HTML extraction helpers. Relative `href` and `src` values require the signed document base URL to canonicalize correctly. - -## The 8 Phases +| Binding | Directory | Runtime requirements | +|---|---|---| +| JavaScript | [`javascript/`](javascript/) | Node.js 22 or newer; `parse5` is installed from `package.json` | +| Go | [`go/`](go/) | Go 1.25 or newer; dependencies are resolved from `go.mod` | +| PHP | [`php/`](php/) | PHP 8.5 or newer with `dom`, `intl`, `mbstring`, `json`, `openssl`, and `sodium`; Composer | +| Python | [`python/`](python/) | Python 3.10 or newer; dependencies include `pywhatwgurl` and `rfc8785` | +| Rust | [`rust/`](rust/) | Rust 1.86 or newer; Cargo uses the committed `Cargo.lock` | -| Phase | What It Does | -|---|---| -| **1. NFKC** | Unicode NFKC normalization — handles ligatures, fullwidth/halfwidth, presentation forms, superscripts, CJK compatibility, Jamo composition | -| **2. Whitespace** | All Unicode whitespace (30+ characters) → ASCII space; collapse runs; trim | -| **3. Quotation Marks** | Curly quotes, guillemets, CJK corner brackets → ASCII straight quotes | -| **4. Dashes** | En dash, em dash, figure dash, non-breaking hyphen → ASCII hyphen-minus | -| **5. Punctuation** | Ellipsis `…` → `...`; minus sign → hyphen-minus | -| **6. Strip Invisibles** | Remove soft hyphens, zero-width spaces, BOM, variation selectors, bidi controls, Arabic tatweel | -| **7. Bidi** | Remove all bidi control characters (rely on HTML `dir` attribute instead) | -| **8. Language-Specific** | Preserve ZWNJ (semantic in Persian/Kurdish), ZWJ (semantic in Indic/emoji), Arabic diacritics, Hebrew nikud | +### JavaScript -## Quick Start +The root package is the installable package. From a checkout, install its +declared dependency and run a direct import: -### JavaScript (Browser / Node.js) +```sh +npm ci +node --input-type=module -e \ + 'import { normalizeText } from "./javascript/index.js"; console.log(normalizeText("A—B"))' +``` -```js -import { normalizeText } from '@htmltrust/canonicalization'; +During the `0.3.0` review, another project can install the current main branch: -const canonical = normalizeText('He said, \u201CHello\u2026\u201D'); -// → 'He said, "Hello..."' +```sh +npm install github:HTMLTrust/htmltrust-canonicalization#main ``` ### Go -```go -import "github.com/HTMLTrust/htmltrust-canonicalization/go" - -canonical := canonicalize.Normalize("He said, \u201CHello\u2026\u201D") -// → "He said, \"Hello...\"" +```sh +cd go +go mod download +go test ./... ``` ### PHP -```php -use HTMLTrust\Canonicalization\Canonicalize; - -$canonical = Canonicalize::normalize("He said, \u{201C}Hello\u{2026}\u{201D}"); -// → 'He said, "Hello..."' +```sh +cd php +composer install --no-interaction +composer test ``` -## Verification Checklist +The PHP API uses PHP 8.5's `Uri\WhatWg\Url` implementation for signed URL +attributes. Older PHP versions do not satisfy the package requirement. -All implementations must produce identical output for these test pairs: - -| Input A | Input B | Same After Normalization? | -|---|---|---| -| `"Hello"` (curly quotes) | `"Hello"` (straight) | ✅ Yes | -| `café` (precomposed) | `café` (combining) | ✅ Yes | -| `find` (fi ligature) | `find` | ✅ Yes | -| `word — word` (em dash) | `word - word` | ✅ Yes | -| `«Bonjour»` (guillemets) | `"Bonjour"` | ✅ Yes | -| `「東京」` (CJK brackets) | `"東京"` | ✅ Yes | -| `می‌خواهم` (with ZWNJ) | `میخواهم` (without) | ❌ No — ZWNJ is semantic | -| `كتـــاب` (with tatweel) | `كتاب` | ✅ Yes | -| `Ａ１` (fullwidth) | `A1` | ✅ Yes | -| `word​word` (with ZWSP) | `wordword` | ✅ Yes | -| `word‌word` (with ZWNJ) | `wordword` | ❌ No — ZWNJ is semantic | - -## Prerequisites - -The root JavaScript binding needs Node.js 22 or newer. The Go binding needs -Go 1.25 or newer. The PHP binding needs PHP 7.2 or newer with `intl`, -`mbstring`, `json`, `openssl`, and `sodium`, plus Composer. The Python -binding needs Python 3.10 or newer and pip. The Rust binding needs Rust 1.74 -or newer. The full conformance command requires all five toolchains. - -## Running tests - -From the repository root, run the language-specific tests as needed: +### Python ```sh -# JavaScript, no install step is needed -node javascript/test.js +python3 -m pip install -e 'python[dev]' +python3 -m pytest -q python/tests +``` + +### Rust -# Go -(cd go && go test -v ./...) +```sh +cargo test --locked --manifest-path rust/Cargo.toml +``` -# PHP -(cd php && composer install --no-interaction && composer test) +## Run the conformance suite -# Python -(cd python && python3 -m pip install -e '.[dev]' && python3 -m pytest) +The conformance suite is the cross-language contract. It reads every JSON +fixture under `conformance/fixtures/` and compares the exact output from each +available runner. -# Rust -(cd rust && cargo test) +```sh +make conformance ``` -Run the public cross-language contract with one command: +The command reports a missing toolchain as `MISSING` and continues with the +other runners. Require all five bindings in CI or before a release: ```sh REQUIRE_ALL_LANGUAGES=1 make conformance ``` -Without `REQUIRE_ALL_LANGUAGES=1`, the runner reports unavailable toolchains -as `SKIP`. Use `make conformance-` when iterating on one binding. -See [`conformance/README.md`](conformance/README.md) for fixture authoring -and update rules. +The current fixture count is derived at run time. To inspect it without +running the bindings: -## Compatibility matrix +```sh +find conformance/fixtures -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l +``` -| Consumer | Compatible release | Canonicalization source | -|---|---|---| -| JavaScript, server, CMS | `v0.2.2` | `79b0d52fecd958f8fc7ade713fe0799ca1e79626` | -| Browser client `@htmltrust/browser-client` | `v0.1.2` | `v0.2.2` | -| Hugo signer | current `main` | Go binding at the `v0.2.2` release commit | +See [`conformance/README.md`](conformance/README.md) for fixture format, +expected errors, and the review process for new cases. -The browser client and server manifests pin the v0.2.2 release archive. Go -consumers should pin the corresponding commit and keep the resulting -pseudo-version in `go.mod`; do not replace it with an unconstrained branch. +## What gets canonicalized -## Companion Repositories +`normalizeText` applies these phases in order: -| Repository | Description | -|---|---| -| [htmltrust-spec](https://github.com/HTMLTrust/htmltrust-spec) | The HTMLTrust specification and paper | -| [htmltrust-server-reference](https://github.com/HTMLTrust/htmltrust-server-reference) | Reference trust directory API server | -| [htmltrust-browser-reference](https://github.com/HTMLTrust/htmltrust-browser-reference) | Reference browser extension | -| [htmltrust-cms-reference](https://github.com/HTMLTrust/htmltrust-cms-reference) | Reference CMS plugins (WordPress, Hugo) | -| [htmltrust-website](https://github.com/HTMLTrust/htmltrust-website) | Project website | +1. Unicode NFKC normalization. +2. Unicode whitespace conversion to ASCII spaces, with runs collapsed. +3. Curly, guillemet, and CJK quotation marks converted to ASCII quotes. +4. Dash and hyphen variants converted to ASCII hyphen-minus. +5. The ellipsis character converted to three periods. +6. Invisible formatting and bidirectional-control characters removed. +7. ZWNJ and ZWJ preserved because they can carry meaning. -## License +`extractCanonicalText` parses HTML, excludes metadata and executable +elements, emits boundaries for block elements, and normalizes signed +`href`, `src`, `alt`, and `aria-label` attributes. Relative `href` and `src` +values require the document base URL. The portable profile rejects source +nesting deeper than 256 elements before canonical traversal. + +`canonicalizeClaims` sorts claim names by UTF-8 byte order, normalizes names and +values, and returns the byte sequence used for signing. The JSON +canonicalization helper applies strict RFC 8785-style serialization to a raw +JSON document. +JavaScript, Go, and PHP expose v1 signing-payload helpers. These functions +derive URL or origin scope, validate the exact UTC timestamp form, and return +the RFC 8785 signing bytes. The older colon-joined binding helpers remain +available for 0.2 compatibility. -This project is licensed under the [PolyForm Noncommercial License 1.0.0](https://polyformproject.org/licenses/noncommercial/1.0.0). You may use, modify, and share the software for any noncommercial purpose with attribution. Commercial use requires a separate agreement with the licensor. +## Development container -## Origin & Contributions +Open the repository in a Dev Container to get Node.js, Go, PHP, Python, and +Rust. `.devcontainer/setup.sh` installs the root JavaScript package, Python +test dependencies, PHP Composer dependencies, and Cargo modules. The setup +script is safe to run again after a dependency change. -HTMLTrust is an idea I (Jason Grey) have been chewing on since 2024. I'm not an academic — I'm an engineer with a day job and a family — so the spec, the reference implementations, and most of this prose have been written with significant help from AI tools acting as research assistant, technical writer, and pair programmer. I wrote the original architectural sketches and reviewed every line; the assistants filled in the gaps and saved me from re-typing the same explanation for the hundredth time. +## Release and compatibility -**Contributions are welcome — human or AI-assisted, doesn't matter to me.** What matters is whether the code, the spec text, or the conformance vectors move the project forward. Open a PR. +Canonical output is protocol data. A change to it requires updates to every +binding and to the conformance fixtures in one change. Consumers that need +the previous published protocol can pin tag `v0.2.2` or commit +`79b0d52fecd958f8fc7ade713fe0799ca1e79626`. Release `0.3.0` contains the +normative v1 parser, URL, resource-limit, and JCS behavior. Tag it after all +five binding jobs pass. -What this project is **not** a forum for: +Go callers must now handle the error returned by `CanonicalizeClaims`. +`CanonicalizeClaimsStrict` remains as an alias with the same fail-closed +behavior. -- Debates about whether AI should be used to write code or specifications. -- Opinions on who is or isn't trustworthy on the web. -- Politics, religion, professional practice, or personal philosophy. +Related repositories: -HTMLTrust is a mechanism — a way for *anyone* to sign content they publish and for *anyone* to decide whom they trust, on their own terms. The project takes no position on what the right answers are; it just provides the tools. If you want to debate the answers, there are entire continents of the internet better suited to it. +- [HTMLTrust specification](https://github.com/HTMLTrust/htmltrust-spec) +- [Reference server](https://github.com/HTMLTrust/htmltrust-server-reference) +- [Reference browser extension](https://github.com/HTMLTrust/htmltrust-browser-reference) +- [Reference CMS plugins](https://github.com/HTMLTrust/htmltrust-cms-reference) + +## License -If this work is useful to you and you'd like to support it, see [GitHub Sponsors](https://github.com/sponsors/jt55401) or the other channels in [`.github/FUNDING.yml`](.github/FUNDING.yml). +This project is licensed under the [PolyForm Noncommercial License 1.0.0](https://polyformproject.org/licenses/noncommercial/1.0.0). diff --git a/compose.test.yml b/compose.test.yml new file mode 100644 index 0000000..d6bf11c --- /dev/null +++ b/compose.test.yml @@ -0,0 +1,87 @@ +services: + javascript: + image: node:22-bookworm + working_dir: /workspace + command: >- + sh -c "npm ci --ignore-scripts --no-audit --no-fund + && node javascript/test.js + && node conformance/runners/run-javascript.mjs" + volumes: + - .:/workspace:ro + - javascript_modules:/workspace/node_modules + - npm_cache:/root/.npm + + go: + image: golang:1.25-bookworm + working_dir: /workspace + command: >- + sh -c "cd go + && /usr/local/go/bin/go mod download + && /usr/local/go/bin/go test ./... + && cd ../conformance/runners + && /usr/local/go/bin/go run ./run-go.go" + environment: + GOCACHE: /cache/build + GOMODCACHE: /cache/modules + volumes: + - .:/workspace:ro + - go_cache:/cache + + php: + build: + context: . + dockerfile: .docker/php/Dockerfile + working_dir: /workspace + command: >- + sh -c "composer install --working-dir=php --no-interaction --prefer-dist --no-progress + && composer test --working-dir=php + && php conformance/runners/run-php.php" + environment: + COMPOSER_CACHE_DIR: /cache/composer + COMPOSER_ROOT_VERSION: 0.3.0 + volumes: + - .:/workspace:ro + - php_vendor:/workspace/php/vendor + - php_cache:/cache + + python: + image: python:3.14-bookworm + working_dir: /workspace + command: >- + sh -c "mkdir -p /build/python + && cp -a python/. /build/python/ + && python -m pip install --disable-pip-version-check --upgrade '/build/python[dev]' + && python -m pytest -q -p no:cacheprovider python/tests + && python tools/gen-test-vectors.py --check + && python conformance/runners/run-python.py" + environment: + PIP_CACHE_DIR: /cache/pip + PYTHONPATH: /usr/local/lib/python3.14/site-packages + volumes: + - .:/workspace:ro + - python_cache:/cache + + rust: + image: rust:1.86-bookworm + working_dir: /workspace + command: >- + sh -c "cargo test --locked --manifest-path rust/Cargo.toml + && cargo test --locked --manifest-path ffi/Cargo.toml + && cargo run --locked --quiet --release --manifest-path conformance/runners/run-rust/Cargo.toml" + environment: + CARGO_HOME: /cache/cargo + CARGO_TARGET_DIR: /cargo-target + volumes: + - .:/workspace:ro + - rust_cache:/cache + - ${HTMLTRUST_CARGO_TARGET_MOUNT:-rust_target}:/cargo-target + +volumes: + javascript_modules: + npm_cache: + go_cache: + php_vendor: + php_cache: + python_cache: + rust_cache: + rust_target: diff --git a/composer.json b/composer.json index f3e45a8..9038216 100644 --- a/composer.json +++ b/composer.json @@ -14,11 +14,16 @@ } }, "require": { - "php": ">=7.2", + "php": ">=8.5", + "ext-dom": "*", "ext-intl": "*", - "ext-mbstring": "*" + "ext-mbstring": "*", + "ext-json": "*", + "ext-openssl": "*", + "ext-sodium": "*", + "root23/php-json-canonicalization": "1.0.1" }, "require-dev": { - "phpunit/phpunit": "^8.0 || ^9.0 || ^10.0" + "phpunit/phpunit": "10.5.64" } } diff --git a/conformance/README.md b/conformance/README.md index 479902e..c0b4f42 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -1,164 +1,168 @@ # Cross-language conformance suite -This directory is the **public contract** between every HTMLTrust -canonicalization binding (JavaScript, Go, PHP, Python, Rust). Every -implementation must produce **byte-identical** output for every fixture -under `fixtures/`. If two languages disagree on a fixture, that's a real -bug; the fixture itself defines the spec. +This directory defines the byte-level contract for the HTMLTrust +Canonicalization bindings. A fixture contains one input and the output that +every binding must produce. A changed canonical output is a protocol change. -## Layout +Status: required for binding changes +Readers: binding contributors and release reviewers -``` -conformance/ - README.md -- this file - run-all.sh -- invokes every runner; non-zero on any divergence - fixtures/ - normalize/ -- normalize_text input/expected pairs - extract/ -- extract_canonical_text input/expected pairs - claims/ -- canonicalize_claims input/expected pairs - runners/ - run-javascript.mjs -- Node 18+ ESM runner - run-go.go -- single-file `go run` runner - run-php.php -- PHP 7.2+ runner (requires ext-intl) - run-python.py -- Python 3.10+ runner - run-rust/ -- tiny Cargo bin crate that vendors the in-tree binding - go.mod -- tells `go run` to use the local `go/` binding +## Run the suite + +Run every unit suite and this conformance suite in containers from a clean +checkout: + +```sh +./scripts/test-in-docker.sh ``` -## Running +From the repository root, install the dependencies for the bindings you want +to run, then run: ```sh -# Run everything (from repo root): make conformance +``` -# Run a single language: +The command runs JavaScript, Go, PHP, Python, and Rust in that order. A +missing executable is reported as `MISSING` and does not fail a local run. +Require every toolchain in CI or before a release: + +```sh +REQUIRE_ALL_LANGUAGES=1 make conformance +``` + +Run one binding while developing: + +```sh make conformance-js make conformance-go make conformance-php make conformance-python make conformance-rust +``` + +The runners also work directly: -# Or invoke a runner directly: -node conformance/runners/run-javascript.mjs -cd conformance/runners && go run ./run-go.go -php conformance/runners/run-php.php +```sh +node conformance/runners/run-javascript.mjs +(cd conformance/runners && go run ./run-go.go) +php conformance/runners/run-php.php python3 conformance/runners/run-python.py -cargo run --release --manifest-path conformance/runners/run-rust/Cargo.toml +cargo run --locked --release --manifest-path conformance/runners/run-rust/Cargo.toml ``` -Every runner prints one line per fixture: +Each runner prints one line per fixture: -``` +```text PASS conformance/fixtures/normalize/basic-ascii.json FAIL conformance/fixtures/normalize/curly-double-quotes.json expected: "\"Hello\"" got: "“Hello”" -SKIP conformance/fixtures/extract/simple-paragraph.json (binding does not implement extract) +PASS conformance/fixtures/extract/url-http-rejected.json (expected error url-policy-violation) ``` -A runner exits 0 if every applicable fixture passes, 1 otherwise. The -`SKIP` status is **never** a failure -- it just means the binding hasn't -implemented that function yet (see [Binding coverage](#binding-coverage) -below). +Exit status `0` means every applicable fixture passed. Exit status `1` means +an output or expected error differed. A missing toolchain becomes exit status +`2` when `REQUIRE_ALL_LANGUAGES=1`. -`run-all.sh` exits 0 only when every available runner exits 0. -Missing toolchains (e.g. no `php` on a CI image) are reported but -don't fail the build by default -- set `REQUIRE_ALL_LANGUAGES=1` to -make them hard-fail. +## Fixture suites -## Fixture format +The directory contains four suites: -Every fixture is a self-contained JSON object: +| Directory | Binding function | Input | +|---|---|---| +| `normalize/` | `normalizeText` | A text string | +| `extract/` | `extractCanonicalText` | An HTML fragment, with optional `baseURL` | +| `claims/` | `canonicalizeClaims` | A JSON object whose values are strings | +| `jcs/` | `canonicalizeJsonDocument` | A raw JSON document string | + +Every fixture is a JSON object with a matching filename and `name` field: ```json { "name": "curly-double-quotes", - "description": "U+201C / U+201D collapse to ASCII double quote U+0022.", + "description": "Curly quotes become ASCII quotation marks.", "input": "“Hello”", "expected": "\"Hello\"" } ``` -- `input` is the raw value passed to the binding function: - - `normalize/` -- a string - - `extract/` -- an HTML fragment as a string - - `claims/` -- a JSON object of `name -> value` pairs serialized as `name:content\n` -- `expected` is the byte-exact output the function must return. -- Both `input` and `expected` may contain literal non-ASCII characters - (JSON allows it). Use Unicode escape sequences (`\uXXXX`) when the - exact code point matters and the literal character would be - ambiguous (combining marks, invisible characters, etc.). +`expected` is compared as a string. The runners encode and compare the UTF-8 +bytes returned by each binding. A fixture may include: -## Authoring new fixtures +- `baseURL` for resolving relative `href` and `src` values. +- `repeat` for testing resource limits without storing a large input file. + String inputs are repeated directly. For a claims object, every string value + is repeated while claim names remain unchanged. +- `error` when the binding must reject the input. The value is a stable error + code such as `resource-limit-exceeded` or `url-policy-violation`. -The recommended flow: +Use `\uXXXX` escapes for invisible or combining characters when the exact code +point matters. Keep the description specific about the rule under test. -1. **Write the fixture with `input` only** and `"expected": ""`. The - filename and the `name` field must match (e.g. `my-case.json` with - `"name": "my-case"`). +## Add or change a fixture -2. **Populate `expected` from a known-good runner.** Python remains a - convenient source of truth for parser-heavy `extract/` cases. The current - JS, Go, PHP, Python, and Rust runners all cover the shared normalize, - extract, and claims suites when their toolchains are installed: +1. Add a JSON file to the suite directory. The filename and `name` must match. +2. Set `expected` to an empty string while writing the case. +3. Generate the expected value from the current Python binding: ```sh python3 conformance/runners/run-python.py --update ``` - Inspect the diff: the new fixture's `expected` is now populated. - -3. **Verify every other runner agrees** without `--update`: +4. Inspect the diff. Run every available binding: ```sh - make conformance + REQUIRE_ALL_LANGUAGES=1 make conformance ``` - If they all PASS, your fixture is consensus. Commit it. +5. If a binding disagrees, fix the binding or document the known divergence. + Do not change `expected` to hide a disagreement. -4. **If a runner diverges**, you've either found a real bug in a - binding or written a fixture that exposes a known-divergent area - (see [Known divergences](#known-divergences) below). Discuss with - the orchestrator before "fixing" anything -- silently editing the - binding to match Python defeats the point of having a conformance - suite. +The update command rewrites all fixture `expected` fields. Review every +changed file before committing. -## Binding coverage +## Fixture count -The five bindings now implement the shared normalize/extract/claims surface. -The runners still report `SKIP` if a future fixture targets a function a -binding cannot run. +The runner counts JSON files from disk and prints the count in its summary. +The same command shows the current total without running a binding: -| Function | JS | Go | PHP | Python | Rust | -|--------------------------|:---:|:---:|:---:|:------:|:----:| -| `normalizeText` | YES | YES | YES | YES | YES | -| `extractCanonicalText` | YES | YES | YES | YES | YES | -| `canonicalizeClaims` | YES | YES | YES | YES | YES | +```sh +find conformance/fixtures -mindepth 2 -maxdepth 2 -type f -name '*.json' | wc -l +``` -## Known divergences +This count includes all four suites, including expected-error cases. -None at the time of the current protocol cleanup. JS, Go, Python, and Rust -agree byte-for-byte on every fixture on the current development machine; PHP -was not installed locally for that run. +## Binding coverage -If divergences appear in the future, add them here with the fixture -name, the diverging language(s), and a one-line root-cause sketch. -**Do not** modify the fixture's `expected` to paper over a divergence --- the whole point of the suite is that it catches drift. +The five in-tree bindings currently implement every suite: -## Fixture inventory +| Function | JavaScript | Go | PHP | Python | Rust | +|---|:---:|:---:|:---:|:---:|:---:| +| `normalizeText` | yes | yes | yes | yes | yes | +| `extractCanonicalText` | yes | yes | yes | yes | yes | +| `canonicalizeClaims` | yes | yes | yes | yes | yes | +| `canonicalizeJsonDocument` | yes | yes | yes | yes | yes | -`normalize/` (22 cases): basic ASCII, empty, whitespace edge cases, -NFKC compatibility forms, curly/CJK/guillemet quotation marks, -dashes, ellipsis, ZWSP stripping, ZWNJ/ZWJ preservation, Arabic -tatweel, BOM, bidi controls. +A future runner may print `SKIP` for a suite that its binding does not +implement. `SKIP` is informational; an implemented fixture that differs is a +failure. -`extract/` (14 cases): paragraphs, inline elements, block boundaries, -nested structure, lists, tables, excluded elements (script/style/meta), -entity decoding, inline anchors, mixed inline formatting, -post-extraction normalization, headings, `br`, signed semantic attributes. +## Files in this directory -`claims/` (7 cases): empty, single claim, multi-claim ordering, value -normalization, name normalization, NFKC inside values, internal newlines. +```text +conformance/ + README.md + run-all.sh + fixtures/{normalize,extract,claims,jcs}/ + runners/ + run-javascript.mjs + run-go.go + run-php.php + run-python.py + run-rust/ +``` -Total: 43 fixtures. +The runner modules use the local bindings. They do not download a published +version of this repository, so a conformance run always tests the checkout +under review. diff --git a/conformance/fixtures/claims/escaping-v1.json b/conformance/fixtures/claims/escaping-v1.json new file mode 100644 index 0000000..09882cd --- /dev/null +++ b/conformance/fixtures/claims/escaping-v1.json @@ -0,0 +1,8 @@ +{ + "name": "escaping-v1", + "description": "Claim names and values escape reverse solidus, colon, and line feed after normalization.", + "input": { + "claim:Path": "C:\\docs:one" + }, + "expected": "claim\\:Path:C\\:\\\\docs\\:one\n" +} diff --git a/conformance/fixtures/claims/non-string-value-rejected.json b/conformance/fixtures/claims/non-string-value-rejected.json new file mode 100644 index 0000000..7b1304b --- /dev/null +++ b/conformance/fixtures/claims/non-string-value-rejected.json @@ -0,0 +1,8 @@ +{ + "name": "non-string-value-rejected", + "description": "Claim maps accept only the string values produced by HTML meta content attributes.", + "input": { + "count": 42 + }, + "error": "claim-malformed" +} diff --git a/conformance/fixtures/claims/resource-count-limit.json b/conformance/fixtures/claims/resource-count-limit.json new file mode 100644 index 0000000..04ba96e --- /dev/null +++ b/conformance/fixtures/claims/resource-count-limit.json @@ -0,0 +1,72 @@ +{ + "name": "resource-count-limit", + "description": "More than 64 direct claims are rejected.", + "input": { + "c00": "v", + "c01": "v", + "c02": "v", + "c03": "v", + "c04": "v", + "c05": "v", + "c06": "v", + "c07": "v", + "c08": "v", + "c09": "v", + "c10": "v", + "c11": "v", + "c12": "v", + "c13": "v", + "c14": "v", + "c15": "v", + "c16": "v", + "c17": "v", + "c18": "v", + "c19": "v", + "c20": "v", + "c21": "v", + "c22": "v", + "c23": "v", + "c24": "v", + "c25": "v", + "c26": "v", + "c27": "v", + "c28": "v", + "c29": "v", + "c30": "v", + "c31": "v", + "c32": "v", + "c33": "v", + "c34": "v", + "c35": "v", + "c36": "v", + "c37": "v", + "c38": "v", + "c39": "v", + "c40": "v", + "c41": "v", + "c42": "v", + "c43": "v", + "c44": "v", + "c45": "v", + "c46": "v", + "c47": "v", + "c48": "v", + "c49": "v", + "c50": "v", + "c51": "v", + "c52": "v", + "c53": "v", + "c54": "v", + "c55": "v", + "c56": "v", + "c57": "v", + "c58": "v", + "c59": "v", + "c60": "v", + "c61": "v", + "c62": "v", + "c63": "v", + "c64": "v" + }, + "error": "resource-limit-exceeded" +} diff --git a/conformance/fixtures/claims/resource-name-limit.json b/conformance/fixtures/claims/resource-name-limit.json new file mode 100644 index 0000000..a1f7f49 --- /dev/null +++ b/conformance/fixtures/claims/resource-name-limit.json @@ -0,0 +1,8 @@ +{ + "name": "resource-name-limit", + "description": "A normalized claim name larger than 4 KiB is rejected.", + "input": { + "nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn": "v" + }, + "error": "resource-limit-exceeded" +} diff --git a/conformance/fixtures/claims/resource-repeat-limit.json b/conformance/fixtures/claims/resource-repeat-limit.json new file mode 100644 index 0000000..ccfadd3 --- /dev/null +++ b/conformance/fixtures/claims/resource-repeat-limit.json @@ -0,0 +1,9 @@ +{ + "name": "resource-repeat-limit", + "description": "Fixture repeat expansion applies to claim values before the 4 KiB field limit.", + "input": { + "claim": "v" + }, + "repeat": 4097, + "error": "resource-limit-exceeded" +} diff --git a/conformance/fixtures/claims/resource-value-limit.json b/conformance/fixtures/claims/resource-value-limit.json new file mode 100644 index 0000000..539e21a --- /dev/null +++ b/conformance/fixtures/claims/resource-value-limit.json @@ -0,0 +1,8 @@ +{ + "name": "resource-value-limit", + "description": "A normalized claim value larger than 4 KiB is rejected.", + "input": { + "claim": "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" + }, + "error": "resource-limit-exceeded" +} diff --git a/conformance/fixtures/extract/escape-at-attribute.json b/conformance/fixtures/extract/escape-at-attribute.json new file mode 100644 index 0000000..acf5c5c --- /dev/null +++ b/conformance/fixtures/extract/escape-at-attribute.json @@ -0,0 +1,6 @@ +{ + "name": "escape-at-attribute", + "description": "Literal U+0040 is doubled inside normalized signed attribute values.", + "input": "

x

", + "expected": "@attr:span:aria-label:Contact @@name\nx" +} diff --git a/conformance/fixtures/extract/escape-at-text.json b/conformance/fixtures/extract/escape-at-text.json new file mode 100644 index 0000000..0346c12 --- /dev/null +++ b/conformance/fixtures/extract/escape-at-text.json @@ -0,0 +1,6 @@ +{ + "name": "escape-at-text", + "description": "Literal U+0040 in normalized text is doubled so text cannot emit an attribute record.", + "input": "

Email @attr:a:href:x and a@b.example

", + "expected": "Email @@attr:a:href:x and a@@b.example" +} diff --git a/conformance/fixtures/extract/escape-at-url.json b/conformance/fixtures/extract/escape-at-url.json new file mode 100644 index 0000000..4ac89b2 --- /dev/null +++ b/conformance/fixtures/extract/escape-at-url.json @@ -0,0 +1,6 @@ +{ + "name": "escape-at-url", + "description": "U+0040 is doubled after an HTTPS URL is parsed and serialized.", + "input": "

x

", + "expected": "@attr:a:href:https://example.org/?email=a@@example.org\nx" +} diff --git a/conformance/fixtures/extract/parser-ambiguous-reference.json b/conformance/fixtures/extract/parser-ambiguous-reference.json new file mode 100644 index 0000000..04a8287 --- /dev/null +++ b/conformance/fixtures/extract/parser-ambiguous-reference.json @@ -0,0 +1,6 @@ +{ + "name": "parser-ambiguous-reference", + "description": "Ambiguous named character references are rejected.", + "input": "

¬it;

", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-bogus-declaration.json b/conformance/fixtures/extract/parser-bogus-declaration.json new file mode 100644 index 0000000..12ae24b --- /dev/null +++ b/conformance/fixtures/extract/parser-bogus-declaration.json @@ -0,0 +1,6 @@ +{ + "name": "parser-bogus-declaration", + "description": "A bogus markup declaration is outside the portable parser profile.", + "input": "x", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-comment-trailing-hyphen.json b/conformance/fixtures/extract/parser-comment-trailing-hyphen.json new file mode 100644 index 0000000..e872148 --- /dev/null +++ b/conformance/fixtures/extract/parser-comment-trailing-hyphen.json @@ -0,0 +1,6 @@ +{ + "name": "parser-comment-trailing-hyphen", + "description": "A comment whose body ends with a hyphen is outside the portable parser profile.", + "input": "

before

after

", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-duplicate-attribute.json b/conformance/fixtures/extract/parser-duplicate-attribute.json new file mode 100644 index 0000000..ffda923 --- /dev/null +++ b/conformance/fixtures/extract/parser-duplicate-attribute.json @@ -0,0 +1,6 @@ +{ + "name": "parser-duplicate-attribute", + "description": "Duplicate source attributes are rejected before a repaired DOM is accepted.", + "input": "

Text

", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-foreign-content.json b/conformance/fixtures/extract/parser-foreign-content.json new file mode 100644 index 0000000..15919fc --- /dev/null +++ b/conformance/fixtures/extract/parser-foreign-content.json @@ -0,0 +1,6 @@ +{ + "name": "parser-foreign-content", + "description": "Foreign-content integration points are outside the portable parser profile.", + "input": "

Text

", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-foreign-object-standalone.json b/conformance/fixtures/extract/parser-foreign-object-standalone.json new file mode 100644 index 0000000..645df6e --- /dev/null +++ b/conformance/fixtures/extract/parser-foreign-object-standalone.json @@ -0,0 +1,6 @@ +{ + "name": "parser-foreign-object-standalone", + "description": "A standalone foreignObject integration element is outside the portable parser profile.", + "input": "

Text

", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-malformed-comment.json b/conformance/fixtures/extract/parser-malformed-comment.json new file mode 100644 index 0000000..73b2a65 --- /dev/null +++ b/conformance/fixtures/extract/parser-malformed-comment.json @@ -0,0 +1,6 @@ +{ + "name": "parser-malformed-comment", + "description": "A comment containing the invalid double-hyphen form is outside the portable parser profile.", + "input": "x", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-misnested-formatting.json b/conformance/fixtures/extract/parser-misnested-formatting.json new file mode 100644 index 0000000..0c1d54c --- /dev/null +++ b/conformance/fixtures/extract/parser-misnested-formatting.json @@ -0,0 +1,6 @@ +{ + "name": "parser-misnested-formatting", + "description": "Misnested formatting elements are outside the portable parser profile.", + "input": "

Text

", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-raw-text-is-data.json b/conformance/fixtures/extract/parser-raw-text-is-data.json new file mode 100644 index 0000000..20933e6 --- /dev/null +++ b/conformance/fixtures/extract/parser-raw-text-is-data.json @@ -0,0 +1,6 @@ +{ + "name": "parser-raw-text-is-data", + "description": "Tag-like strings and ambiguous references inside excluded raw-text elements do not affect portable-profile validation.", + "input": "

before

after

", + "expected": "before\nafter" +} diff --git a/conformance/fixtures/extract/parser-table-foster-parenting.json b/conformance/fixtures/extract/parser-table-foster-parenting.json new file mode 100644 index 0000000..fecab58 --- /dev/null +++ b/conformance/fixtures/extract/parser-table-foster-parenting.json @@ -0,0 +1,6 @@ +{ + "name": "parser-table-foster-parenting", + "description": "Non-space table text that triggers foster parenting is rejected.", + "input": "Text
Cell
", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-table-trailing-foster-text.json b/conformance/fixtures/extract/parser-table-trailing-foster-text.json new file mode 100644 index 0000000..06876ee --- /dev/null +++ b/conformance/fixtures/extract/parser-table-trailing-foster-text.json @@ -0,0 +1,6 @@ +{ + "name": "parser-table-trailing-foster-text", + "description": "Non-space text after a table row would be foster-parented and is rejected.", + "input": "tail
x
", + "error": "parser-profile-unsupported" +} diff --git a/conformance/fixtures/extract/parser-unclosed-comment.json b/conformance/fixtures/extract/parser-unclosed-comment.json new file mode 100644 index 0000000..bc158a9 --- /dev/null +++ b/conformance/fixtures/extract/parser-unclosed-comment.json @@ -0,0 +1,6 @@ +{ + "name": "parser-unclosed-comment", + "description": "An unclosed HTML comment is outside the portable parser profile.", + "input": "|]*>|]*)?\s*/?>`) - tagNameRE = regexp.MustCompile(`(?i)^/=]+)(?:\\s*=\\s*(?:\"([^\"]*)\"|'([^']*)'|([^\\s\"'=<>`]+)))?") blockNameRE = regexp.MustCompile(`(?i)^(` + blockElements + `)$`) @@ -66,61 +49,10 @@ var excludedTags = map[string]bool{ // case-sensitive per the HTML Living Standard. var ( - namedEntityRE = regexp.MustCompile(`&[a-zA-Z][a-zA-Z0-9]*;`) - decimalEntityRE = regexp.MustCompile(`&#([0-9]+);`) - hexEntityRE = regexp.MustCompile(`&#[xX]([0-9a-fA-F]+);`) + namedEntityRE = regexp.MustCompile(`&[a-zA-Z][a-zA-Z0-9]*;`) + unterminatedEntityRE = regexp.MustCompile(`&[a-zA-Z][a-zA-Z0-9]*(?:$|[^a-zA-Z0-9;])`) ) -// c1Replacements maps numeric references in the C1 range (0x80-0x9F) via the -// windows-1252 table, per the HTML5 "numeric character reference end" state. -var c1Replacements = map[rune]rune{ - 0x80: 0x20AC, 0x82: 0x201A, 0x83: 0x0192, 0x84: 0x201E, 0x85: 0x2026, - 0x86: 0x2020, 0x87: 0x2021, 0x88: 0x02C6, 0x89: 0x2030, 0x8A: 0x0160, - 0x8B: 0x2039, 0x8C: 0x0152, 0x8E: 0x017D, 0x91: 0x2018, 0x92: 0x2019, - 0x93: 0x201C, 0x94: 0x201D, 0x95: 0x2022, 0x96: 0x2013, 0x97: 0x2014, - 0x98: 0x02DC, 0x99: 0x2122, 0x9A: 0x0161, 0x9B: 0x203A, 0x9C: 0x0153, - 0x9E: 0x017E, 0x9F: 0x0178, -} - -// numericCharRef applies the HTML5 rules for a parsed numeric reference: -// null/out-of-range/surrogate collapse to U+FFFD; C1 controls map via -// windows-1252; everything else is the literal code point. -func numericCharRef(n int64) string { - if n == 0 || n > 0x10FFFF || (n >= 0xD800 && n <= 0xDFFF) { - return "\uFFFD" - } - if r, ok := c1Replacements[rune(n)]; ok { - return string(r) - } - return string(rune(n)) -} - -func decodeEntities(text string) string { - text = namedEntityRE.ReplaceAllStringFunc(text, func(match string) string { - if v, ok := namedEntities[match]; ok { - return v - } - return match - }) - text = decimalEntityRE.ReplaceAllStringFunc(text, func(match string) string { - m := decimalEntityRE.FindStringSubmatch(match) - n, err := strconv.ParseInt(m[1], 10, 64) - if err != nil { - return "\uFFFD" - } - return numericCharRef(n) - }) - text = hexEntityRE.ReplaceAllStringFunc(text, func(match string) string { - m := hexEntityRE.FindStringSubmatch(match) - n, err := strconv.ParseInt(m[1], 16, 64) - if err != nil { - return "\uFFFD" - } - return numericCharRef(n) - }) - return text -} - // ExtractCanonicalText extracts canonical content from an HTML fragment for // signing or verification. Mirrors the JS extractCanonicalText() reference // implementation: strips excluded elements, emits signed semantic attribute @@ -128,6 +60,9 @@ func decodeEntities(text string) string { // to line feeds, decodes entities, and runs the full text normalization // pipeline. The returned string is trimmed. func ExtractCanonicalText(html string, opts ...Options) (string, error) { + if len(html) > maxResourceBytes || !utf8.ValidString(html) { + return "", fmt.Errorf("resource-limit-exceeded") + } var o Options if len(opts) > 0 { o = opts[0] @@ -137,93 +72,299 @@ func ExtractCanonicalText(html string, opts ...Options) (string, error) { return "", err } + if err := validatePortableHTML(html); err != nil { + return "", err + } + fragment, err := htmlpkg.ParseFragment(strings.NewReader(html), &htmlpkg.Node{Type: htmlpkg.ElementNode, Data: "div", DataAtom: atom.Div}) + if err != nil { + return "", fmt.Errorf("parser-profile-unsupported: %v", err) + } var parts []string - index := 0 - excludedDepth := 0 - matches := htmlTokenRE.FindAllStringIndex(html, -1) - for _, loc := range matches { - if loc[0] > index && excludedDepth == 0 { - appendCanonicalPart(&parts, NormalizeText(decodeEntities(html[index:loc[0]]), o)) - } - token := html[loc[0]:loc[1]] - index = loc[1] + if err := walkHTMLNodes(fragment, &parts, base, o); err != nil { + return "", err + } + result := finalizeCanonicalParts(parts) + if len(result) > maxResourceBytes { + return "", fmt.Errorf("resource-limit-exceeded") + } + return result, nil +} - nameMatch := tagNameRE.FindStringSubmatch(token) - if len(nameMatch) < 2 { +// ExtractClaimsFromSignedSection returns claim metadata from direct child meta +// elements. If the fragment contains a top-level signed-section, that element +// supplies the children; otherwise the fragment is treated as section inner HTML. +func ExtractClaimsFromSignedSection(source string) (map[string]string, error) { + if len(source) > maxResourceBytes || !utf8.ValidString(source) { + return nil, fmt.Errorf("resource-limit-exceeded") + } + if err := validatePortableHTML(source); err != nil { + return nil, err + } + fragment, err := htmlpkg.ParseFragment(strings.NewReader(source), &htmlpkg.Node{Type: htmlpkg.ElementNode, Data: "div", DataAtom: atom.Div}) + if err != nil { + return nil, fmt.Errorf("parser-profile-unsupported: %v", err) + } + children := fragment + for _, node := range fragment { + if node.Type == htmlpkg.ElementNode && strings.EqualFold(node.Data, "signed-section") { + children = children[:0] + for child := node.FirstChild; child != nil; child = child.NextSibling { + children = append(children, child) + } + break + } + } + claims := make(map[string]string) + for _, node := range children { + if node.Type != htmlpkg.ElementNode || !strings.EqualFold(node.Data, "meta") { continue } - name := strings.ToLower(nameMatch[1]) - closing := strings.HasPrefix(strings.TrimSpace(token), "") || voidTags[name] - excluded := excludedTags[name] + attrs := make(map[string]string, len(node.Attr)) + for _, attr := range node.Attr { + attrs[strings.ToLower(attr.Key)] = attr.Val + } + rawName, hasName := attrs["name"] + rawContent, hasContent := attrs["content"] + if !hasName || !hasContent { + return nil, fmt.Errorf("claim-malformed") + } + if len(claims) >= maxClaims { + return nil, fmt.Errorf("resource-limit-exceeded") + } + name, err := NormalizeChecked(rawName) + if err != nil { + return nil, err + } + content, err := NormalizeChecked(rawContent) + if err != nil { + return nil, err + } + name = strings.TrimSpace(name) + content = strings.TrimSpace(content) + if name == "" { + return nil, fmt.Errorf("claim-malformed") + } + if len([]byte(name)) > maxClaimFieldBytes || len([]byte(content)) > maxClaimFieldBytes { + return nil, fmt.Errorf("resource-limit-exceeded") + } + if _, duplicate := claims[name]; duplicate { + return nil, fmt.Errorf("claim-duplicate") + } + claims[name] = content + } + return claims, nil +} - if closing { - if excluded && excludedDepth > 0 { - excludedDepth-- - continue +// x/net/html repairs malformed input and does not expose parser diagnostics. +// Keep a strict source preflight so repaired trees cannot enter the portable +// profile silently. +func validatePortableHTML(source string) error { + stack := []string{} + t := htmlpkg.NewTokenizer(strings.NewReader(source)) + for { + tt := t.Next() + if tt == htmlpkg.ErrorToken { + break + } + switch tt { + case htmlpkg.StartTagToken, htmlpkg.SelfClosingTagToken: + if err := validatePortableReferences(string(t.Raw())); err != nil { + return err } - if excludedDepth > 0 { - continue + if err := validateDuplicateAttributes(string(t.Raw())); err != nil { + return err } - if blockNameRE.MatchString(name) { - appendCanonicalPart(&parts, "\n") + nameBytes, more := t.TagName() + name := strings.ToLower(string(nameBytes)) + if name == "svg" || name == "math" || name == "foreignobject" { + return fmt.Errorf("parser-profile-unsupported") } - continue + seen := map[string]bool{} + for more { + keyBytes, _, next := t.TagAttr() + key := strings.ToLower(string(keyBytes)) + if seen[key] { + return fmt.Errorf("parser-profile-unsupported") + } + seen[key] = true + more = next + } + if tt == htmlpkg.StartTagToken && !voidTags[name] { + if len(stack) >= maxElementDepth { + return fmt.Errorf("resource-limit-exceeded") + } + stack = append(stack, name) + } + case htmlpkg.EndTagToken: + nameBytes, _ := t.TagName() + name := strings.ToLower(string(nameBytes)) + if len(stack) == 0 || stack[len(stack)-1] != name { + return fmt.Errorf("parser-profile-unsupported") + } + stack = stack[:len(stack)-1] + case htmlpkg.TextToken: + rawText := len(stack) > 0 && (stack[len(stack)-1] == "script" || stack[len(stack)-1] == "style" || stack[len(stack)-1] == "iframe") + if !rawText { + if err := validatePortableReferences(string(t.Raw())); err != nil { + return err + } + } + if len(stack) > 0 && stack[len(stack)-1] == "table" && strings.TrimSpace(string(t.Raw())) != "" { + return fmt.Errorf("parser-profile-unsupported") + } + case htmlpkg.CommentToken: + raw := string(t.Raw()) + if !strings.HasPrefix(raw, "") { + return fmt.Errorf("parser-profile-unsupported") + } + body := raw[4 : len(raw)-3] + if strings.Contains(body, "--") || strings.HasSuffix(body, "-") { + return fmt.Errorf("parser-profile-unsupported") + } + case htmlpkg.DoctypeToken: + return fmt.Errorf("parser-profile-unsupported") } + } + if len(stack) != 0 { + return fmt.Errorf("parser-profile-unsupported: unclosed %v", stack) + } + return nil +} - if excluded { - if !selfClosing { - excludedDepth++ - } - continue +func validatePortableReferences(source string) error { + if unterminatedEntityRE.MatchString(source) || hasUnterminatedNumericReference(source) { + return fmt.Errorf("parser-profile-unsupported") + } + for _, loc := range namedEntityRE.FindAllStringIndex(source, -1) { + if _, ok := namedEntities[source[loc[0]:loc[1]]]; !ok { + return fmt.Errorf("parser-profile-unsupported: entity %s", source[loc[0]:loc[1]]) } - if excludedDepth > 0 { + } + return nil +} + +func hasUnterminatedNumericReference(source string) bool { + for i := 0; i+2 < len(source); i++ { + if source[i] != '&' || source[i+1] != '#' { continue } - - attrs := parseAttributes(token) - if err := appendAttributeRecords(&parts, name, attrs, base); err != nil { - return "", err + j := i + 2 + if j < len(source) && (source[j] == 'x' || source[j] == 'X') { + j++ + start := j + for j < len(source) && ((source[j] >= '0' && source[j] <= '9') || + (source[j] >= 'a' && source[j] <= 'f') || (source[j] >= 'A' && source[j] <= 'F')) { + j++ + } + if j == start { + continue + } + } else { + start := j + for j < len(source) && source[j] >= '0' && source[j] <= '9' { + j++ + } + if j == start { + continue + } } - if name == "br" { - appendCanonicalPart(&parts, "\n") + if j >= len(source) || source[j] != ';' { + return true } - if selfClosing && blockNameRE.MatchString(name) { - appendCanonicalPart(&parts, "\n") + } + return false +} + +func validateDuplicateAttributes(token string) error { + body := attrBodyRE.ReplaceAllString(token, "") + body = strings.TrimSuffix(strings.TrimSpace(body), ">") + body = strings.TrimSuffix(strings.TrimSpace(body), "/") + seen := map[string]bool{} + for _, match := range attrRE.FindAllStringSubmatch(body, -1) { + name := strings.ToLower(match[1]) + if seen[name] { + return fmt.Errorf("parser-profile-unsupported") } + seen[name] = true } - if index < len(html) && excludedDepth == 0 { - appendCanonicalPart(&parts, NormalizeText(decodeEntities(html[index:]), o)) + return nil +} + +func walkHTMLNodes(nodes []*htmlpkg.Node, parts *[]string, base *whatwgurl.Url, o Options) error { + for _, node := range nodes { + switch node.Type { + case htmlpkg.TextNode: + normalized, err := NormalizeTextChecked(node.Data, o) + if err != nil { + return err + } + appendCanonicalPart(parts, strings.ReplaceAll(normalized, "@", "@@")) + case htmlpkg.CommentNode, htmlpkg.DoctypeNode: + continue + case htmlpkg.ElementNode: + name := strings.ToLower(node.Data) + if excludedTags[name] { + continue + } + attrs := make(map[string]string, len(node.Attr)) + for _, attr := range node.Attr { + attrs[strings.ToLower(attr.Key)] = attr.Val + } + if err := appendAttributeRecords(parts, name, attrs, base); err != nil { + return err + } + if name == "br" { + appendCanonicalPart(parts, "\n") + } else { + children := make([]*htmlpkg.Node, 0) + for child := node.FirstChild; child != nil; child = child.NextSibling { + children = append(children, child) + } + if err := walkHTMLNodes(children, parts, base, o); err != nil { + return err + } + if blockNameRE.MatchString(name) || name == "signed-section" { + appendCanonicalPart(parts, "\n") + } + } + } } - return finalizeCanonicalParts(parts), nil + return nil } // CanonicalizeClaims serializes a claims map as sorted "name:content\n" // records. Both names and values are pushed through NormalizeText before // serialization so the output is independent of trivial Unicode noise. -// Mirrors the JS canonicalizeClaims() reference implementation. -func CanonicalizeClaims(claims map[string]string) string { - s, _ := CanonicalizeClaimsStrict(claims) - return s -} - -// CanonicalizeClaimsStrict is like CanonicalizeClaims but enforces the draft's -// MUST-fail rules: an empty normalized name is "claim-malformed", and two -// names that normalize to the same value are "claim-duplicate". Names are -// sorted by their UTF-8 byte sequence. -func CanonicalizeClaimsStrict(claims map[string]string) (string, error) { +// It returns an error for the profile's MUST-fail conditions: an empty +// normalized name is "claim-malformed", and two names that normalize to the +// same value are "claim-duplicate". Names are sorted by UTF-8 bytes. +func CanonicalizeClaims(claims map[string]string) (string, error) { + if len(claims) > maxClaims { + return "", fmt.Errorf("resource-limit-exceeded") + } type entry struct{ name, value string } entries := make([]entry, 0, len(claims)) seen := make(map[string]bool, len(claims)) for k, v := range claims { - name := strings.TrimSpace(NormalizeText(k)) - value := strings.TrimSpace(NormalizeText(v)) + name, err := NormalizeChecked(k) + if err != nil { + return "", err + } + value, err := NormalizeChecked(v) + if err != nil { + return "", err + } + name = strings.TrimSpace(name) + value = strings.TrimSpace(value) if name == "" { return "", fmt.Errorf("claim-malformed") } if seen[name] { return "", fmt.Errorf("claim-duplicate") } + if len([]byte(name)) > maxClaimFieldBytes || len([]byte(value)) > maxClaimFieldBytes { + return "", fmt.Errorf("resource-limit-exceeded") + } seen[name] = true entries = append(entries, entry{name, value}) } @@ -232,12 +373,28 @@ func CanonicalizeClaimsStrict(claims map[string]string) (string, error) { }) var b strings.Builder for _, e := range entries { - b.WriteString(e.name) + b.WriteString(escapeClaimField(e.name)) b.WriteByte(':') - b.WriteString(e.value) + b.WriteString(escapeClaimField(e.value)) b.WriteByte('\n') } - return b.String(), nil + result := b.String() + if len(result) > maxResourceBytes { + return "", fmt.Errorf("resource-limit-exceeded") + } + return result, nil +} + +// CanonicalizeClaimsStrict is retained as a compatibility alias. Both public +// entry points enforce the same fail-closed profile. +func CanonicalizeClaimsStrict(claims map[string]string) (string, error) { + return CanonicalizeClaims(claims) +} + +func escapeClaimField(value string) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, ":", `\:`) + return strings.ReplaceAll(value, "\n", `\n`) } func appendCanonicalPart(parts *[]string, value string) { @@ -259,28 +416,7 @@ func finalizeCanonicalParts(parts []string) string { return strings.Trim(text, " \n") } -func parseAttributes(token string) map[string]string { - body := attrBodyRE.ReplaceAllString(token, "") - body = strings.TrimSuffix(strings.TrimSpace(body), ">") - body = strings.TrimSuffix(strings.TrimSpace(body), "/") - attrs := map[string]string{} - for _, m := range attrRE.FindAllStringSubmatch(body, -1) { - if len(m) == 0 || m[1] == "" { - continue - } - value := "" - for _, candidate := range m[2:] { - if candidate != "" { - value = candidate - break - } - } - attrs[strings.ToLower(m[1])] = decodeEntities(value) - } - return attrs -} - -func appendAttributeRecords(parts *[]string, elementName string, attrs map[string]string, base *url.URL) error { +func appendAttributeRecords(parts *[]string, elementName string, attrs map[string]string, base *whatwgurl.Url) error { for _, attrName := range signedAttrs { raw, ok := attrs[attrName] if !ok { @@ -288,19 +424,22 @@ func appendAttributeRecords(parts *[]string, elementName string, attrs map[strin } value := raw if attrName == "href" || attrName == "src" { - if base == nil && !hasURLScheme(value) { - // Relative URL with no base cannot be resolved. The draft - // (§4.3.2) requires a hard failure rather than a silent skip. - return fmt.Errorf("attribute-canonicalization-failed: %s.%s: relative URL with no base", elementName, attrName) - } normalized, err := normalizeURLAttribute(value, base) if err != nil { + if strings.Contains(err.Error(), "url-policy-violation") { + return err + } return fmt.Errorf("attribute-canonicalization-failed: %s.%s: %w", elementName, attrName, err) } value = normalized } else { - value = strings.TrimSpace(NormalizeText(value)) + normalized, err := NormalizeTextChecked(value) + if err != nil { + return fmt.Errorf("attribute-canonicalization-failed: %s.%s: %w", elementName, attrName, err) + } + value = strings.TrimSpace(normalized) } + value = strings.ReplaceAll(value, "@", "@@") if strings.ContainsRune(value, '\n') { return fmt.Errorf("attribute-canonicalization-failed: %s.%s contains newline", elementName, attrName) } @@ -315,151 +454,42 @@ func appendAttributeRecords(parts *[]string, elementName string, attrs map[strin return nil } -func parseBaseURL(raw string) (*url.URL, error) { +func parseBaseURL(raw string) (*whatwgurl.Url, error) { if raw == "" { return nil, nil } - base, err := url.Parse(raw) + base, err := whatwgurl.Parse(raw) if err != nil { - return nil, err + return nil, fmt.Errorf("attribute-canonicalization-failed: %w", err) } - if !base.IsAbs() || base.Host == "" { - return nil, fmt.Errorf("base URL must be absolute") + if base.Scheme() != "https" || base.Hostname() == "" || base.Username() != "" || base.Password() != "" { + return nil, fmt.Errorf("url-policy-violation") } return base, nil } -func hasURLScheme(raw string) bool { - i := strings.IndexByte(raw, ':') - if i <= 0 { - return false - } - for _, r := range raw[:i] { - if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '+' || r == '-' || r == '.' { - continue +// normalizeURLAttribute parses and serializes an href/src with the WHATWG URL algorithm. +func normalizeURLAttribute(raw string, base *whatwgurl.Url) (string, error) { + for _, r := range raw { + if r <= 0x1f || r == 0x7f { + return "", fmt.Errorf("url-policy-violation") } - return false } - return true -} - -// removeDotSegments implements RFC 3986 §5.2.4, matching the WHATWG URL path -// normalization the reference JS/Rust bindings perform via `new URL`. -func removeDotSegments(path string) string { - out := "" - in := path - for len(in) > 0 { - switch { - case strings.HasPrefix(in, "../"): - in = in[3:] - case strings.HasPrefix(in, "./"): - in = in[2:] - case strings.HasPrefix(in, "/./"): - in = "/" + in[3:] - case in == "/.": - in = "/" - case strings.HasPrefix(in, "/../"): - in = "/" + in[4:] - if i := strings.LastIndexByte(out, '/'); i >= 0 { - out = out[:i] - } else { - out = "" - } - case in == "/..": - in = "/" - if i := strings.LastIndexByte(out, '/'); i >= 0 { - out = out[:i] - } else { - out = "" - } - case in == "." || in == "..": - in = "" - default: - start := 0 - if strings.HasPrefix(in, "/") { - start = 1 - } - if idx := strings.IndexByte(in[start:], '/'); idx < 0 { - out += in - in = "" - } else { - out += in[:start+idx] - in = in[start+idx:] - } - } - } - return out -} - -func isASCII(s string) bool { - for i := 0; i < len(s); i++ { - if s[i] >= 0x80 { - return false - } + var u *whatwgurl.Url + var err error + if base != nil { + u, err = base.Parse(raw) + } else { + u, err = whatwgurl.Parse(raw) } - return true -} - -// normalizeURLAttribute serializes an href/src value using the Web (WHATWG) -// URL serializer semantics: lowercase scheme + host, IDNA/punycode host, strip -// default ports, resolve dot-segments, preserve query and fragment. -func normalizeURLAttribute(raw string, base *url.URL) (string, error) { - u, err := url.Parse(strings.TrimSpace(raw)) if err != nil { return "", err } - if base != nil { - u = base.ResolveReference(u) - } - if !u.IsAbs() { - return "", fmt.Errorf("URL must be absolute") - } - if u.Host == "" { - // Opaque URL with no authority (mailto:, tel:, javascript:, data:, - // about:, sms:, ...). The WHATWG URL parser accepts these; serialize - // scheme + opaque remainder verbatim (scheme lowercased), matching - // new URL().href. No host/port/dot-segment normalization applies. - rest := u.Opaque - if u.RawQuery != "" { - rest += "?" + u.RawQuery - } - if u.Fragment != "" { - rest += "#" + u.EscapedFragment() - } - return strings.ToLower(u.Scheme) + ":" + rest, nil - } - if u.User != nil { - return "", fmt.Errorf("userinfo not allowed in signed URL") - } - u.Scheme = strings.ToLower(u.Scheme) - host := strings.ToLower(u.Hostname()) - if !isASCII(host) { - ascii, err := idna.Lookup.ToASCII(host) - if err != nil { - return "", fmt.Errorf("idna: %w", err) - } - host = ascii - } - port := u.Port() - if (u.Scheme == "https" && port == "443") || (u.Scheme == "http" && port == "80") { - port = "" - } - if strings.Contains(host, ":") { - host = "[" + strings.Trim(host, "[]") + "]" - } - if port != "" { - host = net.JoinHostPort(strings.Trim(host, "[]"), port) + if u.Scheme() != "https" || u.Username() != "" || u.Password() != "" { + return "", fmt.Errorf("url-policy-violation") } - u.Host = host - cleaned := removeDotSegments(u.EscapedPath()) - if cleaned == "" { - cleaned = "/" - } - u.RawPath = cleaned - if unesc, err := url.PathUnescape(cleaned); err == nil { - u.Path = unesc - } else { - u.Path = cleaned + if u.Hostname() == "" { + return "", fmt.Errorf("attribute-canonicalization-failed") } - return u.String(), nil + return u.Href(false), nil } diff --git a/go/final_hardening_test.go b/go/final_hardening_test.go new file mode 100644 index 0000000..2bc2c24 --- /dev/null +++ b/go/final_hardening_test.go @@ -0,0 +1,154 @@ +package canonicalize + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestDecodedEndorsementExtensionsAreSignedAndMutable(t *testing.T) { + pem, _, private := newEd25519PEM(t) + var endorsement Endorsement + raw := `{"endorser":"static","endorsement":"sha256:content","algorithm":"ed25519","timestamp":"2026-08-27T12:00:00Z","extension":{"answer":42},"signature":"placeholder"}` + if err := json.Unmarshal([]byte(raw), &endorsement); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + if _, ok := endorsement.Extensions["extension"]; !ok { + t.Fatalf("decoded extension was dropped: %#v", endorsement.Extensions) + } + roundTrip, err := json.Marshal(endorsement) + if err != nil || !strings.Contains(string(roundTrip), `"extension":{"answer":42}`) { + t.Fatalf("decoded extension was not retained by marshal: json=%s err=%v", roundTrip, err) + } + first, err := BuildEndorsementBinding(endorsement) + if err != nil { + t.Fatalf("BuildEndorsementBinding: %v", err) + } + if !strings.Contains(first, `"extension":{"answer":42}`) { + t.Fatalf("decoded extension missing from binding: %s", first) + } + endorsement.Signature = base64.RawStdEncoding.EncodeToString(ed25519.Sign(private, []byte(first))) + if ok, err := VerifyEndorsement(context.Background(), endorsement, []KeyResolver{ + staticKeyResolver{key: &ResolvedKey{PublicKeyPEM: pem, Algorithm: "ed25519"}}, + }); err != nil || !ok { + t.Fatalf("decoded extension endorsement did not verify: ok=%v err=%v", ok, err) + } + value := endorsement.Extensions["extension"].(map[string]any) + value["answer"] = float64(43) + second, err := BuildEndorsementBinding(endorsement) + if err != nil { + t.Fatalf("BuildEndorsementBinding after extension change: %v", err) + } + if first == second || !strings.Contains(second, `"extension":{"answer":43}`) { + t.Fatalf("extension change did not change binding: before=%s after=%s", first, second) + } + if ok, err := VerifyEndorsement(context.Background(), endorsement, []KeyResolver{ + staticKeyResolver{key: &ResolvedKey{PublicKeyPEM: pem, Algorithm: "ed25519"}}, + }); err != nil || ok { + t.Fatalf("changed decoded extension unexpectedly verified: ok=%v err=%v", ok, err) + } +} + +func TestEndorsementExpiryIsStrictAndFailClosed(t *testing.T) { + for _, value := range []string{ + "2026-08-27T12:00:00+00:00", // offsets are not v1 UTC form + "2026-02-29T12:00:00Z", // invalid calendar date + "not-a-timestamp", + } { + if _, ok := parseRFC3339UTC(value); ok { + t.Errorf("parseRFC3339UTC(%q) accepted malformed value", value) + } + if !IsKeyRevoked(&ResolvedKey{Expires: value}) { + t.Errorf("IsKeyRevoked accepted malformed expiry %q", value) + } + } +} + +func TestEndorsementRejectsExplicitEmptyLifecycleFields(t *testing.T) { + for _, field := range []string{"expires", "revokedBy"} { + t.Run(field, func(t *testing.T) { + var endorsement Endorsement + raw := fmt.Sprintf(`{"endorser":"static","endorsement":"sha256:content","algorithm":"ed25519","timestamp":"2026-08-27T12:00:00Z",%q:"","signature":"placeholder"}`, field) + if err := json.Unmarshal([]byte(raw), &endorsement); err != nil { + t.Fatalf("json.Unmarshal: %v", err) + } + if ok, err := VerifyEndorsement(context.Background(), endorsement, nil); err != nil || ok { + t.Fatalf("explicit empty %s unexpectedly verified: ok=%v err=%v", field, ok, err) + } + }) + } +} + +func TestParserPreflightRejectsCommentsAndDeclarations(t *testing.T) { + for _, input := range []string{ + "x", + "x") + if err != nil || got != "x" { + t.Fatalf("valid comment: got %q, err %v", got, err) + } +} + +func TestRemoteKeyResolversBoundResponseBodies(t *testing.T) { + body := strings.Repeat("x", maxRemoteKeyBytes+1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + if _, err := (DirectURLResolver{HTTPClient: srv.Client()}).Resolve(context.Background(), srv.URL); err == nil || !strings.Contains(err.Error(), "resource-limit-exceeded") { + t.Fatalf("oversized direct key response error = %v, want resource-limit-exceeded", err) + } + + client := srv.Client() + client.Transport = rewriteTransport{base: srv.Client().Transport, target: srv.URL} + if _, err := (DidWebResolver{HTTPClient: client}).Resolve(context.Background(), "did:web:example.test"); err == nil || !strings.Contains(err.Error(), "resource-limit-exceeded") { + t.Fatalf("oversized DID key response error = %v, want resource-limit-exceeded", err) + } +} + +func TestDirectURLResolverAcceptsCaseInsensitiveSchemeAndRejectsInvalidUTF8(t *testing.T) { + valid := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"publicKey":"PEM","algorithm":"ed25519"}`)) + })) + defer valid.Close() + upperScheme := strings.ToUpper(valid.URL[:4]) + valid.URL[4:] + key, err := (DirectURLResolver{HTTPClient: valid.Client()}).Resolve(context.Background(), upperScheme) + if err != nil || key == nil { + t.Fatalf("case-insensitive scheme: key=%+v err=%v", key, err) + } + + invalid := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}'}) + })) + defer invalid.Close() + if _, err := (DirectURLResolver{HTTPClient: invalid.Client()}).Resolve(context.Background(), invalid.URL); err == nil || !strings.Contains(err.Error(), "invalid UTF-8") { + t.Fatalf("invalid UTF-8 response error = %v", err) + } +} + +func TestJCSRejectsNestingBeyondLimit(t *testing.T) { + tooDeep := strings.Repeat("[", maxJSONDepth+1) + "0" + strings.Repeat("]", maxJSONDepth+1) + if _, err := CanonicalizeJSONDocument([]byte(tooDeep)); err == nil || !strings.Contains(err.Error(), "resource-limit-exceeded") { + t.Fatalf("too-deep JSON error = %v, want resource-limit-exceeded", err) + } + withinLimit := strings.Repeat("[", maxJSONDepth) + "0" + strings.Repeat("]", maxJSONDepth) + if _, err := CanonicalizeJSONDocument([]byte(withinLimit)); err != nil { + t.Fatalf("JSON at nesting limit rejected: %v", err) + } +} diff --git a/go/go.mod b/go/go.mod index 696258a..f2524d6 100644 --- a/go/go.mod +++ b/go/go.mod @@ -5,3 +5,9 @@ go 1.25.0 require golang.org/x/text v0.39.0 require golang.org/x/net v0.55.0 + +require github.com/gowebpki/jcs v1.0.1 + +require github.com/nlnwa/whatwg-url v0.6.2 + +require github.com/bits-and-blooms/bitset v1.20.0 // indirect diff --git a/go/go.sum b/go/go.sum index 6e88cdf..4691611 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,4 +1,85 @@ +github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= +github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/gowebpki/jcs v1.0.1 h1:Qjzg8EOkrOTuWP7DqQ1FbYtcpEbeTzUoTN9bptp8FOU= +github.com/gowebpki/jcs v1.0.1/go.mod h1:CID1cNZ+sHp1CCpAR8mPf6QRtagFBgPJE0FCUQ6+BrI= +github.com/nlnwa/whatwg-url v0.6.2 h1:jU61lU2ig4LANydbEJmA2nPrtCGiKdtgT0rmMd2VZ/Q= +github.com/nlnwa/whatwg-url v0.6.2/go.mod h1:x0FPXJzzOEieQtsBT/AKvbiBbQ46YlL6Xa7m02M1ECk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/json.go b/go/json.go new file mode 100644 index 0000000..f084ce0 --- /dev/null +++ b/go/json.go @@ -0,0 +1,432 @@ +package canonicalize + +import ( + "fmt" + "sort" + "strconv" + "strings" + "unicode/utf16" + "unicode/utf8" + + jcs "github.com/gowebpki/jcs" +) + +type jsonValue struct { + kind byte + str string + num float64 + arr []*jsonValue + obj []jsonMember +} +type jsonMember struct { + name string + value *jsonValue +} +type strictJSONParser struct { + data []byte + pos int + depth int +} + +const maxJSONDepth = 256 + +func (p *strictJSONParser) fail() error { return fmt.Errorf("invalid JSON document") } +func (p *strictJSONParser) ws() { + for p.pos < len(p.data) && strings.ContainsRune(" \t\r\n", rune(p.data[p.pos])) { + p.pos++ + } +} +func (p *strictJSONParser) parse() (*jsonValue, error) { + v, e := p.value() + if e != nil { + return nil, e + } + p.ws() + if p.pos != len(p.data) { + return nil, p.fail() + } + return v, nil +} +func (p *strictJSONParser) value() (*jsonValue, error) { + p.ws() + if p.pos >= len(p.data) { + return nil, p.fail() + } + switch p.data[p.pos] { + case '"': + s, e := p.string() + return &jsonValue{kind: 's', str: s}, e + case '{': + if p.depth >= maxJSONDepth { + return nil, fmt.Errorf("resource-limit-exceeded") + } + p.depth++ + v, err := p.object() + p.depth-- + return v, err + case '[': + if p.depth >= maxJSONDepth { + return nil, fmt.Errorf("resource-limit-exceeded") + } + p.depth++ + v, err := p.array() + p.depth-- + return v, err + case 't': + if p.literal("true") { + return &jsonValue{kind: 'b', str: "true"}, nil + } + case 'f': + if p.literal("false") { + return &jsonValue{kind: 'b', str: "false"}, nil + } + case 'n': + if p.literal("null") { + return &jsonValue{kind: 'n'}, nil + } + } + n, e := p.number() + if e != nil { + return nil, e + } + return &jsonValue{kind: 'd', num: n}, nil +} +func (p *strictJSONParser) literal(s string) bool { + if strings.HasPrefix(string(p.data[p.pos:]), s) { + p.pos += len(s) + return true + } + return false +} +func hexValue(c byte) (byte, bool) { + switch { + case c >= '0' && c <= '9': + return c - '0', true + case c >= 'a' && c <= 'f': + return c - 'a' + 10, true + case c >= 'A' && c <= 'F': + return c - 'A' + 10, true + } + return 0, false +} +func (p *strictJSONParser) string() (string, error) { + if p.pos >= len(p.data) || p.data[p.pos] != '"' { + return "", p.fail() + } + p.pos++ + var b strings.Builder + for p.pos < len(p.data) { + c := p.data[p.pos] + p.pos++ + if c == '"' { + s := b.String() + if !utf8.ValidString(s) { + return "", p.fail() + } + return s, nil + } + if c < 0x20 { + return "", p.fail() + } + if c != '\\' { + start := p.pos - 1 + for p.pos < len(p.data) && p.data[p.pos] != '"' && p.data[p.pos] != '\\' && p.data[p.pos] >= 0x20 { + p.pos++ + } + b.Write(p.data[start:p.pos]) + continue + } + if p.pos >= len(p.data) { + return "", p.fail() + } + e := p.data[p.pos] + p.pos++ + switch e { + case '"', '\\', '/': + b.WriteByte(e) + case 'b': + b.WriteByte('\b') + case 'f': + b.WriteByte('\f') + case 'n': + b.WriteByte('\n') + case 'r': + b.WriteByte('\r') + case 't': + b.WriteByte('\t') + case 'u': + if p.pos+4 > len(p.data) { + return "", p.fail() + } + var u uint16 + for i := 0; i < 4; i++ { + d, ok := hexValue(p.data[p.pos+i]) + if !ok { + return "", p.fail() + } + u = u<<4 | uint16(d) + } + p.pos += 4 + if u >= 0xd800 && u <= 0xdbff { + if p.pos+6 > len(p.data) || p.data[p.pos] != '\\' || p.data[p.pos+1] != 'u' { + return "", fmt.Errorf("jcs-invalid-surrogate") + } + p.pos += 2 + var lo uint16 + for i := 0; i < 4; i++ { + d, ok := hexValue(p.data[p.pos+i]) + if !ok { + return "", p.fail() + } + lo = lo<<4 | uint16(d) + } + p.pos += 4 + if lo < 0xdc00 || lo > 0xdfff { + return "", fmt.Errorf("jcs-invalid-surrogate") + } + b.WriteRune(utf16.DecodeRune(rune(u), rune(lo))) + } else if u >= 0xdc00 && u <= 0xdfff { + return "", fmt.Errorf("jcs-invalid-surrogate") + } else { + b.WriteRune(rune(u)) + } + default: + return "", p.fail() + } + } + return "", p.fail() +} +func (p *strictJSONParser) number() (float64, error) { + start := p.pos + if p.pos < len(p.data) && p.data[p.pos] == '-' { + p.pos++ + } + if p.pos >= len(p.data) { + return 0, p.fail() + } + if p.data[p.pos] == '0' { + p.pos++ + } else if p.data[p.pos] >= '1' && p.data[p.pos] <= '9' { + for p.pos < len(p.data) && p.data[p.pos] >= '0' && p.data[p.pos] <= '9' { + p.pos++ + } + } else { + return 0, p.fail() + } + if p.pos < len(p.data) && p.data[p.pos] == '.' { + p.pos++ + if p.pos >= len(p.data) || p.data[p.pos] < '0' || p.data[p.pos] > '9' { + return 0, p.fail() + } + for p.pos < len(p.data) && p.data[p.pos] >= '0' && p.data[p.pos] <= '9' { + p.pos++ + } + } + if p.pos < len(p.data) && (p.data[p.pos] == 'e' || p.data[p.pos] == 'E') { + p.pos++ + if p.pos < len(p.data) && (p.data[p.pos] == '+' || p.data[p.pos] == '-') { + p.pos++ + } + if p.pos >= len(p.data) || p.data[p.pos] < '0' || p.data[p.pos] > '9' { + return 0, p.fail() + } + for p.pos < len(p.data) && p.data[p.pos] >= '0' && p.data[p.pos] <= '9' { + p.pos++ + } + } + n, e := strconv.ParseFloat(string(p.data[start:p.pos]), 64) + if e != nil || n != n || n > 1.7976931348623157e308 || n < -1.7976931348623157e308 { + return 0, fmt.Errorf("jcs-number") + } + return n, nil +} +func (p *strictJSONParser) array() (*jsonValue, error) { + p.pos++ + v := &jsonValue{kind: 'a'} + p.ws() + if p.pos < len(p.data) && p.data[p.pos] == ']' { + p.pos++ + return v, nil + } + for { + e, err := p.value() + if err != nil { + return nil, err + } + v.arr = append(v.arr, e) + p.ws() + if p.pos < len(p.data) && p.data[p.pos] == ']' { + p.pos++ + return v, nil + } + if p.pos >= len(p.data) || p.data[p.pos] != ',' { + return nil, p.fail() + } + p.pos++ + } +} +func (p *strictJSONParser) object() (*jsonValue, error) { + p.pos++ + v := &jsonValue{kind: 'o'} + seen := map[string]bool{} + p.ws() + if p.pos < len(p.data) && p.data[p.pos] == '}' { + p.pos++ + return v, nil + } + for { + p.ws() + if p.pos >= len(p.data) || p.data[p.pos] != '"' { + return nil, p.fail() + } + name, err := p.string() + if err != nil { + return nil, err + } + if seen[name] { + return nil, fmt.Errorf("jcs-duplicate-key") + } + seen[name] = true + p.ws() + if p.pos >= len(p.data) || p.data[p.pos] != ':' { + return nil, p.fail() + } + p.pos++ + x, err := p.value() + if err != nil { + return nil, err + } + v.obj = append(v.obj, jsonMember{name, x}) + p.ws() + if p.pos < len(p.data) && p.data[p.pos] == '}' { + p.pos++ + return v, nil + } + if p.pos >= len(p.data) || p.data[p.pos] != ',' { + return nil, p.fail() + } + p.pos++ + } +} +func escapeJSON(s string) string { + var b strings.Builder + b.WriteByte('"') + for _, r := range s { + switch r { + case '"': + b.WriteString(`\"`) + case '\\': + b.WriteString(`\\`) + case '\b': + b.WriteString(`\b`) + case '\f': + b.WriteString(`\f`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + if r < 0x20 { + b.WriteString(fmt.Sprintf(`\u%04x`, r)) + } else { + b.WriteRune(r) + } + } + } + b.WriteByte('"') + return b.String() +} +func serializeJSON(v *jsonValue) (string, error) { + switch v.kind { + case 'n': + return "null", nil + case 'b': + return v.str, nil + case 's': + return escapeJSON(v.str), nil + case 'd': + return jcs.NumberToJSON(v.num) + case 'a': + var b strings.Builder + b.WriteByte('[') + for i, x := range v.arr { + s, e := serializeJSON(x) + if e != nil { + return "", e + } + if i > 0 { + b.WriteByte(',') + } + b.WriteString(s) + } + b.WriteByte(']') + return b.String(), nil + case 'o': + sort.Slice(v.obj, func(i, j int) bool { + return compareUTF16(v.obj[i].name, v.obj[j].name) < 0 + }) + var b strings.Builder + b.WriteByte('{') + for i, m := range v.obj { + s, e := serializeJSON(m.value) + if e != nil { + return "", e + } + if i > 0 { + b.WriteByte(',') + } + b.WriteString(escapeJSON(m.name)) + b.WriteByte(':') + b.WriteString(s) + } + b.WriteByte('}') + return b.String(), nil + } + return "", fmt.Errorf("invalid JSON value") +} + +func compareUTF16(a, b string) int { + aa, bb := utf16.Encode([]rune(a)), utf16.Encode([]rune(b)) + for i := 0; i < len(aa) && i < len(bb); i++ { + if aa[i] < bb[i] { + return -1 + } + if aa[i] > bb[i] { + return 1 + } + } + if len(aa) < len(bb) { + return -1 + } + if len(aa) > len(bb) { + return 1 + } + return 0 +} + +// CanonicalizeJSONDocument validates and canonicalizes one complete JSON document according to RFC 8785. +func CanonicalizeJSONDocument(document []byte) ([]byte, error) { + if len(document) > maxResourceBytes { + return nil, fmt.Errorf("resource-limit-exceeded") + } + if !utf8.Valid(document) { + return nil, fmt.Errorf("jcs-invalid-json") + } + v, e := (&strictJSONParser{data: document}).parse() + if e != nil { + if strings.Contains(e.Error(), "jcs-") || strings.Contains(e.Error(), "resource-limit-exceeded") { + return nil, e + } + return nil, fmt.Errorf("jcs-invalid-json") + } + s, e := serializeJSON(v) + if e != nil { + return nil, e + } + if len(s) > maxResourceBytes { + return nil, fmt.Errorf("resource-limit-exceeded") + } + return []byte(s), nil +} diff --git a/go/resolver.go b/go/resolver.go index caa2b2d..436421b 100644 --- a/go/resolver.go +++ b/go/resolver.go @@ -7,14 +7,62 @@ import ( "fmt" "io" "net/http" + "net/url" + "regexp" "strings" + "time" + "unicode/utf8" ) +const maxRemoteKeyBytes = 64 * 1024 + +var rfc3339UTC = regexp.MustCompile(`^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?Z$`) + // ResolvedKey is the result of a successful keyid resolution. type ResolvedKey struct { - PublicKeyPEM string - Algorithm string - Keyid string + PublicKeyPEM string `json:"publicKeyPem"` + Algorithm string `json:"algorithm"` + Keyid string `json:"keyid"` + // Revoked and Expires are optional lifecycle fields from the key document. + // A revoked key or an expired key MUST NOT be used for verification. + Revoked bool `json:"revoked,omitempty"` + Expires string `json:"expires,omitempty"` +} + +// IsKeyRevoked reports whether a resolved key is revoked or expired. An +// unparseable non-empty expiry is treated as revoked so malformed lifecycle +// metadata cannot keep a key usable. The optional now argument is useful for +// deterministic tests. +func IsKeyRevoked(key *ResolvedKey, now ...time.Time) bool { + if key == nil { + return false + } + if key.Revoked { + return true + } + if key.Expires == "" { + return false + } + when := time.Now() + if len(now) > 0 { + when = now[0] + } + expires, valid := parseRFC3339UTC(key.Expires) + return !valid || !expires.After(when) +} + +// parseRFC3339UTC accepts the v1 lifecycle timestamp form: a valid RFC3339 +// date-time in UTC, with optional fractional seconds. Go's general RFC3339 +// parser also accepts numeric offsets, so the shape is checked first. +func parseRFC3339UTC(value string) (time.Time, bool) { + if !rfc3339UTC.MatchString(value) { + return time.Time{}, false + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, false + } + return parsed, true } // KeyResolver resolves a keyid to a public key. A resolver that does not apply @@ -51,9 +99,10 @@ func httpClient(c *http.Client) *http.Client { // ----- did:web ----- -// DidWebResolver resolves did:web:[:...] keyids by fetching the -// DID document at https:///.well-known/did.json and returning the -// first verificationMethod entry that contains a publicKeyPem field. +// DidWebResolver resolves did:web:[:...] keyids. A bare domain +// fetches https:///.well-known/did.json; a path DID fetches +// https:////did.json. It returns the first currently usable +// verificationMethod entry that contains a publicKeyPem field. type DidWebResolver struct { HTTPClient *http.Client } @@ -67,6 +116,8 @@ type verificationMethod struct { Type string `json:"type"` PublicKeyPem string `json:"publicKeyPem"` Algorithm string `json:"algorithm"` + Revoked bool `json:"revoked"` + Expires string `json:"expires"` } // Resolve implements KeyResolver. @@ -75,14 +126,23 @@ func (r DidWebResolver) Resolve(ctx context.Context, keyid string) (*ResolvedKey return nil, nil } rest := strings.TrimPrefix(keyid, "did:web:") - // did:web allows ":" as path separators after the domain. + // A DID URL fragment identifies a resource in the DID document. It is + // never part of the URL used to retrieve that document. + if suffix := strings.IndexAny(rest, "/?#"); suffix >= 0 { + rest = rest[:suffix] + } + // did:web allows ":" as path separators after the domain. A bare host + // uses the well-known location; a path DID document uses //did.json. parts := strings.Split(rest, ":") domain := parts[0] if domain == "" { return nil, fmt.Errorf("DidWebResolver: empty domain in keyid %q", keyid) } - url := "https://" + domain + "/.well-known/did.json" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + documentURL, err := didWebDocumentURL(domain, parts[1:]) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, documentURL, nil) if err != nil { return nil, err } @@ -92,30 +152,122 @@ func (r DidWebResolver) Resolve(ctx context.Context, keyid string) (*ResolvedKey } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("DidWebResolver: GET %s: status %d", url, resp.StatusCode) + return nil, fmt.Errorf("DidWebResolver: GET %s: status %d", documentURL, resp.StatusCode) } - body, err := io.ReadAll(resp.Body) + body, err := readRemoteKeyBody(resp.Body) if err != nil { return nil, err } + if !utf8.Valid(body) { + return nil, fmt.Errorf("DidWebResolver: invalid UTF-8 response") + } var doc didDocument if err := json.Unmarshal(body, &doc); err != nil { return nil, fmt.Errorf("DidWebResolver: decode did.json: %w", err) } for _, vm := range doc.VerificationMethod { - if vm.PublicKeyPem != "" { + resolved := &ResolvedKey{ + PublicKeyPEM: vm.PublicKeyPem, + Algorithm: vm.Algorithm, + Keyid: keyid, + Revoked: vm.Revoked, + Expires: vm.Expires, + } + // DID resolution skips unusable verification methods. Preserve the + // lifecycle values on usable methods so a later verification cannot + // race an expiry without checking it again. + if vm.PublicKeyPem != "" && !IsKeyRevoked(resolved) { alg := vm.Algorithm if alg == "" { alg = inferAlgorithmFromType(vm.Type) } - return &ResolvedKey{ - PublicKeyPEM: vm.PublicKeyPem, - Algorithm: alg, - Keyid: keyid, - }, nil + resolved.Algorithm = alg + return resolved, nil } } - return nil, fmt.Errorf("DidWebResolver: no verificationMethod with publicKeyPem in %s", url) + // A DID document with no currently usable verification method declines, + // matching the JavaScript resolver and allowing a resolver chain to try + // another source. + return nil, nil +} + +// readRemoteKeyBody reads at most one byte beyond the v1 key-document limit. +// The extra byte lets callers distinguish an exactly-at-limit response from +// an oversized response without ever allocating or buffering an unbounded +// response body. +func readRemoteKeyBody(body io.Reader) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(body, maxRemoteKeyBytes+1)) + if err != nil { + return nil, err + } + if len(data) > maxRemoteKeyBytes { + return nil, fmt.Errorf("resource-limit-exceeded") + } + return data, nil +} + +func didWebDocumentURL(domain string, pathParts []string) (string, error) { + // Keep the authority separate from the path so key material cannot inject + // a query, fragment, credentials, or an alternate host. + // did:web encodes the authority port colon as %3A so it cannot be + // confused with the colon-delimited path segments. + authorityDomain := strings.NewReplacer("%3A", ":", "%3a", ":").Replace(domain) + if strings.Contains(authorityDomain, "%") { + return "", fmt.Errorf("DidWebResolver: invalid domain %q", domain) + } + authority, err := url.Parse("https://" + authorityDomain) + if err != nil || authority.Host != authorityDomain || authority.User != nil || authority.Path != "" || authority.RawQuery != "" || authority.Fragment != "" { + return "", fmt.Errorf("DidWebResolver: invalid domain %q", domain) + } + u := &url.URL{Scheme: "https", Host: authorityDomain} + if len(pathParts) == 0 { + u.Path = "/.well-known/did.json" + return u.String(), nil + } + escaped := make([]string, len(pathParts)) + for i, part := range pathParts { + var err error + escaped[i], err = escapeDidWebPathPart(part) + if err != nil { + return "", err + } + } + rawPath := "/" + strings.Join(escaped, "/") + "/did.json" + path, err := url.PathUnescape(rawPath) + if err != nil { + return "", fmt.Errorf("DidWebResolver: invalid path in domain %q", domain) + } + u.Path = path + u.RawPath = rawPath + return u.String(), nil +} + +func escapeDidWebPathPart(part string) (string, error) { + if part == "" { + return "", fmt.Errorf("DidWebResolver: invalid empty path segment") + } + var escaped strings.Builder + for offset := 0; offset < len(part); { + if part[offset] == '%' { + if offset+2 >= len(part) || !isHexByte(part[offset+1]) || !isHexByte(part[offset+2]) { + return "", fmt.Errorf("DidWebResolver: invalid path escape") + } + escaped.WriteString(part[offset : offset+3]) + offset += 3 + continue + } + next := strings.IndexByte(part[offset:], '%') + if next < 0 { + next = len(part) - offset + } + escaped.WriteString(url.PathEscape(part[offset : offset+next])) + offset += next + } + return escaped.String(), nil +} + +func isHexByte(value byte) bool { + return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' || value >= 'A' && value <= 'F' } func inferAlgorithmFromType(t string) string { @@ -143,15 +295,21 @@ type DirectURLResolver struct { } type directKeyDoc struct { - PublicKey string `json:"publicKey"` - Algorithm string `json:"algorithm"` + PublicKey string `json:"publicKey"` + PublicKeyPEM string `json:"publicKeyPem"` + Key string `json:"key"` + Algorithm string `json:"algorithm"` + Revoked *bool `json:"revoked"` + Expires *string `json:"expires"` } func (r DirectURLResolver) Resolve(ctx context.Context, keyid string) (*ResolvedKey, error) { - if !(strings.HasPrefix(keyid, "https://") || strings.HasPrefix(keyid, "http://")) { + requestURL, err := url.Parse(keyid) + if err != nil || requestURL.Host == "" || !(strings.EqualFold(requestURL.Scheme, "https") || strings.EqualFold(requestURL.Scheme, "http")) { return nil, nil } - return fetchKey(ctx, httpClient(r.HTTPClient), keyid, keyid) + requestURL.Scheme = strings.ToLower(requestURL.Scheme) + return fetchKey(ctx, httpClient(r.HTTPClient), requestURL.String(), keyid) } // ----- trust directory ----- @@ -169,8 +327,12 @@ func (r TrustDirectoryResolver) Resolve(ctx context.Context, keyid string) (*Res } var lastErr error for _, base := range r.BaseURLs { - url := strings.TrimRight(base, "/") + "/keys/" + keyid - key, err := fetchKey(ctx, httpClient(r.HTTPClient), url, keyid) + requestURL, err := directoryKeyURL(base, keyid) + if err != nil { + lastErr = err + continue + } + key, err := fetchKey(ctx, httpClient(r.HTTPClient), requestURL, keyid) if err == nil && key != nil { return key, nil } @@ -182,6 +344,22 @@ func (r TrustDirectoryResolver) Resolve(ctx context.Context, keyid string) (*Res return nil, nil } +func directoryKeyURL(base, keyid string) (string, error) { + u, err := url.Parse(base) + if err != nil || u.Scheme == "" || u.Host == "" || u.RawQuery != "" || u.Fragment != "" { + return "", fmt.Errorf("TrustDirectoryResolver: invalid base URL %q", base) + } + basePath := strings.TrimRight(u.EscapedPath(), "/") + rawPath := basePath + "/keys/" + url.PathEscape(keyid) + path, err := url.PathUnescape(rawPath) + if err != nil { + return "", fmt.Errorf("TrustDirectoryResolver: invalid keyid") + } + u.Path = path + u.RawPath = rawPath + return u.String(), nil +} + // fetchKey GETs `url` and parses either JSON ({publicKey, algorithm}) or a raw // PEM document into a ResolvedKey. The keyid is recorded on the result. func fetchKey(ctx context.Context, client *http.Client, url, keyid string) (*ResolvedKey, error) { @@ -197,10 +375,13 @@ func fetchKey(ctx context.Context, client *http.Client, url, keyid string) (*Res if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("fetchKey: GET %s: status %d", url, resp.StatusCode) } - body, err := io.ReadAll(resp.Body) + body, err := readRemoteKeyBody(resp.Body) if err != nil { return nil, err } + if !utf8.Valid(body) { + return nil, fmt.Errorf("fetchKey: invalid UTF-8 response") + } ct := strings.ToLower(resp.Header.Get("Content-Type")) if strings.Contains(ct, "text/plain") || strings.Contains(ct, "application/x-pem-file") { return &ResolvedKey{ @@ -221,12 +402,28 @@ func fetchKey(ctx context.Context, client *http.Client, url, keyid string) (*Res } return nil, fmt.Errorf("fetchKey: decode %s: %w", url, err) } - if doc.PublicKey == "" { + publicKey := doc.PublicKey + if publicKey == "" { + publicKey = doc.PublicKeyPEM + } + if publicKey == "" { + publicKey = doc.Key + } + if publicKey == "" { return nil, fmt.Errorf("fetchKey: %s: missing publicKey field", url) } return &ResolvedKey{ - PublicKeyPEM: doc.PublicKey, + PublicKeyPEM: publicKey, Algorithm: doc.Algorithm, Keyid: keyid, + Revoked: doc.Revoked != nil && *doc.Revoked, + Expires: optionalString(doc.Expires), }, nil } + +func optionalString(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/go/signature.go b/go/signature.go index 39e2f4d..5d3c7da 100644 --- a/go/signature.go +++ b/go/signature.go @@ -10,17 +10,120 @@ import ( "crypto/x509" "encoding/asn1" "encoding/base64" + "encoding/json" "encoding/pem" "errors" "fmt" "math/big" "net" "net/url" + "regexp" "strings" + "time" + + whatwgurl "github.com/nlnwa/whatwg-url/url" +) + +const ( + SigningProfileV1 = "htmltrust-signature-v1" + CanonicalizationProfileV1 = "htmltrust-c14n-v1" + AttributeProfileV1 = "htmltrust-attrs-v1" + URLProfileV1 = "htmltrust-safe-url-v1" + SigningContextV1 = "https://htmltrust.org/protocol/signed-section" ) -// BuildSignatureBinding returns the canonical signing payload used to compute -// or verify a content signature, as defined in HTMLTrust spec §2.1: +type SigningProfileV1Input struct { + ContentHash string + ClaimsHash string + DocumentURL string + Scope string + KeyID string + Algorithm string + SignedAt string +} + +type signingObjectV1 struct { + Algorithm string `json:"algorithm"` + AttributeProfile string `json:"attributeProfile"` + CanonicalizationProfile string `json:"canonicalizationProfile"` + ClaimsHash string `json:"claimsHash"` + ContentHash string `json:"contentHash"` + Context string `json:"context"` + KeyID string `json:"keyid"` + Location string `json:"location"` + Profile string `json:"profile"` + Scope string `json:"scope"` + SignedAt string `json:"signedAt"` + URLProfile string `json:"urlProfile"` +} + +var signedAtV1Pattern = regexp.MustCompile(`^(?:[0-9]{4})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]Z$`) + +func ValidateSignedAtV1(value string) error { + if !signedAtV1Pattern.MatchString(value) || strings.HasPrefix(value, "0000-") { + return fmt.Errorf("timestamp-invalid") + } + parsed, err := time.Parse("2006-01-02T15:04:05Z", value) + if err != nil || parsed.UTC().Format("2006-01-02T15:04:05Z") != value { + return fmt.Errorf("timestamp-invalid") + } + return nil +} + +func DeriveSigningLocationV1(documentURL, scope string) (string, error) { + u, err := whatwgurl.Parse(documentURL) + if err != nil || u.Scheme() != "https" || u.Hostname() == "" || u.Username() != "" || u.Password() != "" { + return "", fmt.Errorf("origin-not-supported") + } + switch scope { + case "url": + return u.Href(true), nil + case "origin": + return u.Scheme() + "://" + u.Host(), nil + default: + return "", fmt.Errorf("scope-unsupported") + } +} + +// BuildSigningPayloadV1 returns the RFC 8785 bytes fixed by htmltrust-signature-v1. +func BuildSigningPayloadV1(input SigningProfileV1Input) (string, error) { + fields := map[string]string{ + "contentHash": input.ContentHash, "claimsHash": input.ClaimsHash, + "documentURL": input.DocumentURL, "scope": input.Scope, "keyid": input.KeyID, + "algorithm": input.Algorithm, "signedAt": input.SignedAt, + } + for name, value := range fields { + if value == "" || strings.TrimSpace(value) != value { + return "", fmt.Errorf("signing-object-invalid: %s", name) + } + } + if err := ValidateSignedAtV1(input.SignedAt); err != nil { + return "", err + } + location, err := DeriveSigningLocationV1(input.DocumentURL, input.Scope) + if err != nil { + return "", err + } + document := signingObjectV1{ + Algorithm: input.Algorithm, AttributeProfile: AttributeProfileV1, + CanonicalizationProfile: CanonicalizationProfileV1, ClaimsHash: input.ClaimsHash, + ContentHash: input.ContentHash, Context: SigningContextV1, KeyID: input.KeyID, + Location: location, Profile: SigningProfileV1, Scope: input.Scope, + SignedAt: input.SignedAt, URLProfile: URLProfileV1, + } + raw, err := json.Marshal(document) + if err != nil { + return "", err + } + canonical, err := CanonicalizeJSONDocument(raw) + if err != nil { + return "", err + } + return string(canonical), nil +} + +// BuildSignatureBinding returns the legacy 0.2 colon-joined payload. +// New integrations must use BuildSigningPayloadV1. // // {contentHash}:{claimsHash}:{domain}:{signedAt} // @@ -204,3 +307,17 @@ func VerifySignature(message string, signatureB64 string, publicKeyPEM string, a return false, fmt.Errorf("VerifySignature: unsupported algorithm %q", algorithm) } } + +// VerifyResolvedSignature verifies a signature using a resolved key and +// rejects revoked or expired key material before doing any cryptographic work. +// VerifySignature is retained as the legacy PEM-only API; callers that obtain +// keys through a KeyResolver should use this checked form. +func VerifyResolvedSignature(message, signatureB64 string, key *ResolvedKey, algorithm string) (bool, error) { + if key == nil { + return false, errors.New("VerifyResolvedSignature: key is required") + } + if IsKeyRevoked(key) { + return false, nil + } + return VerifySignature(message, signatureB64, key.PublicKeyPEM, algorithm) +} diff --git a/go/vectors_test.go b/go/vectors_test.go index ea13ab3..6f6db53 100644 --- a/go/vectors_test.go +++ b/go/vectors_test.go @@ -18,10 +18,12 @@ func TestEndToEndVectors(t *testing.T) { PublicKeyPem string `json:"publicKeyPem"` } `json:"key"` Input struct { - HTML string `json:"html"` - BaseURL string `json:"baseURL"` - Domain string `json:"domain"` - SignedAt string `json:"signedAt"` + HTML string `json:"html"` + BaseURL string `json:"baseURL"` + DocumentURL string `json:"documentURL"` + Scope string `json:"scope"` + KeyID string `json:"keyid"` + SignedAt string `json:"signedAt"` } `json:"input"` Claims map[string]string `json:"claims"` CanonicalContent string `json:"canonicalContent"` @@ -64,7 +66,11 @@ func TestEndToEndVectors(t *testing.T) { if got := sha(claims); got != v.ClaimsHash { t.Errorf("%s: claimsHash got %s want %s", path, got, v.ClaimsHash) } - payload, err := BuildSignatureBinding(v.ContentHash, v.ClaimsHash, v.Input.Domain, v.Input.SignedAt) + payload, err := BuildSigningPayloadV1(SigningProfileV1Input{ + ContentHash: v.ContentHash, ClaimsHash: v.ClaimsHash, + DocumentURL: v.Input.DocumentURL, Scope: v.Input.Scope, + KeyID: v.Input.KeyID, Algorithm: v.Algorithm, SignedAt: v.Input.SignedAt, + }) if err != nil { t.Fatalf("%s: binding: %v", path, err) } diff --git a/javascript/index.d.ts b/javascript/index.d.ts index a209835..0d9f923 100644 --- a/javascript/index.d.ts +++ b/javascript/index.d.ts @@ -43,6 +43,24 @@ export function extractCanonicalText(html: string, options?: NormalizeOptions): * @returns Canonical serialized string ready to be hashed */ export function canonicalizeClaims(claims: Record): string; +export const SIGNING_PROFILE_V1: Readonly<{ + profile: "htmltrust-signature-v1"; + canonicalizationProfile: "htmltrust-c14n-v1"; + attributeProfile: "htmltrust-attrs-v1"; + urlProfile: "htmltrust-safe-url-v1"; + context: "https://htmltrust.org/protocol/signed-section"; +}>; +export function deriveSigningLocationV1(documentURL: string, scope: "url" | "origin"): string; +export function validateSignedAtV1(value: string): string; +export function buildSigningPayloadV1(parts: { + contentHash: string; + claimsHash: string; + documentURL: string; + scope: "url" | "origin"; + keyid: string; + algorithm: string; + signedAt: string; +}): string; /** Extract direct child `` claims from a signed-section. */ export function extractClaimsFromSignedSection(html: string): Record; @@ -139,6 +157,9 @@ export function buildEndorsementBinding(e: Omit & { si /** Deterministically serialize a JSON value with object keys sorted. */ export function canonicalizeJson(value: unknown): string; +/** Parse and RFC 8785-canonicalize a complete JSON document strictly. */ +export function canonicalizeJsonDocument(document: string): string; + /** Verify a standalone signed endorsement (spec §2.5). */ export function verifyEndorsement( endorsement: Endorsement, diff --git a/javascript/index.js b/javascript/index.js index d5556d3..8e3b445 100644 --- a/javascript/index.js +++ b/javascript/index.js @@ -2,10 +2,29 @@ * HTMLTrust Canonical Text Normalization * Spec: https://github.com/HTMLTrust/htmltrust-canonicalization * - * Zero dependencies. Works in browsers and Node.js. + * Uses parse5 for deterministic HTML parsing. Works in browsers and Node.js. */ import { NAMED_ENTITIES } from "./entities.js"; +import * as parse5 from "parse5"; + +const MAX_RESOURCE_BYTES = 1024 * 1024; +const MAX_CLAIMS = 64; +const MAX_CLAIM_FIELD_BYTES = 4096; +const MAX_ELEMENT_DEPTH = 256; +const MAX_JCS_DEPTH = 256; +const MAX_REMOTE_KEY_BYTES = 64 * 1024; + +function utf8Length(value) { + return new TextEncoder().encode(value).byteLength; +} + +function checkResourceBytes(value, what) { + if (utf8Length(value) > MAX_RESOURCE_BYTES) { + throw new Error("resource-limit-exceeded"); + } + return value; +} // Phase 6: Invisible/formatting characters to strip const STRIP_RE = new RegExp( @@ -68,6 +87,8 @@ const ELLIPSIS_RE = /\u2026/g; * @returns {string} Normalized text */ export function normalizeText(text, options = {}) { + if (typeof text !== "string") throw new TypeError("normalizeText expects a string"); + checkResourceBytes(text, "source"); const { preserveWhitespace = false } = options; // Phase 1: Unicode NFKC normalization @@ -98,6 +119,8 @@ export function normalizeText(text, options = {}) { // Phase 5: Other punctuation text = text.replace(ELLIPSIS_RE, "..."); + checkResourceBytes(text, "output"); + return text; } @@ -122,8 +145,8 @@ const BLOCK_ELEMENTS = // Any remaining HTML tag (inline elements we strip without adding whitespace). const ANY_TAG_RE = /<\/?[a-z][a-z0-9-]*\b[^>]*>/gi; -const HTML_TOKEN_RE = /|]*>|<\/?[a-z][a-z0-9-]*(?:\s[^<>]*)?\s*\/?>/gi; -const TAG_NAME_RE = /^<\/?\s*([a-z][a-z0-9-]*)/i; +const HTML_TOKEN_RE = /|]*>|<\/?[a-z][^\t\n\f\r \/>]*(?:[\t\n\f\r ]+(?:[^>"']+|"[^"]*"|'[^']*')*)?\s*\/?>/gi; +const TAG_NAME_RE = /^<\/?\s*([a-z][^\t\n\f\r \/>]*)/i; const SIGNED_ATTRS = ["href", "src", "alt", "aria-label"]; const VOID_TAGS = new Set([ "area", @@ -182,7 +205,7 @@ function decodeEntities(text) { } function parseAttributes(tag) { const attrs = new Map(); - const body = tag.replace(/^<\/?\s*[a-z][a-z0-9-]*/i, "").replace(/\/?\s*>$/, ""); + const body = tag.replace(/^<\/?\s*[a-z][^\t\n\f\r \/>]*/i, "").replace(/\/?\s*>$/, ""); const attrRe = /([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g; let match; while ((match = attrRe.exec(body))) { @@ -203,23 +226,11 @@ function appendAttributeRecords(parts, elementName, attrs, baseUrl) { if (!attrs.has(attrName)) continue; let value = attrs.get(attrName); if (attrName === "href" || attrName === "src") { - if (!baseUrl && !/^[a-z][a-z0-9+.-]*:/i.test(value)) { - // Relative URL with no base cannot be resolved. The draft (§4.3.2) - // requires a hard failure rather than a silent skip. - throw new Error( - `attribute-canonicalization-failed: ${elementName}.${attrName}`, - ); - } - try { - // `null` base coerces to the invalid string "null"; pass undefined so - // an absolute URL is accepted without a base. - value = new URL(value, baseUrl || undefined).href; - } catch (err) { - throw new Error(`attribute-canonicalization-failed: ${elementName}.${attrName}`); - } + value = normalizeSafeURL(value, baseUrl, elementName, attrName); } else { value = normalizeText(value).trim(); } + value = value.replaceAll("@", "@@"); if (value.includes("\n")) { throw new Error(`attribute-canonicalization-failed: ${elementName}.${attrName}`); } @@ -228,6 +239,24 @@ function appendAttributeRecords(parts, elementName, attrs, baseUrl) { } } +function normalizeSafeURL(value, baseUrl, elementName, attrName) { + // Inspect the parser-decoded value before WHATWG URL preprocessing. URL() + // otherwise silently strips tabs and line feeds. + if (/[\u0000-\u001F\u007F]/u.test(value)) { + throw new Error("url-policy-violation"); + } + try { + const url = new URL(value, baseUrl || undefined); + if (url.protocol !== "https:" || url.username || url.password) { + throw new Error("url-policy-violation"); + } + return url.href; + } catch (error) { + if (error?.message === "url-policy-violation") throw error; + throw new Error(`attribute-canonicalization-failed: ${elementName}.${attrName}`); + } +} + function finalizeCanonicalParts(parts) { return parts .join("") @@ -268,51 +297,137 @@ export function extractCanonicalText(html, options = {}) { if (typeof html !== "string") { throw new TypeError("extractCanonicalText expects a string"); } - + checkResourceBytes(html, "source"); + const fragment = parseHTMLFragment(html); + const baseUrl = validateBaseURL(options.baseUrl); const parts = []; - const baseUrl = options.baseUrl; - let index = 0; - let excludedDepth = 0; - let match; - HTML_TOKEN_RE.lastIndex = 0; - while ((match = HTML_TOKEN_RE.exec(html))) { - if (match.index > index && excludedDepth === 0) { - appendPart(parts, normalizeText(decodeEntities(html.slice(index, match.index)), options)); - } - index = HTML_TOKEN_RE.lastIndex; + walkParsedNode(fragment, parts, baseUrl, options); + const result = finalizeCanonicalParts(parts); + checkResourceBytes(result, "output"); + return result; +} + +function parseHTMLFragment(html) { + validatePortableSource(html); + const errors = []; + const fragment = parse5.parseFragment(html, { + sourceCodeLocationInfo: true, + onParseError(error) { errors.push(error); }, + }); + if (errors.length) throw new Error("parser-profile-unsupported"); + return fragment; +} +function validatePortableSource(source) { + // parse5 recovers misnesting and foster parenting without emitting a parse + // diagnostic, so these source-level checks complement its tokenizer. + const stack = []; + let index = 0; + for (const match of source.matchAll(HTML_TOKEN_RE)) { + const text = source.slice(index, match.index); + // script, style, and iframe are raw-text/escapable-raw-text elements. + // Their bodies are excluded from canonical content, so references there + // must not affect the portable-profile validation of the surrounding + // document. + if (!isRawTextElement(stack.at(-1))) validatePortableReferences(text); + if (stack.at(-1) === "table" && text.trim()) throw new Error("parser-profile-unsupported"); + index = match.index + match[0].length; const token = match[0]; + if (token.startsWith("x'); } catch (error) { + threw = String(error).includes('parser-profile-unsupported'); + } + assert(threw, 'double hyphen in a comment must be rejected'); +}); + +await check('extractCanonicalText accepts qualified tag names and enforces element depth', () => { + assertEq(extractCanonicalText('qualified'), 'qualified'); + const withinLimit = ''.repeat(256) + 'deep' + ''.repeat(256); + assertEq(extractCanonicalText(withinLimit), 'deep'); + let threw = false; + try { + extractCanonicalText(''.repeat(257) + 'too deep' + ''.repeat(257)); + } catch (error) { + threw = String(error).includes('resource-limit-exceeded'); + } + assert(threw, 'qualified element nesting beyond 256 levels must be rejected'); + threw = false; + try { extractCanonicalText('foreign'); } catch (error) { + threw = String(error).includes('parser-profile-unsupported'); + } + assert(threw, 'foreign-content rejection must remain in force'); +}); + +await check('canonicalizeJsonDocument accepts escaped surrogate pairs', () => { + assertEq(canonicalizeJsonDocument('{"music":"\\uD834\\uDD1E"}'), '{"music":"𝄞"}'); + for (const input of [ + '{"music":"\\uD834"}', + '{"music":"\\uDD1E"}', + '{"music":"\\uD834\\u0041"}', + ]) { + let threw = false; + try { canonicalizeJsonDocument(input); } catch (error) { + threw = String(error).includes('jcs-invalid-surrogate'); + } + assert(threw, `invalid surrogate sequence must be rejected: ${input}`); + } +}); + +await check('canonicalizeJsonDocument rejects excessive nesting', () => { + let threw = false; + try { canonicalizeJsonDocument('['.repeat(257) + '0' + ']'.repeat(257)); } catch (error) { + threw = String(error).includes('resource-limit-exceeded'); + } + assert(threw, 'JCS nesting beyond 256 levels must be rejected'); +}); + await check('decodeCanonicalBase64 rejects padded and base64url forms', () => { assertEq(new TextDecoder().decode(decodeCanonicalBase64('Zm9v')), 'foo'); let padded = false; @@ -173,6 +255,18 @@ await check('buildSignatureBinding throws on missing field', () => { assert(threw, 'expected throw on missing field'); }); +await check('buildEndorsementBinding requires non-empty string members', () => { + for (const field of ['endorser', 'endorsement', 'algorithm', 'timestamp']) { + const endorsement = { + endorser: 'a', endorsement: 'b', algorithm: 'ed25519', timestamp: '2026-01-01T00:00:00Z', + }; + endorsement[field] = field === 'endorser' ? 1 : ''; + let threw = false; + try { buildEndorsementBinding(endorsement); } catch { threw = true; } + assert(threw, `${field} must be a non-empty string`); + } +}); + await check('verifySignature ed25519 round-trip', async () => { const { publicKey, privateKey } = generateKeyPairSync('ed25519'); const message = 'hello world'; @@ -370,6 +464,41 @@ await check('didWebResolver fetches did.json and extracts key', async () => { assert(resolved.publicKeyPem.includes('BEGIN PUBLIC KEY'), 'expected PEM'); }); +await check('didWebResolver preserves path escapes and decodes an encoded port', async () => { + let requested; + const resolver = didWebResolver({ + fetch: async (url) => { + requested = url; + return { + ok: true, + headers: { get: (name) => name === 'content-type' ? 'application/json' : null }, + text: async () => JSON.stringify({ verificationMethod: [{ publicKeyPem: edPubPem }] }), + }; + }, + }); + const resolved = await resolver.resolve('did:web:example.com%3A3000:user%2Falice#key-1'); + assert(resolved, 'expected did:web resolution'); + assertEq(requested, 'https://example.com:3000/user%2Falice/did.json'); +}); + +await check('didWebResolver rejects empty userinfo in authority', async () => { + let called = false; + const resolver = didWebResolver({ + fetch: async () => { + called = true; + return { ok: false }; + }, + }); + let rejected = false; + try { + await resolver.resolve('did:web:@example.com'); + } catch (error) { + rejected = String(error).includes('did:web invalid domain'); + } + assert(rejected, 'empty userinfo authority must be rejected'); + assert(!called, 'invalid authority must not invoke fetch'); +}); + await check('directUrlResolver fetches http URL keyid', async () => { const resolved = await resolveKey(`${base}/key.json`, [directUrlResolver()]); assert(resolved, 'expected resolution'); @@ -383,6 +512,36 @@ await check('directUrlResolver accepts vendor JSON media types', async () => { assert(resolved.publicKeyPem.includes('BEGIN PUBLIC KEY'), 'expected parsed key document'); }); +await check('remote key fetchers cap streamed response bodies', async () => { + const chunks = [new Uint8Array(64 * 1024), new Uint8Array(1)]; + let reads = 0; + let cancelled = false; + const response = { + ok: true, + headers: { get: (name) => name === 'content-type' ? 'application/json' : null }, + body: { + getReader() { + return { + async read() { + if (reads >= chunks.length) return { done: true, value: undefined }; + return { done: false, value: chunks[reads++] }; + }, + async cancel() { cancelled = true; }, + }; + }, + }, + }; + let rejected = false; + try { + await directUrlResolver({ fetch: async () => response }).resolve('http://example.test/key.json'); + } catch (error) { + rejected = String(error).includes('resource-limit-exceeded'); + } + assert(rejected, 'oversized streamed key response must fail'); + assert(cancelled, 'oversized response stream must be cancelled'); + assertEq(reads, 2, 'stream should stop at the first oversized chunk'); +}); + await check('directUrlResolver decodes canonical SPKI key documents', async () => { const der = edPub.export({ type: 'spki', format: 'der' }); const encoded = der.toString('base64').replace(/=+$/, ''); @@ -466,6 +625,36 @@ await check('verifyEndorsement fails on tampered hash', async () => { assert(!ok, 'tampered endorsement must not verify'); }); +await check('verifyEndorsement fails closed on expiry and revokedBy lifecycle fields', async () => { + const { publicKey, privateKey } = generateKeyPairSync('ed25519'); + const publicKeyPem = publicKey.export({ type: 'spki', format: 'pem' }); + const keyid = 'did:web:endorsement-lifecycle.example'; + const resolver = { resolve: async () => ({ keyid, publicKeyPem, algorithm: 'ed25519' }) }; + const sign = (unsigned) => ({ + ...unsigned, + signature: nodeSign(null, Buffer.from(buildEndorsementBinding(unsigned)), privateKey) + .toString('base64').replace(/=+$/, ''), + }); + const baseEndorsement = { + endorser: keyid, + endorsement: 'sha256:lifecycle', + timestamp: '2026-04-28T12:00:00Z', + algorithm: 'ed25519', + }; + assert(await verifyEndorsement(sign({ ...baseEndorsement, expires: '2999-01-01T00:00:00Z' }), [resolver]), 'future expiry should verify'); + for (const field of [ + { expires: 'nonsense' }, + { expires: '2020-01-01T00:00:00Z' }, + { expires: '2999-01-01T00:00:00+00:00' }, + { expires: '' }, + { revokedBy: '' }, + { revokedBy: 'did:web:authority.example' }, + { revokedBy: 42 }, + ]) { + assert(!(await verifyEndorsement(sign({ ...baseEndorsement, ...field }), [resolver])), `${Object.keys(field)[0]} must fail closed`); + } +}); + await check('end-to-end test vector reproduces hashes, payload, and signature', async () => { const v = JSON.parse( readFileSync(new URL('../conformance/vectors/vector-01.json', import.meta.url), 'utf8'), @@ -476,10 +665,13 @@ await check('end-to-end test vector reproduces hashes, payload, and signature', const claims = canonicalizeClaims(extractClaimsFromSignedSection(v.input.html)); const contentHash = sha(content); const claimsHash = sha(claims); - const payload = buildSignatureBinding({ + const payload = buildSigningPayloadV1({ contentHash, claimsHash, - domain: v.input.domain, + documentURL: v.input.documentURL, + scope: v.input.scope, + keyid: v.input.keyid, + algorithm: v.algorithm, signedAt: v.input.signedAt, }); assertEq(content, v.canonicalContent, 'canonicalContent'); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..c510ae2 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,43 @@ +{ + "name": "@htmltrust/canonicalization", + "version": "0.3.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@htmltrust/canonicalization", + "version": "0.3.0", + "license": "LicenseRef-PolyForm-Noncommercial-1.0.0", + "dependencies": { + "parse5": "7.3.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + } + } +} diff --git a/package.json b/package.json index b1f69a7..e552ab6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@htmltrust/canonicalization", - "version": "0.2.2", + "version": "0.3.0", "description": "HTMLTrust canonical text normalization, signature verification, and key resolution for browsers and Node.js", "type": "module", "main": "javascript/index.js", @@ -8,6 +8,9 @@ ".": "./javascript/index.js" }, "types": "javascript/index.d.ts", + "engines": { + "node": ">=22" + }, "files": [ "javascript/index.js", "javascript/index.d.ts", @@ -21,5 +24,8 @@ "content-signing" ], "author": "Jason Grey ", - "license": "LicenseRef-PolyForm-Noncommercial-1.0.0" + "license": "LicenseRef-PolyForm-Noncommercial-1.0.0", + "dependencies": { + "parse5": "7.3.0" + } } diff --git a/php/composer.json b/php/composer.json index 20b24b3..04b050d 100644 --- a/php/composer.json +++ b/php/composer.json @@ -14,20 +14,22 @@ } }, "require": { - "php": ">=7.2", + "php": ">=8.5", + "ext-dom": "*", "ext-intl": "*", "ext-mbstring": "*", "ext-json": "*", "ext-openssl": "*", - "ext-sodium": "*" + "ext-sodium": "*", + "root23/php-json-canonicalization": "1.0.1" }, "suggest": { "ext-curl": "Used by the default HttpFetcher for keyid resolution; falls back to file_get_contents when missing." }, "require-dev": { - "phpunit/phpunit": "^8.0 || ^9.0 || ^10.0" + "phpunit/phpunit": "10.5.64" }, "scripts": { - "test": "phpunit --colors=always tests/" + "test": "phpunit --do-not-cache-result --colors=always tests/" } } diff --git a/php/composer.lock b/php/composer.lock new file mode 100644 index 0000000..db06c8d --- /dev/null +++ b/php/composer.lock @@ -0,0 +1,1716 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", + "This file is @generated automatically" + ], + "content-hash": "4bddf54a82628ded373a82fb4d152e17", + "packages": [ + { + "name": "root23/php-json-canonicalization", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/root23/php-json-canonicalization.git", + "reference": "be888e03a171c2b9667265d03924bd6bfc3fe85a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/root23/php-json-canonicalization/zipball/be888e03a171c2b9667265d03924bd6bfc3fe85a", + "reference": "be888e03a171c2b9667265d03924bd6bfc3fe85a", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^v3.27.0", + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9" + }, + "type": "library", + "autoload": { + "psr-4": { + "Root23\\JsonCanonicalizer\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "description": "Serialize data into canonical way, based on RFC-8785.", + "support": { + "issues": "https://github.com/root23/php-json-canonicalization/issues", + "source": "https://github.com/root23/php-json-canonicalization/tree/1.0.1" + }, + "time": "2023-09-27T08:27:25+00:00" + } + ], + "packages-dev": [ + { + "name": "myclabs/deep-copy", + "version": "1.14.0", + "source": { + "type": "git", + "url": "https://github.com/myclabs/DeepCopy.git", + "reference": "8680aa248f8e07bc8fb43f56f0f5fc77a0c96aae" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/8680aa248f8e07bc8fb43f56f0f5fc77a0c96aae", + "reference": "8680aa248f8e07bc8fb43f56f0f5fc77a0c96aae", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "conflict": { + "doctrine/collections": "<1.6.8", + "doctrine/common": "<2.13.3 || >=3 <3.2.2" + }, + "require-dev": { + "doctrine/collections": "^1.6.8", + "doctrine/common": "^2.13.3 || ^3.2.2", + "phpspec/prophecy": "^1.10", + "phpunit/phpunit": "^7.5.20 || ^8.5.23 || ^9.5.13" + }, + "type": "library", + "autoload": { + "files": [ + "src/DeepCopy/deep_copy.php" + ], + "psr-4": { + "DeepCopy\\": "src/DeepCopy/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Create deep copies (clones) of your objects", + "keywords": [ + "clone", + "copy", + "duplicate", + "object", + "object graph" + ], + "support": { + "issues": "https://github.com/myclabs/DeepCopy/issues", + "source": "https://github.com/myclabs/DeepCopy/tree/1.14.0" + }, + "funding": [ + { + "url": "https://github.com/mnapoli", + "type": "github" + } + ], + "time": "2026-08-11T10:17:44+00:00" + }, + { + "name": "nikic/php-parser", + "version": "v5.8.0", + "source": { + "type": "git", + "url": "https://github.com/nikic/PHP-Parser.git", + "reference": "044a6a392ff8ad0d61f14370a5fbbd0a0107152f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/nikic/PHP-Parser/zipball/044a6a392ff8ad0d61f14370a5fbbd0a0107152f", + "reference": "044a6a392ff8ad0d61f14370a5fbbd0a0107152f", + "shasum": "" + }, + "require": { + "ext-json": "*", + "ext-tokenizer": "*", + "php": ">=7.4" + }, + "require-dev": { + "ircmaxell/php-yacc": "^0.0.7", + "phpunit/phpunit": "^9.0" + }, + "bin": [ + "bin/php-parse" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "5.x-dev" + } + }, + "autoload": { + "psr-4": { + "PhpParser\\": "lib/PhpParser" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Nikita Popov" + } + ], + "description": "A PHP parser written in PHP", + "keywords": [ + "parser", + "php" + ], + "support": { + "issues": "https://github.com/nikic/PHP-Parser/issues", + "source": "https://github.com/nikic/PHP-Parser/tree/v5.8.0" + }, + "time": "2026-07-04T14:30:18+00:00" + }, + { + "name": "phar-io/manifest", + "version": "2.0.4", + "source": { + "type": "git", + "url": "https://github.com/phar-io/manifest.git", + "reference": "54750ef60c58e43759730615a392c31c80e23176" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phar-io/manifest/zipball/54750ef60c58e43759730615a392c31c80e23176", + "reference": "54750ef60c58e43759730615a392c31c80e23176", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-libxml": "*", + "ext-phar": "*", + "ext-xmlwriter": "*", + "phar-io/version": "^3.0.1", + "php": "^7.2 || ^8.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.0.x-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + }, + { + "name": "Sebastian Heuer", + "email": "sebastian@phpeople.de", + "role": "Developer" + }, + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "Developer" + } + ], + "description": "Component for reading phar.io manifest information from a PHP Archive (PHAR)", + "support": { + "issues": "https://github.com/phar-io/manifest/issues", + "source": "https://github.com/phar-io/manifest/tree/2.0.4" + }, + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2024-03-03T12:33:53+00:00" + }, + { + "name": "phar-io/version", + "version": "3.2.1", + "source": { + "type": "git", + "url": "https://github.com/phar-io/version.git", + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phar-io/version/zipball/4f7fd7836c6f332bb2933569e566a0d6c4cbed74", + "reference": "4f7fd7836c6f332bb2933569e566a0d6c4cbed74", + "shasum": "" + }, + "require": { + "php": "^7.2 || ^8.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + }, + { + "name": "Sebastian Heuer", + "email": "sebastian@phpeople.de", + "role": "Developer" + }, + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "Developer" + } + ], + "description": "Library for handling version information and constraints", + "support": { + "issues": "https://github.com/phar-io/version/issues", + "source": "https://github.com/phar-io/version/tree/3.2.1" + }, + "time": "2022-02-21T01:04:05+00:00" + }, + { + "name": "phpunit/php-code-coverage", + "version": "10.1.16", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-code-coverage.git", + "reference": "7e308268858ed6baedc8704a304727d20bc07c77" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/7e308268858ed6baedc8704a304727d20bc07c77", + "reference": "7e308268858ed6baedc8704a304727d20bc07c77", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-libxml": "*", + "ext-xmlwriter": "*", + "nikic/php-parser": "^4.19.1 || ^5.1.0", + "php": ">=8.1", + "phpunit/php-file-iterator": "^4.1.0", + "phpunit/php-text-template": "^3.0.1", + "sebastian/code-unit-reverse-lookup": "^3.0.0", + "sebastian/complexity": "^3.2.0", + "sebastian/environment": "^6.1.0", + "sebastian/lines-of-code": "^2.0.2", + "sebastian/version": "^4.0.1", + "theseer/tokenizer": "^1.2.3" + }, + "require-dev": { + "phpunit/phpunit": "^10.1" + }, + "suggest": { + "ext-pcov": "PHP extension that provides line coverage", + "ext-xdebug": "PHP extension that provides line coverage as well as branch and path coverage" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "10.1.x-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", + "homepage": "https://github.com/sebastianbergmann/php-code-coverage", + "keywords": [ + "coverage", + "testing", + "xunit" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", + "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/10.1.16" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-08-22T04:31:57+00:00" + }, + { + "name": "phpunit/php-file-iterator", + "version": "4.1.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-file-iterator.git", + "reference": "a95037b6d9e608ba092da1b23931e537cadc3c3c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/a95037b6d9e608ba092da1b23931e537cadc3c3c", + "reference": "a95037b6d9e608ba092da1b23931e537cadc3c3c", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "FilterIterator implementation that filters files based on a list of suffixes.", + "homepage": "https://github.com/sebastianbergmann/php-file-iterator/", + "keywords": [ + "filesystem", + "iterator" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-file-iterator/issues", + "security": "https://github.com/sebastianbergmann/php-file-iterator/security/policy", + "source": "https://github.com/sebastianbergmann/php-file-iterator/tree/4.1.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-08-31T06:24:48+00:00" + }, + { + "name": "phpunit/php-invoker", + "version": "4.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-invoker.git", + "reference": "f5e568ba02fa5ba0ddd0f618391d5a9ea50b06d7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-invoker/zipball/f5e568ba02fa5ba0ddd0f618391d5a9ea50b06d7", + "reference": "f5e568ba02fa5ba0ddd0f618391d5a9ea50b06d7", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "ext-pcntl": "*", + "phpunit/phpunit": "^10.0" + }, + "suggest": { + "ext-pcntl": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Invoke callables with a timeout", + "homepage": "https://github.com/sebastianbergmann/php-invoker/", + "keywords": [ + "process" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-invoker/issues", + "source": "https://github.com/sebastianbergmann/php-invoker/tree/4.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:56:09+00:00" + }, + { + "name": "phpunit/php-text-template", + "version": "3.0.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-text-template.git", + "reference": "0c7b06ff49e3d5072f057eb1fa59258bf287a748" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/0c7b06ff49e3d5072f057eb1fa59258bf287a748", + "reference": "0c7b06ff49e3d5072f057eb1fa59258bf287a748", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Simple template engine.", + "homepage": "https://github.com/sebastianbergmann/php-text-template/", + "keywords": [ + "template" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-text-template/issues", + "security": "https://github.com/sebastianbergmann/php-text-template/security/policy", + "source": "https://github.com/sebastianbergmann/php-text-template/tree/3.0.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-08-31T14:07:24+00:00" + }, + { + "name": "phpunit/php-timer", + "version": "6.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/php-timer.git", + "reference": "e2a2d67966e740530f4a3343fe2e030ffdc1161d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/e2a2d67966e740530f4a3343fe2e030ffdc1161d", + "reference": "e2a2d67966e740530f4a3343fe2e030ffdc1161d", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "6.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Utility class for timing", + "homepage": "https://github.com/sebastianbergmann/php-timer/", + "keywords": [ + "timer" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/php-timer/issues", + "source": "https://github.com/sebastianbergmann/php-timer/tree/6.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:57:52+00:00" + }, + { + "name": "phpunit/phpunit", + "version": "10.5.64", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/phpunit.git", + "reference": "0e8c1d19cea35ad97d4887f363d07c78e30fbf06" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/0e8c1d19cea35ad97d4887f363d07c78e30fbf06", + "reference": "0e8c1d19cea35ad97d4887f363d07c78e30fbf06", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-filter": "*", + "ext-json": "*", + "ext-libxml": "*", + "ext-mbstring": "*", + "ext-xmlwriter": "*", + "myclabs/deep-copy": "^1.13.4", + "phar-io/manifest": "^2.0.4", + "phar-io/version": "^3.2.1", + "php": ">=8.1", + "phpunit/php-code-coverage": "^10.1.16", + "phpunit/php-file-iterator": "^4.1.0", + "phpunit/php-invoker": "^4.0.0", + "phpunit/php-text-template": "^3.0.1", + "phpunit/php-timer": "^6.0.0", + "sebastian/cli-parser": "^2.0.1", + "sebastian/code-unit": "^2.0.0", + "sebastian/comparator": "^5.0.5", + "sebastian/diff": "^5.1.1", + "sebastian/environment": "^6.1.0", + "sebastian/exporter": "^5.1.4", + "sebastian/global-state": "^6.0.2", + "sebastian/object-enumerator": "^5.0.0", + "sebastian/recursion-context": "^5.0.1", + "sebastian/type": "^4.0.0", + "sebastian/version": "^4.0.1" + }, + "suggest": { + "ext-soap": "To be able to generate mocks based on WSDL files" + }, + "bin": [ + "phpunit" + ], + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "10.5-dev" + } + }, + "autoload": { + "files": [ + "src/Framework/Assert/Functions.php" + ], + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "The PHP Unit Testing framework.", + "homepage": "https://phpunit.de/", + "keywords": [ + "phpunit", + "testing", + "xunit" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/phpunit/issues", + "security": "https://github.com/sebastianbergmann/phpunit/security/policy", + "source": "https://github.com/sebastianbergmann/phpunit/tree/10.5.64" + }, + "funding": [ + { + "url": "https://phpunit.de/sponsoring.html", + "type": "other" + } + ], + "time": "2026-07-06T14:50:35+00:00" + }, + { + "name": "sebastian/cli-parser", + "version": "2.0.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/cli-parser.git", + "reference": "c34583b87e7b7a8055bf6c450c2c77ce32a24084" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/cli-parser/zipball/c34583b87e7b7a8055bf6c450c2c77ce32a24084", + "reference": "c34583b87e7b7a8055bf6c450c2c77ce32a24084", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for parsing CLI options", + "homepage": "https://github.com/sebastianbergmann/cli-parser", + "support": { + "issues": "https://github.com/sebastianbergmann/cli-parser/issues", + "security": "https://github.com/sebastianbergmann/cli-parser/security/policy", + "source": "https://github.com/sebastianbergmann/cli-parser/tree/2.0.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-02T07:12:49+00:00" + }, + { + "name": "sebastian/code-unit", + "version": "2.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/code-unit.git", + "reference": "a81fee9eef0b7a76af11d121767abc44c104e503" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit/zipball/a81fee9eef0b7a76af11d121767abc44c104e503", + "reference": "a81fee9eef0b7a76af11d121767abc44c104e503", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Collection of value objects that represent the PHP code units", + "homepage": "https://github.com/sebastianbergmann/code-unit", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit/issues", + "source": "https://github.com/sebastianbergmann/code-unit/tree/2.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:58:43+00:00" + }, + { + "name": "sebastian/code-unit-reverse-lookup", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/code-unit-reverse-lookup.git", + "reference": "5e3a687f7d8ae33fb362c5c0743794bbb2420a1d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/code-unit-reverse-lookup/zipball/5e3a687f7d8ae33fb362c5c0743794bbb2420a1d", + "reference": "5e3a687f7d8ae33fb362c5c0743794bbb2420a1d", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Looks up which function or method a line of code belongs to", + "homepage": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/", + "support": { + "issues": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/issues", + "source": "https://github.com/sebastianbergmann/code-unit-reverse-lookup/tree/3.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T06:59:15+00:00" + }, + { + "name": "sebastian/comparator", + "version": "5.0.5", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/comparator.git", + "reference": "55dfef806eb7dfeb6e7a6935601fef866f8ca48d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/55dfef806eb7dfeb6e7a6935601fef866f8ca48d", + "reference": "55dfef806eb7dfeb6e7a6935601fef866f8ca48d", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-mbstring": "*", + "php": ">=8.1", + "sebastian/diff": "^5.0", + "sebastian/exporter": "^5.0" + }, + "require-dev": { + "phpunit/phpunit": "^10.5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "5.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Volker Dusch", + "email": "github@wallbash.com" + }, + { + "name": "Bernhard Schussek", + "email": "bschussek@2bepublished.at" + } + ], + "description": "Provides the functionality to compare PHP values for equality", + "homepage": "https://github.com/sebastianbergmann/comparator", + "keywords": [ + "comparator", + "compare", + "equality" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/comparator/issues", + "security": "https://github.com/sebastianbergmann/comparator/security/policy", + "source": "https://github.com/sebastianbergmann/comparator/tree/5.0.5" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/comparator", + "type": "tidelift" + } + ], + "time": "2026-01-24T09:25:16+00:00" + }, + { + "name": "sebastian/complexity", + "version": "3.2.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/complexity.git", + "reference": "68ff824baeae169ec9f2137158ee529584553799" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/complexity/zipball/68ff824baeae169ec9f2137158ee529584553799", + "reference": "68ff824baeae169ec9f2137158ee529584553799", + "shasum": "" + }, + "require": { + "nikic/php-parser": "^4.18 || ^5.0", + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.2-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for calculating the complexity of PHP code units", + "homepage": "https://github.com/sebastianbergmann/complexity", + "support": { + "issues": "https://github.com/sebastianbergmann/complexity/issues", + "security": "https://github.com/sebastianbergmann/complexity/security/policy", + "source": "https://github.com/sebastianbergmann/complexity/tree/3.2.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-12-21T08:37:17+00:00" + }, + { + "name": "sebastian/diff", + "version": "5.1.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/diff.git", + "reference": "c41e007b4b62af48218231d6c2275e4c9b975b2e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/c41e007b4b62af48218231d6c2275e4c9b975b2e", + "reference": "c41e007b4b62af48218231d6c2275e4c9b975b2e", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0", + "symfony/process": "^6.4" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "5.1-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Kore Nordmann", + "email": "mail@kore-nordmann.de" + } + ], + "description": "Diff implementation", + "homepage": "https://github.com/sebastianbergmann/diff", + "keywords": [ + "diff", + "udiff", + "unidiff", + "unified diff" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/diff/issues", + "security": "https://github.com/sebastianbergmann/diff/security/policy", + "source": "https://github.com/sebastianbergmann/diff/tree/5.1.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-02T07:15:17+00:00" + }, + { + "name": "sebastian/environment", + "version": "6.1.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/environment.git", + "reference": "8074dbcd93529b357029f5cc5058fd3e43666984" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/8074dbcd93529b357029f5cc5058fd3e43666984", + "reference": "8074dbcd93529b357029f5cc5058fd3e43666984", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "suggest": { + "ext-posix": "*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "6.1-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Provides functionality to handle HHVM/PHP environments", + "homepage": "https://github.com/sebastianbergmann/environment", + "keywords": [ + "Xdebug", + "environment", + "hhvm" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/environment/issues", + "security": "https://github.com/sebastianbergmann/environment/security/policy", + "source": "https://github.com/sebastianbergmann/environment/tree/6.1.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-23T08:47:14+00:00" + }, + { + "name": "sebastian/exporter", + "version": "5.1.4", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/exporter.git", + "reference": "0735b90f4da94969541dac1da743446e276defa6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/0735b90f4da94969541dac1da743446e276defa6", + "reference": "0735b90f4da94969541dac1da743446e276defa6", + "shasum": "" + }, + "require": { + "ext-mbstring": "*", + "php": ">=8.1", + "sebastian/recursion-context": "^5.0" + }, + "require-dev": { + "phpunit/phpunit": "^10.5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "5.1-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Volker Dusch", + "email": "github@wallbash.com" + }, + { + "name": "Adam Harvey", + "email": "aharvey@php.net" + }, + { + "name": "Bernhard Schussek", + "email": "bschussek@gmail.com" + } + ], + "description": "Provides the functionality to export PHP variables for visualization", + "homepage": "https://www.github.com/sebastianbergmann/exporter", + "keywords": [ + "export", + "exporter" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/exporter/issues", + "security": "https://github.com/sebastianbergmann/exporter/security/policy", + "source": "https://github.com/sebastianbergmann/exporter/tree/5.1.4" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", + "type": "tidelift" + } + ], + "time": "2025-09-24T06:09:11+00:00" + }, + { + "name": "sebastian/global-state", + "version": "6.0.2", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/global-state.git", + "reference": "987bafff24ecc4c9ac418cab1145b96dd6e9cbd9" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/global-state/zipball/987bafff24ecc4c9ac418cab1145b96dd6e9cbd9", + "reference": "987bafff24ecc4c9ac418cab1145b96dd6e9cbd9", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "sebastian/object-reflector": "^3.0", + "sebastian/recursion-context": "^5.0" + }, + "require-dev": { + "ext-dom": "*", + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "6.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Snapshotting of global state", + "homepage": "https://www.github.com/sebastianbergmann/global-state", + "keywords": [ + "global state" + ], + "support": { + "issues": "https://github.com/sebastianbergmann/global-state/issues", + "security": "https://github.com/sebastianbergmann/global-state/security/policy", + "source": "https://github.com/sebastianbergmann/global-state/tree/6.0.2" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2024-03-02T07:19:19+00:00" + }, + { + "name": "sebastian/lines-of-code", + "version": "2.0.2", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/lines-of-code.git", + "reference": "856e7f6a75a84e339195d48c556f23be2ebf75d0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/lines-of-code/zipball/856e7f6a75a84e339195d48c556f23be2ebf75d0", + "reference": "856e7f6a75a84e339195d48c556f23be2ebf75d0", + "shasum": "" + }, + "require": { + "nikic/php-parser": "^4.18 || ^5.0", + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library for counting the lines of code in PHP source code", + "homepage": "https://github.com/sebastianbergmann/lines-of-code", + "support": { + "issues": "https://github.com/sebastianbergmann/lines-of-code/issues", + "security": "https://github.com/sebastianbergmann/lines-of-code/security/policy", + "source": "https://github.com/sebastianbergmann/lines-of-code/tree/2.0.2" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-12-21T08:38:20+00:00" + }, + { + "name": "sebastian/object-enumerator", + "version": "5.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/object-enumerator.git", + "reference": "202d0e344a580d7f7d04b3fafce6933e59dae906" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/object-enumerator/zipball/202d0e344a580d7f7d04b3fafce6933e59dae906", + "reference": "202d0e344a580d7f7d04b3fafce6933e59dae906", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "sebastian/object-reflector": "^3.0", + "sebastian/recursion-context": "^5.0" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "5.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Traverses array structures and object graphs to enumerate all referenced objects", + "homepage": "https://github.com/sebastianbergmann/object-enumerator/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-enumerator/issues", + "source": "https://github.com/sebastianbergmann/object-enumerator/tree/5.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T07:08:32+00:00" + }, + { + "name": "sebastian/object-reflector", + "version": "3.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/object-reflector.git", + "reference": "24ed13d98130f0e7122df55d06c5c4942a577957" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/object-reflector/zipball/24ed13d98130f0e7122df55d06c5c4942a577957", + "reference": "24ed13d98130f0e7122df55d06c5c4942a577957", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "3.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + } + ], + "description": "Allows reflection of object attributes, including inherited and non-public ones", + "homepage": "https://github.com/sebastianbergmann/object-reflector/", + "support": { + "issues": "https://github.com/sebastianbergmann/object-reflector/issues", + "source": "https://github.com/sebastianbergmann/object-reflector/tree/3.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T07:06:18+00:00" + }, + { + "name": "sebastian/recursion-context", + "version": "5.0.2", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/recursion-context.git", + "reference": "5d32fe257a9b39cb63146924d6b4e32a22d4502a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/5d32fe257a9b39cb63146924d6b4e32a22d4502a", + "reference": "5d32fe257a9b39cb63146924d6b4e32a22d4502a", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.5" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "5.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de" + }, + { + "name": "Jeff Welch", + "email": "whatthejeff@gmail.com" + }, + { + "name": "Adam Harvey", + "email": "aharvey@php.net" + } + ], + "description": "Provides functionality to recursively process PHP variables", + "homepage": "https://github.com/sebastianbergmann/recursion-context", + "support": { + "issues": "https://github.com/sebastianbergmann/recursion-context/issues", + "security": "https://github.com/sebastianbergmann/recursion-context/security/policy", + "source": "https://github.com/sebastianbergmann/recursion-context/tree/5.0.2" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/recursion-context", + "type": "tidelift" + } + ], + "time": "2026-08-11T05:27:39+00:00" + }, + { + "name": "sebastian/type", + "version": "4.0.0", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/type.git", + "reference": "462699a16464c3944eefc02ebdd77882bd3925bf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/type/zipball/462699a16464c3944eefc02ebdd77882bd3925bf", + "reference": "462699a16464c3944eefc02ebdd77882bd3925bf", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "phpunit/phpunit": "^10.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Collection of value objects that represent the types of the PHP type system", + "homepage": "https://github.com/sebastianbergmann/type", + "support": { + "issues": "https://github.com/sebastianbergmann/type/issues", + "source": "https://github.com/sebastianbergmann/type/tree/4.0.0" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-03T07:10:45+00:00" + }, + { + "name": "sebastian/version", + "version": "4.0.1", + "source": { + "type": "git", + "url": "https://github.com/sebastianbergmann/version.git", + "reference": "c51fa83a5d8f43f1402e3f32a005e6262244ef17" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/sebastianbergmann/version/zipball/c51fa83a5d8f43f1402e3f32a005e6262244ef17", + "reference": "c51fa83a5d8f43f1402e3f32a005e6262244ef17", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "4.0-dev" + } + }, + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Sebastian Bergmann", + "email": "sebastian@phpunit.de", + "role": "lead" + } + ], + "description": "Library that helps with managing the version number of Git-hosted PHP projects", + "homepage": "https://github.com/sebastianbergmann/version", + "support": { + "issues": "https://github.com/sebastianbergmann/version/issues", + "source": "https://github.com/sebastianbergmann/version/tree/4.0.1" + }, + "funding": [ + { + "url": "https://github.com/sebastianbergmann", + "type": "github" + } + ], + "time": "2023-02-07T11:34:05+00:00" + }, + { + "name": "theseer/tokenizer", + "version": "1.3.1", + "source": { + "type": "git", + "url": "https://github.com/theseer/tokenizer.git", + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/theseer/tokenizer/zipball/b7489ce515e168639d17feec34b8847c326b0b3c", + "reference": "b7489ce515e168639d17feec34b8847c326b0b3c", + "shasum": "" + }, + "require": { + "ext-dom": "*", + "ext-tokenizer": "*", + "ext-xmlwriter": "*", + "php": "^7.2 || ^8.0" + }, + "type": "library", + "autoload": { + "classmap": [ + "src/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "BSD-3-Clause" + ], + "authors": [ + { + "name": "Arne Blankerts", + "email": "arne@blankerts.de", + "role": "Developer" + } + ], + "description": "A small library for converting tokenized PHP source code into XML and potentially other formats", + "support": { + "issues": "https://github.com/theseer/tokenizer/issues", + "source": "https://github.com/theseer/tokenizer/tree/1.3.1" + }, + "funding": [ + { + "url": "https://github.com/theseer", + "type": "github" + } + ], + "time": "2025-11-17T20:03:58+00:00" + } + ], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": {}, + "prefer-stable": false, + "prefer-lowest": false, + "platform": { + "php": ">=8.5", + "ext-dom": "*", + "ext-intl": "*", + "ext-mbstring": "*", + "ext-json": "*", + "ext-openssl": "*", + "ext-sodium": "*" + }, + "platform-dev": {}, + "plugin-api-version": "2.9.0" +} diff --git a/php/src/Canonicalize.php b/php/src/Canonicalize.php index 68dc773..9934da8 100644 --- a/php/src/Canonicalize.php +++ b/php/src/Canonicalize.php @@ -3,7 +3,7 @@ * HTMLTrust Canonical Text Normalization * * Implements all 8 phases of the HTMLTrust canonicalization spec. - * Requires PHP 7.2+ with the intl extension (for Normalizer::normalize). + * Requires PHP 8.5+ with DOM and the built-in WHATWG URL API. * * Spec: https://github.com/HTMLTrust/htmltrust-canonicalization * @@ -16,6 +16,8 @@ class Canonicalize { + private const MAX_RESOURCE_BYTES = 1048576; + /** * Phase 6+7: Invisible/formatting/bidi characters to strip. * Preserves ZWNJ (U+200C) and ZWJ (U+200D) — semantic in Persian, Indic, emoji. @@ -88,6 +90,10 @@ class Canonicalize */ public static function normalizeText(string $text, bool $preserveWhitespace = false): string { + if (strlen($text) > self::MAX_RESOURCE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + // Phase 1: Unicode NFKC normalization. // Handles ligatures, fullwidth/halfwidth, presentation forms, // superscripts, CJK compatibility, Jamo composition, etc. @@ -115,6 +121,10 @@ public static function normalizeText(string $text, bool $preserveWhitespace = fa // Phase 5: Other punctuation. $text = preg_replace(self::ELLIPSIS_PATTERN, '...', $text); + if (strlen($text) > self::MAX_RESOURCE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + return $text; } @@ -162,12 +172,12 @@ public static function normalize(string $text): string private const BLOCK_ELEMENT_NAMES = 'address|article|aside|blockquote|details|dialog|div|dl|fieldset|figcaption' . '|figure|footer|form|h[1-6]|header|hgroup|hr|li|main|nav|ol|p' - . '|pre|section|table|tr|td|th|ul'; + . '|pre|section|signed-section|table|tr|td|th|ul'; /** * Any remaining HTML tag (inline elements stripped without adding whitespace). */ - private const ANY_TAG_PATTERN = '#]*>#i'; + private const ANY_TAG_PATTERN = '#]*\b[^>]*>#i'; /** * Full HTML5 named-entity table lives in Entities::NAMED (generated by @@ -269,72 +279,166 @@ private static function codepointToUtf8(int $cp): string */ public static function extractCanonicalText(string $html, bool $preserveWhitespace = false, ?string $baseUrl = null): string { + self::preflightHtmlSource($html); + // Validate the document base independently of the presence of href or + // src attributes. This keeps malformed bases from being silently + // accepted merely because this fragment has no URL-bearing element. + $baseUrl = self::validateBaseUrl($baseUrl); + if ($html === '') return ''; + $doc = new \DOMDocument('1.0', 'UTF-8'); + $old = libxml_use_internal_errors(true); + // libxml's HTML parser otherwise treats the first fragment element as + // the root and reparents following siblings. The encoding PI also + // prevents its ISO-8859-1 default from corrupting UTF-8 text. + $ok = $doc->loadHTML('' . $html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD); + libxml_clear_errors(); + libxml_use_internal_errors($old); + if (!$ok) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } $parts = []; - $index = 0; - $excludedDepth = 0; - $tokenPattern = '#|]*>|]*)?\s*/?>#i'; - preg_match_all($tokenPattern, $html, $matches, PREG_OFFSET_CAPTURE); + foreach ($doc->childNodes as $child) { + self::walkDomNode($child, $parts, $preserveWhitespace, $baseUrl); + } + return self::finalizeCanonicalParts($parts); + } + private static function preflightHtmlSource(string $html): void + { + if (strlen($html) > self::MAX_RESOURCE_BYTES || preg_match('//u', $html) !== 1) { + throw new \InvalidArgumentException(strlen($html) > self::MAX_RESOURCE_BYTES ? 'resource-limit-exceeded' : 'parser-profile-unsupported'); + } + $scan = preg_replace_callback( + '#(<\s*(script|style|iframe)\b(?:[^>"\']+|"[^"]*"|\'[^\']*\')*>)[\s\S]*?()#i', + static function (array $match): string { return $match[1] . $match[3]; }, + $html + ); + if ($scan === null) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + if (preg_match('#<\s*(?:svg|math|foreignObject)\b#i', $scan)) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + // Reject ambiguous/unterminated references. HTML's error recovery + // would otherwise make `&!` and `A!` differ by parser. + if (preg_match('/&(?:[A-Za-z][A-Za-z0-9]*|#\d+|#[xX][0-9A-Fa-f]+)(?!;)(?=[^A-Za-z0-9]|$)/', $scan)) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + preg_match_all('/&([A-Za-z][A-Za-z0-9]*);/', $scan, $entities); + foreach ($entities[0] as $entity) { + if (!array_key_exists($entity, Entities::NAMED)) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + } + // Validate comments/declarations before libxml can repair them. An + // HTML comment cannot contain `--`, and declarations are outside the + // signed fragment profile. + preg_match_all('//', $scan, $commentEnds); + if (count($commentStarts[0]) !== count($commentEnds[0])) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + preg_match_all('//', $scan, $comments); + foreach ($comments[1] as $comment) { + if (strpos($comment, '--') !== false || str_ends_with($comment, '-')) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + } + $withoutComments = preg_replace('//', '', $scan); + if (!is_string($withoutComments) || preg_match('/|]*(?:[\t\n\f\r ]+(?:[^>"\']+|"[^"]*"|\'[^\']*\')*)?\s*/?>#i', $scan, $matches, PREG_OFFSET_CAPTURE); + $stack = []; + $index = 0; foreach ($matches[0] as $match) { [$token, $offset] = $match; - if ($offset > $index && $excludedDepth === 0) { - self::appendCanonicalPart( - $parts, - self::normalizeText(self::decodeEntities(substr($html, $index, $offset - $index)), $preserveWhitespace) - ); + $text = substr($scan, $index, $offset - $index); + if ($stack && end($stack) === 'table' && trim($text) !== '') { + throw new \InvalidArgumentException('parser-profile-unsupported'); } $index = $offset + strlen($token); - - if (!preg_match('#^]*)#i', $token, $m)) { continue; } - $name = strtolower($nameMatch[1]); + $name = strtolower($m[1]); $trimmed = trim($token); - $closing = strpos($trimmed, '$#', $trimmed) === 1 || self::isVoidElement($name); - $excluded = self::isExcludedElement($name); - - if ($closing) { - if ($excluded && $excludedDepth > 0) { - $excludedDepth--; - continue; - } - if ($excludedDepth > 0) { - continue; - } - if (self::isBlockElement($name)) { - self::appendCanonicalPart($parts, "\n"); + if (strpos($trimmed, ']*|/?>$#i', '', trim($token)); + preg_match_all('#([^\s"\'<>/=]+)(?:\s*=\s*(?:"[^"]*"|\'[^\']*\'|[^\s"\'=<>`]+))?#', $raw, $am); + $seen = []; + foreach ($am[1] as $attr) { + $key = strtolower($attr); + if (isset($seen[$key])) { + throw new \InvalidArgumentException('parser-profile-unsupported'); } - continue; + $seen[$key] = true; } - if ($excludedDepth > 0) { - continue; + if (!self::isVoidElement($name) && preg_match('#/\s*>$#', $trimmed) !== 1) { + if (count($stack) >= 256) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + $stack[] = $name; } + } + $text = substr($scan, $index); + if ($stack && end($stack) === 'table' && trim($text) !== '') { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + if ($stack) { + throw new \InvalidArgumentException('parser-profile-unsupported'); + } + } - self::appendAttributeRecords($parts, $name, self::parseAttributes($token), $baseUrl); - if ($name === 'br') { - self::appendCanonicalPart($parts, "\n"); + private static function walkDomNode(\DOMNode $node, array &$parts, bool $preserveWhitespace, ?string $baseUrl): void + { + if ($node->nodeType === XML_TEXT_NODE) { + self::appendCanonicalPart($parts, str_replace('@', '@@', self::normalizeText($node->nodeValue ?? '', $preserveWhitespace))); + return; + } + if ($node->nodeType !== XML_ELEMENT_NODE) { + return; + } + $name = strtolower($node->localName ?: $node->nodeName); + if (self::isExcludedElement($name)) { + return; + } + $attrs = []; + if ($node->hasAttributes()) { + foreach ($node->attributes as $attr) { + $attrs[strtolower($attr->name)] = $attr->value; } - if ($selfClosing && self::isBlockElement($name)) { - self::appendCanonicalPart($parts, "\n"); + } + self::appendAttributeRecords($parts, $name, $attrs, $baseUrl); + if ($name === 'br') { + self::appendCanonicalPart($parts, "\n"); + } else { + foreach ($node->childNodes as $child) { + self::walkDomNode($child, $parts, $preserveWhitespace, $baseUrl); } } - - if ($index < strlen($html) && $excludedDepth === 0) { - self::appendCanonicalPart( - $parts, - self::normalizeText(self::decodeEntities(substr($html, $index)), $preserveWhitespace) - ); + if (self::isBlockElement($name)) { + self::appendCanonicalPart($parts, "\n"); } - - return self::finalizeCanonicalParts($parts); } /** @@ -353,6 +457,9 @@ private static function appendCanonicalPart(array &$parts, string $value): void private static function finalizeCanonicalParts(array $parts): string { $text = implode('', $parts); + if (strlen($text) > self::MAX_RESOURCE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } while (strpos($text, ' ') !== false) { $text = str_replace(' ', ' ', $text); } @@ -368,7 +475,7 @@ private static function finalizeCanonicalParts(array $parts): string */ private static function parseAttributes(string $token): array { - $body = preg_replace('#^]*#i', '', $token); $body = preg_replace('#/?>$#', '', trim($body)); preg_match_all( '#([^\s"\'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s"\'=<>`]+)))?#', @@ -384,6 +491,29 @@ private static function parseAttributes(string $token): array return $attrs; } + private static function validateBaseUrl(?string $baseUrl): ?string + { + if ($baseUrl === null || $baseUrl === '') { + return null; + } + try { + $base = \Uri\WhatWg\Url::parse($baseUrl); + } catch (\Throwable $e) { + throw new \InvalidArgumentException('attribute-canonicalization-failed'); + } + if ($base === null) { + throw new \InvalidArgumentException('attribute-canonicalization-failed'); + } + if (strtolower($base->getScheme()) !== 'https' + || $base->getAsciiHost() === null + || $base->getAsciiHost() === '' + || $base->getUsername() !== null + || $base->getPassword() !== null) { + throw new \InvalidArgumentException('url-policy-violation'); + } + return $base->toAsciiString(); + } + /** * @param array $parts * @param array $attrs @@ -408,6 +538,7 @@ private static function appendAttributeRecords(array &$parts, string $elementNam if (strpos($value, "\n") !== false) { throw new \InvalidArgumentException('attribute-canonicalization-failed'); } + $value = str_replace('@', '@@', $value); if (!empty($parts)) { $last = $parts[count($parts) - 1]; if ($last !== '' && substr($last, -1) !== ' ' && substr($last, -1) !== "\n") { @@ -420,146 +551,37 @@ private static function appendAttributeRecords(array &$parts, string $elementNam private static function normalizeUrlAttribute(string $value, ?string $baseUrl): string { - $value = trim($value); - if (preg_match('#^[a-z][a-z0-9+.-]*:#i', $value)) { - $absolute = $value; - } elseif ($baseUrl !== null) { - $absolute = self::resolveUrl($value, $baseUrl); - } else { - throw new \InvalidArgumentException('attribute-canonicalization-failed'); - } - - $parts = parse_url($absolute); - if ($parts === false || empty($parts['scheme'])) { - throw new \InvalidArgumentException('attribute-canonicalization-failed'); + for ($i = 0; $i < strlen($value); $i++) { + $ord = ord($value[$i]); + if ($ord <= 0x1F || $ord === 0x7F) { + throw new \InvalidArgumentException('url-policy-violation'); + } } - if (empty($parts['host'])) { - // Opaque URL with no authority (mailto:, tel:, javascript:, data:, - // about:, sms:, ...). The WHATWG URL parser accepts these; serialize - // scheme + opaque remainder verbatim (scheme lowercased), matching - // new URL().href. No host/port/dot-segment normalization applies. - $colon = strpos($absolute, ':'); - return strtolower(substr($absolute, 0, $colon)) . ':' . substr($absolute, $colon + 1); - } - $scheme = strtolower($parts['scheme']); - $host = strtolower($parts['host']); - // IDNA/punycode non-ASCII hosts to match the WHATWG URL serializer. - if (preg_match('/[^\x00-\x7F]/', $host)) { - $ascii = idn_to_ascii($host, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46); - if ($ascii === false) { + $base = null; + if ($baseUrl !== null) { + $base = \Uri\WhatWg\Url::parse($baseUrl); + if ($base === null) { throw new \InvalidArgumentException('attribute-canonicalization-failed'); } - $host = $ascii; - } - $port = isset($parts['port']) ? (int) $parts['port'] : null; - $authority = $host; - if ($port !== null && !(($scheme === 'http' && $port === 80) || ($scheme === 'https' && $port === 443))) { - $authority .= ':' . $port; - } - $path = self::removeDotSegments($parts['path'] ?? '/'); - if ($path === '') { - $path = '/'; - } - $query = isset($parts['query']) ? '?' . $parts['query'] : ''; - $fragment = isset($parts['fragment']) ? '#' . $parts['fragment'] : ''; - return $scheme . '://' . $authority . $path . $query . $fragment; - } - - /** - * RFC 3986 §5.2.4 remove_dot_segments, matching the WHATWG URL path - * normalization the reference JS/Rust bindings perform via `new URL`. - */ - private static function removeDotSegments(string $path): string - { - $out = ''; - $in = $path; - while ($in !== '') { - if (strpos($in, '../') === 0) { - $in = substr($in, 3); - } elseif (strpos($in, './') === 0) { - $in = substr($in, 2); - } elseif (strpos($in, '/./') === 0) { - $in = '/' . substr($in, 3); - } elseif ($in === '/.') { - $in = '/'; - } elseif (strpos($in, '/../') === 0) { - $in = '/' . substr($in, 4); - $pos = strrpos($out, '/'); - $out = $pos !== false ? substr($out, 0, $pos) : ''; - } elseif ($in === '/..') { - $in = '/'; - $pos = strrpos($out, '/'); - $out = $pos !== false ? substr($out, 0, $pos) : ''; - } elseif ($in === '.' || $in === '..') { - $in = ''; - } else { - $start = strpos($in, '/') === 0 ? 1 : 0; - $slash = strpos($in, '/', $start); - if ($slash === false) { - $out .= $in; - $in = ''; - } else { - $out .= substr($in, 0, $slash); - $in = substr($in, $slash); - } + if (strtolower($base->getScheme()) !== 'https' + || $base->getUsername() !== null + || $base->getPassword() !== null) { + throw new \InvalidArgumentException('url-policy-violation'); } } - return $out; - } - - /** - * Resolve a relative reference against a base URL using the RFC 3986 §5.2 - * transform-reference algorithm (the same algorithm the JS/Go/Python/Rust - * bindings get from `new URL` / `ResolveReference` / `urljoin`). The naive - * "strip last segment and append" resolver this replaces mishandled - * fragment-only (`#x`), query-only (`?x`), and empty references, silently - * truncating the path. - */ - private static function resolveUrl(string $relative, string $baseUrl): string - { - $b = parse_url($baseUrl); - if ($b === false || empty($b['scheme']) || empty($b['host'])) { - throw new \InvalidArgumentException('attribute-canonicalization-failed'); - } - $r = parse_url($relative); - if ($r === false) { + $url = \Uri\WhatWg\Url::parse($value, $base); + if ($url === null) { throw new \InvalidArgumentException('attribute-canonicalization-failed'); } - $bAuthority = strtolower($b['host']) - . (isset($b['port']) ? ':' . (int) $b['port'] : ''); - $bPath = $b['path'] ?? ''; - - if (isset($r['host'])) { - // Protocol-relative //host/path form. - $authority = strtolower($r['host']) . (isset($r['port']) ? ':' . (int) $r['port'] : ''); - $path = self::removeDotSegments($r['path'] ?? ''); - $query = $r['query'] ?? null; - } else { - $authority = $bAuthority; - $rPath = $r['path'] ?? ''; - if ($rPath === '') { - $path = $bPath; - $query = $r['query'] ?? ($b['query'] ?? null); - } else { - if ($rPath[0] === '/') { - $path = self::removeDotSegments($rPath); - } else { - // merge(base, ref): base path up to and including last '/' - $merged = ($bPath === '' ? '/' : substr($bPath, 0, strrpos($bPath, '/') + 1)) . $rPath; - $path = self::removeDotSegments($merged); - } - $query = $r['query'] ?? null; - } - } - $scheme = strtolower($b['scheme']); - $out = $scheme . '://' . $authority . $path; - if ($query !== null) { - $out .= '?' . $query; + if (strtolower($url->getScheme()) !== 'https' + || $url->getUsername() !== null + || $url->getPassword() !== null) { + throw new \InvalidArgumentException('url-policy-violation'); } - if (isset($r['fragment'])) { - $out .= '#' . $r['fragment']; + if ($url->getAsciiHost() === null || $url->getAsciiHost() === '') { + throw new \InvalidArgumentException('attribute-canonicalization-failed'); } - return $out; + return $url->toAsciiString(); } private static function isVoidElement(string $name): bool @@ -592,11 +614,20 @@ private static function isBlockElement(string $name): bool */ public static function canonicalizeClaims(array $claims): string { + if (count($claims) > 64) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } $entries = []; $seen = []; foreach ($claims as $name => $value) { + if (!is_string($name) || !is_string($value)) { + throw new \InvalidArgumentException('claim-malformed'); + } $normName = trim(self::normalizeText((string) $name)); - $normValue = trim(self::normalizeText((string) $value)); + $normValue = trim(self::normalizeText($value)); + if (strlen($normName) > 4096 || strlen($normValue) > 4096) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } if ($normName === '') { throw new \InvalidArgumentException('claim-malformed'); } @@ -615,8 +646,265 @@ public static function canonicalizeClaims(array $claims): string $out = ''; foreach ($entries as [$name, $value]) { - $out .= $name . ':' . $value . "\n"; + $escape = static function (string $v): string { + return str_replace(["\\", ":", "\n"], ["\\\\", "\\:", "\\n"], $v); + }; + $out .= $escape($name) . ':' . $escape($value) . "\n"; + } + if (strlen($out) > self::MAX_RESOURCE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); } return $out; } + + /** Extract direct-child claim metadata from a signed-section snapshot. */ + public static function extractClaimsFromSignedSection(string $html): array + { + if ($html === '') return []; + self::preflightHtmlSource($html); + $doc = new \DOMDocument('1.0', 'UTF-8'); + @$doc->loadHTML('' . $html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD); + $section = null; + foreach ($doc->getElementsByTagName('signed-section') as $candidate) { + $section = $candidate; + break; + } + $parent = $section ?: $doc; + $claims = []; + foreach ($parent->childNodes as $child) { + if (!($child instanceof \DOMElement) || strtolower($child->tagName) !== 'meta') continue; + if (!$child->hasAttribute('name') || !$child->hasAttribute('content')) { + throw new \InvalidArgumentException('claim-malformed'); + } + if (count($claims) >= 64) throw new \InvalidArgumentException('resource-limit-exceeded'); + $name = trim(self::normalizeText($child->getAttribute('name'))); + $value = trim(self::normalizeText($child->getAttribute('content'))); + if ($name === '') throw new \InvalidArgumentException('claim-malformed'); + if (strlen($name) > 4096 || strlen($value) > 4096) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + if (array_key_exists($name, $claims)) throw new \InvalidArgumentException('claim-duplicate'); + $claims[$name] = $value; + } + return $claims; + } + + /** Strict RFC 8785 canonicalization of one raw JSON document. */ + public static function canonicalizeJsonDocument(string $document): string + { + if (strlen($document) > self::MAX_RESOURCE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + $pos = 0; + self::scanJsonValue($document, $pos, 0); + self::skipJsonWhitespace($document, $pos); + if ($pos !== strlen($document)) { + throw new \InvalidArgumentException('jcs-invalid-json'); + } + try { + // Decode objects as stdClass so `{}` and objects with numeric + // member names cannot collapse into PHP list arrays. + $value = json_decode($document, false, 512, JSON_THROW_ON_ERROR); + // JSON numbers are IEEE-754 binary64 values in RFC 8785. PHP's + // decoder keeps some of them as integers, which would otherwise + // make the result depend on the host integer width and bypass + // the ECMAScript number serializer used by the package. + $value = self::coerceJsonNumbersToFloat($value); + if (is_object($value) || is_array($value) || is_scalar($value) || $value === null) { + $canonical = self::serializeJcsValue($value); + if (strlen($canonical) > self::MAX_RESOURCE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + return $canonical; + } + } catch (\Throwable $e) { + $message = $e->getMessage(); + if ($message === 'resource-limit-exceeded') { + throw $e; + } + if (strpos($message, 'UTF-8') !== false || strpos($message, 'surrogate') !== false) { + throw new \InvalidArgumentException('jcs-invalid-surrogate', 0, $e); + } + throw new \InvalidArgumentException('jcs-number', 0, $e); + } + throw new \InvalidArgumentException('jcs-invalid-json'); + } + + /** @param mixed $value */ + private static function serializeJcsValue($value): string + { + if ($value === null) return 'null'; + if (is_bool($value)) return $value ? 'true' : 'false'; + if (is_int($value)) return self::serializeJcsNumber((float) $value); + if (is_float($value)) return self::serializeJcsNumber($value); + if (is_string($value)) { + return json_encode($value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + } + if (is_array($value)) { + $items = []; + foreach ($value as $item) { + $items[] = self::serializeJcsValue($item); + } + return '[' . implode(',', $items) . ']'; + } + if (is_object($value)) { + $members = get_object_vars($value); + uksort($members, static function (string $left, string $right): int { + return strcmp( + mb_convert_encoding($left, 'UTF-16BE', 'UTF-8'), + mb_convert_encoding($right, 'UTF-16BE', 'UTF-8') + ); + }); + $items = []; + foreach ($members as $key => $item) { + $items[] = json_encode((string) $key, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) + . ':' . self::serializeJcsValue($item); + } + return '{' . implode(',', $items) . '}'; + } + throw new \InvalidArgumentException('jcs-invalid-json'); + } + + private static function serializeJcsNumber(float $number): string + { + if (is_nan($number) || is_infinite($number)) { + throw new \InvalidArgumentException('jcs-number'); + } + if ($number == 0.0) return '0'; + $previousPrecision = ini_get('serialize_precision'); + // json_encode otherwise inherits a process-wide precision setting, + // which must not affect a signed RFC 8785 payload. + @ini_set('serialize_precision', '-1'); + try { + $encoded = json_encode($number, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); + } finally { + if ($previousPrecision !== false) { + @ini_set('serialize_precision', $previousPrecision); + } + } + if (strpos($encoded, 'e') === false && strpos($encoded, 'E') === false) { + return $encoded; + } + $encoded = strtolower($encoded); + [$mantissa, $exponent] = explode('e', $encoded, 2); + $exponent = (int) $exponent; + $sign = ''; + if ($mantissa[0] === '-') { + $sign = '-'; + $mantissa = substr($mantissa, 1); + } + $digits = str_replace('.', '', $mantissa); + // PHP emits an insignificant trailing zero for integral doubles in + // scientific notation (for example `1.0e-6`). + $digits = rtrim($digits, '0'); + $decimalPosition = 1 + $exponent; + // ECMAScript uses decimal notation for 1e-6 <= abs(x) < 1e21. + if ($exponent >= -6 && $exponent < 21) { + if ($decimalPosition <= 0) { + return $sign . '0.' . str_repeat('0', -$decimalPosition) . $digits; + } + if ($decimalPosition >= strlen($digits)) { + return $sign . $digits . str_repeat('0', $decimalPosition - strlen($digits)); + } + return $sign . substr($digits, 0, $decimalPosition) . '.' . substr($digits, $decimalPosition); + } + $digits = rtrim($digits, '0'); + $mantissa = $digits[0] . (strlen($digits) > 1 ? '.' . substr($digits, 1) : ''); + $expSign = $exponent < 0 ? '-' : '+'; + return $sign . $mantissa . 'e' . $expSign . abs($exponent); + } + + private static function skipJsonWhitespace(string $s, int &$i): void + { + while ($i < strlen($s) && strpos(" \t\r\n", $s[$i]) !== false) $i++; + } + + private static function scanJsonString(string $s, int &$i): string + { + $start = $i++; + $escaped = false; + while ($i < strlen($s)) { + $c = $s[$i++]; + if ($escaped) { $escaped = false; continue; } + if ($c === '\\') { $escaped = true; continue; } + if ($c === '"') { + $decoded = json_decode(substr($s, $start, $i - $start), true); + if (!is_string($decoded)) { + if (preg_match('/\\\\u[dD][89A-Fa-f0-9]{3}/', substr($s, $start, $i - $start))) { + throw new \InvalidArgumentException('jcs-invalid-surrogate'); + } + throw new \InvalidArgumentException('jcs-invalid-json'); + } + return $decoded; + } + if (ord($c) < 0x20) throw new \InvalidArgumentException('jcs-invalid-json'); + } + throw new \InvalidArgumentException('jcs-invalid-json'); + } + + private static function scanJsonValue(string $s, int &$i, int $depth): void + { + self::skipJsonWhitespace($s, $i); + if ($i >= strlen($s)) throw new \InvalidArgumentException('jcs-invalid-json'); + if ($s[$i] === '"') { self::scanJsonString($s, $i); return; } + if ($s[$i] === '{') { + if ($depth >= 256) throw new \InvalidArgumentException('resource-limit-exceeded'); + $i++; self::skipJsonWhitespace($s, $i); $seen = []; + if ($i < strlen($s) && $s[$i] === '}') { $i++; return; } + while (true) { + self::skipJsonWhitespace($s, $i); + if ($i >= strlen($s) || $s[$i] !== '"') throw new \InvalidArgumentException('jcs-invalid-json'); + $key = self::scanJsonString($s, $i); + if (isset($seen[$key])) throw new \InvalidArgumentException('jcs-duplicate-key'); + $seen[$key] = true; + self::skipJsonWhitespace($s, $i); + if ($i >= strlen($s) || $s[$i++] !== ':') throw new \InvalidArgumentException('jcs-invalid-json'); + self::scanJsonValue($s, $i, $depth + 1); self::skipJsonWhitespace($s, $i); + if ($i < strlen($s) && $s[$i] === '}') { $i++; return; } + if ($i >= strlen($s) || $s[$i++] !== ',') throw new \InvalidArgumentException('jcs-invalid-json'); + } + } + if ($s[$i] === '[') { + if ($depth >= 256) throw new \InvalidArgumentException('resource-limit-exceeded'); + $i++; self::skipJsonWhitespace($s, $i); + if ($i < strlen($s) && $s[$i] === ']') { $i++; return; } + while (true) { + self::scanJsonValue($s, $i, $depth + 1); self::skipJsonWhitespace($s, $i); + if ($i < strlen($s) && $s[$i] === ']') { $i++; return; } + if ($i >= strlen($s) || $s[$i++] !== ',') throw new \InvalidArgumentException('jcs-invalid-json'); + } + } + $start = $i; + while ($i < strlen($s) && strpos(" \t\r\n,]}", $s[$i]) === false) $i++; + $token = substr($s, $start, $i - $start); + if (!preg_match('/^(?:true|false|null|-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)$/', $token)) { + throw new \InvalidArgumentException('jcs-invalid-json'); + } + // RFC 8785 delegates number semantics to IEEE-754 binary64. This + // permits integers larger than 2^53 when they have a finite binary64 + // representation, and rounds them exactly as ECMAScript does. + if (is_infinite((float) $token)) { + throw new \InvalidArgumentException('jcs-number'); + } + } + + /** @param mixed $value */ + private static function coerceJsonNumbersToFloat($value) + { + if (is_int($value)) { + return (float) $value; + } + if (is_array($value)) { + foreach ($value as $key => $item) { + $value[$key] = self::coerceJsonNumbersToFloat($item); + } + return $value; + } + if (is_object($value)) { + foreach (get_object_vars($value) as $key => $item) { + $value->{$key} = self::coerceJsonNumbersToFloat($item); + } + } + return $value; + } } diff --git a/php/src/Keys/DidWebResolver.php b/php/src/Keys/DidWebResolver.php index f48a4d7..f0d394c 100644 --- a/php/src/Keys/DidWebResolver.php +++ b/php/src/Keys/DidWebResolver.php @@ -41,7 +41,7 @@ public function resolve(string $keyid): ?ResolvedKey return null; } - $response = ($this->fetcher)($url); + $response = HttpFetcher::validateResponse(($this->fetcher)($url)); if ($response === null) { return null; } @@ -50,6 +50,9 @@ public function resolve(string $keyid): ?ResolvedKey if (!is_array($doc)) { return null; } + if (($doc['deactivated'] ?? false) === true) { + return null; + } $methods = $doc['verificationMethod'] ?? null; if (!is_array($methods)) { @@ -66,7 +69,20 @@ public function resolve(string $keyid): ?ResolvedKey } $algorithm = self::guessAlgorithm($method); - return new ResolvedKey($pem, $algorithm, $keyid); + $revoked = isset($method['revoked']) && is_bool($method['revoked']) + ? $method['revoked'] + : false; + $expires = isset($method['expires']) && is_string($method['expires']) && $method['expires'] !== '' + ? $method['expires'] + : null; + $resolved = new ResolvedKey($pem, $algorithm, $keyid, $revoked, $expires); + // Expired, malformed-expiry, or explicitly revoked verification + // methods are resolution failures. Continue so a later live + // method can still satisfy the DID lookup, matching JS. + if ($resolved->isRevoked()) { + continue; + } + return $resolved; } return null; @@ -87,9 +103,9 @@ private static function didWebToUrl(string $keyid): ?string // Strip any fragment (e.g. did:web:example.com#keys-1) — the fragment // identifies a verificationMethod, but the document URL is the same. - $hash = strpos($rest, '#'); - if ($hash !== false) { - $rest = substr($rest, 0, $hash); + $suffix = strcspn($rest, '/?#'); + if ($suffix < strlen($rest)) { + $rest = substr($rest, 0, $suffix); } $parts = explode(':', $rest); @@ -97,16 +113,59 @@ private static function didWebToUrl(string $keyid): ?string if ($domain === null || $domain === '') { return null; } - // did:web percent-encodes ports as %3A; decode for URL building. - $domain = rawurldecode($domain); + // did:web percent-encodes only the authority's port colon. + $domain = str_ireplace('%3A', ':', $domain); + if (str_contains($domain, '%')) { + return null; + } + try { + $authorityUrl = \Uri\WhatWg\Url::parse('https://' . $domain); + } catch (\Throwable $error) { + return null; + } + if ($authorityUrl === null + || strtolower($authorityUrl->getScheme()) !== 'https' + || $authorityUrl->getAsciiHost() === null + || $authorityUrl->getAsciiHost() === '' + || $authorityUrl->getUsername() !== null + || $authorityUrl->getPassword() !== null + || $authorityUrl->getPath() !== '/' + || $authorityUrl->getQuery() !== null + || $authorityUrl->getFragment() !== null) { + return null; + } + $domain = $authorityUrl->getAsciiHost(); + if ($authorityUrl->getPort() !== null) { + $domain .= ':' . $authorityUrl->getPort(); + } if (count($parts) === 0) { return 'https://' . $domain . '/.well-known/did.json'; } - $path = implode('/', array_map('rawurldecode', $parts)); + $encodedParts = []; + foreach ($parts as $part) { + $encoded = self::encodePathPart($part); + if ($encoded === null) { + return null; + } + $encodedParts[] = $encoded; + } + $path = implode('/', $encodedParts); return 'https://' . $domain . '/' . $path . '/did.json'; } + private static function encodePathPart(string $part): ?string + { + if ($part === '' || preg_match('/%(?![0-9A-Fa-f]{2})/', $part) === 1) { + return null; + } + return preg_replace_callback( + '/%25([0-9A-Fa-f]{2})/', + static fn (array $match): string => '%' . $match[1], + rawurlencode($part) + ); + } + /** * Best-effort algorithm hint from a verificationMethod entry. * The "type" field is conventional but inconsistent across DID diff --git a/php/src/Keys/DirectUrlResolver.php b/php/src/Keys/DirectUrlResolver.php index 6e7bf41..f48936d 100644 --- a/php/src/Keys/DirectUrlResolver.php +++ b/php/src/Keys/DirectUrlResolver.php @@ -34,7 +34,7 @@ public function resolve(string $keyid): ?ResolvedKey return null; } - $response = ($this->fetcher)($keyid); + $response = HttpFetcher::validateResponse(($this->fetcher)($keyid)); if ($response === null) { return null; } @@ -64,7 +64,13 @@ public function resolve(string $keyid): ?ResolvedKey $algorithm = isset($decoded['algorithm']) && is_string($decoded['algorithm']) && $decoded['algorithm'] !== '' ? strtolower($decoded['algorithm']) : 'ed25519'; + $revoked = isset($decoded['revoked']) && is_bool($decoded['revoked']) + ? $decoded['revoked'] + : false; + $expires = isset($decoded['expires']) && is_string($decoded['expires']) && $decoded['expires'] !== '' + ? $decoded['expires'] + : null; - return new ResolvedKey($pem, $algorithm, $keyid); + return new ResolvedKey($pem, $algorithm, $keyid, $revoked, $expires); } } diff --git a/php/src/Keys/HttpFetcher.php b/php/src/Keys/HttpFetcher.php index 95722d6..80b0741 100644 --- a/php/src/Keys/HttpFetcher.php +++ b/php/src/Keys/HttpFetcher.php @@ -17,6 +17,30 @@ final class HttpFetcher { + private const MAX_RESPONSE_BYTES = 64 * 1024; + + /** + * Validate a response returned by either the default or an injected + * fetcher. Injection is useful for tests and alternate transports, but it + * must retain the same response-size bound as the built-in transport. + * + * @param array{body: string, contentType?: string}|null $response + * @return array{body: string, contentType?: string}|null + */ + public static function validateResponse(?array $response): ?array + { + if ($response === null) { + return null; + } + if (!isset($response['body']) || !is_string($response['body'])) { + throw new \InvalidArgumentException('invalid response body'); + } + if (strlen($response['body']) > self::MAX_RESPONSE_BYTES) { + throw new \InvalidArgumentException('resource-limit-exceeded'); + } + return $response; + } + /** * Returns a callable suitable for injection into a KeyResolver: * @@ -38,10 +62,13 @@ public static function default(): callable if (!is_readable($path)) { return null; } - $body = @file_get_contents($path); - if ($body === false) { + $stream = @fopen($path, 'rb'); + if ($stream === false) { return null; } + $body = self::readLimitedStream($stream); + fclose($stream); + if ($body === null) return null; return ['body' => $body, 'contentType' => self::guessContentTypeFromPath($path)]; } @@ -52,9 +79,12 @@ public static function default(): callable if ($handle === false) { return null; } + $body = ''; + $bodyBytes = 0; + $tooLarge = false; curl_setopt_array($handle, [ CURLOPT_URL => $url, - CURLOPT_RETURNTRANSFER => true, + CURLOPT_RETURNTRANSFER => false, CURLOPT_FOLLOWLOCATION => true, CURLOPT_MAXREDIRS => 5, CURLOPT_CONNECTTIMEOUT => 5, @@ -62,16 +92,26 @@ public static function default(): callable CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYHOST => 2, CURLOPT_HTTPHEADER => ['Accept: application/json, application/did+json, application/x-pem-file, */*'], + CURLOPT_WRITEFUNCTION => static function ($handle, string $chunk) use (&$body, &$bodyBytes, &$tooLarge): int { + $length = strlen($chunk); + if ($bodyBytes + $length > self::MAX_RESPONSE_BYTES) { + $tooLarge = true; + return 0; + } + $body .= $chunk; + $bodyBytes += $length; + return $length; + }, ]); - $body = curl_exec($handle); + $ok = curl_exec($handle); $code = (int) curl_getinfo($handle, CURLINFO_HTTP_CODE); $type = (string) curl_getinfo($handle, CURLINFO_CONTENT_TYPE); curl_close($handle); - if ($body === false || $code < 200 || $code >= 300) { + if ($tooLarge || $ok === false || $code < 200 || $code >= 300) { return null; } - return ['body' => (string) $body, 'contentType' => $type]; + return ['body' => $body, 'contentType' => $type]; } // file_get_contents fallback. @@ -85,15 +125,18 @@ public static function default(): callable 'verify_peer_name' => true, ], ]); - $body = @file_get_contents($url, false, $context); - if ($body === false) { + $stream = @fopen($url, 'rb', false, $context); + if ($stream === false) { return null; } + $body = self::readLimitedStream($stream); + fclose($stream); + if ($body === null) return null; $contentType = ''; - // $http_response_header is populated by file_get_contents. - if (isset($http_response_header) && is_array($http_response_header)) { - foreach ($http_response_header as $h) { + $responseHeaders = http_get_last_response_headers(); + if (is_array($responseHeaders)) { + foreach ($responseHeaders as $h) { if (stripos($h, 'content-type:') === 0) { $contentType = trim(substr($h, strlen('content-type:'))); break; @@ -113,4 +156,23 @@ private static function guessContentTypeFromPath(string $path): string default: return ''; } } + + /** Read a response incrementally, retaining at most the protocol limit. */ + private static function readLimitedStream($stream): ?string + { + $body = ''; + $bytes = 0; + while (!feof($stream)) { + $chunk = fread($stream, 8192); + if ($chunk === false) return null; + if ($chunk === '') { + if (feof($stream)) break; + return null; + } + $bytes += strlen($chunk); + if ($bytes > self::MAX_RESPONSE_BYTES) return null; + $body .= $chunk; + } + return $body; + } } diff --git a/php/src/Keys/ResolvedKey.php b/php/src/Keys/ResolvedKey.php index 4edca19..8ba1b39 100644 --- a/php/src/Keys/ResolvedKey.php +++ b/php/src/Keys/ResolvedKey.php @@ -18,10 +18,56 @@ final class ResolvedKey /** @var string The keyid this resolution corresponds to. */ public $keyid; - public function __construct(string $publicKeyPem, string $algorithm, string $keyid) + /** @var bool Whether the key document explicitly revoked this key. */ + public $revoked; + + /** @var ?string RFC3339 expiry supplied by the key document. */ + public $expires; + + public function __construct( + string $publicKeyPem, + string $algorithm, + string $keyid, + bool $revoked = false, + ?string $expires = null + ) { $this->publicKeyPem = $publicKeyPem; $this->algorithm = $algorithm; $this->keyid = $keyid; + $this->revoked = $revoked; + $this->expires = $expires; + } + + /** + * Match the JS lifecycle policy. An explicit boolean revocation always + * wins; an absent expiry is live, while malformed or past expiries are + * treated as revoked so bad directory data cannot extend key lifetime. + */ + public function isRevoked(?\DateTimeImmutable $now = null): bool + { + if ($this->revoked === true) { + return true; + } + if ($this->expires === null || $this->expires === '') { + return false; + } + // Lifecycle timestamps are deliberately narrower than PHP's general + // date parser. Accept only RFC3339's UTC form, including its optional + // fractional seconds, and fail closed on offsets, dates, or parser + // extensions. + if (preg_match('/^((?!0000)\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.\d+)?Z$/D', $this->expires, $parts) !== 1) { + return true; + } + try { + $expiry = new \DateTimeImmutable($this->expires); + } catch (\Exception $e) { + return true; + } + if ($expiry->format('Y-m-d\\TH:i:s') !== $parts[1]) { + return true; + } + $now = $now ?? new \DateTimeImmutable('now'); + return $expiry <= $now; } } diff --git a/php/src/Keys/TrustDirectoryResolver.php b/php/src/Keys/TrustDirectoryResolver.php index fa056a6..3ddba07 100644 --- a/php/src/Keys/TrustDirectoryResolver.php +++ b/php/src/Keys/TrustDirectoryResolver.php @@ -55,7 +55,7 @@ public function resolve(string $keyid): ?ResolvedKey foreach ($this->baseUrls as $base) { $url = rtrim($base, '/') . '/keys/' . rawurlencode($keyid); - $response = ($this->fetcher)($url); + $response = HttpFetcher::validateResponse(($this->fetcher)($url)); if ($response === null) { continue; } @@ -70,8 +70,14 @@ public function resolve(string $keyid): ?ResolvedKey $algorithm = isset($decoded['algorithm']) && is_string($decoded['algorithm']) && $decoded['algorithm'] !== '' ? strtolower($decoded['algorithm']) : 'ed25519'; + $revoked = isset($decoded['revoked']) && is_bool($decoded['revoked']) + ? $decoded['revoked'] + : false; + $expires = isset($decoded['expires']) && is_string($decoded['expires']) && $decoded['expires'] !== '' + ? $decoded['expires'] + : null; - return new ResolvedKey($pem, $algorithm, $keyid); + return new ResolvedKey($pem, $algorithm, $keyid, $revoked, $expires); } return null; diff --git a/php/src/Signature.php b/php/src/Signature.php index 070b1e2..6c65bdd 100644 --- a/php/src/Signature.php +++ b/php/src/Signature.php @@ -2,9 +2,8 @@ /** * HTMLTrust signature binding, verification, and endorsement helpers. * - * Mirrors the JS reference implementation. See htmltrust spec §2.1, §2.2, - * §2.5 for the canonical signing payload, keyid resolution, and - * endorsement formats. + * Mirrors the JS reference implementation for the frozen v1 signing payload, + * key resolution, signature verification, and endorsement formats. * * @package HTMLTrust\Canonicalization */ @@ -19,8 +18,14 @@ class Signature { + public const SIGNING_PROFILE_V1 = 'htmltrust-signature-v1'; + public const CANONICALIZATION_PROFILE_V1 = 'htmltrust-c14n-v1'; + public const ATTRIBUTE_PROFILE_V1 = 'htmltrust-attrs-v1'; + public const URL_PROFILE_V1 = 'htmltrust-safe-url-v1'; + public const SIGNING_CONTEXT_V1 = 'https://htmltrust.org/protocol/signed-section'; + /** - * Build the canonical signing-binding string per spec §2.1: + * Build the legacy 0.2 signing-binding string: * * {content-hash}:{claims-hash}:{domain}:{signed-at} * @@ -28,6 +33,7 @@ class Signature * raises InvalidArgumentException to surface programmer errors early. * * @throws InvalidArgumentException + * @deprecated New integrations must use buildSigningPayloadV1(). */ public static function buildSignatureBinding( string $contentHash, @@ -52,6 +58,109 @@ public static function buildSignatureBinding( return $contentHash . ':' . $claimsHash . ':' . $domain . ':' . $signedAt; } + /** + * Derive the canonical location fixed by htmltrust-signature-v1. + * + * URL scope retains the path and query after WHATWG serialization and + * removes the fragment. Origin scope returns scheme://host[:port]. + * + * @throws InvalidArgumentException + */ + public static function deriveSigningLocationV1(string $documentUrl, string $scope): string + { + $url = \Uri\WhatWg\Url::parse($documentUrl); + if ($url === null + || strtolower((string) $url->getScheme()) !== 'https' + || $url->getAsciiHost() === null + || $url->getAsciiHost() === '' + || $url->getUsername() !== null + || $url->getPassword() !== null) { + throw new InvalidArgumentException('origin-not-supported'); + } + + // Fragments are outside the signed URL scope. All other URL components + // use the WHATWG serializer, matching the JavaScript binding. + $withoutFragment = $url->withFragment(null)->toAsciiString(); + if ($scope === 'url') { + return $withoutFragment; + } + if ($scope !== 'origin') { + throw new InvalidArgumentException('scope-unsupported'); + } + + $originUrl = $url->withPath('/')->withQuery(null)->withFragment(null); + return rtrim($originUrl->toAsciiString(), '/'); + } + + /** Validate the exact UTC timestamp form fixed by v1. */ + public static function validateSignedAtV1(string $value): string + { + if (preg_match('/^(?!0000)\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\dZ$/D', $value) !== 1) { + throw new InvalidArgumentException('timestamp-invalid'); + } + $parsed = \DateTimeImmutable::createFromFormat( + '!Y-m-d\\TH:i:s\\Z', + $value, + new \DateTimeZone('UTC') + ); + if ($parsed === false || $parsed->format('Y-m-d\\TH:i:s\\Z') !== $value) { + throw new InvalidArgumentException('timestamp-invalid'); + } + return $value; + } + + /** + * Build the RFC 8785 signing payload fixed by htmltrust-signature-v1. + * + * @param array $parts + * @throws InvalidArgumentException + */ + public static function buildSigningPayloadV1(array $parts): string + { + $required = [ + 'contentHash', + 'claimsHash', + 'documentURL', + 'scope', + 'keyid', + 'algorithm', + 'signedAt', + ]; + foreach ($required as $name) { + if (!array_key_exists($name, $parts) + || !is_string($parts[$name]) + || $parts[$name] === '' + || trim($parts[$name]) !== $parts[$name]) { + throw new InvalidArgumentException('signing-object-invalid: ' . $name); + } + } + + self::validateSignedAtV1($parts['signedAt']); + $document = [ + 'algorithm' => $parts['algorithm'], + 'attributeProfile' => self::ATTRIBUTE_PROFILE_V1, + 'canonicalizationProfile' => self::CANONICALIZATION_PROFILE_V1, + 'claimsHash' => $parts['claimsHash'], + 'contentHash' => $parts['contentHash'], + 'context' => self::SIGNING_CONTEXT_V1, + 'keyid' => $parts['keyid'], + 'location' => self::deriveSigningLocationV1($parts['documentURL'], $parts['scope']), + 'profile' => self::SIGNING_PROFILE_V1, + 'scope' => $parts['scope'], + 'signedAt' => $parts['signedAt'], + 'urlProfile' => self::URL_PROFILE_V1, + ]; + try { + $encoded = json_encode( + $document, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE + ); + } catch (\JsonException $e) { + throw new InvalidArgumentException('signing-object-invalid', 0, $e); + } + return Canonicalize::canonicalizeJsonDocument($encoded); + } + /** * Validate the legacy-named domain field as a serialized Web origin. * @@ -104,24 +213,68 @@ public static function buildEndorsementBinding($endorsement, ?string $timestamp if ($timestamp === null || $timestamp === '') { throw new InvalidArgumentException('timestamp must be non-empty'); } - return self::canonicalJson([ - 'endorsement' => $endorsement, - 'timestamp' => $timestamp, - ]); + try { + $document = json_encode([ + 'endorsement' => $endorsement, + 'timestamp' => $timestamp, + ], JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + } catch (\JsonException $e) { + throw new InvalidArgumentException('endorsement is not valid UTF-8', 0, $e); + } + return Canonicalize::canonicalizeJsonDocument($document); } /** - * @param array $endorsement + * Canonicalize one endorsement object using strict RFC 8785 rules. + * + * The array form is retained for the original PHP API. Passing raw JSON + * is also supported so duplicate object members can be rejected before + * PHP materializes an object and silently overwrites one of them. + * `signature` is excluded from the signed object, while every other + * extension member remains part of the binding. + * + * @param array|string $endorsement */ - public static function canonicalizeEndorsementDocument(array $endorsement): string + public static function canonicalizeEndorsementDocument($endorsement): string { - unset($endorsement['signature']); + if (is_string($endorsement)) { + // Validate the raw syntax, duplicate names, Unicode scalars, and + // JCS number range before decoding into a PHP object. + Canonicalize::canonicalizeJsonDocument($endorsement); + try { + $endorsement = json_decode($endorsement, false, 512, JSON_THROW_ON_ERROR); + } catch (\JsonException $e) { + throw new InvalidArgumentException('endorsement is not valid JSON', 0, $e); + } + if (!is_object($endorsement)) { + throw new InvalidArgumentException('endorsement must be an object'); + } + unset($endorsement->signature); + $document = $endorsement; + } elseif (is_array($endorsement)) { + unset($endorsement['signature']); + $document = $endorsement; + } else { + throw new InvalidArgumentException('endorsement must be an object'); + } + foreach (['endorser', 'endorsement', 'algorithm', 'timestamp'] as $required) { - if (!isset($endorsement[$required]) || !is_string($endorsement[$required]) || $endorsement[$required] === '') { + $present = is_object($document) ? property_exists($document, $required) : array_key_exists($required, $document); + $value = is_object($document) ? ($document->{$required} ?? null) : ($document[$required] ?? null); + if (!$present || !is_string($value) || $value === '') { throw new InvalidArgumentException("endorsement {$required} must be non-empty"); } } - return self::canonicalJson($endorsement); + + try { + $encoded = json_encode( + $document, + JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE + ); + } catch (\JsonException $e) { + throw new InvalidArgumentException('endorsement is not valid JSON', 0, $e); + } + return Canonicalize::canonicalizeJsonDocument($encoded); } /** @@ -133,6 +286,7 @@ public static function canonicalizeEndorsementDocument(array $endorsement): stri * The 32-byte raw key is extracted from the PEM body. * - "ecdsa": uses openssl_verify with OPENSSL_ALGO_SHA256. * - "rsa": uses openssl_verify with OPENSSL_ALGO_SHA256. + * - "rsa-pss-sha256": RSA-PSS with SHA-256 and a 32-byte salt. * * The signature must be canonical unpadded standard Base64. * @@ -171,6 +325,9 @@ public static function verifySignature( case 'rsa-pkcs1-sha256': return self::verifyOpenssl($message, $signature, $publicKeyPem, OPENSSL_ALGO_SHA256, OPENSSL_KEYTYPE_RSA); + case 'rsa-pss-sha256': + return self::verifyRsaPssSha256($message, $signature, $publicKeyPem); + default: throw new InvalidArgumentException("unsupported signature algorithm: {$algorithm}"); } @@ -199,6 +356,9 @@ public static function verifyEndorsement(array $endorsement, array $resolvers): return false; } } + if (!self::endorsementLifecycleIsValid($endorsement)) { + return false; + } $endorser = $endorsement['endorser']; $signature = $endorsement['signature']; @@ -209,6 +369,10 @@ public static function verifyEndorsement(array $endorsement, array $resolvers): return false; } + if ($resolved->isRevoked()) { + return false; + } + if (!self::algorithmsCompatible($resolved->algorithm, $algoOnWire)) { return false; } @@ -225,6 +389,37 @@ public static function verifyEndorsement(array $endorsement, array $resolvers): // Internal helpers // ------------------------------------------------------------------ + /** Optional endorsement lifecycle fields fail closed when malformed. */ + private static function endorsementLifecycleIsValid(array $endorsement): bool + { + if (array_key_exists('revokedBy', $endorsement)) return false; + if (!array_key_exists('expires', $endorsement)) return true; + if (!is_string($endorsement['expires']) || $endorsement['expires'] === '') return false; + $expiry = self::parseStrictLifecycleExpiry($endorsement['expires']); + if ($expiry === null) return false; + return $expiry > new \DateTimeImmutable('now', new \DateTimeZone('UTC')); + } + + private static function parseStrictLifecycleExpiry(string $value): ?\DateTimeImmutable + { + if (preg_match( + '/^((?!0000)\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d)(?:\.\d+)?Z$/D', + $value, + $parts + ) !== 1) { + return null; + } + try { + $expiry = new \DateTimeImmutable($value, new \DateTimeZone('UTC')); + } catch (\Exception $e) { + return null; + } + if ($expiry->format('Y-m-d\TH:i:s') !== $parts[1]) { + return null; + } + return $expiry; + } + /** * Decode canonical unpadded standard Base64. Returns null on malformed or * non-canonical input. @@ -256,47 +451,6 @@ private static function base64DecodeCanonical(string $input): ?string return $decoded; } - /** - * @param mixed $value - */ - private static function canonicalJson($value): string - { - if (is_array($value)) { - if ($value === [] || array_keys($value) === range(0, count($value) - 1)) { - $items = array_map([self::class, 'canonicalJson'], $value); - return '[' . implode(',', $items) . ']'; - } - uksort($value, static function ($left, $right): int { - return strcmp( - mb_convert_encoding((string) $left, 'UTF-16BE', 'UTF-8'), - mb_convert_encoding((string) $right, 'UTF-16BE', 'UTF-8') - ); - }); - $items = []; - foreach ($value as $key => $item) { - if ($item === null) { - $items[] = json_encode((string) $key, JSON_UNESCAPED_SLASHES) . ':null'; - } else { - $items[] = json_encode((string) $key, JSON_UNESCAPED_SLASHES) . ':' . self::canonicalJson($item); - } - } - return '{' . implode(',', $items) . '}'; - } - if (is_string($value)) { - return json_encode($value, JSON_UNESCAPED_SLASHES); - } - if (is_int($value) || is_float($value)) { - return json_encode($value, JSON_UNESCAPED_SLASHES); - } - if (is_bool($value)) { - return $value ? 'true' : 'false'; - } - if ($value === null) { - return 'null'; - } - throw new InvalidArgumentException('unsupported JSON value'); - } - /** * Verify an Ed25519 signature, given a PEM SubjectPublicKeyInfo or a raw * 32-byte sodium public key. @@ -341,28 +495,29 @@ private static function extractEd25519RawKey(string $publicKey): ?string return $publicKey; } - // PEM path. - if (strpos($publicKey, '-----BEGIN') !== false) { - // Strip header/footer and whitespace, then base64-decode. - $body = preg_replace('/-----BEGIN [^-]+-----|-----END [^-]+-----|\s+/', '', $publicKey); - if ($body === null || $body === '') { - return null; - } - $der = base64_decode($body, true); - if ($der === false) { - return null; - } - // The Ed25519 SubjectPublicKeyInfo DER is 44 bytes; the raw key - // is the trailing 32 bytes regardless of header length, since the - // BIT STRING contents come last in the SPKI structure. - $len = strlen($der); - if ($len < SODIUM_CRYPTO_SIGN_PUBLICKEYBYTES) { - return null; - } - return substr($der, $len - SODIUM_CRYPTO_SIGN_PUBLICKEYBYTES); + // PEM path. Ed25519 SPKI has one exact DER shape: the 12-byte + // SubjectPublicKeyInfo prefix followed by the 32-byte key. Requiring + // the complete structure prevents an unrelated key or arbitrary + // trailing DER bytes from being accepted as an Ed25519 key. + if (preg_match( + '/\A-----BEGIN PUBLIC KEY-----\s*(.*?)\s*-----END PUBLIC KEY-----\s*\z/s', + $publicKey, + $matches + ) !== 1) { + return null; + } + $body = preg_replace('/\s+/', '', $matches[1]); + if (!is_string($body) || $body === '') { + return null; } + $der = base64_decode($body, true); + $prefix = "\x30\x2a\x30\x05\x06\x03\x2b\x65\x70\x03\x21\x00"; + if ($der === false || strlen($der) !== strlen($prefix) + SODIUM_CRYPTO_SIGN_PUBLICKEYBYTES + || substr($der, 0, strlen($prefix)) !== $prefix) { + return null; + } + return substr($der, strlen($prefix)); - return null; } /** @@ -402,6 +557,90 @@ private static function verifyOpenssl( return $result === 1; } + /** + * Verify RSA-PSS(SHA-256, saltLength=32), matching the JS and Go + * implementations. PHP's openssl_verify() does not expose PSS padding + * options, so recover the EMSA-PSS encoded message with the RSA public + * operation and verify EMSA-PSS-VERIFY (RFC 8017 §9.1.2) directly. + */ + private static function verifyRsaPssSha256(string $message, string $signature, string $publicKeyPem): bool + { + if (!function_exists('openssl_public_decrypt') || !function_exists('openssl_pkey_get_public')) { + throw new RuntimeException('ext-openssl is required for rsa verification'); + } + + $key = openssl_pkey_get_public($publicKeyPem); + if ($key === false) { + return false; + } + $details = openssl_pkey_get_details($key); + // OpenSSL 3 exposes RSA-PSS-restricted keys as type -1. Reject those + // here because this implementation cannot inspect their PSS parameter + // restrictions before applying the fixed SHA-256/MGF1/salt policy. + if (!is_array($details) || ($details['type'] ?? null) !== OPENSSL_KEYTYPE_RSA) { + return false; + } + $bits = (int) ($details['bits'] ?? 0); + if ($bits < 512 || strlen($signature) !== (int) ceil($bits / 8)) { + return false; + } + + $encoded = ''; + // NO_PADDING asks OpenSSL for the raw RSA public-operation result. + if (@openssl_public_decrypt($signature, $encoded, $key, OPENSSL_NO_PADDING) !== true) { + return false; + } + + $emBits = $bits - 1; + $emLen = (int) ceil($emBits / 8); + // RSA operations return k octets. For a modulus whose bit length is + // not byte-aligned, EMSA-PSS uses k-1 octets and the leading zero is + // omitted from the encoded message. + if (strlen($encoded) === $emLen + 1 && $encoded[0] === "\0") { + $encoded = substr($encoded, 1); + } + if (strlen($encoded) !== $emLen || $emLen < 32 + 32 + 2) { + return false; + } + + $hLen = 32; + $saltLen = 32; + if (substr($encoded, -1) !== "\xbc") { + return false; + } + $maskedDbLen = $emLen - $hLen - 1; + $maskedDb = substr($encoded, 0, $maskedDbLen); + $hash = substr($encoded, $maskedDbLen, $hLen); + $unusedBits = 8 * $emLen - $emBits; + if ($unusedBits > 0 && (ord($maskedDb[0]) & (0xff << (8 - $unusedBits))) !== 0) { + return false; + } + + $dbMask = self::mgf1Sha256($hash, $maskedDbLen); + $db = $maskedDb ^ $dbMask; + if ($unusedBits > 0) { + $db[0] = chr(ord($db[0]) & (0xff >> $unusedBits)); + } + $paddingLength = $emLen - $hLen - $saltLen - 2; + if (substr($db, 0, $paddingLength) !== str_repeat("\0", $paddingLength) + || ($db[$paddingLength] ?? '') !== "\x01") { + return false; + } + $salt = substr($db, -$saltLen); + $messageHash = hash('sha256', $message, true); + $expectedHash = hash('sha256', str_repeat("\0", 8) . $messageHash . $salt, true); + return hash_equals($expectedHash, $hash); + } + + private static function mgf1Sha256(string $seed, int $length): string + { + $mask = ''; + for ($counter = 0; strlen($mask) < $length; $counter++) { + $mask .= hash('sha256', $seed . pack('N', $counter), true); + } + return substr($mask, 0, $length); + } + private static function verifyEcdsaP1363( string $message, string $signature, diff --git a/php/tests/CanonicalizeClaimsTest.php b/php/tests/CanonicalizeClaimsTest.php index 39dbbc1..6c3322e 100644 --- a/php/tests/CanonicalizeClaimsTest.php +++ b/php/tests/CanonicalizeClaimsTest.php @@ -24,11 +24,11 @@ public function testNormalizesNamesAndValues(): void $this->assertSame("title:\"Hello\"\n", Canonicalize::canonicalizeClaims($claims)); } - public function testStringifiesNonStringValues(): void + public function testRejectsNonStringValues(): void { - $claims = ['count' => 42, 'flag' => true]; - // PHP coerces true to "1", 42 to "42". - $this->assertSame("count:42\nflag:1\n", Canonicalize::canonicalizeClaims($claims)); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('claim-malformed'); + Canonicalize::canonicalizeClaims(['count' => 42]); } public function testEmptyClaimsProducesEmptyString(): void diff --git a/php/tests/CanonicalizeJcsTest.php b/php/tests/CanonicalizeJcsTest.php new file mode 100644 index 0000000..d1ffcd5 --- /dev/null +++ b/php/tests/CanonicalizeJcsTest.php @@ -0,0 +1,59 @@ +assertSame( + '{"a":1e+30,"b":4.5,"z":0,"😀":2,"":1}', + Canonicalize::canonicalizeJsonDocument('{"z":-0,"a":1e30,"b":4.50,"😀":2,"":1}') + ); + } + + public function testNumberFormattingIgnoresSerializePrecision(): void + { + $previous = ini_get('serialize_precision'); + ini_set('serialize_precision', '3'); + try { + $this->assertSame( + '[0,0,5e-324,1e+23,0.000001,333333333.33333325]', + Canonicalize::canonicalizeJsonDocument('[0,-0,5e-324,1e23,1e-6,333333333.33333325]') + ); + } finally { + if ($previous !== false) ini_set('serialize_precision', $previous); + } + } + + public function testRejectsExcessiveJsonNesting(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('resource-limit-exceeded'); + Canonicalize::canonicalizeJsonDocument(str_repeat('[', 257) . '0' . str_repeat(']', 257)); + } + + /** @dataProvider unsafeJsonProvider */ + public function testRejectsUnsafeRawDocument(string $document, string $reason): void + { + try { + Canonicalize::canonicalizeJsonDocument($document); + $this->fail('Expected strict JCS rejection'); + } catch (InvalidArgumentException $error) { + $this->assertStringContainsString($reason, $error->getMessage()); + } + } + + public static function unsafeJsonProvider(): array + { + return [ + ['{"a":1,"a":2}', 'jcs-duplicate-key'], + ['"\\uD800"', 'jcs-invalid-surrogate'], + ['{"n":1e400}', 'jcs-number'], + ]; + } +} diff --git a/php/tests/CanonicalizeTest.php b/php/tests/CanonicalizeTest.php index cb5f54e..dca793c 100644 --- a/php/tests/CanonicalizeTest.php +++ b/php/tests/CanonicalizeTest.php @@ -17,7 +17,7 @@ class CanonicalizeTest extends TestCase * * @return array */ - public function specTestPairsProvider(): array + public static function specTestPairsProvider(): array { return [ 'Curly double quotes → straight' => [ @@ -113,4 +113,52 @@ public function testPreserveWhitespace(): void // Whitespace should NOT be collapsed, but other normalizations apply $this->assertStringContainsString(' ', $result); } + + /** @dataProvider malformedFragmentProvider */ + public function testRejectsParserAmbiguities(string $html, string $reason): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage($reason); + Canonicalize::extractCanonicalText($html); + } + + public static function malformedFragmentProvider(): array + { + return [ + 'standalone foreignObject' => ['text', 'parser-profile-unsupported'], + 'unterminated named reference' => ['

&!

', 'parser-profile-unsupported'], + 'unterminated numeric reference' => ['

A!

', 'parser-profile-unsupported'], + 'unterminated comment' => ['

x', 'parser-profile-unsupported'], + 'table tail foster parenting' => ['tail
x
', 'parser-profile-unsupported'], + ]; + } + + public function testIframeRawTextDoesNotTriggerReferenceAmbiguity(): void + { + $this->assertSame("before\nafter", Canonicalize::extractCanonicalText( + '

before

after

' + )); + } + + public function testRejectsElementNestingBeyondLimit(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('resource-limit-exceeded'); + Canonicalize::extractCanonicalText(str_repeat('
', 257) . 'x' . str_repeat('
', 257)); + } + + public function testColonQualifiedElementsCountTowardNestingLimit(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('resource-limit-exceeded'); + Canonicalize::extractCanonicalText(str_repeat('', 257) . 'x' . str_repeat('', 257)); + } + + public function testInvalidBaseUrlIsRejectedWithoutUrlAttributes(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('attribute-canonicalization-failed'); + Canonicalize::extractCanonicalText('

text

', false, 'not a URL'); + } } diff --git a/php/tests/EndorsementTest.php b/php/tests/EndorsementTest.php index 8ec2de6..724bbda 100644 --- a/php/tests/EndorsementTest.php +++ b/php/tests/EndorsementTest.php @@ -101,6 +101,91 @@ public function testVerifyEndorsementFailsOnMissingFields(): void ], [$resolver])); } + public function testEndorsementBindingPreservesUnicodeExtensions(): void + { + $binding = Signature::canonicalizeEndorsementDocument([ + 'endorser' => 'did:web:例.example', + 'endorsement' => 'sha256:内容😀', + 'timestamp' => '2025-05-01T00:00Z', + 'algorithm' => 'ed25519', + 'extension😀' => ['説明' => 'café', 'value' => "\u{1F469}\u{200D}\u{1F4BB}"], + 'signature' => 'omitted', + ]); + + $this->assertStringContainsString('例.example', $binding); + $this->assertStringContainsString('内容😀', $binding); + $this->assertStringContainsString('extension😀', $binding); + $this->assertStringContainsString('説明', $binding); + $this->assertStringNotContainsString('omitted', $binding); + } + + public function testRawEndorsementBindingRejectsDuplicateMembers(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('jcs-duplicate-key'); + Signature::canonicalizeEndorsementDocument( + '{"endorser":"alice","endorsement":"sha256:X","algorithm":"ed25519",' + . '"timestamp":"2025-05-01T00:00Z","endorsement":"sha256:Y"}' + ); + } + + public function testVerifyEndorsementRejectsRevokedResolvedKey(): void + { + $this->skipIfNoSodium(); + [$endorser, $pem, $secret] = $this->makeEndorser(); + $unsigned = [ + 'endorser' => $endorser, + 'endorsement' => 'sha256:CONTENT', + 'timestamp' => '2025-05-01T00:00Z', + 'algorithm' => 'ed25519', + ]; + $unsigned['signature'] = rtrim(base64_encode( + sodium_crypto_sign_detached(Signature::canonicalizeEndorsementDocument($unsigned), $secret) + ), '='); + + $resolver = new InMemoryResolver([ + $endorser => new ResolvedKey($pem, 'ed25519', $endorser, true), + ]); + $this->assertFalse(Signature::verifyEndorsement($unsigned, [$resolver])); + } + + public function testVerifyEndorsementFailsClosedOnExpiryAndRevokedBy(): void + { + $this->skipIfNoSodium(); + [$endorser, $pem, $secret] = $this->makeEndorser(); + $resolver = new InMemoryResolver([$endorser => new ResolvedKey($pem, 'ed25519', $endorser)]); + $sign = static function (array $unsigned) use ($secret): array { + $unsigned['signature'] = rtrim(base64_encode( + sodium_crypto_sign_detached(Signature::canonicalizeEndorsementDocument($unsigned), $secret) + ), '='); + return $unsigned; + }; + $base = [ + 'endorser' => $endorser, + 'endorsement' => 'sha256:LIFECYCLE', + 'timestamp' => '2025-05-01T00:00:00Z', + 'algorithm' => 'ed25519', + ]; + + $this->assertTrue(Signature::verifyEndorsement( + $sign($base + ['expires' => '2999-01-01T00:00:00Z']), [$resolver] + )); + foreach ([ + ['expires' => 'nonsense'], + ['expires' => '2000-01-01T00:00:00Z'], + ['expires' => '2999-01-01T00:00:00+00:00'], + ['expires' => ''], + ['revokedBy' => ''], + ['revokedBy' => 'did:web:authority.example'], + ['revokedBy' => 42], + ] as $lifecycle) { + $this->assertFalse( + Signature::verifyEndorsement($sign($base + $lifecycle), [$resolver]), + 'malformed or revoked lifecycle field must fail closed' + ); + } + } + // ------------------------------------------------------------------ private function skipIfNoSodium(): void diff --git a/php/tests/Keys/DidWebResolverTest.php b/php/tests/Keys/DidWebResolverTest.php index 96ce3ff..e1f2224 100644 --- a/php/tests/Keys/DidWebResolverTest.php +++ b/php/tests/Keys/DidWebResolverTest.php @@ -71,6 +71,58 @@ public function testResolvesWithPathSegments(): void $this->assertSame('https://example.com/user/alice/did.json', $captured['url']); } + public function testPreservesPercentEncodedPathSegments(): void + { + $captured = ['url' => null]; + $resolver = new DidWebResolver(static function (string $url) use (&$captured): ?array { + $captured['url'] = $url; + return [ + 'body' => json_encode(['verificationMethod' => [['publicKeyPem' => 'PEM']]]), + 'contentType' => 'application/did+json', + ]; + }); + + $this->assertNotNull($resolver->resolve('did:web:example.com:foo%2Fbar')); + $this->assertSame('https://example.com/foo%2Fbar/did.json', $captured['url']); + } + + public function testValidatesAndDecodesEncodedPortAuthority(): void + { + $captured = ['url' => null]; + $resolver = new DidWebResolver(static function (string $url) use (&$captured): ?array { + $captured['url'] = $url; + return [ + 'body' => json_encode(['verificationMethod' => [['publicKeyPem' => 'PEM']]]), + 'contentType' => 'application/did+json', + ]; + }); + + $this->assertNotNull($resolver->resolve('did:web:example.com%3A3000:user')); + $this->assertSame('https://example.com:3000/user/did.json', $captured['url']); + } + + /** @dataProvider invalidAuthorityProvider */ + public function testRejectsInvalidAuthority(string $keyid): void + { + $called = false; + $resolver = new DidWebResolver(static function (string $url) use (&$called): ?array { + $called = true; + return null; + }); + + $this->assertNull($resolver->resolve($keyid)); + $this->assertFalse($called); + } + + public static function invalidAuthorityProvider(): array + { + return [ + 'userinfo' => ['did:web:example.com@evil.com'], + 'nonnumeric port' => ['did:web:example.com%3Aabc'], + 'unexpected escape' => ['did:web:example%2Ecom'], + ]; + } + public function testIgnoresFragment(): void { $captured = ['url' => null]; @@ -99,6 +151,16 @@ public function testReturnsNullOnFetchFailure(): void $this->assertNull($resolver->resolve('did:web:example.com')); } + public function testRejectsOversizedInjectedResponse(): void + { + $resolver = new DidWebResolver(static function (string $url): ?array { + return ['body' => str_repeat('x', 64 * 1024 + 1), 'contentType' => 'application/json']; + }); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('resource-limit-exceeded'); + $resolver->resolve('did:web:example.com'); + } + public function testReturnsNullOnInvalidJson(): void { $fetcher = static function (string $url): ?array { @@ -140,6 +202,60 @@ public function testPicksFirstVerificationMethodWithPem(): void $this->assertSame('A', $resolved->publicKeyPem); } + public function testSkipsRevokedAndExpiredMethods(): void + { + $fetcher = static function (string $url): ?array { + return [ + 'body' => json_encode([ + 'verificationMethod' => [ + ['type' => 'Ed25519VerificationKey2020', 'publicKeyPem' => 'REVOKED', 'revoked' => true], + ['type' => 'Ed25519VerificationKey2020', 'publicKeyPem' => 'EXPIRED', 'expires' => '2000-01-01T00:00:00Z'], + ['type' => 'Ed25519VerificationKey2020', 'publicKeyPem' => 'LIVE', 'expires' => '2999-01-01T00:00:00Z'], + ], + ]), + 'contentType' => 'application/did+json', + ]; + }; + + $resolved = (new DidWebResolver($fetcher))->resolve('did:web:example.com'); + $this->assertNotNull($resolved); + $this->assertSame('LIVE', $resolved->publicKeyPem); + $this->assertFalse($resolved->revoked); + $this->assertSame('2999-01-01T00:00:00Z', $resolved->expires); + } + + public function testDeactivatedDocumentDoesNotResolve(): void + { + $resolver = new DidWebResolver(static function (string $url): ?array { + return [ + 'body' => json_encode([ + 'deactivated' => true, + 'verificationMethod' => [['publicKeyPem' => 'PEM']], + ]), + 'contentType' => 'application/did+json', + ]; + }); + $this->assertNull($resolver->resolve('did:web:example.com')); + } + + public function testMalformedExpiryIsRejectedAndResolverContinues(): void + { + $resolver = new DidWebResolver(static function (string $url): ?array { + return [ + 'body' => json_encode([ + 'verificationMethod' => [ + ['publicKeyPem' => 'BAD', 'expires' => '2026-01-01T00:00:00+00:00'], + ['publicKeyPem' => 'GOOD', 'expires' => '2999-01-01T00:00:00Z'], + ], + ]), + 'contentType' => 'application/did+json', + ]; + }); + $resolved = $resolver->resolve('did:web:example.com'); + $this->assertNotNull($resolved); + $this->assertSame('GOOD', $resolved->publicKeyPem); + } + public function testInfersEcdsaFromMethodType(): void { $fetcher = static function (string $url): ?array { diff --git a/php/tests/Keys/DirectUrlResolverTest.php b/php/tests/Keys/DirectUrlResolverTest.php index 5e9da46..5cf30f4 100644 --- a/php/tests/Keys/DirectUrlResolverTest.php +++ b/php/tests/Keys/DirectUrlResolverTest.php @@ -7,6 +7,7 @@ use PHPUnit\Framework\TestCase; use HTMLTrust\Canonicalization\Keys\DirectUrlResolver; +use HTMLTrust\Canonicalization\Keys\HttpFetcher; class DirectUrlResolverTest extends TestCase { @@ -103,6 +104,16 @@ public function testReturnsNullOnFetchFailure(): void $this->assertNull($resolver->resolve('https://example.com/key.json')); } + public function testRejectsOversizedInjectedResponse(): void + { + $resolver = new DirectUrlResolver(static function (string $url): ?array { + return ['body' => str_repeat('x', 64 * 1024 + 1), 'contentType' => 'application/json']; + }); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('resource-limit-exceeded'); + $resolver->resolve('https://example.com/key.json'); + } + public function testReturnsNullForUnsupportedScheme(): void { $fetcher = static function (string $url): ?array { @@ -129,4 +140,17 @@ public function testReturnsNullWhenJsonHasNoKey(): void $resolver = new DirectUrlResolver($fetcher); $this->assertNull($resolver->resolve('https://example.com/key.json')); } + + public function testDefaultFetcherCapsFileResponseWhileReading(): void + { + $path = tempnam(sys_get_temp_dir(), 'htmltrust-key-'); + $this->assertNotFalse($path); + try { + file_put_contents($path, str_repeat('x', 64 * 1024 + 1)); + $response = (HttpFetcher::default())('file://' . $path); + $this->assertNull($response); + } finally { + @unlink($path); + } + } } diff --git a/php/tests/Keys/TrustDirectoryResolverTest.php b/php/tests/Keys/TrustDirectoryResolverTest.php index edd31fc..2da39ab 100644 --- a/php/tests/Keys/TrustDirectoryResolverTest.php +++ b/php/tests/Keys/TrustDirectoryResolverTest.php @@ -62,6 +62,19 @@ static function (string $url): ?array { $this->assertNull($resolver->resolve('abc123')); } + public function testRejectsOversizedInjectedResponse(): void + { + $resolver = new TrustDirectoryResolver( + ['https://dir.example'], + static function (string $url): ?array { + return ['body' => str_repeat('x', 64 * 1024 + 1), 'contentType' => 'application/json']; + } + ); + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('resource-limit-exceeded'); + $resolver->resolve('abc123'); + } + public function testUrlEncodesKeyid(): void { $captured = ['url' => null]; diff --git a/php/tests/SignatureTest.php b/php/tests/SignatureTest.php index 4bdac83..87df12f 100644 --- a/php/tests/SignatureTest.php +++ b/php/tests/SignatureTest.php @@ -52,7 +52,7 @@ public function testBuildSignatureBindingRejectsEmptyFields(string $contentHash, Signature::buildSignatureBinding($contentHash, $claimsHash, $domain, $signedAt); } - public function emptyFieldProvider(): array + public static function emptyFieldProvider(): array { return [ 'empty contentHash' => ['', 'b', 'https://example.com', 'd'], @@ -62,6 +62,60 @@ public function emptyFieldProvider(): array ]; } + // ------------------------------------------------------------------ + // htmltrust-signature-v1 + // ------------------------------------------------------------------ + + public function testDeriveSigningLocationV1UsesWhatwgSerialization(): void + { + $this->assertSame( + 'https://xn--bcher-kva.example/article?q=1', + Signature::deriveSigningLocationV1('HTTPS://BÜCHER.EXAMPLE:443/a/../article?q=1#part', 'url') + ); + $this->assertSame( + 'https://example.org:8443', + Signature::deriveSigningLocationV1('https://example.org:8443/a?q=1#part', 'origin') + ); + } + + public function testValidateSignedAtV1RejectsNonexistentDate(): void + { + $this->expectException(InvalidArgumentException::class); + Signature::validateSignedAtV1('2026-02-30T12:00:00Z'); + } + + public function testBuildSigningPayloadV1ReproducesFrozenVector(): void + { + $path = dirname(__DIR__, 2) . '/conformance/vectors/vector-01.json'; + $vector = json_decode((string) file_get_contents($path), true, 512, JSON_THROW_ON_ERROR); + $this->assertSame( + $vector['signingPayload'], + Signature::buildSigningPayloadV1([ + 'contentHash' => $vector['contentHash'], + 'claimsHash' => $vector['claimsHash'], + 'documentURL' => $vector['input']['documentURL'], + 'scope' => $vector['input']['scope'], + 'keyid' => $vector['input']['keyid'], + 'algorithm' => $vector['algorithm'], + 'signedAt' => $vector['input']['signedAt'], + ]) + ); + } + + public function testBuildSigningPayloadV1RejectsWhitespacePaddedField(): void + { + $this->expectException(InvalidArgumentException::class); + Signature::buildSigningPayloadV1([ + 'contentHash' => ' sha256:ABC', + 'claimsHash' => 'sha256:DEF', + 'documentURL' => 'https://example.com/', + 'scope' => 'url', + 'keyid' => 'https://keys.example/alice.json', + 'algorithm' => 'ed25519', + 'signedAt' => '2026-01-15T12:00:00Z', + ]); + } + // ------------------------------------------------------------------ // buildEndorsementBinding // ------------------------------------------------------------------ @@ -100,8 +154,8 @@ public function testBuildEndorsementBindingSortsKeysByUtf16CodeUnits(): void $privateUse => 2, $astral => 1, ]); - $astralEncoded = trim((string) json_encode($astral), '"'); - $privateUseEncoded = trim((string) json_encode($privateUse), '"'); + $astralEncoded = trim((string) json_encode($astral, JSON_UNESCAPED_UNICODE), '"'); + $privateUseEncoded = trim((string) json_encode($privateUse, JSON_UNESCAPED_UNICODE), '"'); $this->assertLessThan(strpos($binding, $privateUseEncoded), strpos($binding, $astralEncoded)); } diff --git a/python/README.md b/python/README.md index 333c72e..f91428f 100644 --- a/python/README.md +++ b/python/README.md @@ -1,68 +1,93 @@ -# HTMLTrust Canonicalization -- Python +# HTMLTrust Canonicalization for Python -Python binding for the HTMLTrust canonical text normalization library. Produces byte-identical output to the JavaScript, Go, PHP, and Rust implementations for every test vector in the shared conformance suite. +This package implements the Python binding for `htmltrust-c14n-v1`. It +normalizes text, extracts signed content from HTML, canonicalizes claims, and +canonicalizes raw JSON under RFC 8785. The same shared fixtures run against the +JavaScript, Go, PHP, and Rust bindings. -## Status +Version: `0.3.0` release candidate +Python: 3.10 or newer -Implemented. The shared conformance suite passes for normalization, extraction, and claims. `extract_canonical_text` includes the current signed semantic attribute allowlist (`href`, `src`, `alt`, `aria-label`) when a base URL is available for relative URLs. +## Test a fresh checkout -Out of scope for this package: signature verification and key resolution. Those live in the higher-level HTMLTrust client libraries (and will arrive in a follow-up PR for the Python binding once the JS surface area lands on `main`). +From the repository root, Docker runs the Python unit tests and every shared +conformance fixture: -## Scope +```sh +docker compose -f compose.test.yml run --rm python +``` -This package provides four functions: +Run `./scripts/test-in-docker.sh` to test all five language bindings. -1. **`normalize_text(text: str, preserve_whitespace: bool = False) -> str`** -- applies the 8-phase canonicalization defined in [`../spec.md`](../spec.md) to a UTF-8 string. Mirrors the existing JavaScript/Go/PHP signatures. -2. **`extract_canonical_text(html: str, preserve_whitespace: bool = False, base_url: str | None = None) -> str`** -- parses an HTML fragment with BeautifulSoup, walks the DOM, emits text nodes and signed semantic attributes in document order, and applies `normalize_text` to text/attribute values. -3. **`canonicalize_claims(claims: Mapping[str, object]) -> str`** -- serializes a claim map to the canonical, hashable string used by the `claims-hash` field of the signature binding (each entry normalized, sorted lexically by name, emitted as `name:content\n`). -4. **`extract_claims_from_signed_section(html: str) -> dict[str, str]`** -- extracts all direct child `` claims from a `` or signed-section inner fragment, including `author` and `signed-at`, and rejects duplicate normalized names. +## Install -All three are pure functions: no network, no file I/O, deterministic output for the same input. +Install the package from this checkout: -## Dependencies +```sh +python3 -m pip install -e 'python[dev]' +``` -- `unicodedata` (stdlib) for NFKC normalization -- `beautifulsoup4 >= 4.12` for HTML parsing in `extract_canonical_text` -- No other runtime dependencies +For Python-only development from this directory: -## Conformance +```sh +python3 -m pip install -e '.[dev]' +python3 -m pytest -q +``` -`tests/test_normalize.py` runs all 18 normalization vectors from `javascript/test.js`. `tests/test_extract.py` and `tests/test_claims.py` cover the HTML extraction and claim canonicalization contracts. Output MUST stay byte-identical to the JavaScript / Go / PHP / Rust bindings. +Runtime dependency versions are pinned in `pyproject.toml` because parser and +serializer behavior affects signed bytes. -## Installation +## Public API -```bash -pip install htmltrust-canonicalization -# or for development: -cd python && pip install -e '.[dev]' -``` +- `normalize_text(text, preserve_whitespace=False)` applies the Unicode and + punctuation normalization profile. +- `extract_canonical_text(html, preserve_whitespace=False, base_url=None)` + parses HTML and emits signed text plus semantic attribute records. +- `canonicalize_claims(claims)` emits the sorted and escaped claims byte + sequence. +- `extract_claims_from_signed_section(html)` reads direct-child claim metadata. +- `canonicalize_json_document(document)` validates and canonicalizes one raw + JSON document with duplicate-key detection and IEEE 754 number handling. -## Usage +Each entry point is deterministic and performs no network or file I/O. Text, +HTML, and JSON inputs are limited to 1 MiB. A limit breach raises +`ValueError("resource-limit-exceeded")`. + +## Example ```python from htmltrust_canonicalization import ( - normalize_text, - extract_canonical_text, canonicalize_claims, - extract_claims_from_signed_section, + canonicalize_json_document, + extract_canonical_text, + normalize_text, ) canonical = normalize_text('He said, "Hello…"') -# -> 'He said, "Hello..."' +assert canonical == 'He said, "Hello..."' + +content = extract_canonical_text( + '

Read the paper.

', + base_url='https://example.org/article', +) -from_html = extract_canonical_text('

Hello world!

') -# -> 'Hello world!' +claims = canonicalize_claims({'License': 'CC-BY-4.0'}) +assert claims == 'License:CC-BY-4.0\n' -claims_str = canonicalize_claims({ - 'License': 'CC-BY-4.0', - 'AIAssistance': 'None', -}) -# -> 'AIAssistance:None\nLicense:CC-BY-4.0\n' +payload = canonicalize_json_document('{"z":-0,"a":1e30}') +assert payload == '{"a":1e+30,"z":0}' ``` -## Tests +Relative `href` and `src` attributes require `base_url`. The v1 safe-URL +profile accepts HTTPS URLs and rejects credentials, control characters, and +unsupported schemes. -```bash -pip install -e '.[dev]' -pytest -``` +## Package scope + +Signature verification and key resolution live in the HTMLTrust client and +server packages. This binding produces the canonical byte sequences those +packages hash and sign. + +The normative protocol text is maintained in the +[HTMLTrust specification](https://github.com/HTMLTrust/htmltrust-spec/tree/main/ietf-draft). +The repository's shared fixtures are the executable cross-language contract. diff --git a/python/htmltrust_canonicalization/__init__.py b/python/htmltrust_canonicalization/__init__.py index c60734a..b4fd71a 100644 --- a/python/htmltrust_canonicalization/__init__.py +++ b/python/htmltrust_canonicalization/__init__.py @@ -5,6 +5,7 @@ - extract_canonical_text(html, preserve_whitespace=False) -> str - canonicalize_claims(claims) -> str - extract_claims_from_signed_section(html) -> dict[str, str] + - canonicalize_json_document(document) -> str This binding produces byte-identical output to the JavaScript, Go, PHP, and Rust implementations of the HTMLTrust canonicalization library. @@ -13,12 +14,14 @@ from ._normalize import normalize_text from ._extract import extract_canonical_text from ._claims import canonicalize_claims, extract_claims_from_signed_section +from ._jcs import canonicalize_json_document __all__ = [ "normalize_text", "extract_canonical_text", "canonicalize_claims", "extract_claims_from_signed_section", + "canonicalize_json_document", ] -__version__ = "0.1.0" +__version__ = "0.3.0" diff --git a/python/htmltrust_canonicalization/_claims.py b/python/htmltrust_canonicalization/_claims.py index 5237461..76f6eeb 100644 --- a/python/htmltrust_canonicalization/_claims.py +++ b/python/htmltrust_canonicalization/_claims.py @@ -14,8 +14,22 @@ from ._normalize import normalize_text +_MAX_CLAIMS = 64 +_MAX_CLAIM_BYTES = 4 * 1024 -def canonicalize_claims(claims: Mapping[str, object]) -> str: + +def _claim_field(value: str) -> str: + value = normalize_text(value).strip() + if len(value.encode("utf-8")) > _MAX_CLAIM_BYTES: + raise ValueError("resource-limit-exceeded") + return value + + +def _escape_claim(value: str) -> str: + return value.replace("\\", "\\\\").replace(":", "\\:").replace("\n", "\\n") + + +def canonicalize_claims(claims: Mapping[str, str]) -> str: """Serialize ``claims`` to the canonical, sortable, hashable string form. Each claim name and value is run through ``normalize_text`` so that @@ -23,9 +37,7 @@ def canonicalize_claims(claims: Mapping[str, object]) -> str: sorted lexically by name and emitted as ``name:content\n`` records. Args: - claims: Mapping of claim name to value. Values are coerced to - ``str`` before normalization so callers may pass simple - scalar types. + claims: Mapping of string claim names to string values. Returns: Canonical serialized string ready to be hashed. @@ -35,9 +47,13 @@ def canonicalize_claims(claims: Mapping[str, object]) -> str: entries = [] seen = set() + if len(claims) > _MAX_CLAIMS: + raise ValueError("resource-limit-exceeded") for name, value in claims.items(): - normalized_name = normalize_text(name).strip() - normalized_value = normalize_text(str(value)).strip() + if not isinstance(name, str) or not isinstance(value, str): + raise ValueError("claim-malformed") + normalized_name = _claim_field(name) + normalized_value = _claim_field(value) if not normalized_name: raise ValueError("claim-malformed") if normalized_name in seen: @@ -45,7 +61,7 @@ def canonicalize_claims(claims: Mapping[str, object]) -> str: seen.add(normalized_name) entries.append((normalized_name, normalized_value)) entries.sort(key=lambda nv: nv[0]) - return "".join(f"{name}:{value}\n" for name, value in entries) + return "".join(f"{_escape_claim(name)}:{_escape_claim(value)}\n" for name, value in entries) def extract_claims_from_signed_section(html: str) -> dict[str, str]: @@ -60,17 +76,38 @@ def extract_claims_from_signed_section(html: str) -> dict[str, str]: if not isinstance(html, str): raise TypeError("extract_claims_from_signed_section expects a str") - soup = BeautifulSoup(html, "html.parser") - root = soup.find("signed-section") or soup + # Run the same source-profile checks as content extraction before using + # the recovered HTML5 tree. + from ._extract import _preflight_source + _preflight_source(html) + soup = BeautifulSoup(html, "html5lib") + section = soup.find("signed-section") + root = section if section is not None else soup claims: dict[str, str] = {} seen: set[str] = set() - for child in getattr(root, "children", ()): + children = list(getattr(root, "children", ())) + if section is None: + # html5lib moves top-level metadata into . When callers pass + # an inner signed-section fragment, those head/html wrappers are + # parser scaffolding, so retain only metadata whose intervening + # ancestors are html/head/body. + children = [ + elem for elem in soup.find_all("meta") + if all( + ancestor.name in {"html", "head", "body", "[document]"} + for ancestor in elem.parents + if ancestor.name is not None + ) + ] + for child in children: if not isinstance(child, Tag) or child.name.lower() != "meta": continue if not child.has_attr("name") or not child.has_attr("content"): raise ValueError("claim-malformed") - name = normalize_text(str(child["name"])).strip() - content = normalize_text(str(child["content"])).strip() + if len(claims) >= _MAX_CLAIMS: + raise ValueError("resource-limit-exceeded") + name = _claim_field(str(child["name"])) + content = _claim_field(str(child["content"])) if not name: raise ValueError("claim-malformed") if name in seen: diff --git a/python/htmltrust_canonicalization/_extract.py b/python/htmltrust_canonicalization/_extract.py index 3a96417..2ceda58 100644 --- a/python/htmltrust_canonicalization/_extract.py +++ b/python/htmltrust_canonicalization/_extract.py @@ -10,8 +10,11 @@ from __future__ import annotations +import re + from bs4 import BeautifulSoup, NavigableString, Tag -from urllib.parse import urljoin, urlsplit, urlunsplit +from html5lib.html5parser import HTMLParser +from pywhatwgurl import URL from ._normalize import normalize_text @@ -34,9 +37,55 @@ "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "li", "main", "nav", "ol", "p", "pre", "section", "table", - "tr", "td", "th", "ul", + "tr", "td", "th", "ul", "signed-section", }) _SIGNED_ATTRS = ("href", "src", "alt", "aria-label") +_MAX_SOURCE_BYTES = 1024 * 1024 +_MAX_OUTPUT_BYTES = 1024 * 1024 +_MAX_ELEMENT_DEPTH = 256 +_HTML_TOKEN_RE = re.compile( + r"|]*>|]*(?:[^>\"']+|\"[^\"]*\"|'[^']*')*>", + re.I | re.S, +) +_TAG_NAME_RE = re.compile(r"^]*)", re.I) +_VOID_TAGS = frozenset({ + "area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "param", "source", "track", "wbr", +}) + + +def _preflight_source(html: str) -> None: + """Validate the portable-1 source profile before tree construction. + + html5lib exposes tokenizer/tree-builder diagnostics, which must be + checked before BeautifulSoup receives the recovered tree. A few HTML5 + repairs (notably table foster parenting and foreign content) are not + reported as parse errors by html5lib, so those profile cases are checked + against the source as well. + """ + try: + source_bytes = html.encode("utf-8", "strict") + except UnicodeEncodeError as exc: + raise ValueError("parser-profile-unsupported") from exc + if len(source_bytes) > _MAX_SOURCE_BYTES: + raise ValueError("resource-limit-exceeded") + + # Count source element nesting before handing the document to an HTML5 + # tree builder. This keeps the ceiling independent of BeautifulSoup's + # synthetic html/head/body wrapper nodes and makes the limit effective + # before traversal can recurse. + profile_source = re.sub( + r"(<\s*(script|style|iframe)\b(?:[^>\"']+|\"[^\"]*\"|'[^']*')*>).*?()", + r"\1\3", + html, + flags=re.I | re.S, + ) + _check_source_depth(profile_source) + + parser = HTMLParser(namespaceHTMLElements=False, strict=False) + parser.parseFragment(html) + if parser.errors: + raise ValueError("parser-profile-unsupported") def extract_canonical_text( @@ -69,7 +118,12 @@ def extract_canonical_text( if not isinstance(html, str): raise TypeError("extract_canonical_text expects a str") - soup = BeautifulSoup(html, "html.parser") + _preflight_source(html) + base_url = _validate_base_url(base_url) + # BeautifulSoup's html5lib backend exposes the same HTML5 tree model as + # the diagnostics pass above. The source has already been accepted, so + # no parser repair is silently used as verification input. + soup = BeautifulSoup(html, "html5lib") # Remove excluded elements (and their text content) outright. for tag_name in _EXCLUDED_TAGS: @@ -83,6 +137,58 @@ def extract_canonical_text( return _finalize_parts(text) +def _check_source_depth(source: str) -> None: + """Validate source nesting and reject parser repairs. + + HTML5 parsers implicitly close elements and foster-parent text around a + table. Those repairs are outside the portable profile, so this small + source stack requires explicit matching end tags and checks direct table + text before tree construction. + """ + stack: list[str] = [] + cursor = 0 + for match in _HTML_TOKEN_RE.finditer(source): + text = source[cursor:match.start()] + if stack and stack[-1] == "table" and text.strip(): + raise ValueError("parser-profile-unsupported") + cursor = match.end() + + token = match.group(0) + name_match = _TAG_NAME_RE.match(token) + if not name_match: + continue + name = name_match.group(1).lower() + if token.lstrip().startswith("$", token): + stack.append(name) + if len(stack) > _MAX_ELEMENT_DEPTH: + raise ValueError("resource-limit-exceeded") + + if stack and stack[-1] == "table" and source[cursor:].strip(): + raise ValueError("parser-profile-unsupported") + if stack: + raise ValueError("parser-profile-unsupported") + + +def _validate_base_url(base_url: str | None) -> str | None: + """Validate and serialize the optional document base URL up front.""" + if base_url is None or base_url == "": + return None + try: + base = URL(base_url) + except Exception as exc: + raise ValueError("attribute-canonicalization-failed") from exc + if base.protocol != "https:" or not base.hostname or base.username or base.password: + raise ValueError("url-policy-violation") + return str(base) + + def _walk( node, out: list[str], @@ -100,7 +206,7 @@ def _walk( cls_name = type(child).__name__ if cls_name in ("Comment", "Doctype", "CData", "ProcessingInstruction"): continue - out.append(normalize_text(str(child), preserve_whitespace)) + out.append(_escape_text(normalize_text(str(child), preserve_whitespace))) elif isinstance(child, Tag): name = child.name.lower() if child.name else "" is_block = name in _BLOCK_TAGS @@ -127,95 +233,43 @@ def _append_attribute_records( raw_value = " ".join(str(v) for v in raw_value) value = str(raw_value) if attr_name in ("href", "src"): - if base_url is None and not urlsplit(value).scheme: - # Relative URL with no base cannot be resolved. The draft - # (§4.3.2) requires a hard failure rather than a silent skip. - raise ValueError("attribute-canonicalization-failed") value = _canonicalize_url(value, base_url) else: value = normalize_text(value).strip() if "\n" in value: raise ValueError("attribute-canonicalization-failed") + value = value.replace("@", "@@") if out and out[-1] and not out[-1][-1].isspace(): out.append("\n") out.append(f"@attr:{element_name}:{attr_name}:{value}\n") -def _remove_dot_segments(path: str) -> str: - """RFC 3986 §5.2.4 remove_dot_segments, matching the WHATWG URL path - normalization the reference JS/Rust bindings perform via ``new URL``.""" - out = "" - inp = path - while inp: - if inp.startswith("../"): - inp = inp[3:] - elif inp.startswith("./"): - inp = inp[2:] - elif inp.startswith("/./"): - inp = "/" + inp[3:] - elif inp == "/.": - inp = "/" - elif inp.startswith("/../"): - inp = "/" + inp[4:] - out = out[: out.rfind("/")] if "/" in out else "" - elif inp == "/..": - inp = "/" - out = out[: out.rfind("/")] if "/" in out else "" - elif inp in (".", ".."): - inp = "" - else: - j = inp.find("/", 1) if inp.startswith("/") else inp.find("/") - if j == -1: - out += inp - inp = "" - else: - out += inp[:j] - inp = inp[j:] - return out - - def _canonicalize_url(value: str, base_url: str | None) -> str: - """Canonicalize an href/src value using the Web (WHATWG) URL serializer - semantics: lowercase scheme + host, IDNA/punycode host, strip default - ports, resolve dot-segments, preserve query and fragment. Produces the - same bytes as ``new URL(value, base).href`` for the cases the conformance - vectors exercise (draft §4.3.2).""" + """Parse and serialize an href/src with the WHATWG URL algorithm.""" + # URL preprocessing must not erase controls before policy validation. + if any(ord(ch) <= 0x1F or ord(ch) == 0x7F for ch in value): + raise ValueError("url-policy-violation") try: - absolute = urljoin(base_url or "", value) - parts = urlsplit(absolute) - except Exception as exc: # pragma: no cover - defensive URL parser guard + base = URL(base_url) if base_url is not None else None + except Exception as exc: raise ValueError("attribute-canonicalization-failed") from exc - if not parts.scheme: - raise ValueError("attribute-canonicalization-failed") - if not parts.netloc: - # Opaque URL with no authority (mailto:, tel:, javascript:, data:, - # about:, sms:, geo:, ...). The WHATWG URL parser accepts these; the - # part after "scheme:" is an opaque path that is serialized verbatim - # (scheme lowercased), matching new URL().href. No host/port/dot-segment - # normalization applies. - return urlunsplit((parts.scheme.lower(), "", parts.path, parts.query, parts.fragment)) - if parts.username or parts.password: - raise ValueError("attribute-canonicalization-failed") - hostname = (parts.hostname or "").lower() - if not hostname: + if base is not None: + if base.protocol != "https:" or not base.hostname or base.username or base.password: + raise ValueError("url-policy-violation") + try: + parsed = URL(value, str(base) if base is not None else None) + except Exception as exc: + raise ValueError("attribute-canonicalization-failed") from exc + if parsed.protocol != "https:" or parsed.username or parsed.password: + raise ValueError("url-policy-violation") + if not parsed.hostname: raise ValueError("attribute-canonicalization-failed") - if hostname.isascii(): - netloc = hostname - else: - try: - netloc = hostname.encode("idna").decode("ascii") - except Exception as exc: - raise ValueError("attribute-canonicalization-failed") from exc - if parts.port is not None: - default = (parts.scheme == "http" and parts.port == 80) or ( - parts.scheme == "https" and parts.port == 443 - ) - if not default: - netloc = f"{netloc}:{parts.port}" - path = _remove_dot_segments(parts.path or "/") or "/" - if not path.startswith("/"): - path = "/" + path - return urlunsplit((parts.scheme.lower(), netloc, path, parts.query, parts.fragment)) + return str(parsed) + + +def _escape_text(value: str) -> str: + """Escape commercial-at signs in text-node records (htmltrust-c14n-v1).""" + return value.replace("@", "@@") def _finalize_parts(text: str) -> str: @@ -224,4 +278,7 @@ def _finalize_parts(text: str) -> str: text = text.replace(" \n", "\n").replace("\n ", "\n") while "\n\n" in text: text = text.replace("\n\n", "\n") - return text.strip() + text = text.strip() + if len(text.encode("utf-8")) > _MAX_OUTPUT_BYTES: + raise ValueError("resource-limit-exceeded") + return text diff --git a/python/htmltrust_canonicalization/_jcs.py b/python/htmltrust_canonicalization/_jcs.py new file mode 100644 index 0000000..817c01c --- /dev/null +++ b/python/htmltrust_canonicalization/_jcs.py @@ -0,0 +1,105 @@ +"""Strict raw JSON Canonicalization Scheme (RFC 8785) entry point.""" + +from __future__ import annotations + +import json +import math +from typing import Any + +import rfc8785 + +_MAX_DOCUMENT_BYTES = 1024 * 1024 +_MAX_NESTING_DEPTH = 256 + + +def _enforce_nesting_limit(document: bytes) -> None: + depth = 0 + in_string = False + escaped = False + for byte in document: + if in_string: + if escaped: + escaped = False + elif byte == 0x5C: + escaped = True + elif byte == 0x22: + in_string = False + continue + if byte == 0x22: + in_string = True + elif byte in (0x5B, 0x7B): + depth += 1 + if depth > _MAX_NESTING_DEPTH: + raise ValueError("resource-limit-exceeded") + elif byte in (0x5D, 0x7D): + depth -= 1 + + +def _pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("jcs-duplicate-key") + result[key] = value + return result + + +def _finite(value: str) -> float: + try: + number = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError("jcs-number") from exc + if not math.isfinite(number): + raise ValueError("jcs-number") + return number + + +def _reject_surrogates(value: Any) -> None: + if isinstance(value, str): + if any(0xD800 <= ord(ch) <= 0xDFFF for ch in value): + raise ValueError("jcs-invalid-surrogate") + elif isinstance(value, list): + for item in value: + _reject_surrogates(item) + elif isinstance(value, dict): + for key, item in value.items(): + _reject_surrogates(key) + _reject_surrogates(item) + + +def canonicalize_json_document(document: str | bytes) -> str: + """Parse and canonicalize one raw JSON document using RFC 8785. + + Duplicate members, non-finite/overflowing numbers, invalid JSON, and + lone UTF-16 surrogate code points are rejected before serialization. + """ + if not isinstance(document, (str, bytes, bytearray)): + raise TypeError("canonicalize_json_document expects raw JSON text") + try: + document_bytes = document.encode("utf-8", "strict") if isinstance(document, str) else bytes(document) + document_bytes.decode("utf-8", "strict") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise ValueError("jcs-invalid-surrogate") from exc + if len(document_bytes) > _MAX_DOCUMENT_BYTES: + raise ValueError("resource-limit-exceeded") + _enforce_nesting_limit(document_bytes) + try: + value = json.loads( + document_bytes, + object_pairs_hook=_pairs, + parse_constant=lambda _value: (_ for _ in ()).throw(ValueError("jcs-invalid-json")), + parse_float=_finite, + parse_int=_finite, + ) + except ValueError as exc: + if str(exc) in {"jcs-duplicate-key", "jcs-invalid-json", "jcs-number", "jcs-invalid-surrogate"}: + raise + raise ValueError("jcs-invalid-json") from exc + _reject_surrogates(value) + try: + canonical = rfc8785.dumps(value) + except (ValueError, TypeError, OverflowError) as exc: + raise ValueError("jcs-number") from exc + if len(canonical) > _MAX_DOCUMENT_BYTES: + raise ValueError("resource-limit-exceeded") + return canonical.decode("utf-8") diff --git a/python/htmltrust_canonicalization/_normalize.py b/python/htmltrust_canonicalization/_normalize.py index 84fbdfc..be7c054 100644 --- a/python/htmltrust_canonicalization/_normalize.py +++ b/python/htmltrust_canonicalization/_normalize.py @@ -20,6 +20,7 @@ from typing import Iterable, Union _RangeOrPoint = Union[int, tuple[int, int]] +_MAX_RESOURCE_BYTES = 1024 * 1024 def _build_class(items: Iterable[_RangeOrPoint]) -> str: @@ -163,6 +164,8 @@ def normalize_text(text: str, preserve_whitespace: bool = False) -> str: """ if not isinstance(text, str): raise TypeError("normalize_text expects a str") + if len(text.encode("utf-8")) > _MAX_RESOURCE_BYTES: + raise ValueError("resource-limit-exceeded") # Phase 1: NFKC -- ligatures, fullwidth/halfwidth, presentation forms, # superscripts, CJK compatibility, Jamo composition. @@ -189,4 +192,7 @@ def normalize_text(text: str, preserve_whitespace: bool = False) -> str: # Phase 5: ellipsis. text = _ELLIPSIS_RE.sub("...", text) + if len(text.encode("utf-8")) > _MAX_RESOURCE_BYTES: + raise ValueError("resource-limit-exceeded") + return text diff --git a/python/pyproject.toml b/python/pyproject.toml index 6c6f63f..68471d3 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "htmltrust-canonicalization" -version = "0.1.0" +version = "0.3.0" description = "Canonical text normalization and HTML extraction for HTMLTrust signed content. Byte-identical output to the JavaScript, Go, PHP, and Rust bindings." readme = "README.md" requires-python = ">=3.10" @@ -24,12 +24,16 @@ classifiers = [ "Topic :: Security :: Cryptography", ] dependencies = [ - "beautifulsoup4>=4.12", + "beautifulsoup4==4.15.0", + "html5lib==1.1", + "pywhatwgurl==0.1.2", + "rfc8785==0.1.4", ] [project.optional-dependencies] dev = [ - "pytest>=8.0", + "cryptography==50.0.1", + "pytest==9.1.1", ] [project.urls] diff --git a/python/tests/test_claims.py b/python/tests/test_claims.py index 24af019..e52e37e 100644 --- a/python/tests/test_claims.py +++ b/python/tests/test_claims.py @@ -45,13 +45,9 @@ def test_normalizes_names(): assert out == "odd...name:x\n" -def test_coerces_value_to_string(): - out = canonicalize_claims({"count": 42, "enabled": True}) - # Booleans serialize as "True" / "False" via str(); that's fine for - # this layer -- callers should pre-stringify if they need different - # representations. - assert "count:42" in out - assert "enabled:True" in out +def test_rejects_non_string_value(): + with pytest.raises(ValueError, match="claim-malformed"): + canonicalize_claims({"count": 42}) # type: ignore[dict-item] def test_rejects_non_mapping(): diff --git a/python/tests/test_extract.py b/python/tests/test_extract.py index 7f592cd..10bd22f 100644 --- a/python/tests/test_extract.py +++ b/python/tests/test_extract.py @@ -84,6 +84,25 @@ def test_table_cells_separated(): assert extract_canonical_text(html) == "a\nb\nc\nd" +@pytest.mark.parametrize( + "html", + [ + "

x", + "tail
x
", + ], +) +def test_parser_preflight_rejects_unclosed_and_foster_parented_text(html): + with pytest.raises(ValueError, match="parser-profile-unsupported"): + extract_canonical_text(html) + + +def test_colon_qualified_elements_count_toward_depth_limit(): + nested = "".join("" for _ in range(257)) + closing = "".join("" for _ in range(257)) + with pytest.raises(ValueError, match="resource-limit-exceeded"): + extract_canonical_text(nested + "x" + closing) + + def test_inline_link_no_separator(): """Anchor tags are inline; they must NOT add separators. With a base URL the relative href resolves and emits a signed-attribute record.""" @@ -102,6 +121,11 @@ def test_relative_url_no_base_fails(): extract_canonical_text('

here

') +def test_invalid_base_url_fails_without_url_attributes(): + with pytest.raises(ValueError, match="attribute-canonicalization-failed"): + extract_canonical_text("

x

", base_url="not a URL") + + def test_signed_semantic_attributes_are_canonicalized(): html = ( '

link' diff --git a/python/tests/test_jcs.py b/python/tests/test_jcs.py new file mode 100644 index 0000000..704b5e0 --- /dev/null +++ b/python/tests/test_jcs.py @@ -0,0 +1,34 @@ +import pytest + +from htmltrust_canonicalization import canonicalize_json_document + + +def test_jcs_sorts_utf16_keys_and_numbers(): + assert canonicalize_json_document( + '{"z":-0,"a":1e30,"b":4.50,"😀":2,"":1}' + ) == '{"a":1e+30,"b":4.5,"z":0,"😀":2,"":1}' + + +def test_jcs_uses_binary64_for_large_integer_tokens(): + assert canonicalize_json_document( + '[9007199254740992,295147905179352830000,1424953923781206.25]' + ) == '[9007199254740992,295147905179352830000,1424953923781206.2]' + + +@pytest.mark.parametrize( + ("document", "reason"), + [ + ('{"a":1,"a":2}', "jcs-duplicate-key"), + ('"\\uD800"', "jcs-invalid-surrogate"), + ('{"n":1e400}', "jcs-number"), + ], +) +def test_jcs_rejects_unsafe_raw_json(document, reason): + with pytest.raises(ValueError, match=reason): + canonicalize_json_document(document) + + +def test_jcs_rejects_excessive_nesting(): + document = "[" * 257 + "0" + "]" * 257 + with pytest.raises(ValueError, match="resource-limit-exceeded"): + canonicalize_json_document(document) diff --git a/python/tests/test_normalize.py b/python/tests/test_normalize.py index 44c5947..4863598 100644 --- a/python/tests/test_normalize.py +++ b/python/tests/test_normalize.py @@ -74,6 +74,13 @@ def test_normalize_text_rejects_non_string(): normalize_text(123) # type: ignore[arg-type] +def test_normalize_text_enforces_input_and_output_limits(): + with pytest.raises(ValueError, match="resource-limit-exceeded"): + normalize_text("a" * 1_048_577) + with pytest.raises(ValueError, match="resource-limit-exceeded"): + normalize_text("…" * 349_526) + + def test_zwj_preserved_emoji(): """Family ZWJ sequence must survive normalization.""" family = "\U0001F468‍\U0001F469‍\U0001F467" diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..ba2c0b6 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,970 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cssparser" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.11.3", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12a0bb14ac04a9fcf170d0bbbef949b44cc492f4452bd20c095636956f653642" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "html5ever" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13771afe0e6e846f1e67d038d4cb29998a6779f93c809212e4e9c32efd244d4" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "htmltrust-canonicalization" +version = "0.3.0" +dependencies = [ + "ego-tree", + "scraper", + "serde", + "serde_json", + "serde_json_canonicalizer", + "unicode-normalization", + "url", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ce3abbeba692c8b8441d036ef91aea6df8da2c6b6e21c7e14d3c18e526be45" +dependencies = [ + "log", + "phf 0.11.3", + "phf_codegen 0.11.3", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher 0.3.11", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher 1.0.3", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90460b31bfe1fc07be8262e42c665ad97118d4585869de9345a84d501a9eaf0" +dependencies = [ + "ahash", + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "once_cell", + "selectors", + "tendril", +] + +[[package]] +name = "selectors" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf 0.10.1", + "phf_codegen 0.10.0", + "precomputed-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_json_canonicalizer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe52319a927259afbfa5180c5157cd8167edfd3e8c254f9558c7fef44c5649f2" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + +[[package]] +name = "servo_arc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.11.3", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9e58231..0f73777 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "htmltrust-canonicalization" -version = "0.1.0" +version = "0.3.0" edition = "2021" -rust-version = "1.74" +rust-version = "1.86" description = "Canonical text normalization and HTML extraction for HTMLTrust signed content. Byte-identical output to the JavaScript, Go, PHP, and Python bindings." license = "LicenseRef-PolyForm-Noncommercial-1.0.0" repository = "https://github.com/HTMLTrust/htmltrust-canonicalization" @@ -19,6 +19,9 @@ unicode-normalization = "0.1" scraper = "0.20" ego-tree = "0.6" url = "2" +serde = "1" +serde_json = "1" +serde_json_canonicalizer = "0.3" [dev-dependencies] # pure stdlib tests; nothing extra required. diff --git a/rust/README.md b/rust/README.md index 3af2269..cacd812 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,64 +1,106 @@ -# HTMLTrust Canonicalization -- Rust +# HTMLTrust Canonicalization for Rust -Rust crate for the HTMLTrust canonical text normalization library. Produces byte-identical output to the JavaScript, Go, PHP, and Python implementations for every test vector in the shared conformance suite. +This crate implements the Rust binding for `htmltrust-c14n-v1`. It normalizes +text, extracts signed content from HTML, canonicalizes claims, and +canonicalizes raw JSON under RFC 8785. The shared fixtures require the same +bytes from the JavaScript, Go, PHP, Python, and Rust bindings. -## Status +Version: `0.3.0` release candidate +Rust: 1.86 or newer -Implemented. The 18-case normalization conformance suite from the JavaScript reference (`javascript/test.js`) passes, along with parity tests for `extract_canonical_text` and `canonicalize_claims`. +## Test a fresh checkout -Out of scope for this crate: signature verification and key resolution. Those will arrive in a follow-up PR alongside the Python binding once the JavaScript surface area lands on `main`. +From the repository root, Docker runs the Rust unit tests, native FFI tests, +and every shared conformance fixture: -## Scope - -This crate provides three functions: +```sh +docker compose -f compose.test.yml run --rm rust +``` -1. **`normalize_text(text: &str, preserve_whitespace: bool) -> String`** -- applies the 8-phase canonicalization defined in [`../spec.md`](../spec.md) to a UTF-8 string. -2. **`extract_canonical_text(html: &str) -> String`** -- parses an HTML fragment with `scraper` (html5ever), walks the DOM, emits text nodes and signed semantic attributes in document order, and applies `normalize_text` to text/attribute values. Use `extract_canonical_text_with_base_url` when relative `href` or `src` values need resolution. -3. **`canonicalize_claims(claims: &BTreeMap) -> String`** -- serializes a claim map to the canonical, hashable string used by the `claims-hash` field of the signature binding. +Run `./scripts/test-in-docker.sh` to test all five language bindings. -All three are pure functions: no I/O, deterministic output for the same input. +With Rust installed locally: -## Dependencies +```sh +cargo test --locked --manifest-path rust/Cargo.toml +cargo test --locked --manifest-path ffi/Cargo.toml +``` -- `unicode-normalization` for NFKC -- `scraper` (html5ever-backed) for HTML parsing in `extract_canonical_text` -- `ego-tree` for the DOM walk types re-exported by scraper -- `url` for Web URL parsing of signed semantic `href` and `src` attributes +## Install -## Conformance +Use the repository while `0.3.0` is under review: -`tests/conformance.rs` runs all 18 normalization vectors from `javascript/test.js`, plus `extract_canonical_text` and `canonicalize_claims` parity cases. Output MUST stay byte-identical to the JavaScript / Go / PHP / Python bindings. +```toml +[dependencies] +htmltrust-canonicalization = { git = "https://github.com/HTMLTrust/htmltrust-canonicalization", rev = "" } +``` -## Installation +After the crate is published, use the release series: ```toml [dependencies] -htmltrust-canonicalization = "0.1" +htmltrust-canonicalization = "0.3" ``` -## Usage +## Profile-v1 API + +- `try_normalize_text` normalizes a UTF-8 `str` and enforces the 1 MiB source + and output limits. +- `try_normalize_text_v1` accepts bytes and also rejects invalid UTF-8. +- `try_extract_canonical_text_with_options` parses HTML with explicit + compatibility whitespace and base URL options. Profile-v1 callers use + `preserve_whitespace: false`; the portable profile rejects nesting deeper + than 256 elements. +- `canonicalize_claims_checked` validates, sorts, escapes, and serializes + claim metadata. +- `canonicalize_json_document` validates and canonicalizes one raw JSON + document, including duplicate-key checks. + +The infallible normalization, extraction, and claims functions remain for +`0.2` callers that enforce their own limits. New signing code should use the +fallible functions above. + +## Example ```rust use std::collections::BTreeMap; use htmltrust_canonicalization::{ - normalize_text, extract_canonical_text, canonicalize_claims, + canonicalize_claims_checked, + try_extract_canonical_text_with_options, + try_normalize_text, + ExtractOptions, }; -let canonical = normalize_text("He said, \"Hello\u{2026}\"", false); -// -> "He said, \"Hello...\"" +let canonical = try_normalize_text("He said, \"Hello\u{2026}\"", false)?; +assert_eq!(canonical, "He said, \"Hello...\""); + +let content = try_extract_canonical_text_with_options( + "

Read the paper.

", + ExtractOptions { + preserve_whitespace: false, + base_url: Some("https://example.org/article"), + }, +)?; + +let claims = BTreeMap::from([ + ("License".to_string(), "CC-BY-4.0".to_string()), +]); +let claim_bytes = canonicalize_claims_checked(&claims)?; +assert_eq!(claim_bytes, "License:CC-BY-4.0\n"); +# Ok::<(), String>(()) +``` -let from_html = extract_canonical_text("

Hello world!

"); -// -> "Hello world!" +Relative `href` and `src` attributes require an HTTPS base URL. The safe URL +profile rejects credentials, control characters, and unsupported schemes. -let mut claims = BTreeMap::new(); -claims.insert("License".to_string(), "CC-BY-4.0".to_string()); -claims.insert("AIAssistance".to_string(), "None".to_string()); -let claims_str = canonicalize_claims(&claims); -// -> "AIAssistance:None\nLicense:CC-BY-4.0\n" -``` +## Native FFI -## Tests +The `ffi/` crate exposes length-based `*_v1` functions for normalization and +extraction. Status `0` means success, status `1` returns an allocated UTF-8 +error code, and status `2` reports an invalid pointer. Every valid output +pointer is cleared before input decoding. Release returned byte buffers with +`htmltrust_bytes_free`. -```bash -cargo test -``` +The normative protocol text is maintained in the +[HTMLTrust specification](https://github.com/HTMLTrust/htmltrust-spec/tree/main/ietf-draft). +The repository's shared fixtures are the executable cross-language contract. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index e442d38..d78acde 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -15,11 +15,20 @@ use std::collections::BTreeMap; -use scraper::{node::Node, Html}; use ego_tree::NodeRef; +use scraper::{node::Node, Html}; +use serde::de::{self, Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}; +use serde::ser::{Serialize, Serializer}; use unicode_normalization::UnicodeNormalization; use url::Url; +/// Maximum size of a source document and its canonical output. +pub const MAX_DOCUMENT_BYTES: usize = 1024 * 1024; +const MAX_ELEMENT_DEPTH: usize = 256; +const INVALID_UTF8: &str = "invalid-utf8"; +const PARSER_UNSUPPORTED: &str = "parser-profile-unsupported"; +const RESOURCE_LIMIT: &str = "resource-limit-exceeded"; + // --------------------------------------------------------------------------- // Codepoint ranges, mirroring the JS reference regex character classes // byte-for-byte. Inclusive ranges. Single codepoints expressed as @@ -147,7 +156,10 @@ pub fn normalize_text(text: &str, preserve_whitespace: bool) -> String { let nfkc: String = text.nfkc().collect(); // Phases 6 + 7: strip invisible / formatting / bidi characters. - let stripped: String = nfkc.chars().filter(|&c| !in_ranges(c, STRIP_RANGES)).collect(); + let stripped: String = nfkc + .chars() + .filter(|&c| !in_ranges(c, STRIP_RANGES)) + .collect(); // Phase 2: whitespace normalization. let ws: String = if preserve_whitespace { @@ -187,6 +199,31 @@ pub fn normalize_text(text: &str, preserve_whitespace: bool) -> String { out } +/// Fallible byte-oriented normalization entry point for profile-v1 callers. +/// +/// The source and normalized UTF-8 output are each limited to +/// [`MAX_DOCUMENT_BYTES`]. The legacy [`normalize_text`] wrapper remains +/// available for callers that already enforce their own limits. +pub fn try_normalize_text(text: &str, preserve_whitespace: bool) -> Result { + if text.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + let result = normalize_text(text, preserve_whitespace); + if result.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + Ok(result) +} + +/// Fallible byte-oriented normalization entry point for profile-v1 callers. +pub fn try_normalize_text_v1(text: &[u8], preserve_whitespace: bool) -> Result { + if text.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + let text = std::str::from_utf8(text).map_err(|_| INVALID_UTF8.to_string())?; + try_normalize_text(text, preserve_whitespace) +} + /// Extract canonical text from an HTML fragment. /// /// Implements the HTML -> canonical text extraction defined in spec §2.1 @@ -196,37 +233,372 @@ pub fn normalize_text(text: &str, preserve_whitespace: bool) -> String { /// # Arguments /// /// * `html` -- HTML fragment to canonicalize. +/// * `options` -- extraction options, including `preserve_whitespace` and an +/// optional HTTPS base URL for relative signed attributes. /// /// # Returns /// /// Canonical text, ready to be hashed. Trimmed of leading/trailing /// whitespace. pub fn extract_canonical_text(html: &str) -> String { - extract_canonical_text_with_base_url(html, None) + extract_canonical_text_with_options(html, ExtractOptions::default()) } /// Extract canonical text from an HTML fragment, resolving relative signed /// semantic URL attributes against `base_url` when supplied. pub fn extract_canonical_text_with_base_url(html: &str, base_url: Option<&str>) -> String { - try_extract_canonical_text_with_base_url(html, base_url) + extract_canonical_text_with_options( + html, + ExtractOptions { + base_url, + ..ExtractOptions::default() + }, + ) +} + +/// Extraction options. `preserve_whitespace` is passed to text-node +/// normalization, matching the JavaScript `preserveWhitespace` option. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct ExtractOptions<'a> { + pub preserve_whitespace: bool, + pub base_url: Option<&'a str>, +} + +/// Extract canonical text with explicit options. +pub fn extract_canonical_text_with_options(html: &str, options: ExtractOptions<'_>) -> String { + try_extract_canonical_text_with_options(html, options) .expect("attribute-canonicalization-failed") } +/// Fallible extraction entry point with explicit options. Source HTML, base +/// URL, and canonical output are each limited to +/// [`MAX_DOCUMENT_BYTES`]. +pub fn try_extract_canonical_text(html: &str) -> Result { + try_extract_canonical_text_with_options(html, ExtractOptions::default()) +} + /// Fallible form of [`extract_canonical_text_with_base_url`]. pub fn try_extract_canonical_text_with_base_url( html: &str, base_url: Option<&str>, ) -> Result { + try_extract_canonical_text_with_options( + html, + ExtractOptions { + base_url, + ..ExtractOptions::default() + }, + ) +} + +/// Fallible extraction entry point with explicit options. +pub fn try_extract_canonical_text_with_options( + html: &str, + options: ExtractOptions<'_>, +) -> Result { + if html.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + preflight_source(html)?; let document = Html::parse_fragment(html); - let base = match base_url { - Some(raw) => Some(Url::parse(raw).map_err(|_| "attribute-canonicalization-failed".to_string())?), + // html5ever deliberately repairs malformed HTML. The portable profile + // cannot sign a repaired tree, so every diagnostic is a hard failure. + if !document.errors.is_empty() { + return Err(PARSER_UNSUPPORTED.to_string()); + } + let base = match options.base_url { + Some(raw) => { + if raw.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + let parsed = + Url::parse(raw).map_err(|_| "attribute-canonicalization-failed".to_string())?; + if parsed.scheme() != "https" + || !parsed.username().is_empty() + || parsed.password().is_some() + { + return Err("url-policy-violation".to_string()); + } + Some(parsed) + } None => None, }; let mut out = String::new(); - walk(document.tree.root(), &mut out, base.as_ref())?; + walk( + document.tree.root(), + &mut out, + base.as_ref(), + options.preserve_whitespace, + )?; + + if out.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + let result = finalize_parts(&out, options.preserve_whitespace); + if result.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + Ok(result) +} + +/// Fallible, profile-v1 entry point accepting raw UTF-8 bytes. No lossy +/// decoding is performed, which makes the API suitable for FFI callers. +pub fn try_extract_canonical_text_v1( + html: &[u8], + base_url: Option<&[u8]>, +) -> Result { + if html.len() > MAX_DOCUMENT_BYTES { + return Err(RESOURCE_LIMIT.to_string()); + } + if base_url.is_some_and(|base| base.len() > MAX_DOCUMENT_BYTES) { + return Err(RESOURCE_LIMIT.to_string()); + } + let html = std::str::from_utf8(html).map_err(|_| INVALID_UTF8.to_string())?; + let base = match base_url { + Some(bytes) => Some(std::str::from_utf8(bytes).map_err(|_| INVALID_UTF8.to_string())?), + None => None, + }; + try_extract_canonical_text_with_base_url(html, base) +} - Ok(finalize_parts(&out)) +/// Alias emphasizing that the input is a byte string. +pub fn try_extract_canonical_text_bytes( + html: &[u8], + base_url: Option<&[u8]>, +) -> Result { + try_extract_canonical_text_v1(html, base_url) +} + +/// Extract claim metadata from direct child `meta` elements of the first +/// signed section. Without a wrapper, the fragment is treated as section +/// inner HTML and parser-created html/head/body nodes are ignored. +pub fn extract_claims_from_signed_section(html: &str) -> Result, String> { + if html.len() > MAX_DOCUMENT_BYTES { + return Err("resource-limit-exceeded".to_string()); + } + preflight_source(html)?; + let document = Html::parse_fragment(html); + if !document.errors.is_empty() { + return Err(PARSER_UNSUPPORTED.to_string()); + } + let root = document.tree.root(); + let section = root.descendants().find( + |node| matches!(node.value(), Node::Element(element) if element.name() == "signed-section"), + ); + let candidates: Vec<_> = match section { + Some(node) => node.children().collect(), + None => root + .descendants() + .filter(|node| { + if !matches!(node.value(), Node::Element(element) if element.name() == "meta") { + return false; + } + let mut parent = node.parent(); + while let Some(ancestor) = parent { + match ancestor.value() { + Node::Document | Node::Fragment => return true, + Node::Element(element) + if matches!(element.name(), "html" | "head" | "body") => {} + _ => return false, + } + parent = ancestor.parent(); + } + true + }) + .collect(), + }; + let mut claims = BTreeMap::new(); + for node in candidates { + let Node::Element(element) = node.value() else { + continue; + }; + if element.name() != "meta" { + continue; + } + let raw_name = element + .attr("name") + .ok_or_else(|| "claim-malformed".to_string())?; + let raw_content = element + .attr("content") + .ok_or_else(|| "claim-malformed".to_string())?; + if claims.len() >= 64 { + return Err("resource-limit-exceeded".to_string()); + } + let name = normalize_text(raw_name, false).trim().to_string(); + let content = normalize_text(raw_content, false).trim().to_string(); + if name.is_empty() { + return Err("claim-malformed".to_string()); + } + if name.len() > 4096 || content.len() > 4096 { + return Err("resource-limit-exceeded".to_string()); + } + if claims.insert(name, content).is_some() { + return Err("claim-duplicate".to_string()); + } + } + Ok(claims) +} + +fn preflight_source(html: &str) -> Result<(), String> { + let lower = html.to_ascii_lowercase(); + // A small source-level stack catches EOF-implied closes and lets us reject + // malformed nesting before html5ever has a chance to repair it. It is + // intentionally conservative: all non-void starts require an explicit + // matching end tag in the signed profile. + const VOID: &[&str] = &[ + "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", + "source", "track", "wbr", + ]; + let bytes = html.as_bytes(); + let mut stack: Vec = Vec::new(); + let mut i = 0; + while i < bytes.len() { + let raw_close = stack.last().and_then(|name| { + matches!( + name.as_str(), + "script" | "style" | "iframe" | "xmp" | "noembed" + ) + .then(|| format!("") else { + return Err(PARSER_UNSUPPORTED.to_string()); + }; + let comment = &html[i + 4..i + 4 + end]; + if comment.contains("--") || comment.ends_with('-') { + return Err(PARSER_UNSUPPORTED.to_string()); + } + i += end + 7; + continue; + } + let mut j = i + 1; + if j >= bytes.len() { + return Err(PARSER_UNSUPPORTED.to_string()); + } + let closing = bytes[j] == b'/'; + if closing { + j += 1; + } + while j < bytes.len() && bytes[j].is_ascii_whitespace() { + j += 1; + } + let name_start = j; + while j < bytes.len() + && !bytes[j].is_ascii_whitespace() + && bytes[j] != b'/' + && bytes[j] != b'>' + { + j += 1; + } + if j == name_start { + // declarations and processing instructions are left to html5ever; + // malformed ones will produce a parser diagnostic. + i += 1; + continue; + } + let name = html[name_start..j].to_ascii_lowercase(); + if !closing && matches!(name.as_str(), "svg" | "math" | "foreignobject") { + return Err(PARSER_UNSUPPORTED.to_string()); + } + let mut quote = 0u8; + let mut end = j; + while end < bytes.len() { + match (quote, bytes[end]) { + (0, b'\'' | b'"') => quote = bytes[end], + (q, c) if q == c => quote = 0, + (0, b'>') => break, + _ => {} + } + end += 1; + } + if end == bytes.len() { + return Err(PARSER_UNSUPPORTED.to_string()); + } + if closing { + if stack.pop().as_deref() != Some(name.as_str()) { + return Err(PARSER_UNSUPPORTED.to_string()); + } + } else if !VOID.contains(&name.as_str()) && !html[i..=end].ends_with("/>") { + if stack.len() >= MAX_ELEMENT_DEPTH { + return Err(RESOURCE_LIMIT.to_string()); + } + stack.push(name); + } + i = end + 1; + } + if !stack.is_empty() { + return Err(PARSER_UNSUPPORTED.to_string()); + } + Ok(()) +} + +fn reference_end(bytes: &[u8], start: usize) -> Option { + if start >= bytes.len() || !(bytes[start].is_ascii_alphanumeric() || bytes[start] == b'#') { + return None; + } + let mut i = start; + while i < bytes.len() && !matches!(bytes[i], b';' | b'<' | b'>' | b' ' | b'\t' | b'\r' | b'\n') + { + i += 1; + } + if bytes.get(i) == Some(&b';') { + Some(i + 1) + } else { + Some(i) + } +} + +fn recognized_reference(reference: &str) -> bool { + // Numeric references are unambiguous only with a semicolon, and the + // parser is the authority for validity/range handling. + if reference.starts_with("&#") { + let doc = Html::parse_fragment(&format!("{reference}")); + return doc.errors.is_empty(); + } + // Compare the parser's result against the literal. Unknown names remain + // unchanged; names that are only a prefix (for example `¬it;`) are + // rejected because html5ever changes the source but does not consume the + // complete named reference. + let doc = Html::parse_fragment(&format!("{reference}")); + if !doc.errors.is_empty() { + return false; + } + let mut text = String::new(); + for node in doc.tree.root().descendants() { + if let Node::Text(t) = node.value() { + text.push_str(&t.text); + } + } + text != reference } fn is_excluded_tag(name: &str) -> bool { @@ -268,6 +640,7 @@ fn is_block_tag(name: &str) -> bool { | "p" | "pre" | "section" + | "signed-section" | "table" | "tr" | "td" @@ -276,7 +649,12 @@ fn is_block_tag(name: &str) -> bool { ) } -fn walk<'a>(root: NodeRef<'a, Node>, out: &mut String, base_url: Option<&Url>) -> Result<(), String> { +fn walk<'a>( + root: NodeRef<'a, Node>, + out: &mut String, + base_url: Option<&Url>, + preserve_whitespace: bool, +) -> Result<(), String> { // Iterative depth-first walk with an explicit heap stack, equivalent to the // natural recursion but bounded by heap rather than the call stack. Real-world // DOMs can nest deeply enough to overflow a native thread stack (a latent @@ -300,7 +678,10 @@ fn walk<'a>(root: NodeRef<'a, Node>, out: &mut String, base_url: Option<&Url>) - match item { Work::CloseBlock => out.push('\n'), Work::Enter(node) => match node.value() { - Node::Text(t) => out.push_str(&normalize_text(&t.text, false)), + Node::Text(t) => out.push_str(&escape_at_signs(&normalize_text( + &t.text, + preserve_whitespace, + ))), Node::Element(e) => { let name = e.name(); if is_excluded_tag(name) { @@ -344,7 +725,7 @@ fn append_attribute_records( let value = if attr == "href" || attr == "src" { canonicalize_url(raw, base_url)? } else { - normalize_text(raw, false).trim().to_string() + escape_at_signs(normalize_text(raw, false).trim()) }; if value.contains('\n') { return Err("attribute-canonicalization-failed".to_string()); @@ -364,6 +745,12 @@ fn append_attribute_records( } fn canonicalize_url(raw: &str, base_url: Option<&Url>) -> Result { + // URL parsers generally strip C0 controls as part of their recovery. The + // signed profile must inspect the decoded HTML attribute first so that a + // reference such as ` ` cannot silently change its meaning. + if raw.chars().any(|c| c.is_control()) { + return Err("url-policy-violation".to_string()); + } // The `url` crate is a WHATWG URL implementation: parsing already // lowercases scheme + host, punycodes IDN hosts, resolves dot-segments, // strips default ports, and preserves query + fragment. @@ -380,15 +767,26 @@ fn canonicalize_url(raw: &str, base_url: Option<&Url>) -> Result .map_err(|_| "attribute-canonicalization-failed".to_string())? } }; - Ok(parsed.to_string()) + if parsed.scheme() != "https" || !parsed.username().is_empty() || parsed.password().is_some() { + return Err("url-policy-violation".to_string()); + } + Ok(escape_at_signs(&parsed.to_string())) +} + +fn escape_at_signs(value: &str) -> String { + value.replace('@', "@@") } -fn finalize_parts(text: &str) -> String { +fn finalize_parts(text: &str, _preserve_whitespace: bool) -> String { let mut text = text.to_string(); while text.contains(" ") { text = text.replace(" ", " "); } - while text.contains(" \n") || text.contains("\n ") || text.contains("\t\n") || text.contains("\n\t") { + while text.contains(" \n") + || text.contains("\n ") + || text.contains("\t\n") + || text.contains("\n\t") + { text = text.replace(" \n", "\n"); text = text.replace("\n ", "\n"); text = text.replace("\t\n", "\n"); @@ -424,7 +822,13 @@ pub fn canonicalize_claims(claims: &BTreeMap) -> String { entries.sort_by(|a, b| a.0.cmp(&b.0)); entries .into_iter() - .map(|(k, v)| format!("{}:{}\n", k, v)) + .map(|(k, v)| { + format!( + "{}:{}\n", + escape_claim_component(&k), + escape_claim_component(&v) + ) + }) .collect::() } @@ -432,9 +836,10 @@ pub fn canonicalize_claims(claims: &BTreeMap) -> String { /// an empty normalized name is `claim-malformed`, and two names that /// normalize to the same value are `claim-duplicate`. Names are compared and /// sorted by their UTF-8 byte sequence (`String` ordering). -pub fn canonicalize_claims_checked( - claims: &BTreeMap, -) -> Result { +pub fn canonicalize_claims_checked(claims: &BTreeMap) -> Result { + if claims.len() > 64 { + return Err("resource-limit-exceeded".to_string()); + } let mut entries: Vec<(String, String)> = Vec::with_capacity(claims.len()); let mut seen: std::collections::BTreeSet = std::collections::BTreeSet::new(); for (k, v) in claims { @@ -443,14 +848,248 @@ pub fn canonicalize_claims_checked( if name.is_empty() { return Err("claim-malformed".to_string()); } + if name.len() > 4096 || value.len() > 4096 { + return Err("resource-limit-exceeded".to_string()); + } if !seen.insert(name.clone()) { return Err("claim-duplicate".to_string()); } entries.push((name, value)); } entries.sort_by(|a, b| a.0.cmp(&b.0)); - Ok(entries + let result: String = entries .into_iter() - .map(|(k, v)| format!("{}:{}\n", k, v)) - .collect::()) + .map(|(k, v)| { + format!( + "{}:{}\n", + escape_claim_component(&k), + escape_claim_component(&v) + ) + }) + .collect(); + if result.len() > MAX_DOCUMENT_BYTES { + return Err("resource-limit-exceeded".to_string()); + } + Ok(result) +} + +fn escape_claim_component(value: &str) -> String { + // Ordering matters: escape the escape character before introducing any + // escapes for the other delimiters. + value + .replace('\\', "\\\\") + .replace(':', "\\:") + .replace('\n', "\\n") +} + +/// Canonicalize one raw JSON document according to RFC 8785 (JCS). +/// +/// Parsing is done with a duplicate-preserving serde visitor before values are +/// handed to `serde_json_canonicalizer`; serde_json::Value would silently keep +/// only the last duplicate object member. JSON strings are not normalized. +pub fn canonicalize_json_document(raw: &[u8]) -> Result { + if raw.len() > MAX_DOCUMENT_BYTES { + return Err("resource-limit-exceeded".to_string()); + } + enforce_json_nesting_limit(raw)?; + if has_lone_surrogate_escape(raw) { + return Err("jcs-invalid-surrogate".to_string()); + } + let mut de = serde_json::Deserializer::from_slice(raw); + let value = StrictJson::deserialize(&mut de).map_err(map_json_error)?; + de.end().map_err(map_json_error)?; + let output = serde_json_canonicalizer::to_string(&value) + .map_err(|e| format!("jcs-invalid-json: {e}"))?; + if output.len() > MAX_DOCUMENT_BYTES { + return Err("resource-limit-exceeded".to_string()); + } + Ok(output) +} + +fn enforce_json_nesting_limit(raw: &[u8]) -> Result<(), String> { + const MAX_NESTING_DEPTH: usize = 256; + let mut depth = 0usize; + let mut in_string = false; + let mut escaped = false; + for byte in raw { + if in_string { + if escaped { + escaped = false; + } else if *byte == b'\\' { + escaped = true; + } else if *byte == b'"' { + in_string = false; + } + continue; + } + match *byte { + b'"' => in_string = true, + b'[' | b'{' => { + depth += 1; + if depth > MAX_NESTING_DEPTH { + return Err("resource-limit-exceeded".to_string()); + } + } + b']' | b'}' => depth = depth.saturating_sub(1), + _ => {} + } + } + Ok(()) +} + +fn map_json_error(error: serde_json::Error) -> String { + let msg = error.to_string(); + if msg.contains("surrogate") { + "jcs-invalid-surrogate".to_string() + } else if msg.contains("number out of range") || msg.contains("invalid number") { + "jcs-number".to_string() + } else if msg.contains("duplicate object key") { + "jcs-duplicate-key".to_string() + } else { + format!("jcs-invalid-json: {msg}") + } +} + +fn hex4(bytes: &[u8]) -> Option { + if bytes.len() < 4 { + return None; + } + let mut value = 0u16; + for &c in &bytes[..4] { + value = value.checked_mul(16)?.checked_add(match c { + b'0'..=b'9' => (c - b'0') as u16, + b'a'..=b'f' => (c - b'a' + 10) as u16, + b'A'..=b'F' => (c - b'A' + 10) as u16, + _ => return None, + })?; + } + Some(value) +} + +fn has_lone_surrogate_escape(raw: &[u8]) -> bool { + let mut i = 0; + while i + 5 < raw.len() { + // A slash preceded by another slash is the escaped literal `\\`, not + // the start of a Unicode escape. Count the run to distinguish the two + // cases without pulling in a second JSON parser. + let mut slash_run = 0; + let mut p = i; + while p > 0 && raw[p - 1] == b'\\' { + slash_run += 1; + p -= 1; + } + if raw[i] == b'\\' && raw[i + 1] == b'u' && slash_run % 2 == 0 { + if let Some(value) = hex4(&raw[i + 2..i + 6]) { + if (0xD800..=0xDBFF).contains(&value) { + let paired = i + 11 < raw.len() + && raw[i + 6] == b'\\' + && raw[i + 7] == b'u' + && hex4(&raw[i + 8..i + 12]) + .is_some_and(|v| (0xDC00..=0xDFFF).contains(&v)); + if !paired { + return true; + } + i += 12; + continue; + } + if (0xDC00..=0xDFFF).contains(&value) { + return true; + } + } + } + i += 1; + } + false +} + +#[derive(Debug)] +enum StrictJson { + Null, + Bool(bool), + Number(f64), + String(String), + Array(Vec), + Object(BTreeMap), +} + +impl Serialize for StrictJson { + fn serialize(&self, serializer: S) -> Result { + match self { + Self::Null => serializer.serialize_none(), + Self::Bool(v) => serializer.serialize_bool(*v), + Self::Number(v) => serializer.serialize_f64(*v), + Self::String(v) => serializer.serialize_str(v), + Self::Array(v) => v.serialize(serializer), + Self::Object(v) => v.serialize(serializer), + } + } +} + +struct StrictVisitor; + +impl<'de> Visitor<'de> for StrictVisitor { + type Value = StrictJson; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a JSON value") + } + fn visit_unit(self) -> Result { + Ok(StrictJson::Null) + } + fn visit_bool(self, v: bool) -> Result { + Ok(StrictJson::Bool(v)) + } + fn visit_str(self, v: &str) -> Result { + Ok(StrictJson::String(v.to_owned())) + } + fn visit_string(self, v: String) -> Result { + Ok(StrictJson::String(v)) + } + fn visit_i64(self, v: i64) -> Result { + number(v as f64, self) + } + fn visit_u64(self, v: u64) -> Result { + number(v as f64, self) + } + fn visit_f64(self, v: f64) -> Result { + number(v, self) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut values = Vec::new(); + while let Some(v) = seq.next_element_seed(StrictSeed)? { + values.push(v); + } + Ok(StrictJson::Array(values)) + } + fn visit_map>(self, mut map: A) -> Result { + let mut values = BTreeMap::new(); + while let Some(key) = map.next_key::()? { + if values.contains_key(&key) { + return Err(de::Error::custom("duplicate object key")); + } + values.insert(key, map.next_value_seed(StrictSeed)?); + } + Ok(StrictJson::Object(values)) + } +} + +fn number(v: f64, _visitor: StrictVisitor) -> Result { + if v.is_finite() { + Ok(StrictJson::Number(v)) + } else { + Err(E::custom("number out of range")) + } +} + +struct StrictSeed; +impl<'de> de::DeserializeSeed<'de> for StrictSeed { + type Value = StrictJson; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(StrictVisitor) + } +} +impl<'de> Deserialize<'de> for StrictJson { + fn deserialize>(d: D) -> Result { + d.deserialize_any(StrictVisitor) + } } diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index ac895a6..51cb2db 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -7,9 +7,20 @@ use std::collections::BTreeMap; use htmltrust_canonicalization::{ - canonicalize_claims, extract_canonical_text, normalize_text, + canonicalize_claims, canonicalize_json_document, extract_canonical_text, + extract_canonical_text_with_options, extract_claims_from_signed_section, normalize_text, + try_extract_canonical_text, try_normalize_text, ExtractOptions, MAX_DOCUMENT_BYTES, }; +#[test] +fn jcs_rejects_excessive_nesting() { + let document = format!("{}0{}", "[".repeat(257), "]".repeat(257)); + assert_eq!( + canonicalize_json_document(document.as_bytes()), + Err("resource-limit-exceeded".to_string()) + ); +} + /// One conformance vector. `(input_a, input_b, should_match, description)`. type Case = (&'static str, &'static str, bool, &'static str); @@ -111,6 +122,55 @@ fn preserve_whitespace_skips_collapse() { assert_eq!(normalize_text(src, true), src); } +#[test] +fn extraction_options_use_shared_finalization() { + let options = ExtractOptions { + preserve_whitespace: true, + base_url: None, + }; + assert_eq!( + extract_canonical_text_with_options("
line1\n    line2\t\tline3
", options), + "line1\nline2\t\tline3", + ); + assert_eq!( + extract_canonical_text_with_options("

a b

", ExtractOptions::default()), + "a b", + ); +} + +#[test] +fn fallible_text_apis_enforce_source_and_output_limits() { + let source = "x".repeat(MAX_DOCUMENT_BYTES + 1); + assert_eq!( + try_normalize_text(&source, false), + Err("resource-limit-exceeded".into()) + ); + assert_eq!( + try_extract_canonical_text(&source), + Err("resource-limit-exceeded".into()) + ); + + // Ellipsis expansion makes a source below the limit produce an oversized + // canonical output, which must also be rejected. + let expanding = "…".repeat(MAX_DOCUMENT_BYTES / 2); + assert_eq!( + try_normalize_text(&expanding, false), + Err("resource-limit-exceeded".into()) + ); +} + +#[test] +fn fallible_text_apis_reject_invalid_utf8() { + assert_eq!( + htmltrust_canonicalization::try_normalize_text_v1(b"\xff", false), + Err("invalid-utf8".into()) + ); + assert_eq!( + htmltrust_canonicalization::try_extract_canonical_text_v1(b"\xff", None), + Err("invalid-utf8".into()) + ); +} + #[test] fn idempotent_for_typical_input() { let src = "\u{201C}Caf\u{00E9}\u{2014}test\u{2026}\u{201D}"; @@ -143,6 +203,14 @@ fn extract_excluded_elements_removed() { assert_eq!(extract_canonical_text(html), "before\nafter"); } +#[test] +fn extract_rejects_malformed_comments() { + assert_eq!( + try_extract_canonical_text("x"), + Err("parser-profile-unsupported".to_string()) + ); +} + #[test] fn extract_entity_decoding() { assert_eq!( @@ -199,6 +267,29 @@ fn extract_relative_url_no_base_fails() { assert!(err.contains("attribute-canonicalization-failed")); } +#[test] +fn extracts_direct_child_claims() { + let claims = extract_claims_from_signed_section( + r#"
"#, + ) + .unwrap(); + assert_eq!(claims.len(), 2); + assert_eq!(claims.get("author").map(String::as_str), Some("Alice")); + assert_eq!( + claims.get("signed-at").map(String::as_str), + Some("2026-08-27T12:00:00Z") + ); +} + +#[test] +fn rejects_duplicate_extracted_claim_names() { + let error = extract_claims_from_signed_section( + r#""#, + ) + .unwrap_err(); + assert_eq!(error, "claim-duplicate"); +} + #[test] fn claims_empty() { let claims: BTreeMap = BTreeMap::new(); diff --git a/scripts/test-in-docker.sh b/scripts/test-in-docker.sh new file mode 100755 index 0000000..f2ad0eb --- /dev/null +++ b/scripts/test-in-docker.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +if ! command -v docker >/dev/null 2>&1 || ! docker compose version >/dev/null 2>&1; then + echo "Docker with the Compose plugin is required." >&2 + exit 2 +fi + +CHECKOUT_ID="$(printf '%s' "$REPO_ROOT" | cksum | awk '{print $1}')" +COMPOSE_PROJECT="htmltrust-c14n-${CHECKOUT_ID}" +if [[ -n "${HTMLTRUST_TEST_SESSION_ID:-}" ]]; then + if [[ ! "$HTMLTRUST_TEST_SESSION_ID" =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then + echo "HTMLTRUST_TEST_SESSION_ID must use lowercase letters, digits, underscores, or hyphens." >&2 + exit 2 + fi + COMPOSE_PROJECT="${COMPOSE_PROJECT}-${HTMLTRUST_TEST_SESSION_ID}" +fi +COMPOSE_FILE="$REPO_ROOT/compose.test.yml" + +# Docker cannot create a nested volume target beneath the read-only checkout +# bind mount. Create empty host mountpoints for dependency volumes, then remove +# the empty directories after Compose unmounts them. +MOUNTPOINTS=("$REPO_ROOT/node_modules" "$REPO_ROOT/php/vendor") +CREATED_MOUNTPOINTS=() +for mountpoint in "${MOUNTPOINTS[@]}"; do + if [[ ! -d "$mountpoint" ]]; then + mkdir -p "$mountpoint" + CREATED_MOUNTPOINTS+=("$mountpoint") + fi +done + +cleanup_mountpoints() { + for mountpoint in "${CREATED_MOUNTPOINTS[@]}"; do + rmdir "$mountpoint" 2>/dev/null || true + done +} +trap cleanup_mountpoints EXIT + +for service in javascript go php python rust; do + echo + echo "Running ${service} tests" + docker compose --project-name "$COMPOSE_PROJECT" --file "$COMPOSE_FILE" run --rm "$service" +done + +echo +echo "All five bindings passed their unit and conformance tests." diff --git a/tools/gen-test-vectors.py b/tools/gen-test-vectors.py index bcf7563..fa3d31e 100644 --- a/tools/gen-test-vectors.py +++ b/tools/gen-test-vectors.py @@ -4,15 +4,26 @@ content/claims hash -> §5 signing payload -> Ed25519 signature). Every signer and verifier MUST reproduce these bytes. -Run: uv run --with cryptography python tools/gen-test-vectors.py +Run after installing ``python[dev]`` and ``cryptography``: + python tools/gen-test-vectors.py + python tools/gen-test-vectors.py --check """ -import base64, hashlib, json, sys, pathlib +import argparse, base64, hashlib, json, sys, pathlib sys.path.insert(0, "python") -from htmltrust_canonicalization import extract_canonical_text, extract_claims_from_signed_section, canonicalize_claims +from htmltrust_canonicalization import ( + canonicalize_claims, + canonicalize_json_document, + extract_canonical_text, + extract_claims_from_signed_section, +) +from pywhatwgurl import URL from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from cryptography.hazmat.primitives import serialization ROOT = pathlib.Path(".") +ARGS = argparse.ArgumentParser(description=__doc__.splitlines()[0]) +ARGS.add_argument("--check", action="store_true", help="fail if the checked-in vector is stale") +args = ARGS.parse_args() def b64(b): return base64.b64encode(b).decode().rstrip("=") def sha256_hash(bs): return "sha256:" + b64(hashlib.sha256(bs).digest()) @@ -25,15 +36,15 @@ def sha256_hash(bs): return "sha256:" + b64(hashlib.sha256(bs).digest()) pub_raw = pk.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) pub_pem = pk.public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo).decode() -DOMAIN = "https://example.com" # serialized Web origin BASE_URL = "https://example.com/essays/engines" # signed document URL +DOCUMENT_URL = "HTTPS://EXAMPLE.COM:443/essays/engines#analysis" SIGNED_AT = "2026-01-15T12:00:00Z" HTML = ( "" '' f'' - '' + '' "

On Analytical Engines

" "

The engine weaves algebraic patterns — just as the loom weaves flowers.

" '

See the notes and ' @@ -43,23 +54,44 @@ def sha256_hash(bs): return "sha256:" + b64(hashlib.sha256(bs).digest()) content = extract_canonical_text(HTML, base_url=BASE_URL) content_hash = sha256_hash(content.encode("utf-8")) -claims_map = extract_claims_from_signed_section(HTML) +claims_map = dict(sorted(extract_claims_from_signed_section(HTML).items())) claims_str = canonicalize_claims(claims_map) claims_hash = sha256_hash(claims_str.encode("utf-8")) -payload = f"{content_hash}:{claims_hash}:{DOMAIN}:{SIGNED_AT}" +location_url = URL(DOCUMENT_URL) +location_url.hash = "" +signing_object = { + "algorithm": "ed25519", + "attributeProfile": "htmltrust-attrs-v1", + "canonicalizationProfile": "htmltrust-c14n-v1", + "claimsHash": claims_hash, + "contentHash": content_hash, + "context": "https://htmltrust.org/protocol/signed-section", + "keyid": "https://keys.example/alice-2026.json", + "location": str(location_url), + "profile": "htmltrust-signature-v1", + "scope": "url", + "signedAt": SIGNED_AT, + "urlProfile": "htmltrust-safe-url-v1", +} +payload = canonicalize_json_document(json.dumps(signing_object, ensure_ascii=False)) signature = b64(sk.sign(payload.encode("utf-8"))) vector = { - "description": "HTMLTrust end-to-end test vector 01 (Ed25519). Every signer " - "and verifier MUST reproduce contentHash, claimsHash, " - "signingPayload and signature exactly.", + "description": "HTMLTrust htmltrust-signature-v1 end-to-end Ed25519 vector.", "algorithm": "ed25519", "key": { "seedHex": SEED.hex(), "publicKeyRawHex": pub_raw.hex(), "publicKeyPem": pub_pem, }, - "input": {"html": HTML, "baseURL": BASE_URL, "domain": DOMAIN, "signedAt": SIGNED_AT}, + "input": { + "html": HTML, + "baseURL": BASE_URL, + "documentURL": DOCUMENT_URL, + "scope": "url", + "keyid": "https://keys.example/alice-2026.json", + "signedAt": SIGNED_AT, + }, "canonicalContent": content, "contentHash": content_hash, "claims": claims_map, @@ -69,7 +101,13 @@ def sha256_hash(bs): return "sha256:" + b64(hashlib.sha256(bs).digest()) "signature": signature, } out = ROOT / "conformance" / "vectors" / "vector-01.json" -out.write_text(json.dumps(vector, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +rendered = json.dumps(vector, ensure_ascii=False, indent=2) + "\n" +if args.check: + if out.read_text(encoding="utf-8") != rendered: + raise SystemExit(f"stale vector: run {pathlib.Path(__file__).as_posix()}") + print("vector is current:", out) + raise SystemExit(0) +out.write_text(rendered, encoding="utf-8") print("wrote", out) print(" contentHash :", content_hash) print(" claimsHash :", claims_hash)