From b9a4d03d7dd6899d98d6318676cf86964b95e596 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:32:43 -0400 Subject: [PATCH 01/33] build(deps): update dependencies, toolchain, and quality gates Rust: - quick-xml 0.42, zip 8 (deflate decode backend only), scraper 0.27, time >= 0.3.47, rsfulmen =0.2.0 - jsonschema 0.57 without default features; test validators build an offline registry - base64 0.23 (dev) - Track Cargo.lock; cargo invocations in Makefile, CI and scripts use --locked - Pin toolchain 1.94.1 in rust-toolchain.toml; MSRV stays 1.88.0 with a CI job that builds and tests on it TypeScript bindings: - napi 3, napi-derive 3, @napi-rs/cli 3, TypeScript 6 - tsconfig moves to Node16 module resolution with explicit node types - Track package-lock.json Gates: - make check now includes cargo-audit; cargo-deny runs every check - deny.toml: sources deny, graph targets for shipped platforms, bans on OpenSSL/native-tls and quick-xml < 0.41 - Direct clippy runs alongside goneat v0.6.0; sfetch pinned to v0.4.11 and bootstrap installer arguments corrected - CI actions moved to current majors; runner image pinned by digest; cbindgen, cargo-deny and cargo-audit versions pinned Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .github/workflows/ci.yml | 21 +- .github/workflows/go-bindings-prep.yml | 34 +- .github/workflows/go-bindings.yml | 30 +- .github/workflows/release.yml | 36 +- .github/workflows/typescript-bindings.yml | 8 +- .../workflows/typescript-napi-prebuilds.yml | 54 +- .github/workflows/typescript-npm-publish.yml | 2 +- .github/workflows/validate-release.yml | 10 +- .gitignore | 4 +- Cargo.lock | 1984 ++++++++++++++++ Cargo.toml | 10 +- Makefile | 50 +- .../typescript/docprims/native/Cargo.toml | 6 +- .../typescript/docprims/package-lock.json | 2037 +++++++++++++++++ bindings/typescript/docprims/package.json | 29 +- bindings/typescript/docprims/tsconfig.json | 5 +- crates/docprims-cli/Cargo.toml | 4 +- crates/docprims-cli/tests/golden_v0.rs | 37 +- crates/docprims-ooxml/Cargo.toml | 2 +- crates/docprims-ooxml/src/common.rs | 4 +- crates/docprims-ooxml/src/docx.rs | 22 +- crates/docprims-ooxml/src/lib.rs | 31 +- crates/docprims-ooxml/src/pptx.rs | 30 +- crates/docprims-ooxml/src/xlsx.rs | 58 +- crates/docprims-text/Cargo.toml | 2 +- crates/docprims-text/src/lib.rs | 31 +- crates/docprims-text/src/xml.rs | 22 +- deny.toml | 29 +- ffi/docprims-ffi/Cargo.toml | 2 +- rust-toolchain.toml | 4 + scripts/build-go-ffi-lib.sh | 2 +- 31 files changed, 4319 insertions(+), 281 deletions(-) create mode 100644 Cargo.lock create mode 100644 bindings/typescript/docprims/package-lock.json create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c389657..e694d69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,14 +7,16 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy - name: Install cargo-deny - run: cargo install cargo-deny@0.18.3 --locked + run: cargo install cargo-deny@0.20.2 --locked + - name: Install cargo-audit + run: cargo install cargo-audit@0.22.2 --locked - name: Install cbindgen - run: cargo install cbindgen --locked + run: cargo install cbindgen@0.29.4 --locked - name: Quality gates run: make check - name: Verify generated C header is up to date @@ -22,11 +24,20 @@ jobs: make cbindgen git diff --exit-code ffi/docprims-ffi/docprims.h diff -u ffi/docprims-ffi/docprims.h bindings/go/docprims/include/docprims.h + msrv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: 1.88.0 + - name: Build and test on MSRV + run: make msrv test-go-vendored: runs-on: macos-14 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version: "1.22.x" - name: Run Go tests (vendored libs) diff --git a/.github/workflows/go-bindings-prep.yml b/.github/workflows/go-bindings-prep.yml index f771277..e81bc79 100644 --- a/.github/workflows/go-bindings-prep.yml +++ b/.github/workflows/go-bindings-prep.yml @@ -17,7 +17,7 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Validate version matches VERSION file id: version run: | @@ -34,7 +34,7 @@ jobs: name: Build FFI (Linux) runs-on: ubuntu-latest container: - image: ghcr.io/fulmenhq/goneat-tools-runner-glibc:v0.3.1 + image: ghcr.io/fulmenhq/goneat-tools-runner-glibc@sha256:afcffa00283c765928dfef40966c282f2ca190653cf4443f634cb8f67c1d51ab # v0.5.6 options: --user root needs: validate strategy: @@ -53,7 +53,7 @@ jobs: glibc: "" artifact_suffix: linux-arm64-musl steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Verify cargo-zigbuild is available shell: bash run: | @@ -105,7 +105,7 @@ jobs: cp "target/${TARGET_DIR}/release/libdocprims_ffi.so" "dist/ffi/${{ matrix.artifact_suffix }}/" fi - name: Upload FFI artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ffi-${{ matrix.artifact_suffix }} path: dist/ffi/${{ matrix.artifact_suffix }} @@ -116,14 +116,14 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: "11.0" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: targets: aarch64-apple-darwin - name: Build FFI (arm64) run: | set -euo pipefail - cargo build --release --target aarch64-apple-darwin -p docprims-ffi + cargo build --release --target aarch64-apple-darwin -p docprims-ffi --locked - name: Package FFI run: | set -euo pipefail @@ -135,7 +135,7 @@ jobs: # This allows consumers to set rpath or place dylib alongside executable. install_name_tool -id "@rpath/libdocprims_ffi.dylib" dist/ffi/darwin-arm64/libdocprims_ffi.dylib - name: Upload FFI artifact (darwin-arm64) - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ffi-darwin-arm64 path: dist/ffi/darwin-arm64 @@ -144,7 +144,7 @@ jobs: runs-on: windows-latest needs: validate steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: targets: x86_64-pc-windows-gnu @@ -159,7 +159,7 @@ jobs: shell: msys2 {0} run: | set -euo pipefail - cargo build --release --target x86_64-pc-windows-gnu -p docprims-ffi + cargo build --release --target x86_64-pc-windows-gnu -p docprims-ffi --locked - name: Package FFI shell: bash run: | @@ -169,7 +169,7 @@ jobs: cp target/x86_64-pc-windows-gnu/release/docprims_ffi.dll dist/ffi/windows-amd64/ cp target/x86_64-pc-windows-gnu/release/libdocprims_ffi.dll.a dist/ffi/windows-amd64/ - name: Upload FFI artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ffi-windows-amd64 path: dist/ffi/windows-amd64 @@ -177,18 +177,18 @@ jobs: name: Generate C Header runs-on: ubuntu-latest container: - image: ghcr.io/fulmenhq/goneat-tools-runner-glibc:v0.3.1 + image: ghcr.io/fulmenhq/goneat-tools-runner-glibc@sha256:afcffa00283c765928dfef40966c282f2ca190653cf4443f634cb8f67c1d51ab # v0.5.6 options: --user root needs: validate steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Generate C header shell: bash run: | set -euo pipefail cbindgen --config cbindgen.toml --crate docprims-ffi --output docprims.h - name: Upload header - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: c-header path: docprims.h @@ -200,16 +200,16 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - name: Download all FFI artifacts - uses: actions/download-artifact@v4.1.3 + uses: actions/download-artifact@v8 with: pattern: ffi-* path: ffi-libs - name: Download C header - uses: actions/download-artifact@v4.1.3 + uses: actions/download-artifact@v8 with: name: c-header path: ffi-libs @@ -258,7 +258,7 @@ jobs: echo "Go bindings updated for $VERSION" ls -la bindings/go/docprims/lib/*/ || true - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v8 with: token: ${{ secrets.GITHUB_TOKEN }} base: main diff --git a/.github/workflows/go-bindings.yml b/.github/workflows/go-bindings.yml index f4639f8..1d4f836 100644 --- a/.github/workflows/go-bindings.yml +++ b/.github/workflows/go-bindings.yml @@ -6,15 +6,15 @@ jobs: go-test-linux-amd64: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.88.0 - - uses: actions/setup-go@v5 + toolchain: 1.94.1 + - uses: actions/setup-go@v7 with: go-version: "1.22.x" - name: Install cbindgen - run: cargo install cbindgen --locked + run: cargo install cbindgen@0.29.4 --locked - name: Build FFI and run Go tests (glibc) run: | make go-test @@ -22,8 +22,8 @@ jobs: go-test-darwin-arm64-vendored: runs-on: macos-14 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 with: go-version: "1.22.x" - name: Run Go tests (vendored darwin-arm64) @@ -33,26 +33,26 @@ jobs: go-test-darwin-arm64-shared: runs-on: macos-14 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.88.0 - - uses: actions/setup-go@v5 + toolchain: 1.94.1 + - uses: actions/setup-go@v7 with: go-version: "1.22.x" - name: Install cbindgen - run: cargo install cbindgen --locked + run: cargo install cbindgen@0.29.4 --locked - name: Build FFI and run Go tests (shared) run: make go-test-shared go-test-linux-amd64-musl: if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.88.0 - - uses: actions/setup-go@v5 + toolchain: 1.94.1 + - uses: actions/setup-go@v7 with: go-version: "1.22.x" - name: Install toolchain @@ -60,11 +60,11 @@ jobs: sudo apt-get update sudo apt-get install -y musl-tools rustup target add x86_64-unknown-linux-musl - cargo install cbindgen --locked + cargo install cbindgen@0.29.4 --locked - name: Build musl static lib and run Go tests run: | make cbindgen - cargo build --release -p docprims-ffi --target x86_64-unknown-linux-musl + cargo build --release -p docprims-ffi --target x86_64-unknown-linux-musl --locked mkdir -p bindings/go/docprims/include cp ffi/docprims-ffi/docprims.h bindings/go/docprims/include/docprims.h mkdir -p bindings/go/docprims/lib/local/linux-amd64-musl diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 26a86bf..f732e2d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Validate VERSION matches tag id: version run: | @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest needs: validate-version steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Verify Go module contains prebuilt libs run: | set -euo pipefail @@ -91,7 +91,7 @@ jobs: name: Build Linux runs-on: ubuntu-latest container: - image: ghcr.io/fulmenhq/goneat-tools-runner-glibc:v0.3.1 + image: ghcr.io/fulmenhq/goneat-tools-runner-glibc@sha256:afcffa00283c765928dfef40966c282f2ca190653cf4443f634cb8f67c1d51ab # v0.5.6 options: --user root needs: validate-version strategy: @@ -110,7 +110,7 @@ jobs: glibc: "" artifact_suffix: linux-arm64-musl steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Build CLI run: | if [ -n "${{ matrix.glibc }}" ]; then @@ -148,7 +148,7 @@ jobs: VERSION=${{ needs.validate-version.outputs.version }} tar -czf docprims-${VERSION}-${{ matrix.artifact_suffix }}.tar.gz -C dist/cli . - name: Upload CLI artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: cli-${{ matrix.artifact_suffix }} path: docprims-${{ needs.validate-version.outputs.version }}-${{ matrix.artifact_suffix }}.tar.gz @@ -159,12 +159,12 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: "11.0" steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: targets: aarch64-apple-darwin - name: Build CLI - run: cargo build --release --target aarch64-apple-darwin -p docprims-cli + run: cargo build --release --target aarch64-apple-darwin -p docprims-cli --locked - name: Package CLI run: | mkdir -p dist/cli @@ -172,7 +172,7 @@ jobs: cp LICENSE-MIT LICENSE-APACHE dist/cli/ tar -czf docprims-${{ needs.validate-version.outputs.version }}-darwin-arm64.tar.gz -C dist/cli . - name: Upload CLI artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: cli-darwin-arm64 path: docprims-${{ needs.validate-version.outputs.version }}-darwin-arm64.tar.gz @@ -181,7 +181,7 @@ jobs: runs-on: windows-latest needs: validate-version steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@stable with: targets: x86_64-pc-windows-gnu @@ -194,7 +194,7 @@ jobs: path-type: inherit - name: Build CLI (GNU target) shell: msys2 {0} - run: cargo build --release --target x86_64-pc-windows-gnu -p docprims-cli + run: cargo build --release --target x86_64-pc-windows-gnu -p docprims-cli --locked - name: Package CLI shell: bash run: | @@ -203,7 +203,7 @@ jobs: cp LICENSE-MIT LICENSE-APACHE dist/cli/ cd dist/cli && 7z a -tzip ../../docprims-${{ needs.validate-version.outputs.version }}-windows-amd64.zip . - name: Upload CLI artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: cli-windows-amd64 path: docprims-${{ needs.validate-version.outputs.version }}-windows-amd64.zip @@ -211,15 +211,15 @@ jobs: name: Generate C Header runs-on: ubuntu-latest container: - image: ghcr.io/fulmenhq/goneat-tools-runner-glibc:v0.3.1 + image: ghcr.io/fulmenhq/goneat-tools-runner-glibc@sha256:afcffa00283c765928dfef40966c282f2ca190653cf4443f634cb8f67c1d51ab # v0.5.6 options: --user root needs: validate-version steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Generate C header run: cbindgen --config cbindgen.toml --crate docprims-ffi --output docprims.h - name: Upload header - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: c-header path: docprims.h @@ -236,14 +236,14 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Download all CLI artifacts - uses: actions/download-artifact@v4.1.3 + uses: actions/download-artifact@v8 with: pattern: cli-* path: release-cli - name: Download C header - uses: actions/download-artifact@v4.1.3 + uses: actions/download-artifact@v8 with: name: c-header path: release-header @@ -262,7 +262,7 @@ jobs: ls -la release/ - name: Create Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: release/* generate_release_notes: true diff --git a/.github/workflows/typescript-bindings.yml b/.github/workflows/typescript-bindings.yml index de0e8cd..34366d9 100644 --- a/.github/workflows/typescript-bindings.yml +++ b/.github/workflows/typescript-bindings.yml @@ -13,14 +13,14 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v7 - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: node-version: "22" - uses: dtolnay/rust-toolchain@stable - name: Install deps and run tests run: | cd bindings/typescript/docprims - npm install + npm install --omit=optional npm run test:ci test-typescript-alpine: name: Test TypeScript Bindings (Alpine/musl) @@ -36,8 +36,8 @@ jobs: sh -lc ' set -eu apk add --no-cache bash curl build-base python3 ca-certificates - curl -fsSL https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.88.0 + curl -fsSL https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.94.1 . "$HOME/.cargo/env" - npm install + npm install --omit=optional npm run test:ci ' diff --git a/.github/workflows/typescript-napi-prebuilds.yml b/.github/workflows/typescript-napi-prebuilds.yml index 1be9d59..e3183a8 100644 --- a/.github/workflows/typescript-napi-prebuilds.yml +++ b/.github/workflows/typescript-napi-prebuilds.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: '22' - name: Install Rust @@ -27,9 +27,11 @@ jobs: rustup target add \ x86_64-unknown-linux-musl \ aarch64-unknown-linux-musl + - name: Install cargo-zigbuild (pinned; napi --cross-compile would otherwise fetch latest) + run: cargo install cargo-zigbuild --locked --version 0.23.4 - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm install --omit=optional - name: Build .node files (zig) working-directory: bindings/typescript/docprims shell: bash @@ -39,21 +41,21 @@ jobs: mkdir -p prebuilds # glibc baseline (2.17) for broad compatibility - npx napi build --cargo-cwd native --release --platform --js false --strip \ - --target x86_64-unknown-linux-gnu --zig --zig-abi-suffix 2.17 + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip \ + --target x86_64-unknown-linux-gnu --use-napi-cross mv docprims.linux-x64-gnu.node prebuilds/ - npx napi build --cargo-cwd native --release --platform --js false --strip \ - --target x86_64-unknown-linux-musl --zig + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip \ + --target x86_64-unknown-linux-musl --cross-compile mv docprims.linux-x64-musl.node prebuilds/ # arm64-gnu built natively on ubuntu-latest-arm64-s (build-linux-arm64 job) - npx napi build --cargo-cwd native --release --platform --js false --strip \ - --target aarch64-unknown-linux-musl --zig + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip \ + --target aarch64-unknown-linux-musl --cross-compile mv docprims.linux-arm64-musl.node prebuilds/ - name: Upload prebuilds - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ts-prebuilds-linux path: bindings/typescript/docprims/prebuilds/*.node @@ -63,14 +65,14 @@ jobs: steps: - uses: actions/checkout@v7 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: '22' - name: Install Rust uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm install --omit=optional - name: Build .node file working-directory: bindings/typescript/docprims shell: bash @@ -78,10 +80,10 @@ jobs: set -euo pipefail rm -f docprims.*.node mkdir -p prebuilds - npx napi build --cargo-cwd native --release --platform --js false --strip + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip mv docprims.linux-arm64-gnu.node prebuilds/ - name: Upload prebuild - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ts-prebuilds-linux-arm64-gnu-native path: bindings/typescript/docprims/prebuilds/*.node @@ -91,14 +93,14 @@ jobs: steps: - uses: actions/checkout@v7 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: '22' - name: Install Rust uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm install --omit=optional - name: Build .node file working-directory: bindings/typescript/docprims shell: bash @@ -106,10 +108,10 @@ jobs: set -euo pipefail rm -f docprims.*.node mkdir -p prebuilds - npx napi build --cargo-cwd native --release --platform --js false --strip + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip mv docprims.darwin-arm64.node prebuilds/ - name: Upload prebuild - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ts-prebuilds-darwin-arm64 path: bindings/typescript/docprims/prebuilds/*.node @@ -119,14 +121,14 @@ jobs: steps: - uses: actions/checkout@v7 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: '22' - name: Install Rust uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm install --omit=optional - name: Build .node file working-directory: bindings/typescript/docprims shell: pwsh @@ -134,10 +136,10 @@ jobs: $ErrorActionPreference = 'Stop' Remove-Item -Force docprims.*.node -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path prebuilds | Out-Null - npx napi build --cargo-cwd native --release --platform --js false --strip + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip Move-Item docprims.win32-x64-msvc.node prebuilds/ - name: Upload prebuild - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ts-prebuilds-win32-x64-msvc path: bindings/typescript/docprims/prebuilds/*.node @@ -148,14 +150,14 @@ jobs: steps: - uses: actions/checkout@v7 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: '22' - name: Install Rust uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm install --omit=optional - name: Build JS dist (root package) working-directory: bindings/typescript/docprims run: npm run clean && npm run build && npm run build:native @@ -166,7 +168,7 @@ jobs: set -euo pipefail rm -f dist/native/docprims.*.node || true - name: Download prebuilds - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: ts-prebuilds-* path: downloaded-prebuilds @@ -176,7 +178,7 @@ jobs: run: | set -euo pipefail rm -rf npm - npx napi create-npm-dir -t . + npx napi create-npm-dirs - name: Remove unsupported platform packages working-directory: bindings/typescript/docprims shell: bash @@ -201,7 +203,7 @@ jobs: cp "$f" "$dest" done - name: Upload npm dir - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: ts-npm-dir path: bindings/typescript/docprims/npm diff --git a/.github/workflows/typescript-npm-publish.yml b/.github/workflows/typescript-npm-publish.yml index 98ce517..8ef0ee7 100644 --- a/.github/workflows/typescript-npm-publish.yml +++ b/.github/workflows/typescript-npm-publish.yml @@ -36,7 +36,7 @@ jobs: # it causes a git fetch conflict. The tag is already checked out. - uses: actions/checkout@v7 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@v7 with: node-version: '24' registry-url: 'https://registry.npmjs.org' diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index b71e17b..bb01c8b 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v7 with: go-version: '1.22' - name: Test Go module is fetchable @@ -43,14 +43,14 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ inputs.tag }} - - uses: actions/setup-node@v6 + - uses: actions/setup-node@v7 with: node-version: "22" - uses: dtolnay/rust-toolchain@stable - name: Build and test TypeScript bindings run: | cd bindings/typescript/docprims - npm install + npm install --omit=optional npm run test:ci validate-typescript-musl: name: Validate TypeScript Bindings (Alpine/musl) @@ -70,7 +70,7 @@ jobs: shell: bash run: | set -euo pipefail - curl -fsSL https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.88.0 + curl -fsSL https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.94.1 . "$HOME/.cargo/env" rustc -V cargo -V @@ -80,5 +80,5 @@ jobs: run: | set -euo pipefail . "$HOME/.cargo/env" - npm install + npm install --omit=optional npm run test:ci diff --git a/.gitignore b/.gitignore index 8ccbfe5..0bcb68f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Rust /target/ -Cargo.lock # IDE .idea/ @@ -45,8 +44,7 @@ bindings/typescript/**/node_modules/ bindings/typescript/**/dist/ bindings/typescript/**/dist-test/ bindings/typescript/**/.npmrc -bindings/typescript/**/package-lock.json -package-lock.json +/package-lock.json # Test artifacts *.profraw diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..5bf2730 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1984 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "convert_case" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cssparser" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a" +dependencies = [ + "quote", + "syn 3.0.6", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" + +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "docprims-cli" +version = "0.1.5" +dependencies = [ + "assert_cmd", + "base64", + "clap", + "docprims-core", + "docprims-ooxml", + "docprims-text", + "jsonschema", + "predicates", + "rsfulmen", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "docprims-core" +version = "0.1.5" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "docprims-ffi" +version = "0.1.5" +dependencies = [ + "base64", + "docprims-core", + "docprims-ooxml", + "docprims-text", + "serde", + "serde_json", +] + +[[package]] +name = "docprims-ooxml" +version = "0.1.5" +dependencies = [ + "docprims-core", + "encoding_rs", + "jsonschema", + "quick-xml", + "serde_json", + "zip", +] + +[[package]] +name = "docprims-text" +version = "0.1.5" +dependencies = [ + "docprims-core", + "jsonschema", + "pulldown-cmark", + "quick-xml", + "scraper", + "serde_json", +] + +[[package]] +name = "docprims-ts-napi" +version = "0.1.5" +dependencies = [ + "docprims-core", + "docprims-ooxml", + "docprims-text", + "napi", + "napi-build", + "napi-derive", + "serde", + "serde_json", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b04dc5a38e4f151a79d9f2451ae6037fb6eaf5cba34771f44781f80e508498e3" + +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fancy-regex" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "zlib-rs", +] + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fraction" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e246562084dde8ebbcc943b261c406ce4f68e5032ec28029a251a47d6a295500" +dependencies = [ + "num", + "num-bigint", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "html5ever" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8" +dependencies = [ + "log", + "markup5ever", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "jsonschema" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71160ed5f6dbe36a2d6be79f4ca03ee09a99d2866f340faed49458913449aefa" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom", + "itoa", + "jsonschema-regex", + "jsonschema-value", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "strum", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d2120d8466ffcdc1b3be4b88ff0e9191bf5b6b09b1b1af1eef9aff8f465ea3" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "jsonschema-value" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fbfa40a42415369d940b3848f3ce08099f1f1ac04d73e8580f4d652617c4f44" +dependencies = [ + "ahash", + "bytecount", + "fraction", + "getrandom", + "num-cmp", + "num-traits", + "serde_json", + "zmij", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "markup5ever" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + +[[package]] +name = "napi" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b12638855cec016784a7e17cc4ecbdee7ca8b0a3456aa50640d0ee5ed1a42f8" +dependencies = [ + "bitflags", + "ctor", + "futures", + "libc", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash", +] + +[[package]] +name = "napi-build" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0941983559cd62bfa7f04646a16428967a847365adeef27711e4e3927046bbed" + +[[package]] +name = "napi-derive" +version = "3.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1d232c24dc6754979b27efdf2047a5849efc7c92c1245de87e6bedbb9f5c3c" +dependencies = [ + "convert_case", + "ctor", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "napi-derive-backend" +version = "6.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6856fe1686867da90ed0dc292d7c0c87a695eb7c578b913360f33619ee01e3" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "semver", + "syn 2.0.119", +] + +[[package]] +name = "napi-sys" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6801fc65ba4d7035636e93635e281113f00410401a237dcd4f55b1565ead2115" +dependencies = [ + "libloading", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "pulldown-cmark-escape", + "unicase", +] + +[[package]] +name = "pulldown-cmark-escape" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "referencing" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06f6798be4fed305e74df8b1fb90cf59c6b8cc17aa3febeb1830c0c6b627b3e" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom", + "hashbrown", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rsfulmen" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58499c875a8706aa72494fa86e518b6c407893ce1eb6378d387d985ace39b13f" +dependencies = [ + "ctrlc", + "libc", + "once_cell", + "serde", + "serde_yaml_ng", + "signal-hook", + "thiserror 1.0.69", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdd0be4d296f048bfb06dd01bbc80ef789ddd2e55583e8d2e6b804942abfabc2" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + +[[package]] +name = "selectors" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adfa1c298912827b8a28b223b3b874357397ae706e6190acd9bf28cee99114d" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tendril" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08" +dependencies = [ + "new_debug_unreachable", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web_atoms" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e8fafad82b542ff3717217ecdc736231b59e387768c9630123b4ce4d2db44" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595f56e044df4f46a0c9a626f65c3d99eb8488f7e8a8baa12dd76326d9710bf2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index 253306d..95291cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,23 +23,23 @@ docprims-text = { path = "crates/docprims-text" } docprims-ooxml = { path = "crates/docprims-ooxml" } # Fulmen Ecosystem -rsfulmen = { version = "=0.1.2", default-features = false, features = ["foundry-core"] } +rsfulmen = { version = "=0.2.0", default-features = false, features = ["foundry-core"] } # Serialization serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" # Time -time = { version = "0.3", features = ["formatting", "serde"] } +time = { version = "0.3.47", features = ["formatting", "serde"] } # XML parsing (GPL-free) -quick-xml = "0.39" +quick-xml = "0.42" # ZIP archive handling (GPL-free) -zip = { version = "2", default-features = false, features = ["deflate"] } +zip = { version = "8", default-features = false, features = ["deflate-flate2-zlib-rs"] } # HTML parsing (GPL-free) -scraper = "0.25" +scraper = "0.27" # Markdown parsing (GPL-free) pulldown-cmark = "0.13" diff --git a/Makefile b/Makefile index b970b4c..d00d294 100644 --- a/Makefile +++ b/Makefile @@ -29,8 +29,8 @@ VERSION := $(shell cat VERSION 2>/dev/null || echo "dev") BIN_DIR := $(CURDIR)/bin # Pinned tool versions for reproducibility -SFETCH_VERSION := latest -GONEAT_VERSION ?= v0.5.1 +SFETCH_VERSION := v0.4.11 +GONEAT_VERSION ?= v0.6.0 # Tool paths SFETCH = $(shell [ -x "$(BIN_DIR)/sfetch" ] && echo "$(BIN_DIR)/sfetch" || command -v sfetch 2>/dev/null) @@ -38,6 +38,10 @@ GONEAT = $(shell command -v goneat 2>/dev/null) # Rust toolchain CARGO = cargo +# Build against the committed Cargo.lock; fail rather than re-resolve. +CARGO_LOCKED = --locked +# Minimum supported Rust version (workspace rust-version); verified by `make msrv`. +MSRV = 1.88.0 # ----------------------------------------------------------------------------- # Default and Help @@ -82,7 +86,7 @@ help: ## Show available targets @echo " deny Run cargo-deny license and advisory checks" @echo " audit Run cargo-audit security scan" @echo " miri Run Miri UB detection on unsafe code (nightly)" - @echo " msrv Verify build with MSRV (Rust 1.88.0)" + @echo " msrv Verify build and tests with MSRV (Rust $(MSRV))" @echo " npm-publish-prereqs-check Verify npm trusted publishing runtime guard" @echo " check-windows Cross-check Windows targets (no SDK required)" @echo "" @@ -122,7 +126,7 @@ bootstrap: ## Install required tools (sfetch -> goneat -> tools) @mkdir -p "$(BIN_DIR)" @if [ ! -x "$(BIN_DIR)/sfetch" ] && ! command -v sfetch >/dev/null 2>&1; then \ echo "[..] Installing sfetch (trust anchor)..."; \ - curl -fsSL https://github.com/3leaps/sfetch/releases/download/$(SFETCH_VERSION)/install-sfetch.sh | bash -s -- --dest "$(BIN_DIR)"; \ + curl -fsSL https://github.com/3leaps/sfetch/releases/download/$(SFETCH_VERSION)/install-sfetch.sh | bash -s -- --dir "$(BIN_DIR)" --tag $(SFETCH_VERSION) --yes; \ else \ echo "[ok] sfetch already installed"; \ fi @@ -206,12 +210,12 @@ tools: ## Verify external tools are available # Quality Gates # ----------------------------------------------------------------------------- -check: fmt-check lint test deny ## Run all quality checks +check: fmt-check lint test deny audit ## Run all quality checks @echo "[ok] All quality checks passed" test: ## Run test suite @echo "Running tests..." - $(CARGO) test --workspace + $(CARGO) test --workspace $(CARGO_LOCKED) @echo "[ok] Tests passed" fmt: ## Format code (goneat assess or cargo fmt) @@ -234,15 +238,14 @@ fmt-check: ## Check formatting without modifying fi @echo "[ok] Formatting check passed" -lint: ## Run linting (goneat assess or cargo clippy) +lint: ## Run linting (goneat assess + cargo clippy) @echo "Linting..." @bash scripts/check-npm-trusted-publish-runtime.sh @if command -v goneat >/dev/null 2>&1; then \ goneat assess --categories lint; \ - else \ - echo "[--] goneat not found, using cargo clippy"; \ - $(CARGO) clippy --workspace --all-targets -- -D warnings; \ fi + @# goneat v0.6.0 does not run the Rust lint pass; clippy runs directly as the gap-filler. + $(CARGO) clippy --workspace --all-targets $(CARGO_LOCKED) -- -D warnings @echo "[ok] Linting passed" npm-publish-prereqs-check: ## Verify npm trusted publishing runtime guard @@ -251,15 +254,12 @@ npm-publish-prereqs-check: ## Verify npm trusted publishing runtime guard deny: ## Run cargo-deny license and advisory checks @echo "Running cargo-deny..." @if command -v cargo-deny >/dev/null 2>&1; then \ - cargo-deny check bans licenses sources; \ + cargo-deny --locked check; \ else \ echo "[!!] cargo-deny not found (run 'make bootstrap')"; \ exit 1; \ fi @echo "[ok] cargo-deny passed" -# NOTE: advisories check temporarily skipped due to cargo-deny CVSS 4.0 parsing issue -# See: https://github.com/EmbarkStudios/cargo-deny/issues/804 -# Re-enable with `cargo-deny check` when fixed upstream audit: ## Run cargo-audit security scan @echo "Running cargo-audit..." @@ -283,14 +283,14 @@ miri: ## Run Miri to detect undefined behavior in unsafe code (requires nightly) fi @echo "[ok] Miri passed" -msrv: ## Verify build with Minimum Supported Rust Version (1.88.0) - @echo "Checking MSRV (1.88.0)..." - @if rustup run 1.88.0 cargo --version >/dev/null 2>&1; then \ - rustup run 1.88.0 cargo build --workspace && \ - rustup run 1.88.0 cargo test --workspace; \ +msrv: ## Verify build and tests with the Minimum Supported Rust Version + @echo "Checking MSRV ($(MSRV))..." + @if rustup run $(MSRV) cargo --version >/dev/null 2>&1; then \ + CARGO_TARGET_DIR=target/msrv rustup run $(MSRV) cargo build --workspace $(CARGO_LOCKED) && \ + CARGO_TARGET_DIR=target/msrv rustup run $(MSRV) cargo test --workspace $(CARGO_LOCKED); \ else \ - echo "[!!] Rust 1.88.0 not installed. Install with:"; \ - echo " rustup install 1.88.0"; \ + echo "[!!] Rust $(MSRV) not installed. Install with:"; \ + echo " rustup install $(MSRV)"; \ exit 1; \ fi @echo "[ok] MSRV check passed" @@ -336,17 +336,17 @@ deps-check: ## Check dependencies for cooling violations build: ## Build all crates (debug) @echo "Building (debug)..." - $(CARGO) build --workspace + $(CARGO) build --workspace $(CARGO_LOCKED) @echo "[ok] Build complete" build-release: ## Build all crates (release) @echo "Building (release)..." - $(CARGO) build --workspace --release + $(CARGO) build --workspace --release $(CARGO_LOCKED) @echo "[ok] Release build complete" build-ffi: cbindgen ## Build FFI library with C header @echo "Building FFI library..." - $(CARGO) build --package docprims-ffi --release + $(CARGO) build --package docprims-ffi --release $(CARGO_LOCKED) @echo "[ok] FFI build complete" @echo "Library: target/release/libdocprims_ffi.*" @echo "Header: ffi/docprims-ffi/docprims.h" @@ -375,7 +375,7 @@ GO_BINDINGS_DIR := bindings/go/docprims build-local-go: cbindgen ## Build FFI for local Go development @echo "Building FFI for local Go development..." - $(CARGO) build --release -p docprims-ffi + $(CARGO) build --release -p docprims-ffi $(CARGO_LOCKED) @# Sync header and local static lib into Go module layout @PLATFORM=""; \ UNAME_S="$$(uname -s)"; \ diff --git a/bindings/typescript/docprims/native/Cargo.toml b/bindings/typescript/docprims/native/Cargo.toml index 5e9aa1c..42d5206 100644 --- a/bindings/typescript/docprims/native/Cargo.toml +++ b/bindings/typescript/docprims/native/Cargo.toml @@ -12,8 +12,8 @@ description = "Node-API addon for @3leaps/docprims (TypeScript bindings)" crate-type = ["cdylib"] [dependencies] -napi = { version = "2.16.16", default-features = false, features = ["napi8"] } -napi-derive = "2.16.13" +napi = { version = "3", default-features = false, features = ["napi8"] } +napi-derive = "3" serde = { workspace = true, features = ["derive"] } serde_json.workspace = true @@ -22,4 +22,4 @@ docprims-text.workspace = true docprims-ooxml.workspace = true [build-dependencies] -napi-build = "2.0.1" +napi-build = "2.3" diff --git a/bindings/typescript/docprims/package-lock.json b/bindings/typescript/docprims/package-lock.json new file mode 100644 index 0000000..5021dc1 --- /dev/null +++ b/bindings/typescript/docprims/package-lock.json @@ -0,0 +1,2037 @@ +{ + "name": "@3leaps/docprims", + "version": "0.1.5", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@3leaps/docprims", + "version": "0.1.5", + "license": "MIT OR Apache-2.0", + "devDependencies": { + "@napi-rs/cli": "^3.10.5", + "@types/node": "^22.20.4", + "typescript": "^6.0.3" + }, + "engines": { + "node": ">=22" + }, + "optionalDependencies": { + "@3leaps/docprims-darwin-arm64": "0.1.5", + "@3leaps/docprims-linux-arm64-gnu": "0.1.5", + "@3leaps/docprims-linux-arm64-musl": "0.1.5", + "@3leaps/docprims-linux-x64-gnu": "0.1.5", + "@3leaps/docprims-linux-x64-musl": "0.1.5", + "@3leaps/docprims-win32-x64-msvc": "0.1.5" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", + "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.3", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", + "dev": true, + "license": "MIT", + "optional": true, + "peer": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@inquirer/ansi": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-2.0.8.tgz", + "integrity": "sha512-WpQM+Ti6Z40EFwwt+uL2p4UabT+W179zHp6HhLVOzfbwnVn05IPO/eXIZXGNqcT1jbQ15SujNLzQ39k4QPPxBQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-5.2.5.tgz", + "integrity": "sha512-bRt8J8m+Fot9CXv+zNQGXUq2ET0MggR1fPz7v6edN6MFYmsbfGnMmkmWZJEegMKqrAC8ej/o1sqisHZXZJMAfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.8", + "@inquirer/core": "^12.0.3", + "@inquirer/figures": "^2.0.9", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/confirm": { + "version": "6.3.2", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-6.3.2.tgz", + "integrity": "sha512-Xvr/0HggjddPtGppuqVmxhTw+Hr8PvsZ/k0HmOEaAqQEt80OITNkFWnsdNmyT0/eM4Ab+iJLx2R8rctlEyfSVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^12.0.3", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "12.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-12.0.3.tgz", + "integrity": "sha512-wsSy0sznmXwkty+2PzZwx00Cazc/E0r0B7mAzdGROz2Ct+DFZXaK7WDjGZvgjRldxH5ZhFVfF2lgkYrqgOw2KA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.8", + "@inquirer/figures": "^2.0.9", + "@inquirer/type": "4.1.1", + "cli-width": "^4.1.0", + "fast-wrap-ansi": "^0.2.0", + "mute-stream": "^3.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/editor": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-5.3.3.tgz", + "integrity": "sha512-YsKkS2q63IiLtaDK/9nqzdComN97SDQrmKiyNggN+ceP4ty+Z6VwyTz3FpjeUWeW1Efss2xHFKCC9sx7hnrsxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^12.0.3", + "@inquirer/external-editor": "^3.0.5", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/expand": { + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-5.1.5.tgz", + "integrity": "sha512-uHuXLmXW+TtIfT/9vSBotypAkqn1n34Ul+CLGPos/xANyO4Ff5xZzkYhbKR4NEcfVK4a9mHQOpwVZzluSHFRGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^12.0.3", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-3.0.5.tgz", + "integrity": "sha512-f3QQJRIX5ZEneBHNUIuPjmbdzHnmRFJA8r2dkcb8q+OM5Uv5KtnuAttQumnrjcBVBM3mcTX1CkmtAkU58VRZxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.2" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-2.0.9.tgz", + "integrity": "sha512-EAWgUTGQ/Umgga51dE3B2PUHbufuXarDfg86uVgoSgNHNNQnyFKcOrQLWVqYMghuSyHh8+2HUH0Js9cTC1WAdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + } + }, + "node_modules/@inquirer/input": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-5.1.6.tgz", + "integrity": "sha512-HtcJhB2QFVXbLuJ5S3syhNbTUVxYvwqV4VRBDkQceBloC9bmTViUoRFP5PbSaDZb3HzfPmpuU/gG4ybVBz4FHA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^12.0.3", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/number": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-4.2.3.tgz", + "integrity": "sha512-6Yuwh1NGSbu1Lo4N1EWjXs1jKRntLg/ZCwhmeorEHde90v1XxAozdbd4Iu30eOQLW+6h1hp2O9ujNfLSbTPJnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^12.0.3", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/password": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-5.2.2.tgz", + "integrity": "sha512-W9zYdyzogK+6110mqwaSJWCBu2yA5Q/OfnGSjjZB1bNpHlmUozXxTl0+QOZBNeVd6Qo81/qT75gW05gLAtITxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.8", + "@inquirer/core": "^12.0.3", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/prompts": { + "version": "8.7.2", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-8.7.2.tgz", + "integrity": "sha512-QoRB4wFIjgH5iOhSjoIKMkTvSHDuV+O3OITlIqAYO0oK5x364GJILXiMBvlPiE+klg7Xx9tq5XVqQHcGUDYYPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/checkbox": "^5.2.5", + "@inquirer/confirm": "^6.3.2", + "@inquirer/editor": "^5.3.3", + "@inquirer/expand": "^5.1.5", + "@inquirer/input": "^5.1.6", + "@inquirer/number": "^4.2.3", + "@inquirer/password": "^5.2.2", + "@inquirer/rawlist": "^5.3.5", + "@inquirer/search": "^4.3.3", + "@inquirer/select": "^5.2.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/rawlist": { + "version": "5.3.5", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-5.3.5.tgz", + "integrity": "sha512-1oHky1ONfCOwNrnkQGDE1oaSij/3fI6HFMSf2H/WsGO2lEyDX9My82iggITSy9ddSZ8yk8j9v41OI0fVoSIoaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^12.0.3", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/search": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-4.3.3.tgz", + "integrity": "sha512-fyuIU1Nbpvwlikjg3gXwJFDI11+EFjqQ7P+iByfmivIKQ1vmaykNrD/vy5unHuUqUpsOsnvJ25//tPF7E/RBRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/core": "^12.0.3", + "@inquirer/figures": "^2.0.9", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/select": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-5.2.5.tgz", + "integrity": "sha512-9kc15hr8r/kI+3DO/xLog5nOzTz1jqsHXa6JBFzmQKhkoJ8Slda1I1L/uD8ZSZ9tF1yp79wwXe7mclvX1rqR2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.8", + "@inquirer/core": "^12.0.3", + "@inquirer/figures": "^2.0.9", + "@inquirer/type": "4.1.1" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/type": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-4.1.1.tgz", + "integrity": "sha512-yJoHYrMnxIsJZCY+0Vb66Dy3he3kL3e2wOBKhoSwWWAzZAY82emlxwgprCtp6yRixvNRNq9ztfRWQYPNr3Go7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^20.17.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@napi-rs/cli": { + "version": "3.10.5", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-3.10.5.tgz", + "integrity": "sha512-GF4WXmLLQk3uQvkPuLCOSH50DN4GUKZw1zHlMDh4rRF5TWpfOj3a8DkZ2PWCTwpPDMD4ex05sAjrCEwNdukEjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@inquirer/prompts": "^8.5.2", + "@napi-rs/cross-toolchain": "^1.0.3", + "@napi-rs/wasm-tools": "^1.1.0", + "@octokit/rest": "^22.0.1", + "clipanion": "^4.0.0-rc.4", + "colorette": "^2.0.20", + "es-toolkit": "^1.47.0", + "js-yaml": "^5.0.0", + "obug": "^3.0.0", + "semver": "^7.8.2", + "typanion": "^3.14.0", + "typescript": "^6.0.3" + }, + "bin": { + "napi": "dist/cli.js", + "napi-raw": "cli.mjs" + }, + "engines": { + "node": "^20.17.0 || ^22.13.0 || >= 23.5.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", + "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4", + "emnapi": "^1.7.1 || ^2.0.0-alpha.4" + }, + "peerDependenciesMeta": { + "@emnapi/core": { + "optional": true + }, + "@emnapi/runtime": { + "optional": true + }, + "emnapi": { + "optional": true + } + } + }, + "node_modules/@napi-rs/cross-toolchain": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@napi-rs/cross-toolchain/-/cross-toolchain-1.0.3.tgz", + "integrity": "sha512-ENPfLe4937bsKVTDA6zdABx4pq9w0tHqRrJHyaGxgaPq03a2Bd1unD5XSKjXJjebsABJ+MjAv1A2OvCgK9yehg==", + "dev": true, + "license": "MIT", + "workspaces": [ + ".", + "arm64/*", + "x64/*" + ], + "dependencies": { + "@napi-rs/lzma": "^1.4.5", + "@napi-rs/tar": "^1.1.0", + "debug": "^4.4.1" + }, + "peerDependencies": { + "@napi-rs/cross-toolchain-arm64-target-aarch64": "^1.0.3", + "@napi-rs/cross-toolchain-arm64-target-armv7": "^1.0.3", + "@napi-rs/cross-toolchain-arm64-target-ppc64le": "^1.0.3", + "@napi-rs/cross-toolchain-arm64-target-s390x": "^1.0.3", + "@napi-rs/cross-toolchain-arm64-target-x86_64": "^1.0.3", + "@napi-rs/cross-toolchain-x64-target-aarch64": "^1.0.3", + "@napi-rs/cross-toolchain-x64-target-armv7": "^1.0.3", + "@napi-rs/cross-toolchain-x64-target-ppc64le": "^1.0.3", + "@napi-rs/cross-toolchain-x64-target-s390x": "^1.0.3", + "@napi-rs/cross-toolchain-x64-target-x86_64": "^1.0.3" + }, + "peerDependenciesMeta": { + "@napi-rs/cross-toolchain-arm64-target-aarch64": { + "optional": true + }, + "@napi-rs/cross-toolchain-arm64-target-armv7": { + "optional": true + }, + "@napi-rs/cross-toolchain-arm64-target-ppc64le": { + "optional": true + }, + "@napi-rs/cross-toolchain-arm64-target-s390x": { + "optional": true + }, + "@napi-rs/cross-toolchain-arm64-target-x86_64": { + "optional": true + }, + "@napi-rs/cross-toolchain-x64-target-aarch64": { + "optional": true + }, + "@napi-rs/cross-toolchain-x64-target-armv7": { + "optional": true + }, + "@napi-rs/cross-toolchain-x64-target-ppc64le": { + "optional": true + }, + "@napi-rs/cross-toolchain-x64-target-s390x": { + "optional": true + }, + "@napi-rs/cross-toolchain-x64-target-x86_64": { + "optional": true + } + } + }, + "node_modules/@napi-rs/lzma": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma/-/lzma-1.4.5.tgz", + "integrity": "sha512-zS5LuN1OBPAyZpda2ZZgYOEDC+xecUdAGnrvbYzjnLXkrq/OBC3B9qcRvlxbDR3k5H/gVfvef1/jyUqPknqjbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/lzma-android-arm-eabi": "1.4.5", + "@napi-rs/lzma-android-arm64": "1.4.5", + "@napi-rs/lzma-darwin-arm64": "1.4.5", + "@napi-rs/lzma-darwin-x64": "1.4.5", + "@napi-rs/lzma-freebsd-x64": "1.4.5", + "@napi-rs/lzma-linux-arm-gnueabihf": "1.4.5", + "@napi-rs/lzma-linux-arm64-gnu": "1.4.5", + "@napi-rs/lzma-linux-arm64-musl": "1.4.5", + "@napi-rs/lzma-linux-ppc64-gnu": "1.4.5", + "@napi-rs/lzma-linux-riscv64-gnu": "1.4.5", + "@napi-rs/lzma-linux-s390x-gnu": "1.4.5", + "@napi-rs/lzma-linux-x64-gnu": "1.4.5", + "@napi-rs/lzma-linux-x64-musl": "1.4.5", + "@napi-rs/lzma-wasm32-wasi": "1.4.5", + "@napi-rs/lzma-win32-arm64-msvc": "1.4.5", + "@napi-rs/lzma-win32-ia32-msvc": "1.4.5", + "@napi-rs/lzma-win32-x64-msvc": "1.4.5" + } + }, + "node_modules/@napi-rs/lzma-android-arm-eabi": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-android-arm-eabi/-/lzma-android-arm-eabi-1.4.5.tgz", + "integrity": "sha512-Up4gpyw2SacmyKWWEib06GhiDdF+H+CCU0LAV8pnM4aJIDqKKd5LHSlBht83Jut6frkB0vwEPmAkv4NjQ5u//Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-android-arm64": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-android-arm64/-/lzma-android-arm64-1.4.5.tgz", + "integrity": "sha512-uwa8sLlWEzkAM0MWyoZJg0JTD3BkPknvejAFG2acUA1raXM8jLrqujWCdOStisXhqQjZ2nDMp3FV6cs//zjfuQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-darwin-arm64": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-darwin-arm64/-/lzma-darwin-arm64-1.4.5.tgz", + "integrity": "sha512-0Y0TQLQ2xAjVabrMDem1NhIssOZzF/y/dqetc6OT8mD3xMTDtF8u5BqZoX3MyPc9FzpsZw4ksol+w7DsxHrpMA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-darwin-x64": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-darwin-x64/-/lzma-darwin-x64-1.4.5.tgz", + "integrity": "sha512-vR2IUyJY3En+V1wJkwmbGWcYiT8pHloTAWdW4pG24+51GIq+intst6Uf6D/r46citObGZrlX0QvMarOkQeHWpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-freebsd-x64": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-freebsd-x64/-/lzma-freebsd-x64-1.4.5.tgz", + "integrity": "sha512-XpnYQC5SVovO35tF0xGkbHYjsS6kqyNCjuaLQ2dbEblFRr5cAZVvsJ/9h7zj/5FluJPJRDojVNxGyRhTp4z2lw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-arm-gnueabihf": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-arm-gnueabihf/-/lzma-linux-arm-gnueabihf-1.4.5.tgz", + "integrity": "sha512-ic1ZZMoRfRMwtSwxkyw4zIlbDZGC6davC9r+2oX6x9QiF247BRqqT94qGeL5ZP4Vtz0Hyy7TEViWhx5j6Bpzvw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-arm64-gnu": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-arm64-gnu/-/lzma-linux-arm64-gnu-1.4.5.tgz", + "integrity": "sha512-asEp7FPd7C1Yi6DQb45a3KPHKOFBSfGuJWXcAd4/bL2Fjetb2n/KK2z14yfW8YC/Fv6x3rBM0VAZKmJuz4tysg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-arm64-musl": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-arm64-musl/-/lzma-linux-arm64-musl-1.4.5.tgz", + "integrity": "sha512-yWjcPDgJ2nIL3KNvi4536dlT/CcCWO0DUyEOlBs/SacG7BeD6IjGh6yYzd3/X1Y3JItCbZoDoLUH8iB1lTXo3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-ppc64-gnu": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-ppc64-gnu/-/lzma-linux-ppc64-gnu-1.4.5.tgz", + "integrity": "sha512-0XRhKuIU/9ZjT4WDIG/qnX7Xz7mSQHYZo9Gb3MP2gcvBgr6BA4zywQ9k3gmQaPn9ECE+CZg2V7DV7kT+x2pUMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-riscv64-gnu": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-riscv64-gnu/-/lzma-linux-riscv64-gnu-1.4.5.tgz", + "integrity": "sha512-QrqDIPEUUB23GCpyQj/QFyMlr8SGxxyExeZz9OWFnHfb70kXdTLWrHS/hEI1Ru+lSbQ/6xRqeoGyQ4Aqdg+/RA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-s390x-gnu": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-s390x-gnu/-/lzma-linux-s390x-gnu-1.4.5.tgz", + "integrity": "sha512-k8RVM5aMhW86E9H0QXdquwojew4H3SwPxbRVbl49/COJQWCUjGi79X6mYruMnMPEznZinUiT1jgKbFo2A00NdA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.4.5.tgz", + "integrity": "sha512-6rMtBgnIq2Wcl1rQdZsnM+rtCcVCbws1nF8S2NzaUsVaZv8bjrPiAa0lwg4Eqnn1d9lgwqT+cZgm5m+//K08Kw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-musl": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-musl/-/lzma-linux-x64-musl-1.4.5.tgz", + "integrity": "sha512-eiadGBKi7Vd0bCArBUOO/qqRYPHt/VQVvGyYvDFt6C2ZSIjlD+HuOl+2oS1sjf4CFjK4eDIog6EdXnL0NE6iyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-wasm32-wasi": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-wasm32-wasi/-/lzma-wasm32-wasi-1.4.5.tgz", + "integrity": "sha512-+VyHHlr68dvey6fXc2hehw9gHVFIW3TtGF1XkcbAu65qVXsA9D/T+uuoRVqhE+JCyFHFrO0ixRbZDRK1XJt1sA==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^1.0.3" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@napi-rs/lzma-win32-arm64-msvc": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-win32-arm64-msvc/-/lzma-win32-arm64-msvc-1.4.5.tgz", + "integrity": "sha512-eewnqvIyyhHi3KaZtBOJXohLvwwN27gfS2G/YDWdfHlbz1jrmfeHAmzMsP5qv8vGB+T80TMHNkro4kYjeh6Deg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-win32-ia32-msvc": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-win32-ia32-msvc/-/lzma-win32-ia32-msvc-1.4.5.tgz", + "integrity": "sha512-OeacFVRCJOKNU/a0ephUfYZ2Yt+NvaHze/4TgOwJ0J0P4P7X1mHzN+ig9Iyd74aQDXYqc7kaCXA2dpAOcH87Cg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/lzma-win32-x64-msvc": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-win32-x64-msvc/-/lzma-win32-x64-msvc-1.4.5.tgz", + "integrity": "sha512-T4I1SamdSmtyZgDXGAGP+y5LEK5vxHUFwe8mz6D4R7Sa5/WCxTcCIgPJ9BD7RkpO17lzhlaM2vmVvMy96Lvk9Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar/-/tar-1.1.1.tgz", + "integrity": "sha512-p6q2HhUc5vwH1CNwfOcrhLoxfgn8ust8Sqlfx+sA4VzAcp1cMbvbkl99tZZlDqOjCHgQNSiTfk/yWPjl/D42qA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/tar-android-arm-eabi": "1.1.1", + "@napi-rs/tar-android-arm64": "1.1.1", + "@napi-rs/tar-darwin-arm64": "1.1.1", + "@napi-rs/tar-darwin-x64": "1.1.1", + "@napi-rs/tar-freebsd-x64": "1.1.1", + "@napi-rs/tar-linux-arm-gnueabihf": "1.1.1", + "@napi-rs/tar-linux-arm64-gnu": "1.1.1", + "@napi-rs/tar-linux-arm64-musl": "1.1.1", + "@napi-rs/tar-linux-ppc64-gnu": "1.1.1", + "@napi-rs/tar-linux-s390x-gnu": "1.1.1", + "@napi-rs/tar-linux-x64-gnu": "1.1.1", + "@napi-rs/tar-linux-x64-musl": "1.1.1", + "@napi-rs/tar-wasm32-wasi": "1.1.1", + "@napi-rs/tar-win32-arm64-msvc": "1.1.1", + "@napi-rs/tar-win32-ia32-msvc": "1.1.1", + "@napi-rs/tar-win32-x64-msvc": "1.1.1" + } + }, + "node_modules/@napi-rs/tar-android-arm-eabi": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-android-arm-eabi/-/tar-android-arm-eabi-1.1.1.tgz", + "integrity": "sha512-cAhnA10cSusAUbcE9HtjQY/tZ9BH/0w2sKtRcQc94TzIlnm7QSr1htJSd/PPrbWNPtrv1orXb2CkrHlVlbnlHA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-android-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-android-arm64/-/tar-android-arm64-1.1.1.tgz", + "integrity": "sha512-EslUWHCDBY/g5abTPBiHLsMaML4GagV0TXLm5WL9hAjx/DDtlxz9fegMb77RJ+f7nFLOIsUxF/3QWFvgOT0sMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-darwin-arm64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-darwin-arm64/-/tar-darwin-arm64-1.1.1.tgz", + "integrity": "sha512-+A42/6ES5G9CQ35BOwzwA+WBjLID28r2jNPgc0dteD2hhClIhng0mva7D2ujUlXBNmgNOsr1LHn3stA4uTf4NQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-darwin-x64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-darwin-x64/-/tar-darwin-x64-1.1.1.tgz", + "integrity": "sha512-RYtE8w1dkEvj8hSJCDV5Jw0Rz2i13fsM7u893zv5O9n/4Ad5GNsw/f4RQ7/0YGSFaenkVxqPFrjmEvUHlKzsrg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-freebsd-x64": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-freebsd-x64/-/tar-freebsd-x64-1.1.1.tgz", + "integrity": "sha512-rEepBvCJUwcuvUYkY83e8aot8RsR5Jcnal4PsG3tbWGKW1yAvcXhyMXf0fN6ZGpVRZFnB+FJqDyBxvsCPEXKhw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-linux-arm-gnueabihf": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-linux-arm-gnueabihf/-/tar-linux-arm-gnueabihf-1.1.1.tgz", + "integrity": "sha512-an1bJdfyhI5FpZYyTQ20mrqwR+a676i8GkaYc4Uy12dH/a7TJIfrK6Qa2Gm46arZvxUvx56qxoRKXbpOjUPvwA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-linux-arm64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-linux-arm64-gnu/-/tar-linux-arm64-gnu-1.1.1.tgz", + "integrity": "sha512-w++Vtx36T2yHTKws7GVnmHHcUT1ybB59xLWSh9A8bwEpJVG4dG7Qub9mFe5cpcbfrJ+XP2mKKxC3oUJSunK3iQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-linux-arm64-musl": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-linux-arm64-musl/-/tar-linux-arm64-musl-1.1.1.tgz", + "integrity": "sha512-Rh6UFhNtj3i4deJHOBINFIeRL0072mgbeyuK5rl1HokKnNoMKx8qKIZNEzBTTqpogMfDHWGvzyTQdnVxes5dpA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-linux-ppc64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-linux-ppc64-gnu/-/tar-linux-ppc64-gnu-1.1.1.tgz", + "integrity": "sha512-Cp+AxFbv9zcyAXtnzQi0OzmgDnQgy2w9D4Ubr+iwzMtVgJcztzcEoCcCrN1k2ATdEB01LX2Vb49IaocGOZhC9Q==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-linux-s390x-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-linux-s390x-gnu/-/tar-linux-s390x-gnu-1.1.1.tgz", + "integrity": "sha512-ZyscC3SYKTBWyDRYjLOKAd5TyJ7q0KACRdQ8bWrb3rgrra1CCIJD66CsGTH6Dh0AVSdfLwZ8MfIIXU6+14BMjQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-linux-x64-gnu": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-linux-x64-gnu/-/tar-linux-x64-gnu-1.1.1.tgz", + "integrity": "sha512-LlIv+zg4fiOQge9LQX/ieBdRWE2fhVDjCTHxnunZkbugNmdhdelxWf1RpZb/6ZujWpNF4LPu4N/MW7ygg2oYAQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-linux-x64-musl": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-linux-x64-musl/-/tar-linux-x64-musl-1.1.1.tgz", + "integrity": "sha512-gZBeoKLjanOVj55qk4EMu13P2i9M0SuINmlGQkOxm1niIJofexzddHUYtqO5o/5QqtyL8lADmAcZplLILMLhHA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-wasm32-wasi": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-wasm32-wasi/-/tar-wasm32-wasi-1.1.1.tgz", + "integrity": "sha512-rwtQ1Mdt/ft6g6I54fJzbUeLspl4yTwj6I3UJ6mitKnrN42soJkcDrdh3Y/FGvlpqZTad2YMQ96fGJl3EtAm2Q==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.11.2", + "@emnapi/runtime": "1.11.2", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@napi-rs/tar-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.2.tgz", + "integrity": "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/tar-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/tar-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/tar-win32-arm64-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-win32-arm64-msvc/-/tar-win32-arm64-msvc-1.1.1.tgz", + "integrity": "sha512-30PVp1AehRpfwxmv5wI4cg0yj3WmWBsZ+1QnLGnvEELu7Eu/+dhNU0nrmhI7VfPgLwSRK2eg9DQTB3tP7Wv9bA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-win32-ia32-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-win32-ia32-msvc/-/tar-win32-ia32-msvc-1.1.1.tgz", + "integrity": "sha512-aI3/rmz+izUChiSeaPxcasAOxhf3FpJNuIHMXlxS/vpW+HIxUsSDR5+XV61PEG5DL4L/75iENVUxmSGM5l2yaw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/tar-win32-x64-msvc": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@napi-rs/tar-win32-x64-msvc/-/tar-win32-x64-msvc-1.1.1.tgz", + "integrity": "sha512-yJsB2IsrODQVLKbm2Fg1nHiVRbEj49mSPbj4x7JPZWJI0jGVPjohE2Sif0FBbx8OxsVoUODvS0BwksZZ8jl/OA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.4.tgz", + "integrity": "sha512-AJxoUD2/15ESHbvpcyjU274nsAPLuOtPHCk0vKJM5pj//Fg/B1FXNWjPnXTT9PymCYYiHo4zPj0ZomXBKhoy7g==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", + "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" + } + }, + "node_modules/@napi-rs/wasm-tools": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools/-/wasm-tools-1.1.0.tgz", + "integrity": "sha512-VjHyKEqXAwYZK+HY7iJctYvRm3TFEbaQxeZwvAG1QRkoo1a39phMY8J6x9tUEqJI03W6MysB8F2jacI6wvcx+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12.22.0" + }, + "optionalDependencies": { + "@napi-rs/wasm-tools-android-arm-eabi": "1.1.0", + "@napi-rs/wasm-tools-android-arm64": "1.1.0", + "@napi-rs/wasm-tools-darwin-arm64": "1.1.0", + "@napi-rs/wasm-tools-darwin-x64": "1.1.0", + "@napi-rs/wasm-tools-freebsd-x64": "1.1.0", + "@napi-rs/wasm-tools-linux-arm64-gnu": "1.1.0", + "@napi-rs/wasm-tools-linux-arm64-musl": "1.1.0", + "@napi-rs/wasm-tools-linux-x64-gnu": "1.1.0", + "@napi-rs/wasm-tools-linux-x64-musl": "1.1.0", + "@napi-rs/wasm-tools-wasm32-wasi": "1.1.0", + "@napi-rs/wasm-tools-win32-arm64-msvc": "1.1.0", + "@napi-rs/wasm-tools-win32-ia32-msvc": "1.1.0", + "@napi-rs/wasm-tools-win32-x64-msvc": "1.1.0" + } + }, + "node_modules/@napi-rs/wasm-tools-android-arm-eabi": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-android-arm-eabi/-/wasm-tools-android-arm-eabi-1.1.0.tgz", + "integrity": "sha512-p6J8PB59I8d/XItXB/go5JH6nKW+xIbpzaL43EBTV0hi7mrS/Z4gs+MsB04ZrlqZN29BdZV8fChRyasuXLhRaA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-android-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-android-arm64/-/wasm-tools-android-arm64-1.1.0.tgz", + "integrity": "sha512-lWoKN3suypeBSCIRPIw+++sH9V2K6nQkhtdt1opu7XY3v9JwLs6Gw063HWRqkNjphlYpkd/Qy8XcfSPGbJj7nQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-darwin-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-darwin-arm64/-/wasm-tools-darwin-arm64-1.1.0.tgz", + "integrity": "sha512-jfw5vyNDUf6oe0kP8lMveFN9U7cLk1cUosS7uMIfw/xmqmopYfKQ198DAx2g/6aEF7Tm+CqER2gpMpYKui30LA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-darwin-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-darwin-x64/-/wasm-tools-darwin-x64-1.1.0.tgz", + "integrity": "sha512-R+pjeudAB7BYdH1vKkOJM61Tfv5jB6uXkxmFscYd+KKpdUpWBlNG+s4hr0w4i1rMBM91VhIAETZn2pz+MDHK9A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-freebsd-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-freebsd-x64/-/wasm-tools-freebsd-x64-1.1.0.tgz", + "integrity": "sha512-hQJTe+aazrT++Vgm6I4lUd9099ItUCFYdd+aKg6Ys6nax6d/cZ1barDLTwA2lwOoVDsXMekJI/FOL6ZvVlIYBg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-linux-arm64-gnu": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-linux-arm64-gnu/-/wasm-tools-linux-arm64-gnu-1.1.0.tgz", + "integrity": "sha512-1TAXJxUHsWGar90k3W/MknavvBMwOWzjh7Q6Spxo8twRcWJbBD5Kow/Q2KhhDq5hxh2sKGDXn3uLc1tdtz4WUg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-linux-arm64-musl": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-linux-arm64-musl/-/wasm-tools-linux-arm64-musl-1.1.0.tgz", + "integrity": "sha512-7rw3nlubTjNAVRH2LwphCxHy1b/N2/TerXocQ6XRn4Q+buaY1Z7P/hbdALy1i1ex2yfOU2Xcij7ib7ZLi/lKfw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-linux-x64-gnu": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-linux-x64-gnu/-/wasm-tools-linux-x64-gnu-1.1.0.tgz", + "integrity": "sha512-1sel0t9MRjI/tdT89M8Dd6gPfANeeFP24Xa46R11WeHNwhjsXXZh+xUk50uWCRTSGcaCy3ugm3AMK/lmHYQJkg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-linux-x64-musl": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-linux-x64-musl/-/wasm-tools-linux-x64-musl-1.1.0.tgz", + "integrity": "sha512-o2jH5AMfor4EKF2HII1LBnMQxoWu7+usPifTEY8Zk6e9OiSi4EJkAXf9v3ANlX7TI2V/cUEV34OEW7r10GiVIA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-wasm32-wasi": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-wasm32-wasi/-/wasm-tools-wasm32-wasi-1.1.0.tgz", + "integrity": "sha512-s6YDtDR1UWrsqJPtaxf+JLYLceWVyn3l8OpQYElHkDhf3Qfz9R6Ba3S0OgznTBv38L5/TIHysQ9Q4yO73Z0csg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.9.2", + "@emnapi/runtime": "1.9.2", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@napi-rs/wasm-tools-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", + "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/wasm-tools-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/wasm-tools-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@napi-rs/wasm-tools-win32-arm64-msvc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-win32-arm64-msvc/-/wasm-tools-win32-arm64-msvc-1.1.0.tgz", + "integrity": "sha512-x+NuxbG84VxU68tU8w7Rf5lSyq0l584M6dVlke5DTweHYFZoMyeqkpbwEq+qsyAX6ivfipK8xRsmFwamb5uDnA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-win32-ia32-msvc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-win32-ia32-msvc/-/wasm-tools-win32-ia32-msvc-1.1.0.tgz", + "integrity": "sha512-mdD96QDEp70SX67rXFTY6c725nVYeqEEjyDqzzbNh6u1APj7CI7IMNpMmvE75XbCRl4C2MHZVU4U6AWdAzvyQQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@napi-rs/wasm-tools-win32-x64-msvc": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-tools-win32-x64-msvc/-/wasm-tools-win32-x64-msvc-1.1.0.tgz", + "integrity": "sha512-bVVjuvhlyVX++3eJXfDR63cXdw1ay5QYac6iq0MKQw8wZARInTM+bXCtByDT4fzVFI3+7ZthYb/ERWRdBNIqgQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.22.0" + } + }, + "node_modules/@octokit/auth-token": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-6.0.0.tgz", + "integrity": "sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/core": { + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-7.0.8.tgz", + "integrity": "sha512-L7y8eYc+AwxGr2PWI4WFt1VG4TiJ66c26BD16mXpYIlXxG0SMigM1+m4aTSlYyBr5BlQsGAlz8uDCoZN4SEMcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/auth-token": "^6.0.0", + "@octokit/graphql": "^9.0.5", + "@octokit/request": "^10.0.16", + "@octokit/request-error": "^7.1.2", + "@octokit/types": "^18.0.0", + "before-after-hook": "^4.0.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/endpoint": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-11.0.5.tgz", + "integrity": "sha512-iXa654H3yFafF/ieHkukfbgWo2rmXD2ceD0ZOtrPhw1bc3FDch1d9N/TNs0FQ1/cIbwb7kspUX8jzIs8nzb9DQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^18.0.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/graphql": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-9.0.5.tgz", + "integrity": "sha512-bt/hm03LeU6Vy7FwTrkkC9p3XGT/lBwClglMqxBSe5/q0E5CdJTXeAqEI0vlw89/LF/G6tryTIH8HirZ3prMVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/request": "^10.0.16", + "@octokit/types": "^18.0.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "29.0.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-29.0.1.tgz", + "integrity": "sha512-9qWOMFNxxLokERcms42rU0PTLqQmVs7g5E41TI4mCOxmpFayD1rfC7XxOL55cG9MBZLFlC31BrR37myMKardwg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-14.0.0.tgz", + "integrity": "sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^16.0.0" + }, + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, + "node_modules/@octokit/plugin-request-log": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-6.0.0.tgz", + "integrity": "sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "17.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-17.0.0.tgz", + "integrity": "sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^16.0.0" + }, + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "27.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-27.0.0.tgz", + "integrity": "sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "16.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-16.0.0.tgz", + "integrity": "sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^27.0.0" + } + }, + "node_modules/@octokit/request": { + "version": "10.0.16", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-10.0.16.tgz", + "integrity": "sha512-A0zWGjHzISIb+9ccG8s0dq7LKO5zVpJLRICjgUb+sJxEWqn8RUHB1rD3AE51+PECvXHIxqZ1VVvs4fHTSD9nUQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^11.0.5", + "@octokit/request-error": "^7.1.2", + "@octokit/types": "^18.0.0", + "content-type": "^3.0.0", + "json-with-bigint": "^3.5.12", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/request-error": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-7.1.2.tgz", + "integrity": "sha512-XZRuT3xZ84D3gYErI1DZvhJ33dCWVV6uzBtWkaBB4TvA/L6eOeTZodxLFVB44bBEEo3vEx7y00UfX1tBLrtLRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^18.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/rest": { + "version": "22.0.1", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-22.0.1.tgz", + "integrity": "sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/core": "^7.0.6", + "@octokit/plugin-paginate-rest": "^14.0.0", + "@octokit/plugin-request-log": "^6.0.0", + "@octokit/plugin-rest-endpoint-methods": "^17.0.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/@octokit/types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-18.0.0.tgz", + "integrity": "sha512-l6bAF43PNxkJp6g+W4PjoUSSkxHomXw2nOum5CTftJz1NlV3vu93NImgOYtLf6CbBUb5j+fiuzW0PPQ5JTSvZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^29.0.1" + } + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", + "integrity": "sha512-W3c4gRigFS0T/Ma4qIYF3GDAc5AQdHb1yL5znJT1Zv1YaD9Kitx656wBjvr19qbiosmZT8lWDM5BEMynUqX65A==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/before-after-hook": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-4.0.0.tgz", + "integrity": "sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/chardet": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.2.0.tgz", + "integrity": "sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/clipanion": { + "version": "4.0.0-rc.4", + "resolved": "https://registry.npmjs.org/clipanion/-/clipanion-4.0.0-rc.4.tgz", + "integrity": "sha512-CXkMQxU6s9GklO/1f714dkKBMu1lopS1WFF0B8o4AxPykR1hpozxSiUZ5ZUeBjfPgCWqbcNOtZVFhB8Lkfp1+Q==", + "dev": true, + "license": "MIT", + "workspaces": [ + "website" + ], + "dependencies": { + "typanion": "^3.8.0" + }, + "peerDependencies": { + "typanion": "*" + } + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/content-type": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-3.1.1.tgz", + "integrity": "sha512-GW4qUsfFo59d0HbUibDlWv5wPz+vAAcaTWbKIuKCf0JkC7wWkSyf8f13IpXn5JkeMlB2P8iTSSIjwXljorg2vA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=22" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/es-toolkit": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.52.0.tgz", + "integrity": "sha512-XTNEJQh1tY1ZJVcf6ayP/2n4ZPyaHlW2FWs7xvw5ddPuhUVjLD3olQVQS7kf58JbAB48iL0uL/jerTrjtV3lDA==", + "dev": true, + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks", + "tests/types", + "tests/browser-compat" + ] + }, + "node_modules/fast-string-truncated-width": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", + "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", + "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^3.0.2" + } + }, + "node_modules/fast-wrap-ansi": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", + "integrity": "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-string-width": "^3.0.2" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/js-yaml": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.4.2.tgz", + "integrity": "sha512-m+aqu+LwO1O6sIopafj8HUVl5aawITwZQe/yHpMCKjaWBaA/d07B/QdMb3529REftiU+RMMHL3Vlsw3hON7vWg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.mjs" + } + }, + "node_modules/json-with-bigint": { + "version": "3.5.12", + "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.12.tgz", + "integrity": "sha512-uwbF/wSSuOgC7qqlq27Xp5B6a2MHVug3t0idZdTqu0JnlFvgJuH7ju+KAk/J06C7GfhoYy2gnb9wz2INqcne7w==", + "dev": true, + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/mute-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-3.0.0.tgz", + "integrity": "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==", + "dev": true, + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/obug": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/obug/-/obug-3.0.0.tgz", + "integrity": "sha512-5vvB5+W7ePv+p3uqxi+RcW1XAzLW0/hxt3/4X4Lc4qHudzOhmBiBwOY6DRob4WnanAEGvNcLjF+KNOufrUoEQw==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/typanion": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/typanion/-/typanion-3.14.0.tgz", + "integrity": "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==", + "dev": true, + "license": "MIT", + "workspaces": [ + "website" + ] + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/universal-user-agent": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.3.tgz", + "integrity": "sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/bindings/typescript/docprims/package.json b/bindings/typescript/docprims/package.json index b6c879c..1ecf65e 100644 --- a/bindings/typescript/docprims/package.json +++ b/bindings/typescript/docprims/package.json @@ -20,22 +20,19 @@ "node": ">=22" }, "napi": { - "name": "docprims", - "triples": { - "defaults": false, - "additional": [ - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "aarch64-apple-darwin", - "x86_64-pc-windows-msvc" - ] - } + "binaryName": "docprims", + "targets": [ + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] }, "scripts": { "clean": "node scripts/clean.js", - "build:native": "napi build --cargo-cwd native --release --platform --strip dist/native", + "build:native": "napi build --manifest-path native/Cargo.toml --release --platform --strip --output-dir dist/native", "build": "tsc -p tsconfig.build.json", "build:test": "tsc -p tsconfig.test.json", "test": "npm run clean && npm run build && npm run build:native && npm run build:test && node scripts/run-tests.js", @@ -52,8 +49,8 @@ "@3leaps/docprims-win32-x64-msvc": "0.1.5" }, "devDependencies": { - "@napi-rs/cli": "^2.18.4", - "@types/node": "^22.10.0", - "typescript": "^5.6.3" + "@napi-rs/cli": "^3.10.5", + "@types/node": "^22.20.4", + "typescript": "^6.0.3" } } diff --git a/bindings/typescript/docprims/tsconfig.json b/bindings/typescript/docprims/tsconfig.json index 42a6e50..01bdfbc 100644 --- a/bindings/typescript/docprims/tsconfig.json +++ b/bindings/typescript/docprims/tsconfig.json @@ -1,8 +1,8 @@ { "compilerOptions": { "target": "ES2022", - "module": "CommonJS", - "moduleResolution": "Node", + "module": "Node16", + "moduleResolution": "Node16", "declaration": true, "declarationMap": true, "sourceMap": true, @@ -10,6 +10,7 @@ "esModuleInterop": true, "forceConsistentCasingInFileNames": true, "skipLibCheck": true, + "types": ["node"], "outDir": "dist", "rootDir": "src" } diff --git a/crates/docprims-cli/Cargo.toml b/crates/docprims-cli/Cargo.toml index 7868b5d..87fc1f5 100644 --- a/crates/docprims-cli/Cargo.toml +++ b/crates/docprims-cli/Cargo.toml @@ -30,5 +30,5 @@ tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } [dev-dependencies] assert_cmd = "2.0" predicates = "3.1" -base64 = "0.22" -jsonschema = "0.32" +base64 = "0.23" +jsonschema = { version = "0.57", default-features = false } diff --git a/crates/docprims-cli/tests/golden_v0.rs b/crates/docprims-cli/tests/golden_v0.rs index 12e200d..d07a154 100644 --- a/crates/docprims-cli/tests/golden_v0.rs +++ b/crates/docprims-cli/tests/golden_v0.rs @@ -1,7 +1,6 @@ use base64::engine::general_purpose::STANDARD; use base64::Engine; use docprims_core::{DocprimsExtract, ExtractLimits}; -use jsonschema::Resource; use std::io::Cursor; use std::path::{Path, PathBuf}; @@ -29,22 +28,26 @@ fn v0_validator() -> jsonschema::Validator { )) .unwrap(); - jsonschema::draft202012::options() - .with_resources( - [ - ( - "https://schemas.3leaps.dev/docprims/extract/v0/docprims-block.schema.json", - Resource::from_contents(block_schema).unwrap(), - ), - ( - "https://schemas.3leaps.dev/docprims/extract/v0/docprims-location.schema.json", - Resource::from_contents(loc_schema).unwrap(), - ), - ] - .into_iter(), - ) - .build(&root_schema) - .unwrap() + { + let registry = jsonschema::Registry::new() + .add( + "https://schemas.3leaps.dev/docprims/extract/v0/docprims-block.schema.json", + block_schema, + ) + .unwrap() + .add( + "https://schemas.3leaps.dev/docprims/extract/v0/docprims-location.schema.json", + loc_schema, + ) + .unwrap() + .prepare() + .unwrap(); + jsonschema::draft202012::options() + .offline() + .with_registry(®istry) + .build(&root_schema) + .unwrap() + } } fn read_text_fixture(path: &str) -> String { diff --git a/crates/docprims-ooxml/Cargo.toml b/crates/docprims-ooxml/Cargo.toml index 6248c62..17c2c92 100644 --- a/crates/docprims-ooxml/Cargo.toml +++ b/crates/docprims-ooxml/Cargo.toml @@ -21,5 +21,5 @@ zip = { workspace = true } encoding_rs = { workspace = true } [dev-dependencies] -jsonschema = "0.32" +jsonschema = { version = "0.57", default-features = false } serde_json = { workspace = true } diff --git a/crates/docprims-ooxml/src/common.rs b/crates/docprims-ooxml/src/common.rs index 255874c..32264b5 100644 --- a/crates/docprims-ooxml/src/common.rs +++ b/crates/docprims-ooxml/src/common.rs @@ -179,9 +179,7 @@ mod tests { /// /// Handles predefined XML entities (lt, gt, amp, apos, quot) and numeric /// character references (&#NNN; or &#xHHH;). -pub fn resolve_entity(entity: &[u8]) -> Option<&'static str> { - let name = std::str::from_utf8(entity).ok()?; - +pub fn resolve_entity(name: &str) -> Option<&'static str> { // Try predefined entities first if let Some(resolved) = resolve_xml_entity(name) { return Some(resolved); diff --git a/crates/docprims-ooxml/src/docx.rs b/crates/docprims-ooxml/src/docx.rs index 12f3593..9cd7311 100644 --- a/crates/docprims-ooxml/src/docx.rs +++ b/crates/docprims-ooxml/src/docx.rs @@ -145,17 +145,17 @@ fn extract_text_from_document(xml: &str) -> Result { Ok(Event::Start(e)) => { let local_name = e.local_name(); match local_name.as_ref() { - b"p" => { + "p" => { // Paragraph start - add newline if we have content if !text.is_empty() && !text.ends_with('\n') { text.push('\n'); } } - b"br" => { + "br" => { // Line break text.push('\n'); } - b"tab" => { + "tab" => { // Tab character text.push('\t'); } @@ -163,9 +163,7 @@ fn extract_text_from_document(xml: &str) -> Result { } } Ok(Event::Text(e)) => { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); text.push_str(&decoded); } Ok(Event::GeneralRef(e)) => { @@ -196,20 +194,20 @@ fn extract_paragraphs(xml: &str) -> Result> { loop { match reader.read_event_into(&mut buf) { Ok(Event::Start(e)) => match e.local_name().as_ref() { - b"p" => { + "p" => { in_p = true; cur.clear(); } - b"br" if in_p => { + "br" if in_p => { cur.push('\n'); } - b"tab" if in_p => { + "tab" if in_p => { cur.push('\t'); } _ => {} }, Ok(Event::End(e)) => { - if e.local_name().as_ref() == b"p" { + if e.local_name().as_ref() == "p" { in_p = false; let t = cur.trim().to_string(); if !t.is_empty() { @@ -220,9 +218,7 @@ fn extract_paragraphs(xml: &str) -> Result> { } Ok(Event::Text(e)) => { if in_p { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); cur.push_str(&decoded); } } diff --git a/crates/docprims-ooxml/src/lib.rs b/crates/docprims-ooxml/src/lib.rs index f8e5a57..836018e 100644 --- a/crates/docprims-ooxml/src/lib.rs +++ b/crates/docprims-ooxml/src/lib.rs @@ -197,7 +197,6 @@ pub fn extract_pptx_v0_reader( #[cfg(test)] pub(crate) mod test_support { use docprims_core::DocprimsExtract; - use jsonschema::Resource; use std::io::{Cursor, Write}; use zip::write::SimpleFileOptions; @@ -217,20 +216,26 @@ pub(crate) mod test_support { )) .unwrap(); - let validator = jsonschema::draft202012::options() - .with_resources([ - ( + let validator = { + let registry = jsonschema::Registry::new() + .add( "https://schemas.3leaps.dev/docprims/extract/v0/docprims-block.schema.json", - Resource::from_contents(block_schema).unwrap(), - ), - ( + block_schema, + ) + .unwrap() + .add( "https://schemas.3leaps.dev/docprims/extract/v0/docprims-location.schema.json", - Resource::from_contents(loc_schema).unwrap(), - ), - ] - .into_iter()) - .build(&root_schema) - .unwrap(); + loc_schema, + ) + .unwrap() + .prepare() + .unwrap(); + jsonschema::draft202012::options() + .offline() + .with_registry(®istry) + .build(&root_schema) + .unwrap() + }; let value = serde_json::to_value(extract).unwrap(); let errors: Vec<_> = validator.iter_errors(&value).collect(); diff --git a/crates/docprims-ooxml/src/pptx.rs b/crates/docprims-ooxml/src/pptx.rs index bdc74c4..68718f3 100644 --- a/crates/docprims-ooxml/src/pptx.rs +++ b/crates/docprims-ooxml/src/pptx.rs @@ -183,10 +183,10 @@ fn parse_presentation_slide_rids(xml: &str) -> Result> { loop { match reader.read_event_into(&mut buf) { Ok(Event::Empty(e)) | Ok(Event::Start(e)) => { - if e.local_name().as_ref() == b"sldId" { + if e.local_name().as_ref() == "sldId" { for attr in e.attributes().flatten() { - if attr.key.as_ref() == b"r:id" { - rids.push(String::from_utf8_lossy(&attr.value).to_string()); + if attr.key.as_ref() == "r:id" { + rids.push(attr.value.to_string()); } } } @@ -215,15 +215,13 @@ fn parse_relationships(xml: &str) -> Result> { loop { match reader.read_event_into(&mut buf) { Ok(Event::Empty(e)) | Ok(Event::Start(e)) => { - if e.local_name().as_ref() == b"Relationship" { + if e.local_name().as_ref() == "Relationship" { let mut id: Option = None; let mut target: Option = None; for attr in e.attributes().flatten() { match attr.key.as_ref() { - b"Id" => id = Some(String::from_utf8_lossy(&attr.value).to_string()), - b"Target" => { - target = Some(String::from_utf8_lossy(&attr.value).to_string()) - } + "Id" => id = Some(attr.value.to_string()), + "Target" => target = Some(attr.value.to_string()), _ => {} } } @@ -282,7 +280,7 @@ fn extract_slide_text(xml: &str) -> Result { match reader.read_event_into(&mut buf) { Ok(Event::Start(e)) => { let local_name = e.local_name(); - if local_name.as_ref() == b"p" { + if local_name.as_ref() == "p" { // Paragraph start - flush and prepare for new paragraph let trimmed = current_part.trim(); if !trimmed.is_empty() { @@ -293,7 +291,7 @@ fn extract_slide_text(xml: &str) -> Result { } Ok(Event::End(e)) => { let local_name = e.local_name(); - if local_name.as_ref() == b"p" { + if local_name.as_ref() == "p" { // Paragraph end - flush current content let trimmed = current_part.trim(); if !trimmed.is_empty() { @@ -303,9 +301,7 @@ fn extract_slide_text(xml: &str) -> Result { } } Ok(Event::Text(e)) => { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); current_part.push_str(&decoded); } Ok(Event::GeneralRef(e)) => { @@ -342,7 +338,7 @@ fn extract_slide_paragraphs(xml: &str) -> Result> { loop { match reader.read_event_into(&mut buf) { Ok(Event::Start(e)) => { - if e.local_name().as_ref() == b"p" { + if e.local_name().as_ref() == "p" { let trimmed = current.trim(); if !trimmed.is_empty() { paras.push(trimmed.to_string()); @@ -351,7 +347,7 @@ fn extract_slide_paragraphs(xml: &str) -> Result> { } } Ok(Event::End(e)) => { - if e.local_name().as_ref() == b"p" { + if e.local_name().as_ref() == "p" { let trimmed = current.trim(); if !trimmed.is_empty() { paras.push(trimmed.to_string()); @@ -360,9 +356,7 @@ fn extract_slide_paragraphs(xml: &str) -> Result> { } } Ok(Event::Text(e)) => { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); current.push_str(&decoded); } Ok(Event::GeneralRef(e)) => { diff --git a/crates/docprims-ooxml/src/xlsx.rs b/crates/docprims-ooxml/src/xlsx.rs index 0513175..23c8312 100644 --- a/crates/docprims-ooxml/src/xlsx.rs +++ b/crates/docprims-ooxml/src/xlsx.rs @@ -223,15 +223,13 @@ fn parse_workbook_sheets(xml: &str) -> Result> { loop { match reader.read_event_into(&mut buf) { Ok(Event::Empty(e)) | Ok(Event::Start(e)) => { - if e.local_name().as_ref() == b"sheet" { + if e.local_name().as_ref() == "sheet" { let mut name: Option = None; let mut rid: Option = None; for attr in e.attributes().flatten() { match attr.key.as_ref() { - b"name" => { - name = Some(String::from_utf8_lossy(&attr.value).to_string()) - } - b"r:id" => rid = Some(String::from_utf8_lossy(&attr.value).to_string()), + "name" => name = Some(attr.value.to_string()), + "r:id" => rid = Some(attr.value.to_string()), _ => {} } } @@ -270,15 +268,13 @@ fn parse_relationships(xml: &str) -> Result> { loop { match reader.read_event_into(&mut buf) { Ok(Event::Empty(e)) | Ok(Event::Start(e)) => { - if e.local_name().as_ref() == b"Relationship" { + if e.local_name().as_ref() == "Relationship" { let mut id: Option = None; let mut target: Option = None; for attr in e.attributes().flatten() { match attr.key.as_ref() { - b"Id" => id = Some(String::from_utf8_lossy(&attr.value).to_string()), - b"Target" => { - target = Some(String::from_utf8_lossy(&attr.value).to_string()) - } + "Id" => id = Some(attr.value.to_string()), + "Target" => target = Some(attr.value.to_string()), _ => {} } } @@ -339,23 +335,21 @@ fn parse_shared_strings(xml: &str) -> Result> { match reader.read_event_into(&mut buf) { Ok(Event::Start(e)) => { let local_name = e.local_name(); - if local_name.as_ref() == b"si" { + if local_name.as_ref() == "si" { in_si = true; current_string.clear(); } } Ok(Event::End(e)) => { let local_name = e.local_name(); - if local_name.as_ref() == b"si" { + if local_name.as_ref() == "si" { strings.push(current_string.trim().to_string()); in_si = false; } } Ok(Event::Text(e)) => { if in_si { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); current_string.push_str(&decoded); } } @@ -398,21 +392,21 @@ fn extract_sheet_rows(xml: &str, shared_strings: &[String]) -> Result match e.local_name().as_ref() { - b"row" => { + "row" => { in_row = true; row_text.clear(); row_index = None; for attr in e.attributes().flatten() { - if attr.key.as_ref() == b"r" { - row_index = String::from_utf8_lossy(&attr.value).parse::().ok(); + if attr.key.as_ref() == "r" { + row_index = attr.value.parse::().ok(); } } } - b"c" => { + "c" => { cell_type = None; for attr in e.attributes().flatten() { - if attr.key.as_ref() == b"t" { - cell_type = Some(String::from_utf8_lossy(&attr.value).to_string()); + if attr.key.as_ref() == "t" { + cell_type = Some(attr.value.to_string()); } } cell_value.clear(); @@ -420,7 +414,7 @@ fn extract_sheet_rows(xml: &str, shared_strings: &[String]) -> Result {} }, Ok(Event::End(e)) => match e.local_name().as_ref() { - b"c" => { + "c" => { if in_row { let value = resolve_cell_value(cell_value.trim(), &cell_type, shared_strings); @@ -432,7 +426,7 @@ fn extract_sheet_rows(xml: &str, shared_strings: &[String]) -> Result { + "row" => { in_row = false; let t = row_text.trim_end().to_string(); if !t.is_empty() { @@ -444,9 +438,7 @@ fn extract_sheet_rows(xml: &str, shared_strings: &[String]) -> Result {} }, Ok(Event::Text(e)) => { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); cell_value.push_str(&decoded); } Ok(Event::GeneralRef(e)) => { @@ -485,17 +477,17 @@ fn extract_sheet_text(xml: &str, shared_strings: &[String]) -> Result { Ok(Event::Start(e)) => { let local_name = e.local_name(); match local_name.as_ref() { - b"row" => { + "row" => { if !text.is_empty() && !text.ends_with('\n') { text.push('\n'); } } - b"c" => { + "c" => { // Cell - check type attribute cell_type = None; for attr in e.attributes().flatten() { - if attr.key.as_ref() == b"t" { - cell_type = Some(String::from_utf8_lossy(&attr.value).to_string()); + if attr.key.as_ref() == "t" { + cell_type = Some(attr.value.to_string()); } } cell_value.clear(); @@ -505,7 +497,7 @@ fn extract_sheet_text(xml: &str, shared_strings: &[String]) -> Result { } Ok(Event::End(e)) => { let local_name = e.local_name(); - if local_name.as_ref() == b"c" { + if local_name.as_ref() == "c" { // Output cell value let value = resolve_cell_value(cell_value.trim(), &cell_type, shared_strings); if !value.is_empty() { @@ -517,9 +509,7 @@ fn extract_sheet_text(xml: &str, shared_strings: &[String]) -> Result { } } Ok(Event::Text(e)) => { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); cell_value.push_str(&decoded); } Ok(Event::GeneralRef(e)) => { diff --git a/crates/docprims-text/Cargo.toml b/crates/docprims-text/Cargo.toml index a09dbb6..2663754 100644 --- a/crates/docprims-text/Cargo.toml +++ b/crates/docprims-text/Cargo.toml @@ -21,5 +21,5 @@ scraper = { workspace = true, optional = true } quick-xml = { workspace = true, optional = true } [dev-dependencies] -jsonschema = "0.32" +jsonschema = { version = "0.57", default-features = false } serde_json = { workspace = true } diff --git a/crates/docprims-text/src/lib.rs b/crates/docprims-text/src/lib.rs index ddd329a..be4a641 100644 --- a/crates/docprims-text/src/lib.rs +++ b/crates/docprims-text/src/lib.rs @@ -126,7 +126,6 @@ pub fn extract_xml_str(content: &str) -> Result { #[cfg(test)] pub(crate) mod test_support { use docprims_core::DocprimsExtract; - use jsonschema::Resource; pub(crate) fn assert_v0_schema_valid(extract: &DocprimsExtract) { let root_schema: serde_json::Value = serde_json::from_str(include_str!( @@ -144,20 +143,26 @@ pub(crate) mod test_support { )) .unwrap(); - let validator = jsonschema::draft202012::options() - .with_resources([ - ( + let validator = { + let registry = jsonschema::Registry::new() + .add( "https://schemas.3leaps.dev/docprims/extract/v0/docprims-block.schema.json", - Resource::from_contents(block_schema).unwrap(), - ), - ( + block_schema, + ) + .unwrap() + .add( "https://schemas.3leaps.dev/docprims/extract/v0/docprims-location.schema.json", - Resource::from_contents(loc_schema).unwrap(), - ), - ] - .into_iter()) - .build(&root_schema) - .unwrap(); + loc_schema, + ) + .unwrap() + .prepare() + .unwrap(); + jsonschema::draft202012::options() + .offline() + .with_registry(®istry) + .build(&root_schema) + .unwrap() + }; let value = serde_json::to_value(extract).unwrap(); let errors: Vec<_> = validator.iter_errors(&value).collect(); diff --git a/crates/docprims-text/src/xml.rs b/crates/docprims-text/src/xml.rs index a29f5e0..ee349aa 100644 --- a/crates/docprims-text/src/xml.rs +++ b/crates/docprims-text/src/xml.rs @@ -22,15 +22,12 @@ pub fn extract(content: &str) -> Result { loop { match reader.read_event_into(&mut buf) { Ok(Event::Text(e)) => { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); current_part.push_str(&decoded); } Ok(Event::GeneralRef(e)) => { // Resolve entity reference (e.g., "lt" -> "<", "amp" -> "&") - let entity_name = std::str::from_utf8(&e) - .map_err(|e| DocprimsError::Parse(format!("Invalid entity encoding: {}", e)))?; + let entity_name: &str = &e; if let Some(resolved) = resolve_xml_entity(entity_name) { current_part.push_str(resolved); @@ -43,8 +40,8 @@ pub fn extract(content: &str) -> Result { // Unknown entities are silently dropped } Ok(Event::CData(e)) => { - let cdata_content = String::from_utf8_lossy(&e); - current_part.push_str(&cdata_content); + let cdata_content: &str = &e; + current_part.push_str(cdata_content); } Ok(Event::Start(_) | Event::End(_)) => { // Element boundaries - flush current text part @@ -94,14 +91,11 @@ pub fn extract_v0_str( loop { match reader.read_event_into(&mut buf) { Ok(Event::Text(e)) => { - let decoded = e - .decode() - .map_err(|e| DocprimsError::Parse(format!("XML decode error: {}", e)))?; + let decoded = e.into_inner(); current_part.push_str(&decoded); } Ok(Event::GeneralRef(e)) => { - let entity_name = std::str::from_utf8(&e) - .map_err(|e| DocprimsError::Parse(format!("Invalid entity encoding: {}", e)))?; + let entity_name: &str = &e; if let Some(resolved) = resolve_xml_entity(entity_name) { current_part.push_str(resolved); @@ -112,8 +106,8 @@ pub fn extract_v0_str( } } Ok(Event::CData(e)) => { - let cdata_content = String::from_utf8_lossy(&e); - current_part.push_str(&cdata_content); + let cdata_content: &str = &e; + current_part.push_str(cdata_content); } Ok(Event::Start(_) | Event::End(_)) => { let trimmed = current_part.trim(); diff --git a/deny.toml b/deny.toml index f231f21..f8a4bf7 100644 --- a/deny.toml +++ b/deny.toml @@ -3,9 +3,20 @@ # # Run: cargo deny check +# Evaluate the graph for every platform docprims ships artifacts for, so +# target-specific dependencies are checked too. +[graph] +targets = [ + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-pc-windows-gnu", +] + [advisories] -db-path = "~/.cargo/advisory-db" -db-urls = ["https://github.com/rustsec/advisory-db"] ignore = [] [licenses] @@ -32,15 +43,23 @@ multiple-versions = "warn" wildcards = "allow" highlight = "all" -# Deny GPL and LGPL - this is the core mission of docprims deny = [ + # Deny GPL and LGPL - this is the core mission of docprims { name = "poppler-rs" }, # GPL { name = "mupdf" }, # AGPL { name = "mupdf-sys" }, # AGPL { name = "lopdf", version = "<0.30" }, # Older versions had issues + + # TLS: rustls only; no OpenSSL or platform TLS in any graph + { name = "openssl" }, + { name = "openssl-sys" }, + { name = "native-tls" }, + + # Minimum parser versions for untrusted input + { name = "quick-xml", version = "<0.41" }, ] [sources] -unknown-registry = "warn" -unknown-git = "warn" +unknown-registry = "deny" +unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] diff --git a/ffi/docprims-ffi/Cargo.toml b/ffi/docprims-ffi/Cargo.toml index 9ad5949..3e2e3f0 100644 --- a/ffi/docprims-ffi/Cargo.toml +++ b/ffi/docprims-ffi/Cargo.toml @@ -19,4 +19,4 @@ serde_json = { workspace = true } serde = { workspace = true } [dev-dependencies] -base64 = "0.22" +base64 = "0.23" diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..ef450a1 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.94.1" +components = ["rustfmt", "clippy"] +profile = "minimal" diff --git a/scripts/build-go-ffi-lib.sh b/scripts/build-go-ffi-lib.sh index fd4f5b2..083896a 100755 --- a/scripts/build-go-ffi-lib.sh +++ b/scripts/build-go-ffi-lib.sh @@ -35,7 +35,7 @@ esac echo "[docprims] building FFI static lib for $PLATFORM" make cbindgen -cargo build --release -p docprims-ffi +cargo build --release -p docprims-ffi --locked mkdir -p "bindings/go/docprims/include" cp "ffi/docprims-ffi/docprims.h" "bindings/go/docprims/include/docprims.h" From a79516059a328d27d6fb2ebbbd100acadfe883a1 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:33:07 -0400 Subject: [PATCH 02/33] fix(ooxml): bound decompressed bytes per part and per archive Archive part reads now cap the bytes actually decompressed instead of relying on the size declared in the archive header, and buffer pre-allocation no longer follows the declared size. Archives carry a total decompression budget (4x the per-part limit) shared by every part read from them; exceeding either limit is a ResourceLimit error. open_archive now returns an OoxmlArchive wrapper that holds the budget. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- crates/docprims-ooxml/src/common.rs | 302 ++++++++++++++++++++++++++-- crates/docprims-ooxml/src/pptx.rs | 4 +- crates/docprims-ooxml/src/xlsx.rs | 4 +- 3 files changed, 286 insertions(+), 24 deletions(-) diff --git a/crates/docprims-ooxml/src/common.rs b/crates/docprims-ooxml/src/common.rs index 32264b5..3033645 100644 --- a/crates/docprims-ooxml/src/common.rs +++ b/crates/docprims-ooxml/src/common.rs @@ -8,35 +8,81 @@ use zip::ZipArchive; use encoding_rs::Encoding; -/// Maximum decompressed size to prevent zip bombs (100 MB default). +/// Maximum decompressed size of any single archive part (100 MB default). pub const MAX_DECOMPRESSED_SIZE: u64 = 100 * 1024 * 1024; +/// Maximum decompressed size summed over every part read from one archive. +pub const MAX_TOTAL_DECOMPRESSED_SIZE: u64 = 4 * MAX_DECOMPRESSED_SIZE; + /// Maximum number of files in archive to prevent DoS. pub const MAX_ARCHIVE_FILES: usize = 10_000; +/// Upper bound on buffer pre-allocation. The header-declared size is untrusted, +/// so it only sizes the initial buffer up to this cap. +const MAX_PREALLOC: u64 = 1024 * 1024; + +/// A ZIP archive opened for OOXML extraction. +/// +/// Carries a decompression budget shared by every part read from it, so an +/// archive of many individually legal parts cannot inflate without bound. +pub struct OoxmlArchive { + zip: ZipArchive, + part_limit: u64, + remaining: u64, +} + /// Open a ZIP archive with safety limits. -pub fn open_archive(reader: R) -> Result> { - let archive = ZipArchive::new(reader) +pub fn open_archive(reader: R) -> Result> { + open_archive_with_limits(reader, MAX_DECOMPRESSED_SIZE, MAX_TOTAL_DECOMPRESSED_SIZE) +} + +fn open_archive_with_limits( + reader: R, + part_limit: u64, + total_limit: u64, +) -> Result> { + let zip = ZipArchive::new(reader) .map_err(|e| DocprimsError::Malformed(format!("Invalid ZIP archive: {}", e)))?; // Check file count limit - if archive.len() > MAX_ARCHIVE_FILES { + if zip.len() > MAX_ARCHIVE_FILES { return Err(DocprimsError::ResourceLimit(format!( "Archive contains too many files: {} (max {})", - archive.len(), + zip.len(), MAX_ARCHIVE_FILES ))); } - Ok(archive) + Ok(OoxmlArchive { + zip, + part_limit, + remaining: total_limit, + }) } -/// Read a file from the archive with size limits. +/// Read a file from the archive, bounding the bytes actually decompressed. +/// +/// The declared uncompressed size in the archive header is attacker-controlled +/// and is used only as an early reject; the read itself is capped at the +/// smaller of the per-part limit and the archive's remaining budget. pub fn read_archive_file( - archive: &mut ZipArchive, + archive: &mut OoxmlArchive, name: &str, ) -> Result> { - let file = match archive.by_name(name) { + let part_too_large = || { + DocprimsError::ResourceLimit(format!( + "File {} too large: decompressed size exceeds {} bytes", + name, archive.part_limit + )) + }; + let total_too_large = || { + DocprimsError::ResourceLimit(format!( + "Archive too large: total decompressed size exceeds budget at {}", + name + )) + }; + + let file = match archive.zip.by_name(name) { Ok(f) => f, Err(zip::result::ZipError::FileNotFound) => return Ok(None), Err(e) => { @@ -47,22 +93,45 @@ pub fn read_archive_file( } }; - // Check decompressed size - let size = file.size(); - if size > MAX_DECOMPRESSED_SIZE { - return Err(DocprimsError::ResourceLimit(format!( - "File {} too large: {} bytes (max {})", - name, size, MAX_DECOMPRESSED_SIZE - ))); + let declared = file.size(); + if declared > archive.part_limit { + return Err(part_too_large()); + } + if declared > archive.remaining { + return Err(total_too_large()); } - let mut content = Vec::with_capacity(size as usize); - let mut reader = std::io::BufReader::new(file); + let limit = archive.part_limit.min(archive.remaining); + let Some(content) = read_bounded(file, name, limit, declared)? else { + return Err(if limit < archive.part_limit { + total_too_large() + } else { + part_too_large() + }); + }; + archive.remaining -= content.len() as u64; + + Ok(Some(decode_xml_bytes(&content)?)) +} + +/// Read at most `limit` bytes. Returns `None`, without reading further, if the +/// stream holds more. `size_hint` sizes the initial buffer, capped at +/// `MAX_PREALLOC`. +fn read_bounded( + reader: R, + name: &str, + limit: u64, + size_hint: u64, +) -> Result>> { + let mut content = Vec::with_capacity(size_hint.min(limit).min(MAX_PREALLOC) as usize); reader + .take(limit.saturating_add(1)) .read_to_end(&mut content) .map_err(|e| DocprimsError::Malformed(format!("Error reading {}: {}", name, e)))?; - - Ok(Some(decode_xml_bytes(&content)?)) + if content.len() as u64 > limit { + return Ok(None); + } + Ok(Some(content)) } fn decode_xml_bytes(bytes: &[u8]) -> Result { @@ -143,6 +212,199 @@ fn sniff_xml_decl_encoding(bytes: &[u8]) -> Option { #[cfg(test)] mod tests { use super::*; + use std::io::{Cursor, Write}; + use zip::write::SimpleFileOptions; + + /// Build a single-entry deflated archive. If `declared_size` is set, the + /// uncompressed-size fields in both the local and central headers are + /// rewritten to that value, as a hostile archive would. + fn archive_with(name: &str, content: &[u8], declared_size: Option) -> Vec { + let mut bytes = archive_of(&[(name, content)]); + if let Some(size) = declared_size { + patch_uncompressed_size(&mut bytes, size); + } + bytes + } + + fn archive_of(entries: &[(&str, &[u8])]) -> Vec { + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + let opts = + SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, content) in entries { + w.start_file(*name, opts).unwrap(); + w.write_all(content).unwrap(); + } + w.finish().unwrap().into_inner() + } + + fn patch_uncompressed_size(bytes: &mut [u8], size: u32) { + let mut patched = 0; + for i in 0..bytes.len().saturating_sub(4) { + let offset = match bytes[i..i + 4] { + [0x50, 0x4b, 0x03, 0x04] => 22, // local file header + [0x50, 0x4b, 0x01, 0x02] => 24, // central directory header + _ => continue, + }; + bytes[i + offset..i + offset + 4].copy_from_slice(&size.to_le_bytes()); + patched += 1; + } + assert_eq!(patched, 2, "expected one local and one central header"); + } + + fn read(bytes: Vec, limit: u64) -> Result> { + let mut archive = open_archive_with_limits(Cursor::new(bytes), limit, u64::MAX).unwrap(); + read_archive_file(&mut archive, "part.xml") + } + + #[test] + fn lying_header_cannot_bypass_decompression_limit() { + let content = vec![b'a'; 64 * 1024]; + let bytes = archive_with("part.xml", &content, Some(100)); + match read(bytes, 16 * 1024) { + Err(DocprimsError::ResourceLimit(msg)) => assert!(msg.contains("exceeds 16384")), + other => panic!("expected ResourceLimit, got {other:?}"), + } + } + + #[test] + fn decompression_limit_is_inclusive() { + let limit = 4096; + let at_limit = archive_with("part.xml", &vec![b'a'; limit], None); + assert_eq!(read(at_limit, limit as u64).unwrap().unwrap().len(), limit); + + let over = archive_with("part.xml", &vec![b'a'; limit + 1], None); + assert!(matches!( + read(over, limit as u64), + Err(DocprimsError::ResourceLimit(_)) + )); + } + + #[test] + fn declared_size_over_limit_is_rejected_before_reading() { + let bytes = archive_with("part.xml", b"", Some(u32::MAX)); + match read(bytes, 1024) { + // Content is 4 bytes, so only the declared size can trigger this. + Err(DocprimsError::ResourceLimit(_)) => {} + other => panic!("expected ResourceLimit, got {other:?}"), + } + } + + #[test] + fn understated_size_within_limit_reads_full_content() { + // A header that under-reports must not truncate what we extract. + let content = "".to_string() + &"y".repeat(10_000) + ""; + let bytes = archive_with("part.xml", content.as_bytes(), Some(10)); + assert_eq!(read(bytes, 1024 * 1024).unwrap().unwrap(), content); + } + + /// Endless source that fails the test if pulled past `allowed` bytes. + struct Tripwire { + pulled: u64, + allowed: u64, + } + + impl Read for Tripwire { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + assert!( + self.pulled + buf.len() as u64 <= self.allowed, + "read past the limit: {} bytes already pulled", + self.pulled + ); + buf.fill(b'a'); + self.pulled += buf.len() as u64; + Ok(buf.len()) + } + } + + #[test] + fn bounded_read_stops_at_limit_on_endless_stream() { + let limit = 8192; + let src = Tripwire { + pulled: 0, + allowed: limit + 1, + }; + assert!(read_bounded(src, "part.xml", limit, 10).unwrap().is_none()); + } + + #[test] + fn bounded_read_ignores_huge_size_hint_for_allocation() { + let content = read_bounded(&b""[..], "part.xml", MAX_DECOMPRESSED_SIZE, u64::MAX) + .unwrap() + .unwrap(); + assert_eq!(content, b""); + assert!(content.capacity() <= MAX_PREALLOC as usize); + } + + #[test] + fn missing_part_is_none_not_error() { + let bytes = archive_with("other.xml", b"", None); + assert!(read(bytes, 1024).unwrap().is_none()); + } + + fn three_parts(each: usize) -> Vec { + let body = vec![b'a'; each]; + archive_of(&[("p1.xml", &body), ("p2.xml", &body), ("p3.xml", &body)]) + } + + #[test] + fn aggregate_budget_spans_parts_that_are_individually_legal() { + // Each part is under the per-part limit; together they exceed the total. + let mut archive = + open_archive_with_limits(Cursor::new(three_parts(3000)), 4096, 8192).unwrap(); + assert!(read_archive_file(&mut archive, "p1.xml").unwrap().is_some()); + assert!(read_archive_file(&mut archive, "p2.xml").unwrap().is_some()); + match read_archive_file(&mut archive, "p3.xml") { + Err(DocprimsError::ResourceLimit(msg)) => assert!(msg.contains("total"), "{msg}"), + other => panic!("expected aggregate ResourceLimit, got {other:?}"), + } + } + + #[test] + fn aggregate_budget_holds_when_headers_understate_sizes() { + let mut bytes = three_parts(3000); + let mut patched = 0; + for i in 0..bytes.len() - 4 { + let offset = match bytes[i..i + 4] { + [0x50, 0x4b, 0x03, 0x04] => 22, + [0x50, 0x4b, 0x01, 0x02] => 24, + _ => continue, + }; + bytes[i + offset..i + offset + 4].copy_from_slice(&1u32.to_le_bytes()); + patched += 1; + } + assert_eq!(patched, 6); + let mut archive = open_archive_with_limits(Cursor::new(bytes), 4096, 8192).unwrap(); + read_archive_file(&mut archive, "p1.xml").unwrap(); + read_archive_file(&mut archive, "p2.xml").unwrap(); + assert!(matches!( + read_archive_file(&mut archive, "p3.xml"), + Err(DocprimsError::ResourceLimit(_)) + )); + } + + #[test] + fn declared_size_over_remaining_budget_is_rejected_before_reading() { + // Content is 4 bytes, so only the declared size can trigger this. + let bytes = archive_with("part.xml", b"", Some(5000)); + let mut archive = open_archive_with_limits(Cursor::new(bytes), 8192, 4096).unwrap(); + match read_archive_file(&mut archive, "part.xml") { + Err(DocprimsError::ResourceLimit(msg)) => assert!(msg.contains("total"), "{msg}"), + other => panic!("expected aggregate ResourceLimit, got {other:?}"), + } + } + + #[test] + fn missing_parts_do_not_consume_budget() { + let mut archive = + open_archive_with_limits(Cursor::new(three_parts(3000)), 4096, 6000).unwrap(); + for _ in 0..10 { + assert!(read_archive_file(&mut archive, "absent.xml") + .unwrap() + .is_none()); + } + read_archive_file(&mut archive, "p1.xml").unwrap(); + read_archive_file(&mut archive, "p2.xml").unwrap(); + } #[test] fn sniff_xml_decl_encoding_handles_whitespace() { diff --git a/crates/docprims-ooxml/src/pptx.rs b/crates/docprims-ooxml/src/pptx.rs index 68718f3..d6939dc 100644 --- a/crates/docprims-ooxml/src/pptx.rs +++ b/crates/docprims-ooxml/src/pptx.rs @@ -1,6 +1,6 @@ //! PPTX (PowerPoint) text extraction. -use crate::common::{open_archive, read_archive_file, resolve_entity}; +use crate::common::{open_archive, read_archive_file, resolve_entity, OoxmlArchive}; use docprims_core::{ DocprimsBlock, DocprimsByteRange, DocprimsDocument, DocprimsError, DocprimsExtract, DocprimsGenerator, DocprimsQuality, DocprimsSource, ExtractLimits, ExtractedText, Result, @@ -156,7 +156,7 @@ pub fn extract_v0( )) } -fn enumerate_slides(archive: &mut zip::ZipArchive) -> Result> { +fn enumerate_slides(archive: &mut OoxmlArchive) -> Result> { let pres_xml = read_archive_file(archive, PRESENTATION_PATH)? .ok_or_else(|| DocprimsError::Malformed("Missing ppt/presentation.xml".to_string()))?; let rels_xml = read_archive_file(archive, PRESENTATION_RELS_PATH)?.ok_or_else(|| { diff --git a/crates/docprims-ooxml/src/xlsx.rs b/crates/docprims-ooxml/src/xlsx.rs index 23c8312..66fd464 100644 --- a/crates/docprims-ooxml/src/xlsx.rs +++ b/crates/docprims-ooxml/src/xlsx.rs @@ -1,6 +1,6 @@ //! XLSX (Excel) text extraction. -use crate::common::{open_archive, read_archive_file, resolve_entity}; +use crate::common::{open_archive, read_archive_file, resolve_entity, OoxmlArchive}; use docprims_core::{ DocprimsBlock, DocprimsByteRange, DocprimsDocument, DocprimsError, DocprimsExtract, DocprimsGenerator, DocprimsQuality, DocprimsSource, ExtractLimits, ExtractedText, Result, @@ -191,7 +191,7 @@ pub fn extract_v0( )) } -fn enumerate_sheets(archive: &mut zip::ZipArchive) -> Result> { +fn enumerate_sheets(archive: &mut OoxmlArchive) -> Result> { let workbook_xml = read_archive_file(archive, WORKBOOK_PATH)? .ok_or_else(|| DocprimsError::Malformed("Missing xl/workbook.xml".to_string()))?; let rels_xml = read_archive_file(archive, WORKBOOK_RELS_PATH)?.ok_or_else(|| { From efe2787faeff979f0790a48f8de8f42ce97011b9 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:33:17 -0400 Subject: [PATCH 03/33] test: add malformed-archive and cross-surface contract suites - docprims-ooxml: malformed, truncated and non-archive inputs across all three formats; entry-count limit; attribute-count scaling; deep nesting; out-of-range shared strings; entity decoding; non-ASCII text and block byte ranges; understated entry sizes - TypeScript: v0 goldens through the native addon; Uint8Array and offset views; error-code mapping; loader error without an addon - Go: v0 goldens through cgo; error-code mapping including zero-length input; per-thread error messages under concurrent calls Fixtures are generated in-test. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .../go/docprims/docprims_contract_test.go | 135 ++++++++++ .../typescript/docprims/test/contract.test.ts | 140 ++++++++++ .../docprims-ooxml/tests/hostile_archives.rs | 243 ++++++++++++++++++ 3 files changed, 518 insertions(+) create mode 100644 bindings/go/docprims/docprims_contract_test.go create mode 100644 bindings/typescript/docprims/test/contract.test.ts create mode 100644 crates/docprims-ooxml/tests/hostile_archives.rs diff --git a/bindings/go/docprims/docprims_contract_test.go b/bindings/go/docprims/docprims_contract_test.go new file mode 100644 index 0000000..00b9cd1 --- /dev/null +++ b/bindings/go/docprims/docprims_contract_test.go @@ -0,0 +1,135 @@ +package docprims + +import ( + "encoding/json" + "errors" + "fmt" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" +) + +// The Go surface must reproduce the committed v0 goldens exactly: the same +// contract the Rust CLI and TypeScript surfaces are held to. +func TestExtractBytesMatchesV0Goldens(t *testing.T) { + cases := []struct{ golden, fixture, uri string }{ + {"markdown-simple", "testdata/fixtures/text/simple.md", "testdata/fixtures/text/simple.md"}, + {"html-simple", "testdata/fixtures/text/simple.html", "testdata/fixtures/text/simple.html"}, + {"xml-simple", "testdata/fixtures/text/simple.xml", "testdata/fixtures/text/simple.xml"}, + {"docx-mini", "testdata/fixtures/ooxml/mini.docx.b64", "testdata/fixtures/ooxml/mini.docx"}, + {"xlsx-mini", "testdata/fixtures/ooxml/mini.xlsx.b64", "testdata/fixtures/ooxml/mini.xlsx"}, + {"pptx-mini", "testdata/fixtures/ooxml/mini.pptx.b64", "testdata/fixtures/ooxml/mini.pptx"}, + } + for _, c := range cases { + t.Run(c.golden, func(t *testing.T) { + var data []byte + if strings.HasSuffix(c.fixture, ".b64") { + data = readB64Fixture(c.fixture) + } else { + data = readFixture(c.fixture) + } + out, err := ExtractBytes(c.uri, data, nil) + if err != nil { + t.Fatalf("ExtractBytes: %v", err) + } + + var got, want map[string]any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("output is not JSON: %v", err) + } + wantBytes := readFixture(filepath.Join("testdata", "golden", "v0", c.golden+".json")) + if err := json.Unmarshal(wantBytes, &want); err != nil { + t.Fatal(err) + } + // Goldens pin the contract, not the library version. + got["generator"].(map[string]any)["version"] = want["generator"].(map[string]any)["version"] + if !reflect.DeepEqual(got, want) { + t.Fatalf("output differs from golden\n got: %s", out) + } + }) + } +} + +func errorCode(t *testing.T, err error) uint32 { + t.Helper() + var e *Error + if !errors.As(err, &e) { + t.Fatalf("expected *docprims.Error, got %T: %v", err, err) + } + return e.Code +} + +func TestErrorCodesAcrossFailureClasses(t *testing.T) { + docx := readB64Fixture("testdata/fixtures/ooxml/mini.docx.b64") + cases := []struct { + name string + uri string + data []byte + opts *Options + code uint32 + }{ + {"not an archive", "mem://x.docx", []byte("plain text"), nil, 60}, + // Zero-length input crosses the cgo boundary as a nil pointer. + {"empty input", "mem://x.docx", nil, nil, 60}, + {"truncated archive", "mem://x.pptx", docx[:len(docx)/2], nil, 60}, + {"unknown extension", "mem://x.bin", []byte("x"), nil, 60}, + {"missing extension", "mem://noext", []byte("x"), nil, 60}, + {"invalid utf-8 text", "mem://x.md", []byte{'#', ' ', 0xff, 0xfe}, nil, 60}, + {"input over limit", "mem://x.md", []byte("# heading"), &Options{Limits: &Limits{MaxInputBytes: 1}}, 70}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + out, err := ExtractBytes(c.uri, c.data, c.opts) + if err == nil { + t.Fatalf("expected error, got output %s", out) + } + if got := errorCode(t, err); got != c.code { + t.Fatalf("code = %d, want %d (%v)", got, c.code, err) + } + if err.Error() == "" { + t.Fatal("error carries no message") + } + }) + } +} + +// Error detail lives in thread-local FFI state. Concurrent callers must each +// see their own message, and a success must not surface a stale failure. +func TestConcurrentCallsKeepErrorsSeparate(t *testing.T) { + const workers = 32 + var wg sync.WaitGroup + failures := make(chan string, workers*20) + for w := 0; w < workers; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + for i := 0; i < 20; i++ { + ext := fmt.Sprintf("e%dx%d", w, i) + _, err := ExtractBytes("mem://file."+ext, []byte("x"), nil) + if err == nil || !strings.Contains(err.Error(), ext) { + failures <- fmt.Sprintf("worker %d iter %d: want message naming %q, got %v", w, i, ext, err) + } + out, err := ExtractBytes("mem://ok.md", []byte("# ok\n"), nil) + if err != nil || !strings.Contains(string(out), `"ok`) { + failures <- fmt.Sprintf("worker %d iter %d: success call failed: %v", w, i, err) + } + } + }(w) + } + wg.Wait() + close(failures) + for f := range failures { + t.Error(f) + } +} + +func TestVersionReporting(t *testing.T) { + if Version() == "" { + t.Fatal("Version() is empty") + } + if got := ABIVersion(); got != 1 { + t.Fatalf("ABIVersion() = %d, want 1", got) + } +} diff --git a/bindings/typescript/docprims/test/contract.test.ts b/bindings/typescript/docprims/test/contract.test.ts new file mode 100644 index 0000000..e5e8767 --- /dev/null +++ b/bindings/typescript/docprims/test/contract.test.ts @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { + abiVersion, + DocprimsError, + DocprimsErrorCode, + extractBytes, + extractFile, + version, +} from "../src/index"; +import { loadNativeBinding } from "../src/native"; + +function packageRoot(): string { + let current = __dirname; + for (let i = 0; i < 10; i++) { + if (fs.existsSync(path.join(current, "package.json"))) return current; + current = path.dirname(current); + } + throw new Error("package.json not found"); +} + +const repoRoot = path.resolve(packageRoot(), "..", "..", ".."); + +function fixtureBytes(rel: string): Buffer { + const p = path.join(repoRoot, rel); + if (!rel.endsWith(".b64")) return fs.readFileSync(p); + return Buffer.from(fs.readFileSync(p, "utf8").replace(/\s+/g, ""), "base64"); +} + +function expectCode(fn: () => unknown, code: DocprimsErrorCode): void { + assert.throws(fn, (e: unknown) => { + assert.ok(e instanceof DocprimsError, `not a DocprimsError: ${e}`); + assert.equal(e.code, code, `code ${e.code} (${e.message})`); + return true; + }); +} + +// The native addon must reproduce the committed v0 goldens exactly: the same +// contract the Rust CLI and FFI surfaces are held to. +const goldens: Array<[string, string, string]> = [ + ["markdown-simple", "testdata/fixtures/text/simple.md", "testdata/fixtures/text/simple.md"], + ["html-simple", "testdata/fixtures/text/simple.html", "testdata/fixtures/text/simple.html"], + ["xml-simple", "testdata/fixtures/text/simple.xml", "testdata/fixtures/text/simple.xml"], + ["docx-mini", "testdata/fixtures/ooxml/mini.docx.b64", "testdata/fixtures/ooxml/mini.docx"], + ["xlsx-mini", "testdata/fixtures/ooxml/mini.xlsx.b64", "testdata/fixtures/ooxml/mini.xlsx"], + ["pptx-mini", "testdata/fixtures/ooxml/mini.pptx.b64", "testdata/fixtures/ooxml/mini.pptx"], +]; + +for (const [name, fixture, uri] of goldens) { + test(`extractBytes() reproduces the ${name} v0 golden`, () => { + const expected = JSON.parse( + fs.readFileSync(path.join(repoRoot, "testdata", "golden", "v0", `${name}.json`), "utf8"), + ); + const got = extractBytes(uri, fixtureBytes(fixture)); + // Goldens pin the contract, not the library version. + got.generator.version = expected.generator.version; + assert.deepEqual(got, expected); + }); +} + +test("extractBytes() accepts a plain Uint8Array, not only Buffer", () => { + const u8 = new TextEncoder().encode("# Title\n\nbody text\n"); + assert.ok(!Buffer.isBuffer(u8)); + assert.ok(extractBytes("mem://x.md", u8).document.text.includes("body text")); +}); + +test("extractBytes() honours the byteOffset of a view into a larger buffer", () => { + const docx = fixtureBytes("testdata/fixtures/ooxml/mini.docx.b64"); + const whole = extractBytes("mem://x.docx", docx); + + // Surround the archive with junk so a binding that ignored the view's + // offset or length would see a corrupt archive. + const backing = Buffer.alloc(docx.length + 64, 0x5a); + docx.copy(backing, 32); + const view = new Uint8Array(backing.buffer, backing.byteOffset + 32, docx.length); + assert.deepEqual(extractBytes("mem://x.docx", view), whole); +}); + +test("malformed and unrecognised inputs map to DataInvalid", () => { + const docx = fixtureBytes("testdata/fixtures/ooxml/mini.docx.b64"); + expectCode(() => extractBytes("mem://x.docx", Buffer.from("not a zip")), DocprimsErrorCode.DataInvalid); + expectCode(() => extractBytes("mem://x.docx", new Uint8Array(0)), DocprimsErrorCode.DataInvalid); + expectCode( + () => extractBytes("mem://x.xlsx", docx.subarray(0, docx.length >> 1)), + DocprimsErrorCode.DataInvalid, + ); + expectCode(() => extractBytes("mem://x.bin", Buffer.from("x")), DocprimsErrorCode.DataInvalid); + expectCode(() => extractBytes("mem://noext", Buffer.from("x")), DocprimsErrorCode.DataInvalid); + expectCode( + () => extractBytes("mem://x.md", Buffer.from([0x23, 0x20, 0xff, 0xfe])), + DocprimsErrorCode.DataInvalid, + ); +}); + +test("invalid options are a usage error, not silently defaulted", () => { + expectCode( + () => + extractBytes("mem://x.md", Buffer.from("# x"), { + limits: { max_blocks: -1 }, + }), + DocprimsErrorCode.Usage, + ); +}); + +test("file-path errors map to Io and ResourceLimit", () => { + const missing = path.join(os.tmpdir(), `docprims-missing-${process.pid}.md`); + expectCode(() => extractFile(missing), DocprimsErrorCode.Io); + + const md = path.join(repoRoot, "testdata", "fixtures", "text", "simple.md"); + expectCode( + () => extractFile(md, { limits: { max_input_bytes: 1 } }), + DocprimsErrorCode.ResourceLimit, + ); +}); + +test("version() and abiVersion() describe the loaded addon", () => { + const pkg = JSON.parse(fs.readFileSync(path.join(packageRoot(), "package.json"), "utf8")); + assert.equal(version(), pkg.version); + assert.equal(abiVersion(), 1); +}); + +test("loader explains a missing native addon instead of crashing", (t) => { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), "docprims-noaddon-")); + let platformPkgInstalled = false; + try { + loadNativeBinding(empty); + platformPkgInstalled = true; + } catch (e) { + assert.ok(e instanceof Error); + assert.match(e.message, /Failed to load docprims native addon/); + assert.match(e.message, /@3leaps\/docprims-/); + } + if (platformPkgInstalled) { + t.skip("platform package installed; absent-addon path not reachable here"); + } +}); diff --git a/crates/docprims-ooxml/tests/hostile_archives.rs b/crates/docprims-ooxml/tests/hostile_archives.rs new file mode 100644 index 0000000..67cd947 --- /dev/null +++ b/crates/docprims-ooxml/tests/hostile_archives.rs @@ -0,0 +1,243 @@ +//! Behavior of the OOXML extractors on hostile and malformed archives. +//! +//! Every fixture is generated in-test from benign content. The properties +//! checked are the ones an embedding host relies on: malformed input yields +//! an error rather than a panic, resource use stays bounded, and text is +//! neither lost nor double-decoded. + +use std::io::{Cursor, Write}; +use std::time::{Duration, Instant}; + +use docprims_core::{DocprimsError, ExtractLimits}; +use docprims_ooxml::{ + extract_docx_reader, extract_docx_v0_reader, extract_pptx_reader, extract_xlsx_reader, +}; +use zip::write::SimpleFileOptions; + +const DECL: &str = r#""#; +const W_NS: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main""#; +const S_NS: &str = r#"xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main""#; +const R_NS: &str = + r#"xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships""#; + +fn zip_of(parts: &[(&str, &str)]) -> Vec { + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + let opts = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, body) in parts { + w.start_file(*name, opts).unwrap(); + w.write_all(body.as_bytes()).unwrap(); + } + w.finish().unwrap().into_inner() +} + +fn docx(body: &str) -> Vec { + let doc = format!("{DECL}{body}"); + zip_of(&[("word/document.xml", &doc)]) +} + +fn xlsx(sheet_data: &str, shared: &[&str]) -> Vec { + let workbook = format!( + r#"{DECL}"# + ); + let rels = format!( + r#"{DECL}"# + ); + let sst: String = shared + .iter() + .map(|s| format!("{s}")) + .collect(); + let sst = format!("{DECL}{sst}"); + let sheet = format!("{DECL}{sheet_data}"); + zip_of(&[ + ("xl/workbook.xml", &workbook), + ("xl/_rels/workbook.xml.rels", &rels), + ("xl/sharedStrings.xml", &sst), + ("xl/worksheets/sheet1.xml", &sheet), + ]) +} + +fn para(text: &str) -> String { + format!(r#"{text}"#) +} + +// --- archive-level failures: error, never panic -------------------------- + +#[test] +fn non_archive_inputs_are_malformed_for_every_format() { + let good = docx(¶("hello")); + let cases: [(&str, Vec); 3] = [ + ("empty", Vec::new()), + ("not a zip", b"plain text, no archive here".to_vec()), + ("truncated", good[..good.len() / 2].to_vec()), + ]; + for (label, bytes) in cases { + for (fmt, result) in [ + ( + "docx", + extract_docx_reader(Cursor::new(bytes.clone())).err(), + ), + ( + "xlsx", + extract_xlsx_reader(Cursor::new(bytes.clone())).err(), + ), + ( + "pptx", + extract_pptx_reader(Cursor::new(bytes.clone())).err(), + ), + ] { + assert!( + matches!(result, Some(DocprimsError::Malformed(_))), + "{fmt}/{label}: expected Malformed, got {result:?}" + ); + } + } +} + +#[test] +fn archive_without_main_part_is_an_error() { + let bytes = zip_of(&[("[Content_Types].xml", "")]); + assert!(extract_docx_reader(Cursor::new(bytes.clone())).is_err()); + assert!(extract_xlsx_reader(Cursor::new(bytes.clone())).is_err()); + assert!(extract_pptx_reader(Cursor::new(bytes)).is_err()); +} + +#[test] +fn archive_entry_count_is_limited() { + let names: Vec = (0..10_001).map(|i| format!("f{i}")).collect(); + let parts: Vec<(&str, &str)> = names.iter().map(|n| (n.as_str(), "")).collect(); + let err = extract_docx_reader(Cursor::new(zip_of(&parts))).unwrap_err(); + assert!( + matches!(err, DocprimsError::ResourceLimit(_)), + "got {err:?}" + ); +} + +// --- XML-level robustness ------------------------------------------------- + +/// Best-of-3 extraction time for one cell carrying `n` distinct attributes. +fn time_unique_attributes(n: usize) -> Duration { + let attrs: String = (0..n).map(|i| format!(r#" a{i}="v""#)).collect(); + let row = format!(r#"0"#); + let bytes = xlsx(&row, &["kept"]); + (0..3) + .map(|_| { + let start = Instant::now(); + let out = extract_xlsx_reader(Cursor::new(bytes.clone())).unwrap(); + let elapsed = start.elapsed(); + assert_eq!(out.content.trim(), "kept"); + elapsed + }) + .min() + .unwrap() +} + +#[test] +fn many_unique_attributes_on_one_element_scale_linearly() { + // Quadruple the attribute count: linear work grows ~4x, a quadratic + // duplicate-attribute check ~16x. A ratio is used rather than an absolute + // bound so host speed and build profile do not decide the outcome. + let small = time_unique_attributes(25_000); + let large = time_unique_attributes(100_000); + let ratio = large.as_secs_f64() / small.as_secs_f64().max(1e-3); + assert!( + ratio < 10.0, + "4x attributes took {ratio:.1}x as long ({small:?} -> {large:?})" + ); + // Hang guard only; the ratio above is the real check. + assert!(large < Duration::from_secs(60), "took {large:?}"); +} + +#[test] +fn deeply_nested_elements_do_not_exhaust_the_stack() { + let depth = 200_000; + let body = format!( + "{}{}{}", + "".repeat(depth), + para("deep"), + "".repeat(depth) + ); + let out = extract_docx_reader(Cursor::new(docx(&body))).unwrap(); + assert_eq!(out.content.trim(), "deep"); +} + +#[test] +fn unterminated_document_xml_does_not_panic() { + let doc = format!("{DECL}cut off"); + let bytes = zip_of(&[("word/document.xml", &doc)]); + // Either a parse error or best-effort partial text is acceptable; a panic + // across the FFI/napi boundary is not. + let _ = extract_docx_reader(Cursor::new(bytes)); +} + +#[test] +fn shared_string_index_out_of_range_does_not_panic() { + let row = r#"9918446744073709551616-1"#; + let out = extract_xlsx_reader(Cursor::new(xlsx(row, &["only"]))).unwrap(); + assert!( + !out.content.contains("only"), + "unrelated shared string leaked: {:?}", + out.content + ); +} + +// --- text fidelity -------------------------------------------------------- + +#[test] +fn escaped_markup_is_decoded_exactly_once() { + let out = extract_docx_reader(Cursor::new(docx(¶( + "a &lt;b&gt; <c> "d"", + )))) + .unwrap(); + assert_eq!(out.content.trim(), r#"a <b> "d""#); +} + +#[test] +fn non_ascii_text_survives_and_offsets_stay_on_char_boundaries() { + let text = "caf\u{e9} \u{65e5}\u{672c} \u{1f600} end"; + let bytes = docx(&(para(text) + ¶("second"))); + let out = + extract_docx_v0_reader(Cursor::new(bytes), "mem.docx", ExtractLimits::default()).unwrap(); + let doc_text = &out.document.text; + assert!(doc_text.contains(text), "got {doc_text:?}"); + + let value = serde_json::to_value(&out).unwrap(); + for block in value["document"]["blocks"].as_array().unwrap() { + let range = &block["doc_text_range"]; + let (start, end) = ( + range["start_byte"].as_u64().unwrap() as usize, + range["end_byte"].as_u64().unwrap() as usize, + ); + let slice = doc_text + .get(start..end) + .unwrap_or_else(|| panic!("range {start}..{end} not on char boundaries")); + assert_eq!(slice, block["text"].as_str().unwrap()); + } +} + +#[test] +fn text_split_across_runs_is_joined_in_order() { + let body = r#"Hello, world"#; + let out = extract_docx_reader(Cursor::new(docx(body))).unwrap(); + assert_eq!(out.content.trim(), "Hello, world"); +} + +#[test] +fn understated_entry_size_does_not_truncate_text() { + // The declared uncompressed size is advisory; a header that under-reports + // must not silently drop content. + let long = "x".repeat(50_000); + let mut bytes = docx(¶(&long)); + let mut patched = 0; + for i in 0..bytes.len() - 4 { + let off = match bytes[i..i + 4] { + [0x50, 0x4b, 0x03, 0x04] => 22, + [0x50, 0x4b, 0x01, 0x02] => 24, + _ => continue, + }; + bytes[i + off..i + off + 4].copy_from_slice(&64u32.to_le_bytes()); + patched += 1; + } + assert_eq!(patched, 2); + let out = extract_docx_reader(Cursor::new(bytes)).unwrap(); + assert_eq!(out.content.trim(), long); +} From 4cc27d6bb2835058f19f7507ab8b9e18eb81bf74 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:37:38 -0400 Subject: [PATCH 04/33] refactor!: remove unenforced extraction timeout option Remove `ExtractOptions::timeout_ms` and the CLI `--timeout-ms` flag, which were accepted but never applied. Update the architecture threat table to list the limits that are enforced. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- crates/docprims-cli/src/main.rs | 4 ---- crates/docprims-core/src/lib.rs | 3 --- docs/architecture/overview.md | 6 +++--- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/crates/docprims-cli/src/main.rs b/crates/docprims-cli/src/main.rs index 8808937..c39eb2c 100644 --- a/crates/docprims-cli/src/main.rs +++ b/crates/docprims-cli/src/main.rs @@ -91,10 +91,6 @@ struct ExtractArgs { /// Maximum input file size in bytes. #[arg(long, value_name = "BYTES", default_value = "104857600")] max_input_bytes: usize, - - /// Extraction timeout in milliseconds. - #[arg(long, value_name = "MS", default_value = "30000")] - timeout_ms: u64, } #[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/docprims-core/src/lib.rs b/crates/docprims-core/src/lib.rs index de72aa2..a7fa68a 100644 --- a/crates/docprims-core/src/lib.rs +++ b/crates/docprims-core/src/lib.rs @@ -175,9 +175,6 @@ pub struct ExtractOptions { /// Maximum output size in bytes (0 = unlimited) pub max_output_size: usize, - - /// Timeout for extraction in milliseconds (0 = unlimited) - pub timeout_ms: u64, } /// Resource limits for extraction. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index d05bd6b..913fc42 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -296,11 +296,11 @@ docprims is classified as **security-sensitive** because it parses untrusted inp | Threat | Mitigation | |--------|------------| -| Zip bombs | Decompression ratio limits, file count limits | -| XML bombs (billion laughs) | Entity expansion disabled, depth limits | +| Zip bombs | Per-part and per-archive decompressed-size limits (bytes actually read), file count limits | +| XML bombs (billion laughs) | No DTD or custom entity expansion; iterative (non-recursive) parsing | | Path traversal | Archive path validation, no `..` allowed | | Memory exhaustion | `max_input_bytes`, `max_output_bytes` limits | -| CPU exhaustion | `timeout_ms`, iteration limits | +| CPU exhaustion | Linear-time parsing; input, output, block and decompression limits. No wall-clock timeout. | | Malformed input crashes | `Result` everywhere, no panics on bad input | ### Resource Limits (ExtractLimits) From 6902d2a8c7f7a0a21f5362b7c4a85c94f34646d3 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:43:15 -0400 Subject: [PATCH 05/33] fix(ooxml): resolve numeric character references in DOCX, XLSX and PPTX Decimal and hexadecimal character references in OOXML text are now resolved instead of dropped, matching the XML text extractor. References that do not name a Unicode scalar value are still dropped. This changes extracted text for documents that contain numeric character references. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- crates/docprims-ooxml/src/common.rs | 22 +++++--- crates/docprims-ooxml/src/docx.rs | 4 +- crates/docprims-ooxml/src/pptx.rs | 4 +- crates/docprims-ooxml/src/xlsx.rs | 6 +-- .../docprims-ooxml/tests/hostile_archives.rs | 50 +++++++++++++++++++ 5 files changed, 72 insertions(+), 14 deletions(-) diff --git a/crates/docprims-ooxml/src/common.rs b/crates/docprims-ooxml/src/common.rs index 3033645..86e326d 100644 --- a/crates/docprims-ooxml/src/common.rs +++ b/crates/docprims-ooxml/src/common.rs @@ -440,16 +440,24 @@ mod tests { /// Resolve an XML entity reference to its string value. /// /// Handles predefined XML entities (lt, gt, amp, apos, quot) and numeric -/// character references (&#NNN; or &#xHHH;). -pub fn resolve_entity(name: &str) -> Option<&'static str> { - // Try predefined entities first +/// character references (&#NNN; or &#xHHH;). A numeric reference resolves +/// when it names a Unicode scalar value; surrogates, out-of-range values and +/// malformed references resolve to `None`, as do unknown named entities. +pub fn resolve_entity(name: &str) -> Option> { if let Some(resolved) = resolve_xml_entity(name) { - return Some(resolved); + return Some(Cow::Borrowed(resolved)); } + parse_numeric_entity(name).map(|c| Cow::Owned(c.to_string())) +} - // For numeric entities, we'd need to return an owned String, - // but for OOXML content this is rarely needed - None +fn parse_numeric_entity(entity: &str) -> Option { + let s = entity.strip_prefix('#')?; + let code = if let Some(hex) = s.strip_prefix('x').or_else(|| s.strip_prefix('X')) { + u32::from_str_radix(hex, 16).ok()? + } else { + s.parse::().ok()? + }; + char::from_u32(code) } /// OOXML namespace constants. diff --git a/crates/docprims-ooxml/src/docx.rs b/crates/docprims-ooxml/src/docx.rs index 9cd7311..fb69293 100644 --- a/crates/docprims-ooxml/src/docx.rs +++ b/crates/docprims-ooxml/src/docx.rs @@ -168,7 +168,7 @@ fn extract_text_from_document(xml: &str) -> Result { } Ok(Event::GeneralRef(e)) => { if let Some(resolved) = resolve_entity(&e) { - text.push_str(resolved); + text.push_str(&resolved); } } Ok(Event::Eof) => break, @@ -225,7 +225,7 @@ fn extract_paragraphs(xml: &str) -> Result> { Ok(Event::GeneralRef(e)) => { if in_p { if let Some(resolved) = resolve_entity(&e) { - cur.push_str(resolved); + cur.push_str(&resolved); } } } diff --git a/crates/docprims-ooxml/src/pptx.rs b/crates/docprims-ooxml/src/pptx.rs index d6939dc..9486ca9 100644 --- a/crates/docprims-ooxml/src/pptx.rs +++ b/crates/docprims-ooxml/src/pptx.rs @@ -306,7 +306,7 @@ fn extract_slide_text(xml: &str) -> Result { } Ok(Event::GeneralRef(e)) => { if let Some(resolved) = resolve_entity(&e) { - current_part.push_str(resolved); + current_part.push_str(&resolved); } } Ok(Event::Eof) => { @@ -361,7 +361,7 @@ fn extract_slide_paragraphs(xml: &str) -> Result> { } Ok(Event::GeneralRef(e)) => { if let Some(resolved) = resolve_entity(&e) { - current.push_str(resolved); + current.push_str(&resolved); } } Ok(Event::Eof) => { diff --git a/crates/docprims-ooxml/src/xlsx.rs b/crates/docprims-ooxml/src/xlsx.rs index 66fd464..9984426 100644 --- a/crates/docprims-ooxml/src/xlsx.rs +++ b/crates/docprims-ooxml/src/xlsx.rs @@ -356,7 +356,7 @@ fn parse_shared_strings(xml: &str) -> Result> { Ok(Event::GeneralRef(e)) => { if in_si { if let Some(resolved) = resolve_entity(&e) { - current_string.push_str(resolved); + current_string.push_str(&resolved); } } } @@ -443,7 +443,7 @@ fn extract_sheet_rows(xml: &str, shared_strings: &[String]) -> Result { if let Some(resolved) = resolve_entity(&e) { - cell_value.push_str(resolved); + cell_value.push_str(&resolved); } } Ok(Event::Eof) => break, @@ -514,7 +514,7 @@ fn extract_sheet_text(xml: &str, shared_strings: &[String]) -> Result { } Ok(Event::GeneralRef(e)) => { if let Some(resolved) = resolve_entity(&e) { - cell_value.push_str(resolved); + cell_value.push_str(&resolved); } } Ok(Event::Eof) => break, diff --git a/crates/docprims-ooxml/tests/hostile_archives.rs b/crates/docprims-ooxml/tests/hostile_archives.rs index 67cd947..681f9b0 100644 --- a/crates/docprims-ooxml/tests/hostile_archives.rs +++ b/crates/docprims-ooxml/tests/hostile_archives.rs @@ -241,3 +241,53 @@ fn understated_entry_size_does_not_truncate_text() { let out = extract_docx_reader(Cursor::new(bytes)).unwrap(); assert_eq!(out.content.trim(), long); } + +// --- numeric character references ------------------------------------------ + +fn pptx(slide_text: &str) -> Vec { + const P_NS: &str = r#"xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main""#; + let pres = format!( + r#"{DECL}"# + ); + let rels = format!( + r#"{DECL}"# + ); + let slide = format!( + r#"{DECL}{slide_text}"# + ); + zip_of(&[ + ("ppt/presentation.xml", &pres), + ("ppt/_rels/presentation.xml.rels", &rels), + ("ppt/slides/slide1.xml", &slide), + ]) +} + +const CHAR_REFS: &str = "café 日本 😀 &lt;"; +const CHAR_REFS_DECODED: &str = "caf\u{e9} \u{65e5}\u{672c} \u{1f600} <"; + +#[test] +fn numeric_character_references_resolve_in_every_format() { + let docx_out = extract_docx_reader(Cursor::new(docx(¶(CHAR_REFS)))).unwrap(); + assert_eq!(docx_out.content.trim(), CHAR_REFS_DECODED); + + let row = r#"0"#; + let xlsx_out = extract_xlsx_reader(Cursor::new(xlsx(row, &[CHAR_REFS]))).unwrap(); + assert_eq!(xlsx_out.content.trim(), CHAR_REFS_DECODED); + + let inline = + format!(r#"{CHAR_REFS}"#); + let inline_out = extract_xlsx_reader(Cursor::new(xlsx(&inline, &[]))).unwrap(); + assert_eq!(inline_out.content.trim(), CHAR_REFS_DECODED); + + let pptx_out = extract_pptx_reader(Cursor::new(pptx(CHAR_REFS))).unwrap(); + assert_eq!(pptx_out.content.trim(), CHAR_REFS_DECODED); +} + +#[test] +fn unresolvable_character_references_are_dropped() { + // Policy (shared with the text/xml extractor): a reference that does not + // name a Unicode scalar value is dropped; surrounding text is kept. + let refs = "a�b�c�d&#xZZ;e&#;f&unknown;g"; + let out = extract_docx_reader(Cursor::new(docx(¶(refs)))).unwrap(); + assert_eq!(out.content.trim(), "abcdefg"); +} From a531813f326cc880a4d1b3ea88f95d608d6141dc Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:44:13 -0400 Subject: [PATCH 06/33] docs(changelog): note OOXML numeric character reference resolution Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index faed71e..8064093 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **OOXML numeric character references**: DOCX, XLSX and PPTX extraction now resolves numeric character references (`é`, `😀`) instead of dropping them, matching the XML extractor. Extracted text changes for documents that contain such references; references that do not name a Unicode scalar value are still dropped. + ## [0.1.5] - 2026-07-07 ### Added From ec85fb960732d1b72616d1791ec5aec54bdccb37 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:47:33 -0400 Subject: [PATCH 07/33] fix(xml): resolve only XML 1.0 characters from character references XML and OOXML extraction now share one rule for numeric character references: a reference resolves only if it names an XML 1.0 Char (tab, line feed, carriage return, U+0020-U+D7FF, U+E000-U+FFFD, U+10000-U+10FFFF). Other references, including NUL and other control characters, are dropped. The rule lives in docprims_core::xml. This changes extracted text for documents that contain such references. Markdown and HTML handling is unchanged and pinned by tests. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- CHANGELOG.md | 2 +- Cargo.lock | 1 + crates/docprims-cli/Cargo.toml | 1 + crates/docprims-cli/tests/char_refs.rs | 118 +++++++++++++++++++++++++ crates/docprims-core/src/lib.rs | 1 + crates/docprims-core/src/xml.rs | 79 +++++++++++++++++ crates/docprims-ooxml/src/common.rs | 16 +--- crates/docprims-text/src/xml.rs | 14 +-- 8 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 crates/docprims-cli/tests/char_refs.rs create mode 100644 crates/docprims-core/src/xml.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8064093..d8ff8ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **OOXML numeric character references**: DOCX, XLSX and PPTX extraction now resolves numeric character references (`é`, `😀`) instead of dropping them, matching the XML extractor. Extracted text changes for documents that contain such references; references that do not name a Unicode scalar value are still dropped. +- **Numeric character references**: DOCX, XLSX, PPTX and XML extraction resolve a numeric character reference (`é`, `😀`) when it names a character allowed by XML 1.0 (tab, line feed, carriage return, U+0020–U+D7FF, U+E000–U+FFFD, U+10000–U+10FFFF); any other reference, including `�` and other control characters, is dropped. OOXML extraction previously dropped all numeric references and XML extraction previously kept control characters, so extracted text changes for documents that contain such references. ## [0.1.5] - 2026-07-07 diff --git a/Cargo.lock b/Cargo.lock index 5bf2730..812f042 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -360,6 +360,7 @@ dependencies = [ "serde_json", "tracing", "tracing-subscriber", + "zip", ] [[package]] diff --git a/crates/docprims-cli/Cargo.toml b/crates/docprims-cli/Cargo.toml index 87fc1f5..2b0d836 100644 --- a/crates/docprims-cli/Cargo.toml +++ b/crates/docprims-cli/Cargo.toml @@ -32,3 +32,4 @@ assert_cmd = "2.0" predicates = "3.1" base64 = "0.23" jsonschema = { version = "0.57", default-features = false } +zip = { workspace = true } diff --git a/crates/docprims-cli/tests/char_refs.rs b/crates/docprims-cli/tests/char_refs.rs new file mode 100644 index 0000000..b264455 --- /dev/null +++ b/crates/docprims-cli/tests/char_refs.rs @@ -0,0 +1,118 @@ +//! Character references resolve identically on every XML-based path. +//! +//! The XML text extractor and the OOXML extractors share one rule: a numeric +//! character reference resolves only if it names an XML 1.0 `Char`; anything +//! else is dropped. Markdown and HTML follow their own specs and are pinned +//! here so a change to either is deliberate. + +use std::io::{Cursor, Write}; + +use docprims_core::ExtractLimits; + +const W_NS: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main""#; + +fn via_xml(body: &str) -> String { + let xml = format!("{body}"); + docprims_text::xml::extract_v0_str(&xml, "mem.xml", ExtractLimits::default()) + .unwrap() + .document + .text +} + +fn via_docx(body: &str) -> String { + let doc = format!( + r#"{body}"# + ); + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + w.start_file( + "word/document.xml", + zip::write::SimpleFileOptions::default(), + ) + .unwrap(); + w.write_all(doc.as_bytes()).unwrap(); + let bytes = w.finish().unwrap().into_inner(); + docprims_ooxml::extract_docx_v0_reader(Cursor::new(bytes), "mem.docx", ExtractLimits::default()) + .unwrap() + .document + .text +} + +fn via_markdown(body: &str) -> String { + docprims_text::markdown::extract_v0_str(body, "mem.md", ExtractLimits::default()) + .unwrap() + .document + .text +} + +fn via_html(body: &str) -> String { + let html = format!("

{body}

"); + docprims_text::html::extract_v0_str(&html, "mem.html", ExtractLimits::default()) + .unwrap() + .document + .text +} + +/// (reference, text expected between `a` and `b` on the XML-based paths) +const CASES: &[(&str, &str)] = &[ + ("�", ""), + ("", ""), + ("", ""), + ("", ""), + (" ", "\t"), + (" ", "\n"), + (" ", "\r"), + (" ", " "), + ("퟿", "\u{d7ff}"), + ("�", ""), + ("", "\u{e000}"), + ("�", "\u{fffd}"), + ("￾", ""), + ("￿", ""), + ("𐀀", "\u{10000}"), + ("􏿿", "\u{10ffff}"), + ("�", ""), + ("é", "\u{e9}"), +]; + +#[test] +fn xml_and_ooxml_resolve_character_references_identically() { + for (reference, expected) in CASES { + let body = format!("a{reference}b"); + let want = format!("a{expected}b"); + assert_eq!(via_xml(&body), want, "xml path: {reference}"); + assert_eq!(via_docx(&body), want, "docx path: {reference}"); + } +} + +// Current behaviour, pinned. Literal (unescaped) control characters are not +// well-formed XML; the parser passes them through on both XML-based paths. +#[test] +fn literal_control_characters_pass_through_both_xml_paths() { + let body = "a\u{1}b\u{1b}c"; + assert_eq!(via_xml(body), body); + assert_eq!(via_docx(body), body); +} + +// Current behaviour, pinned. Markdown and HTML follow the HTML numeric +// character reference rules: NUL becomes U+FFFD, other controls and +// noncharacters are passed through. +#[test] +fn markdown_and_html_character_reference_handling_is_pinned() { + for (reference, expected) in [ + ("�", "\u{fffd}"), + ("", "\u{1}"), + ("", "\u{1b}"), + (" ", "\t"), + ("￾", "\u{fffe}"), + ("é", "\u{e9}"), + ] { + let body = format!("a{reference}b"); + let want = format!("a{expected}b"); + assert_eq!( + via_markdown(&body).trim_end(), + want, + "markdown: {reference}" + ); + assert_eq!(via_html(&body).trim_end(), want, "html: {reference}"); + } +} diff --git a/crates/docprims-core/src/lib.rs b/crates/docprims-core/src/lib.rs index a7fa68a..6e26d22 100644 --- a/crates/docprims-core/src/lib.rs +++ b/crates/docprims-core/src/lib.rs @@ -5,6 +5,7 @@ //! This crate provides the shared foundation used by format-specific extractors //! (docprims-text, docprims-ooxml, etc.). +pub mod xml; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; use std::path::Path; diff --git a/crates/docprims-core/src/xml.rs b/crates/docprims-core/src/xml.rs new file mode 100644 index 0000000..0e01561 --- /dev/null +++ b/crates/docprims-core/src/xml.rs @@ -0,0 +1,79 @@ +//! XML character rules shared by the extractors that parse XML. + +/// Whether `c` is an XML 1.0 `Char` (XML 1.0 §2.2): tab, line feed, carriage +/// return, and the ranges U+0020–U+D7FF, U+E000–U+FFFD and U+10000–U+10FFFF. +pub fn is_xml_char(c: char) -> bool { + matches!(c, + '\t' | '\n' | '\r' + | '\u{20}'..='\u{D7FF}' + | '\u{E000}'..='\u{FFFD}' + | '\u{10000}'..='\u{10FFFF}') +} + +/// Resolve the body of a numeric character reference (`#233` or `#xE9`, i.e. +/// the text between `&` and `;`). +/// +/// Returns `None` if the reference is malformed or does not name an XML 1.0 +/// `Char`; callers drop such references. +pub fn resolve_char_ref(reference: &str) -> Option { + let s = reference.strip_prefix('#')?; + let code = match s.strip_prefix('x').or_else(|| s.strip_prefix('X')) { + Some(hex) => u32::from_str_radix(hex, 16).ok()?, + None => s.parse::().ok()?, + }; + char::from_u32(code).filter(|&c| is_xml_char(c)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolves_decimal_and_hex_references_to_xml_chars() { + assert_eq!(resolve_char_ref("#233"), Some('\u{e9}')); + assert_eq!(resolve_char_ref("#xE9"), Some('\u{e9}')); + assert_eq!(resolve_char_ref("#XE9"), Some('\u{e9}')); + assert_eq!(resolve_char_ref("#x1F600"), Some('\u{1f600}')); + assert_eq!(resolve_char_ref("#9"), Some('\t')); + assert_eq!(resolve_char_ref("#xA"), Some('\n')); + assert_eq!(resolve_char_ref("#xD"), Some('\r')); + } + + #[test] + fn boundaries_of_the_xml_char_ranges() { + for (reference, allowed) in [ + ("#x1F", false), + ("#x20", true), + ("#xD7FF", true), + ("#xD800", false), + ("#xDFFF", false), + ("#xE000", true), + ("#xFFFD", true), + ("#xFFFE", false), + ("#xFFFF", false), + ("#x10000", true), + ("#x10FFFF", true), + ("#x110000", false), + ] { + assert_eq!( + resolve_char_ref(reference).is_some(), + allowed, + "{reference}" + ); + } + } + + #[test] + fn control_characters_other_than_tab_lf_cr_are_rejected() { + for code in (0u32..0x20).filter(|c| ![0x9, 0xA, 0xD].contains(c)) { + assert_eq!(resolve_char_ref(&format!("#{code}")), None, "#{code}"); + } + } + + #[test] + fn malformed_references_are_rejected() { + for reference in ["", "#", "#x", "#xZZ", "#-1", "#99999999999", "amp", "x41"] { + assert_eq!(resolve_char_ref(reference), None, "{reference:?}"); + } + } +} diff --git a/crates/docprims-ooxml/src/common.rs b/crates/docprims-ooxml/src/common.rs index 86e326d..a252bf4 100644 --- a/crates/docprims-ooxml/src/common.rs +++ b/crates/docprims-ooxml/src/common.rs @@ -441,23 +441,13 @@ mod tests { /// /// Handles predefined XML entities (lt, gt, amp, apos, quot) and numeric /// character references (&#NNN; or &#xHHH;). A numeric reference resolves -/// when it names a Unicode scalar value; surrogates, out-of-range values and -/// malformed references resolve to `None`, as do unknown named entities. +/// only when it names an XML 1.0 `Char` (see `docprims_core::xml`); other +/// numeric references and unknown named entities resolve to `None`. pub fn resolve_entity(name: &str) -> Option> { if let Some(resolved) = resolve_xml_entity(name) { return Some(Cow::Borrowed(resolved)); } - parse_numeric_entity(name).map(|c| Cow::Owned(c.to_string())) -} - -fn parse_numeric_entity(entity: &str) -> Option { - let s = entity.strip_prefix('#')?; - let code = if let Some(hex) = s.strip_prefix('x').or_else(|| s.strip_prefix('X')) { - u32::from_str_radix(hex, 16).ok()? - } else { - s.parse::().ok()? - }; - char::from_u32(code) + docprims_core::xml::resolve_char_ref(name).map(|c| Cow::Owned(c.to_string())) } /// OOXML namespace constants. diff --git a/crates/docprims-text/src/xml.rs b/crates/docprims-text/src/xml.rs index ee349aa..193b839 100644 --- a/crates/docprims-text/src/xml.rs +++ b/crates/docprims-text/src/xml.rs @@ -33,7 +33,7 @@ pub fn extract(content: &str) -> Result { current_part.push_str(resolved); } else if entity_name.starts_with('#') { // Numeric character reference (&#NNN; or &#xHHH;) - if let Some(ch) = parse_numeric_entity(entity_name) { + if let Some(ch) = docprims_core::xml::resolve_char_ref(entity_name) { current_part.push(ch); } } @@ -100,7 +100,7 @@ pub fn extract_v0_str( if let Some(resolved) = resolve_xml_entity(entity_name) { current_part.push_str(resolved); } else if entity_name.starts_with('#') { - if let Some(ch) = parse_numeric_entity(entity_name) { + if let Some(ch) = docprims_core::xml::resolve_char_ref(entity_name) { current_part.push(ch); } } @@ -223,16 +223,6 @@ pub fn extract_v0_str( } /// Parse a numeric character reference like "#65" or "#x41" to a char. -fn parse_numeric_entity(entity: &str) -> Option { - let s = entity.strip_prefix('#')?; - let code = if let Some(hex) = s.strip_prefix('x').or_else(|| s.strip_prefix('X')) { - u32::from_str_radix(hex, 16).ok()? - } else { - s.parse::().ok()? - }; - char::from_u32(code) -} - #[cfg(test)] mod tests { use super::*; From 06ce0104960d487bdf30976815f489261e90487f Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:53:26 -0400 Subject: [PATCH 08/33] fix: limit extracted text to XML 1.0 characters on every format Every extractor now drops characters outside the XML 1.0 Char set from its text before block byte ranges are computed, so literal control bytes and HTML or Markdown references to them no longer reach the output, and block ranges still slice the document text exactly. The legacy ExtractedText API applies the same filter. This changes extracted text for documents that contain such characters. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- CHANGELOG.md | 2 +- crates/docprims-cli/tests/char_refs.rs | 148 ++++++++++++++++++++++--- crates/docprims-core/src/lib.rs | 4 +- crates/docprims-core/src/xml.rs | 11 ++ crates/docprims-ooxml/src/docx.rs | 3 + crates/docprims-ooxml/src/pptx.rs | 3 + crates/docprims-ooxml/src/xlsx.rs | 3 +- crates/docprims-text/src/html.rs | 2 + crates/docprims-text/src/markdown.rs | 5 +- crates/docprims-text/src/xml.rs | 6 +- 10 files changed, 163 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8ff8ea..85a59ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Numeric character references**: DOCX, XLSX, PPTX and XML extraction resolve a numeric character reference (`é`, `😀`) when it names a character allowed by XML 1.0 (tab, line feed, carriage return, U+0020–U+D7FF, U+E000–U+FFFD, U+10000–U+10FFFF); any other reference, including `�` and other control characters, is dropped. OOXML extraction previously dropped all numeric references and XML extraction previously kept control characters, so extracted text changes for documents that contain such references. +- **XML 1.0 characters only**: Extracted text from every format (Markdown, HTML, XML, DOCX, XLSX, PPTX) now contains only characters allowed by XML 1.0 (tab, line feed, carriage return, U+0020–U+D7FF, U+E000–U+FFFD, U+10000–U+10FFFF). Numeric character references (`é`, `😀`) resolve when they name such a character, including in DOCX, XLSX and PPTX, which previously dropped all numeric references. Other characters, whether written literally or as references (for example NUL, ESC and other control characters, U+FFFE, U+FFFF), are dropped before block byte ranges are computed. Extracted text changes for documents that contain such characters. ## [0.1.5] - 2026-07-07 diff --git a/crates/docprims-cli/tests/char_refs.rs b/crates/docprims-cli/tests/char_refs.rs index b264455..7929838 100644 --- a/crates/docprims-cli/tests/char_refs.rs +++ b/crates/docprims-cli/tests/char_refs.rs @@ -1,9 +1,10 @@ -//! Character references resolve identically on every XML-based path. +//! Extracted text contains only XML 1.0 characters, on every format. //! -//! The XML text extractor and the OOXML extractors share one rule: a numeric -//! character reference resolves only if it names an XML 1.0 `Char`; anything -//! else is dropped. Markdown and HTML follow their own specs and are pinned -//! here so a change to either is deliberate. +//! Numeric character references resolve only if they name an XML 1.0 `Char` +//! (shared by the XML and OOXML extractors). Separately, every extractor drops +//! non-XML characters from its text before block byte ranges are computed, so +//! literal control bytes and HTML/Markdown references to them never reach the +//! output and provenance ranges still slice the text exactly. use std::io::{Cursor, Write}; @@ -84,26 +85,24 @@ fn xml_and_ooxml_resolve_character_references_identically() { } } -// Current behaviour, pinned. Literal (unescaped) control characters are not -// well-formed XML; the parser passes them through on both XML-based paths. #[test] -fn literal_control_characters_pass_through_both_xml_paths() { +fn literal_control_characters_are_dropped_on_xml_paths() { let body = "a\u{1}b\u{1b}c"; - assert_eq!(via_xml(body), body); - assert_eq!(via_docx(body), body); + assert_eq!(via_xml(body), "abc"); + assert_eq!(via_docx(body), "abc"); } -// Current behaviour, pinned. Markdown and HTML follow the HTML numeric -// character reference rules: NUL becomes U+FFFD, other controls and -// noncharacters are passed through. +// Markdown and HTML decode references per the HTML spec (NUL becomes U+FFFD, +// which is an XML character and is kept); the final-text filter then drops +// anything outside the XML character set. #[test] -fn markdown_and_html_character_reference_handling_is_pinned() { +fn markdown_and_html_output_contains_only_xml_characters() { for (reference, expected) in [ ("�", "\u{fffd}"), - ("", "\u{1}"), - ("", "\u{1b}"), + ("", ""), + ("", ""), (" ", "\t"), - ("￾", "\u{fffe}"), + ("￾", ""), ("é", "\u{e9}"), ] { let body = format!("a{reference}b"); @@ -115,4 +114,119 @@ fn markdown_and_html_character_reference_handling_is_pinned() { ); assert_eq!(via_html(&body).trim_end(), want, "html: {reference}"); } + assert_eq!(via_markdown("a\u{1b}[31mred\n").trim_end(), "a[31mred"); + assert_eq!(via_html("a\u{1b}[31mred").trim_end(), "a[31mred"); +} + +fn zip_parts(parts: &[(&str, &str)]) -> Vec { + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (name, body) in parts { + w.start_file(*name, zip::write::SimpleFileOptions::default()) + .unwrap(); + w.write_all(body.as_bytes()).unwrap(); + } + w.finish().unwrap().into_inner() +} + +/// Every block's byte range slices the document text to exactly the block's +/// text, and no non-XML character survives anywhere. +fn assert_clean_and_consistent(label: &str, extract: &docprims_core::DocprimsExtract) { + let doc = &extract.document.text; + assert!( + doc.chars().all(docprims_core::xml::is_xml_char), + "{label}: non-XML char in {doc:?}" + ); + assert!(!extract.document.blocks.is_empty(), "{label}: no blocks"); + for block in &extract.document.blocks { + let r = &block.doc_text_range; + let slice = doc + .get(r.start_byte..r.end_byte) + .unwrap_or_else(|| panic!("{label}: range {r:?} invalid for {doc:?}")); + assert_eq!(slice, block.text, "{label}: block {}", block.id); + assert!(!block.text.is_empty(), "{label}: empty block {}", block.id); + } +} + +#[test] +fn block_ranges_slice_filtered_text_on_every_format() { + let l = ExtractLimits::default; + // Each input has a clean block, a block with control bytes mid-text, and a + // block made only of control bytes (which must vanish, not leave a gap). + let md = "first\n\nmid\u{1}dle \u{1b}[0m\n\n\u{7}\u{8}\n\nlast\n"; + let e = docprims_text::markdown::extract_v0_str(md, "m.md", l()).unwrap(); + assert_clean_and_consistent("markdown", &e); + assert_eq!(e.document.text, "first\nmiddle [0m\nlast"); + + let html = "

first

mid\u{1}dle \u{1b}bold

\u{7}

last

"; + let e = docprims_text::html::extract_v0_str(html, "h.html", l()).unwrap(); + assert_clean_and_consistent("html", &e); + assert_eq!(e.document.text, "first\nmiddle bold\nlast"); + + let xml = "firstmid\u{1}dle\u{7}last"; + let e = docprims_text::xml::extract_v0_str(xml, "x.xml", l()).unwrap(); + assert_clean_and_consistent("xml", &e); + assert_eq!(e.document.text, "first middle last"); + + let p = |t: &str| format!(r#"{t}"#); + let docx = zip_parts(&[( + "word/document.xml", + &format!( + "{}{}{}{}", + p("first"), + p("mid\u{1}dle "), + p("\u{7}"), + p("last") + ), + )]); + let e = docprims_ooxml::extract_docx_v0_reader(Cursor::new(docx), "d.docx", l()).unwrap(); + assert_clean_and_consistent("docx", &e); + assert_eq!(e.document.text, "first\nmiddle\nlast"); + + const S: &str = r#"xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main""#; + const R: &str = + r#"xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships""#; + let xlsx = zip_parts(&[ + ( + "xl/workbook.xml", + &format!( + r#""# + ), + ), + ( + "xl/_rels/workbook.xml.rels", + r#""#, + ), + ( + "xl/sharedStrings.xml", + &format!("mid\u{1}dle\u{7}"), + ), + ( + "xl/worksheets/sheet1.xml", + &format!( + r#"first01last"# + ), + ), + ]); + let e = docprims_ooxml::extract_xlsx_v0_reader(Cursor::new(xlsx), "s.xlsx", l()).unwrap(); + assert_clean_and_consistent("xlsx", &e); + assert_eq!(e.document.text, "first\nmiddle\nlast"); + + const P: &str = r#"xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main""#; + let ap = |t: &str| format!("{t}"); + let pptx = zip_parts(&[ + ("ppt/presentation.xml", &format!(r#""#)), + ("ppt/_rels/presentation.xml.rels", r#""#), + ("ppt/slides/slide1.xml", &format!("{}{}{}{}", ap("first"), ap("mid\u{1}dle"), ap("\u{7}"), ap("last"))), + ]); + let e = docprims_ooxml::extract_pptx_v0_reader(Cursor::new(pptx), "p.pptx", l()).unwrap(); + assert_clean_and_consistent("pptx", &e); + assert_eq!(e.document.text, "first\nmiddle\nlast"); +} + +#[test] +fn legacy_extract_api_output_is_filtered_too() { + let out = docprims_text::xml::extract("a\u{1}b").unwrap(); + assert_eq!(out.content, "ab"); + let out = docprims_text::markdown::extract("a\u{1b}b\n").unwrap(); + assert!(!out.content.contains('\u{1b}'), "{:?}", out.content); } diff --git a/crates/docprims-core/src/lib.rs b/crates/docprims-core/src/lib.rs index 6e26d22..ec46dff 100644 --- a/crates/docprims-core/src/lib.rs +++ b/crates/docprims-core/src/lib.rs @@ -95,7 +95,7 @@ impl ExtractedText { /// Create a new ExtractedText with complete extraction. pub fn complete(content: String) -> Self { Self { - content, + content: xml::retain_xml_chars(content), metadata: None, quality: ExtractionQuality::Complete, } @@ -104,7 +104,7 @@ impl ExtractedText { /// Create a new ExtractedText with partial extraction. pub fn partial(content: String, reason: String) -> Self { Self { - content, + content: xml::retain_xml_chars(content), metadata: None, quality: ExtractionQuality::Partial { reason }, } diff --git a/crates/docprims-core/src/xml.rs b/crates/docprims-core/src/xml.rs index 0e01561..fb4f5e6 100644 --- a/crates/docprims-core/src/xml.rs +++ b/crates/docprims-core/src/xml.rs @@ -10,6 +10,17 @@ pub fn is_xml_char(c: char) -> bool { | '\u{10000}'..='\u{10FFFF}') } +/// Remove every character that is not an XML 1.0 `Char`. +/// +/// Applied to extracted text before block byte ranges are computed, so the +/// output of every extractor contains only XML 1.0 characters. +pub fn retain_xml_chars(text: String) -> String { + if text.chars().all(is_xml_char) { + return text; + } + text.chars().filter(|&c| is_xml_char(c)).collect() +} + /// Resolve the body of a numeric character reference (`#233` or `#xE9`, i.e. /// the text between `&` and `;`). /// diff --git a/crates/docprims-ooxml/src/docx.rs b/crates/docprims-ooxml/src/docx.rs index fb69293..a5db720 100644 --- a/crates/docprims-ooxml/src/docx.rs +++ b/crates/docprims-ooxml/src/docx.rs @@ -48,6 +48,9 @@ pub fn extract_v0( let mut block_index = 0usize; for (p_idx, p) in paragraphs.into_iter().enumerate() { + // Paragraphs arrive trimmed; filtering removes no whitespace, so + // re-trimming yields the same result as filtering before the trim. + let p = docprims_core::xml::retain_xml_chars(p).trim().to_string(); if p.is_empty() { continue; } diff --git a/crates/docprims-ooxml/src/pptx.rs b/crates/docprims-ooxml/src/pptx.rs index 9486ca9..e282226 100644 --- a/crates/docprims-ooxml/src/pptx.rs +++ b/crates/docprims-ooxml/src/pptx.rs @@ -66,6 +66,9 @@ pub fn extract_v0( let paras = extract_slide_paragraphs(&xml)?; for (p_idx, p) in paras.into_iter().enumerate() { + // Paragraphs arrive trimmed; filtering removes no whitespace, so + // re-trimming yields the same result as filtering before the trim. + let p = docprims_core::xml::retain_xml_chars(p).trim().to_string(); if p.is_empty() { continue; } diff --git a/crates/docprims-ooxml/src/xlsx.rs b/crates/docprims-ooxml/src/xlsx.rs index 9984426..551045b 100644 --- a/crates/docprims-ooxml/src/xlsx.rs +++ b/crates/docprims-ooxml/src/xlsx.rs @@ -97,7 +97,8 @@ pub fn extract_v0( }; let rows = extract_sheet_rows(&sheet_xml, &shared_strings)?; - for row in rows { + for mut row in rows { + row.text = docprims_core::xml::retain_xml_chars(row.text); if row.text.is_empty() { continue; } diff --git a/crates/docprims-text/src/html.rs b/crates/docprims-text/src/html.rs index dc1b63a..8536860 100644 --- a/crates/docprims-text/src/html.rs +++ b/crates/docprims-text/src/html.rs @@ -167,6 +167,8 @@ fn extract_block_text(el: &ElementRef<'_>) -> String { } if let Some(t) = node.value().as_text() { + let t = docprims_core::xml::retain_xml_chars(t.to_string()); + let t = t.as_str(); if is_pre { out.push_str(t); } else { diff --git a/crates/docprims-text/src/markdown.rs b/crates/docprims-text/src/markdown.rs index 2daa242..085e027 100644 --- a/crates/docprims-text/src/markdown.rs +++ b/crates/docprims-text/src/markdown.rs @@ -111,7 +111,10 @@ pub fn extract_v0_str( cur_kind: &mut Option, cur_text: &mut String| { if let Some(kind) = cur_kind.take() { - let t = cur_text.trim_end().to_string(); + // Filtering removes no whitespace, so re-trimming equals filtering first. + let t = docprims_core::xml::retain_xml_chars(cur_text.trim_end().to_string()) + .trim_end() + .to_string(); cur_text.clear(); if !t.is_empty() { raw_blocks.push((kind, t)); diff --git a/crates/docprims-text/src/xml.rs b/crates/docprims-text/src/xml.rs index 193b839..86c455d 100644 --- a/crates/docprims-text/src/xml.rs +++ b/crates/docprims-text/src/xml.rs @@ -110,7 +110,8 @@ pub fn extract_v0_str( current_part.push_str(cdata_content); } Ok(Event::Start(_) | Event::End(_)) => { - let trimmed = current_part.trim(); + let filtered = docprims_core::xml::retain_xml_chars(current_part.clone()); + let trimmed = filtered.trim(); if !trimmed.is_empty() { if raw_parts.len() < limits.max_blocks { raw_parts.push(trimmed.to_string()); @@ -122,7 +123,8 @@ pub fn extract_v0_str( current_part.clear(); } Ok(Event::Eof) => { - let trimmed = current_part.trim(); + let filtered = docprims_core::xml::retain_xml_chars(current_part.clone()); + let trimmed = filtered.trim(); if !trimmed.is_empty() { if raw_parts.len() < limits.max_blocks { raw_parts.push(trimmed.to_string()); From e940899743c9fb70d36d6ed19eb84ec34e2bc2b6 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 18:57:04 -0400 Subject: [PATCH 09/33] fix: drop DEL and C1 controls from extracted text Extracted text now also excludes U+007F-U+009F, which XML 1.0 allows but discourages. The exclusion is a separate predicate (docprims_core::xml::is_output_char), so is_xml_char still matches XML 1.0 exactly. HTML references in that range that map to Windows-1252 characters are kept as those characters. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- CHANGELOG.md | 2 +- crates/docprims-cli/tests/char_refs.rs | 61 ++++++++++++++++++++------ crates/docprims-core/src/lib.rs | 4 +- crates/docprims-core/src/xml.rs | 34 +++++++++++--- crates/docprims-ooxml/src/docx.rs | 4 +- crates/docprims-ooxml/src/pptx.rs | 4 +- crates/docprims-ooxml/src/xlsx.rs | 2 +- crates/docprims-text/src/html.rs | 2 +- crates/docprims-text/src/markdown.rs | 2 +- crates/docprims-text/src/xml.rs | 4 +- 10 files changed, 91 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85a59ee..934bee5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **XML 1.0 characters only**: Extracted text from every format (Markdown, HTML, XML, DOCX, XLSX, PPTX) now contains only characters allowed by XML 1.0 (tab, line feed, carriage return, U+0020–U+D7FF, U+E000–U+FFFD, U+10000–U+10FFFF). Numeric character references (`é`, `😀`) resolve when they name such a character, including in DOCX, XLSX and PPTX, which previously dropped all numeric references. Other characters, whether written literally or as references (for example NUL, ESC and other control characters, U+FFFE, U+FFFF), are dropped before block byte ranges are computed. Extracted text changes for documents that contain such characters. +- **Extracted text character set**: Extracted text from every format (Markdown, HTML, XML, DOCX, XLSX, PPTX) contains only XML 1.0 characters, excluding DEL and C1 controls (U+007F–U+009F); other characters are dropped before block byte ranges are computed. Markdown and HTML keep their parsers' U+FFFD replacement for NUL. Numeric character references (`é`, `😀`) resolve under the same rule, including in DOCX, XLSX and PPTX, which previously dropped all numeric references. Extracted text changes for documents that contain such characters or references. ## [0.1.5] - 2026-07-07 diff --git a/crates/docprims-cli/tests/char_refs.rs b/crates/docprims-cli/tests/char_refs.rs index 7929838..fdb2118 100644 --- a/crates/docprims-cli/tests/char_refs.rs +++ b/crates/docprims-cli/tests/char_refs.rs @@ -1,8 +1,9 @@ -//! Extracted text contains only XML 1.0 characters, on every format. +//! Extracted text contains only XML 1.0 characters and no DEL or C1 controls, +//! on every format. //! //! Numeric character references resolve only if they name an XML 1.0 `Char` //! (shared by the XML and OOXML extractors). Separately, every extractor drops -//! non-XML characters from its text before block byte ranges are computed, so +//! non-XML characters, DEL and C1 controls from its text before block byte ranges are computed, so //! literal control bytes and HTML/Markdown references to them never reach the //! output and provenance ranges still slice the text exactly. @@ -73,6 +74,11 @@ const CASES: &[(&str, &str)] = &[ ("􏿿", "\u{10ffff}"), ("�", ""), ("é", "\u{e9}"), + // XML Chars that are DEL or C1 controls: resolved, then dropped. + ("", ""), + ("…", ""), + ("›", ""), + (" ", "\u{a0}"), ]; #[test] @@ -96,7 +102,7 @@ fn literal_control_characters_are_dropped_on_xml_paths() { // which is an XML character and is kept); the final-text filter then drops // anything outside the XML character set. #[test] -fn markdown_and_html_output_contains_only_xml_characters() { +fn markdown_and_html_output_excludes_control_characters() { for (reference, expected) in [ ("�", "\u{fffd}"), ("", ""), @@ -104,6 +110,8 @@ fn markdown_and_html_output_contains_only_xml_characters() { (" ", "\t"), ("￾", ""), ("é", "\u{e9}"), + ("", ""), + ("", ""), ] { let body = format!("a{reference}b"); let want = format!("a{expected}b"); @@ -114,8 +122,31 @@ fn markdown_and_html_output_contains_only_xml_characters() { ); assert_eq!(via_html(&body).trim_end(), want, "html: {reference}"); } - assert_eq!(via_markdown("a\u{1b}[31mred\n").trim_end(), "a[31mred"); - assert_eq!(via_html("a\u{1b}[31mred").trim_end(), "a[31mred"); + // HTML maps references in U+0080-U+009F to Windows-1252 characters where + // one is defined ("€", "…", "›"); Markdown does not, so they are C1 + // controls there and are dropped. Either way no C1 control is emitted. + for (reference, html_expected) in [ + ("€", "\u{20ac}"), + ("…", "\u{2026}"), + ("›", "\u{203a}"), + ] { + let body = format!("a{reference}b"); + assert_eq!( + via_markdown(&body).trim_end(), + "ab", + "markdown: {reference}" + ); + assert_eq!( + via_html(&body).trim_end(), + format!("a{html_expected}b"), + "html: {reference}" + ); + } + assert_eq!( + via_markdown("a\u{1b}[31mred\u{9b}0m\n").trim_end(), + "a[31mred0m" + ); + assert_eq!(via_html("a\u{1b}[31mred\u{9b}0m").trim_end(), "a[31mred0m"); } fn zip_parts(parts: &[(&str, &str)]) -> Vec { @@ -133,8 +164,8 @@ fn zip_parts(parts: &[(&str, &str)]) -> Vec { fn assert_clean_and_consistent(label: &str, extract: &docprims_core::DocprimsExtract) { let doc = &extract.document.text; assert!( - doc.chars().all(docprims_core::xml::is_xml_char), - "{label}: non-XML char in {doc:?}" + doc.chars().all(docprims_core::xml::is_output_char), + "{label}: disallowed char in {doc:?}" ); assert!(!extract.document.blocks.is_empty(), "{label}: no blocks"); for block in &extract.document.blocks { @@ -152,17 +183,19 @@ fn block_ranges_slice_filtered_text_on_every_format() { let l = ExtractLimits::default; // Each input has a clean block, a block with control bytes mid-text, and a // block made only of control bytes (which must vanish, not leave a gap). - let md = "first\n\nmid\u{1}dle \u{1b}[0m\n\n\u{7}\u{8}\n\nlast\n"; + let md = "first\n\nmid\u{1}d\u{7f}l\u{85}\u{9b}e \u{1b}[0m\n\n\u{7}\u{8}\n\nlast\n"; let e = docprims_text::markdown::extract_v0_str(md, "m.md", l()).unwrap(); assert_clean_and_consistent("markdown", &e); assert_eq!(e.document.text, "first\nmiddle [0m\nlast"); - let html = "

first

mid\u{1}dle \u{1b}bold

\u{7}

last

"; + let html = + "

first

mid\u{1}d\u{7f}l\u{85}\u{9b}e \u{1b}bold

\u{7}

last

"; let e = docprims_text::html::extract_v0_str(html, "h.html", l()).unwrap(); assert_clean_and_consistent("html", &e); assert_eq!(e.document.text, "first\nmiddle bold\nlast"); - let xml = "firstmid\u{1}dle\u{7}last"; + let xml = + "firstmid\u{1}d\u{7f}l\u{85}\u{9b}e\u{7}last"; let e = docprims_text::xml::extract_v0_str(xml, "x.xml", l()).unwrap(); assert_clean_and_consistent("xml", &e); assert_eq!(e.document.text, "first middle last"); @@ -173,7 +206,7 @@ fn block_ranges_slice_filtered_text_on_every_format() { &format!( "{}{}{}{}", p("first"), - p("mid\u{1}dle "), + p("mid\u{1}d\u{7f}l\u{85}\u{9b}e "), p("\u{7}"), p("last") ), @@ -198,7 +231,9 @@ fn block_ranges_slice_filtered_text_on_every_format() { ), ( "xl/sharedStrings.xml", - &format!("mid\u{1}dle\u{7}"), + &format!( + "mid\u{1}d\u{7f}l\u{85}\u{9b}e\u{7}" + ), ), ( "xl/worksheets/sheet1.xml", @@ -216,7 +251,7 @@ fn block_ranges_slice_filtered_text_on_every_format() { let pptx = zip_parts(&[ ("ppt/presentation.xml", &format!(r#""#)), ("ppt/_rels/presentation.xml.rels", r#""#), - ("ppt/slides/slide1.xml", &format!("{}{}{}{}", ap("first"), ap("mid\u{1}dle"), ap("\u{7}"), ap("last"))), + ("ppt/slides/slide1.xml", &format!("{}{}{}{}", ap("first"), ap("mid\u{1}d\u{7f}l\u{85}\u{9b}e"), ap("\u{7}"), ap("last"))), ]); let e = docprims_ooxml::extract_pptx_v0_reader(Cursor::new(pptx), "p.pptx", l()).unwrap(); assert_clean_and_consistent("pptx", &e); diff --git a/crates/docprims-core/src/lib.rs b/crates/docprims-core/src/lib.rs index ec46dff..5ad6d29 100644 --- a/crates/docprims-core/src/lib.rs +++ b/crates/docprims-core/src/lib.rs @@ -95,7 +95,7 @@ impl ExtractedText { /// Create a new ExtractedText with complete extraction. pub fn complete(content: String) -> Self { Self { - content: xml::retain_xml_chars(content), + content: xml::retain_output_chars(content), metadata: None, quality: ExtractionQuality::Complete, } @@ -104,7 +104,7 @@ impl ExtractedText { /// Create a new ExtractedText with partial extraction. pub fn partial(content: String, reason: String) -> Self { Self { - content: xml::retain_xml_chars(content), + content: xml::retain_output_chars(content), metadata: None, quality: ExtractionQuality::Partial { reason }, } diff --git a/crates/docprims-core/src/xml.rs b/crates/docprims-core/src/xml.rs index fb4f5e6..427cf39 100644 --- a/crates/docprims-core/src/xml.rs +++ b/crates/docprims-core/src/xml.rs @@ -10,15 +10,24 @@ pub fn is_xml_char(c: char) -> bool { | '\u{10000}'..='\u{10FFFF}') } -/// Remove every character that is not an XML 1.0 `Char`. +/// Whether `c` may appear in extracted text: an XML 1.0 `Char` that is not +/// DEL or a C1 control (U+007F–U+009F). XML 1.0 allows those but discourages +/// them, and they act as terminal controls. +pub fn is_output_char(c: char) -> bool { + is_xml_char(c) && !matches!(c, '\u{7F}'..='\u{9F}') +} + +/// Remove every character for which [`is_output_char`] is false. /// /// Applied to extracted text before block byte ranges are computed, so the -/// output of every extractor contains only XML 1.0 characters. -pub fn retain_xml_chars(text: String) -> String { - if text.chars().all(is_xml_char) { +/// output of every extractor contains no C0 controls other than tab, line +/// feed and carriage return, no DEL, no C1 controls, and nothing outside the +/// XML 1.0 character set. +pub fn retain_output_chars(text: String) -> String { + if text.chars().all(is_output_char) { return text; } - text.chars().filter(|&c| is_xml_char(c)).collect() + text.chars().filter(|&c| is_output_char(c)).collect() } /// Resolve the body of a numeric character reference (`#233` or `#xE9`, i.e. @@ -81,6 +90,21 @@ mod tests { } } + #[test] + fn output_chars_exclude_del_and_c1_but_keep_neighbours() { + for c in ['\u{7F}', '\u{80}', '\u{85}', '\u{9B}', '\u{9F}'] { + assert!(is_xml_char(c), "{c:?} is an XML Char"); + assert!(!is_output_char(c), "{c:?}"); + } + for c in ['\u{7E}', '\u{A0}', '\t', '\n', '\r', 'é'] { + assert!(is_output_char(c), "{c:?}"); + } + assert_eq!( + retain_output_chars("a\u{7f}b\u{9b}31mc\u{1b}d".into()), + "ab31mcd" + ); + } + #[test] fn malformed_references_are_rejected() { for reference in ["", "#", "#x", "#xZZ", "#-1", "#99999999999", "amp", "x41"] { diff --git a/crates/docprims-ooxml/src/docx.rs b/crates/docprims-ooxml/src/docx.rs index a5db720..1a2355c 100644 --- a/crates/docprims-ooxml/src/docx.rs +++ b/crates/docprims-ooxml/src/docx.rs @@ -50,7 +50,9 @@ pub fn extract_v0( for (p_idx, p) in paragraphs.into_iter().enumerate() { // Paragraphs arrive trimmed; filtering removes no whitespace, so // re-trimming yields the same result as filtering before the trim. - let p = docprims_core::xml::retain_xml_chars(p).trim().to_string(); + let p = docprims_core::xml::retain_output_chars(p) + .trim() + .to_string(); if p.is_empty() { continue; } diff --git a/crates/docprims-ooxml/src/pptx.rs b/crates/docprims-ooxml/src/pptx.rs index e282226..2d84927 100644 --- a/crates/docprims-ooxml/src/pptx.rs +++ b/crates/docprims-ooxml/src/pptx.rs @@ -68,7 +68,9 @@ pub fn extract_v0( for (p_idx, p) in paras.into_iter().enumerate() { // Paragraphs arrive trimmed; filtering removes no whitespace, so // re-trimming yields the same result as filtering before the trim. - let p = docprims_core::xml::retain_xml_chars(p).trim().to_string(); + let p = docprims_core::xml::retain_output_chars(p) + .trim() + .to_string(); if p.is_empty() { continue; } diff --git a/crates/docprims-ooxml/src/xlsx.rs b/crates/docprims-ooxml/src/xlsx.rs index 551045b..7c5ff65 100644 --- a/crates/docprims-ooxml/src/xlsx.rs +++ b/crates/docprims-ooxml/src/xlsx.rs @@ -98,7 +98,7 @@ pub fn extract_v0( let rows = extract_sheet_rows(&sheet_xml, &shared_strings)?; for mut row in rows { - row.text = docprims_core::xml::retain_xml_chars(row.text); + row.text = docprims_core::xml::retain_output_chars(row.text); if row.text.is_empty() { continue; } diff --git a/crates/docprims-text/src/html.rs b/crates/docprims-text/src/html.rs index 8536860..49c9a78 100644 --- a/crates/docprims-text/src/html.rs +++ b/crates/docprims-text/src/html.rs @@ -167,7 +167,7 @@ fn extract_block_text(el: &ElementRef<'_>) -> String { } if let Some(t) = node.value().as_text() { - let t = docprims_core::xml::retain_xml_chars(t.to_string()); + let t = docprims_core::xml::retain_output_chars(t.to_string()); let t = t.as_str(); if is_pre { out.push_str(t); diff --git a/crates/docprims-text/src/markdown.rs b/crates/docprims-text/src/markdown.rs index 085e027..382d760 100644 --- a/crates/docprims-text/src/markdown.rs +++ b/crates/docprims-text/src/markdown.rs @@ -112,7 +112,7 @@ pub fn extract_v0_str( cur_text: &mut String| { if let Some(kind) = cur_kind.take() { // Filtering removes no whitespace, so re-trimming equals filtering first. - let t = docprims_core::xml::retain_xml_chars(cur_text.trim_end().to_string()) + let t = docprims_core::xml::retain_output_chars(cur_text.trim_end().to_string()) .trim_end() .to_string(); cur_text.clear(); diff --git a/crates/docprims-text/src/xml.rs b/crates/docprims-text/src/xml.rs index 86c455d..b093310 100644 --- a/crates/docprims-text/src/xml.rs +++ b/crates/docprims-text/src/xml.rs @@ -110,7 +110,7 @@ pub fn extract_v0_str( current_part.push_str(cdata_content); } Ok(Event::Start(_) | Event::End(_)) => { - let filtered = docprims_core::xml::retain_xml_chars(current_part.clone()); + let filtered = docprims_core::xml::retain_output_chars(current_part.clone()); let trimmed = filtered.trim(); if !trimmed.is_empty() { if raw_parts.len() < limits.max_blocks { @@ -123,7 +123,7 @@ pub fn extract_v0_str( current_part.clear(); } Ok(Event::Eof) => { - let filtered = docprims_core::xml::retain_xml_chars(current_part.clone()); + let filtered = docprims_core::xml::retain_output_chars(current_part.clone()); let trimmed = filtered.trim(); if !trimmed.is_empty() { if raw_parts.len() < limits.max_blocks { From fc5862cb97d3a144b2760873c635a34e2a06ff36 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 19:09:51 -0400 Subject: [PATCH 10/33] feat: add docprims facade crate Add a `docprims` crate as the entry point for the library. It re-exports the extract/v0 types and provides extraction by file or bytes with format dispatch by extension (`Format`), behind per-format features (`markdown`, `html`, `xml`, `docx`, `xlsx`, `pptx`, grouped as `text` and `ooxml`). The command-line tool moves into this crate behind a `cli` feature; the binary name and output are unchanged. `docprims-cli` is removed. Workspace targets build with all features, and `make lean-lib-check` verifies library builds pull no CLI dependencies and no dependencies of disabled formats. `docprims-ooxml` now builds with any subset of its format features. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- Cargo.lock | 2 +- Cargo.toml | 6 +- Makefile | 24 +- .../typescript/docprims/native/Cargo.toml | 4 +- crates/docprims-cli/Cargo.toml | 35 --- crates/docprims-core/src/xml.rs | 10 + crates/docprims-ooxml/src/lib.rs | 13 + .../docprims-ooxml/tests/hostile_archives.rs | 2 + crates/docprims-text/src/lib.rs | 4 + crates/docprims/Cargo.toml | 72 +++++ crates/docprims/src/lib.rs | 250 ++++++++++++++++++ crates/{docprims-cli => docprims}/src/main.rs | 40 +-- .../tests/char_refs.rs | 32 +-- .../tests/golden_v0.rs | 18 +- ffi/docprims-ffi/Cargo.toml | 4 +- scripts/check-lean-lib.sh | 58 ++++ 16 files changed, 459 insertions(+), 115 deletions(-) delete mode 100644 crates/docprims-cli/Cargo.toml create mode 100644 crates/docprims/Cargo.toml create mode 100644 crates/docprims/src/lib.rs rename crates/{docprims-cli => docprims}/src/main.rs (93%) rename crates/{docprims-cli => docprims}/tests/char_refs.rs (87%) rename crates/{docprims-cli => docprims}/tests/golden_v0.rs (86%) create mode 100755 scripts/check-lean-lib.sh diff --git a/Cargo.lock b/Cargo.lock index 812f042..17b6917 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -344,7 +344,7 @@ dependencies = [ ] [[package]] -name = "docprims-cli" +name = "docprims" version = "0.1.5" dependencies = [ "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 95291cd..1c98525 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = [ "crates/docprims-core", "crates/docprims-text", "crates/docprims-ooxml", - "crates/docprims-cli", + "crates/docprims", "ffi/docprims-ffi", "bindings/typescript/docprims/native", ] @@ -19,8 +19,8 @@ authors = ["3leaps Architecture Council"] [workspace.dependencies] docprims-core = { path = "crates/docprims-core" } -docprims-text = { path = "crates/docprims-text" } -docprims-ooxml = { path = "crates/docprims-ooxml" } +docprims-text = { path = "crates/docprims-text", default-features = false } +docprims-ooxml = { path = "crates/docprims-ooxml", default-features = false } # Fulmen Ecosystem rsfulmen = { version = "=0.2.0", default-features = false, features = ["foundry-core"] } diff --git a/Makefile b/Makefile index d00d294..4ef062a 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,8 @@ GONEAT = $(shell command -v goneat 2>/dev/null) CARGO = cargo # Build against the committed Cargo.lock; fail rather than re-resolve. CARGO_LOCKED = --locked +# Build every feature (including the `cli` binary) in workspace-wide targets. +CARGO_ALL = --all-features # Minimum supported Rust version (workspace rust-version); verified by `make msrv`. MSRV = 1.88.0 @@ -210,12 +212,12 @@ tools: ## Verify external tools are available # Quality Gates # ----------------------------------------------------------------------------- -check: fmt-check lint test deny audit ## Run all quality checks +check: fmt-check lint test lean-lib-check deny audit ## Run all quality checks @echo "[ok] All quality checks passed" test: ## Run test suite @echo "Running tests..." - $(CARGO) test --workspace $(CARGO_LOCKED) + $(CARGO) test --workspace $(CARGO_ALL) $(CARGO_LOCKED) @echo "[ok] Tests passed" fmt: ## Format code (goneat assess or cargo fmt) @@ -245,12 +247,16 @@ lint: ## Run linting (goneat assess + cargo clippy) goneat assess --categories lint; \ fi @# goneat v0.6.0 does not run the Rust lint pass; clippy runs directly as the gap-filler. - $(CARGO) clippy --workspace --all-targets $(CARGO_LOCKED) -- -D warnings + $(CARGO) clippy --workspace --all-targets $(CARGO_ALL) $(CARGO_LOCKED) -- -D warnings @echo "[ok] Linting passed" npm-publish-prereqs-check: ## Verify npm trusted publishing runtime guard @bash scripts/check-npm-trusted-publish-runtime.sh +lean-lib-check: ## Verify library builds of the docprims crate pull no CLI or unused format dependencies + @bash scripts/check-lean-lib.sh + @echo "[ok] Lean library dependency check passed" + deny: ## Run cargo-deny license and advisory checks @echo "Running cargo-deny..." @if command -v cargo-deny >/dev/null 2>&1; then \ @@ -286,8 +292,8 @@ miri: ## Run Miri to detect undefined behavior in unsafe code (requires nightly) msrv: ## Verify build and tests with the Minimum Supported Rust Version @echo "Checking MSRV ($(MSRV))..." @if rustup run $(MSRV) cargo --version >/dev/null 2>&1; then \ - CARGO_TARGET_DIR=target/msrv rustup run $(MSRV) cargo build --workspace $(CARGO_LOCKED) && \ - CARGO_TARGET_DIR=target/msrv rustup run $(MSRV) cargo test --workspace $(CARGO_LOCKED); \ + CARGO_TARGET_DIR=target/msrv rustup run $(MSRV) cargo build --workspace $(CARGO_ALL) $(CARGO_LOCKED) && \ + CARGO_TARGET_DIR=target/msrv rustup run $(MSRV) cargo test --workspace $(CARGO_ALL) $(CARGO_LOCKED); \ else \ echo "[!!] Rust $(MSRV) not installed. Install with:"; \ echo " rustup install $(MSRV)"; \ @@ -309,7 +315,7 @@ check-windows-msvc: ## Cross-check Windows MSVC target (type checking only) echo "[..] Installing x86_64-pc-windows-msvc target..."; \ rustup target add x86_64-pc-windows-msvc; \ fi - $(CARGO) check --target x86_64-pc-windows-msvc --workspace + $(CARGO) check --target x86_64-pc-windows-msvc --workspace $(CARGO_ALL) @echo "[ok] Windows MSVC check passed" check-windows-gnu: ## Cross-check Windows GNU target (type checking only) @@ -318,7 +324,7 @@ check-windows-gnu: ## Cross-check Windows GNU target (type checking only) echo "[..] Installing x86_64-pc-windows-gnu target..."; \ rustup target add x86_64-pc-windows-gnu; \ fi - $(CARGO) check --target x86_64-pc-windows-gnu --workspace + $(CARGO) check --target x86_64-pc-windows-gnu --workspace $(CARGO_ALL) @echo "[ok] Windows GNU check passed" deps-check: ## Check dependencies for cooling violations @@ -336,12 +342,12 @@ deps-check: ## Check dependencies for cooling violations build: ## Build all crates (debug) @echo "Building (debug)..." - $(CARGO) build --workspace $(CARGO_LOCKED) + $(CARGO) build --workspace $(CARGO_ALL) $(CARGO_LOCKED) @echo "[ok] Build complete" build-release: ## Build all crates (release) @echo "Building (release)..." - $(CARGO) build --workspace --release $(CARGO_LOCKED) + $(CARGO) build --workspace --release $(CARGO_ALL) $(CARGO_LOCKED) @echo "[ok] Release build complete" build-ffi: cbindgen ## Build FFI library with C header diff --git a/bindings/typescript/docprims/native/Cargo.toml b/bindings/typescript/docprims/native/Cargo.toml index 42d5206..957266a 100644 --- a/bindings/typescript/docprims/native/Cargo.toml +++ b/bindings/typescript/docprims/native/Cargo.toml @@ -18,8 +18,8 @@ serde = { workspace = true, features = ["derive"] } serde_json.workspace = true docprims-core.workspace = true -docprims-text.workspace = true -docprims-ooxml.workspace = true +docprims-text = { workspace = true, features = ["markdown", "html", "xml"] } +docprims-ooxml = { workspace = true, features = ["docx", "xlsx", "pptx"] } [build-dependencies] napi-build = "2.3" diff --git a/crates/docprims-cli/Cargo.toml b/crates/docprims-cli/Cargo.toml deleted file mode 100644 index 2b0d836..0000000 --- a/crates/docprims-cli/Cargo.toml +++ /dev/null @@ -1,35 +0,0 @@ -[package] -name = "docprims-cli" -description = "CLI for docprims document text extraction" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -authors.workspace = true - -[[bin]] -name = "docprims" -path = "src/main.rs" - -[dependencies] -docprims-core = { workspace = true } -docprims-text = { workspace = true } -docprims-ooxml = { workspace = true } -clap = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } - -# Fulmen ecosystem -rsfulmen = { workspace = true } - -# Observability -tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } - -[dev-dependencies] -assert_cmd = "2.0" -predicates = "3.1" -base64 = "0.23" -jsonschema = { version = "0.57", default-features = false } -zip = { workspace = true } diff --git a/crates/docprims-core/src/xml.rs b/crates/docprims-core/src/xml.rs index 427cf39..9d7b55a 100644 --- a/crates/docprims-core/src/xml.rs +++ b/crates/docprims-core/src/xml.rs @@ -105,6 +105,16 @@ mod tests { ); } + #[test] + fn legacy_extracted_text_is_filtered() { + assert_eq!( + crate::ExtractedText::complete("a\u{1}b\u{9b}c".into()).content, + "abc" + ); + let partial = crate::ExtractedText::partial("a\u{1b}b".into(), "limit".into()); + assert_eq!(partial.content, "ab"); + } + #[test] fn malformed_references_are_rejected() { for reference in ["", "#", "#x", "#xZZ", "#-1", "#99999999999", "amp", "x41"] { diff --git a/crates/docprims-ooxml/src/lib.rs b/crates/docprims-ooxml/src/lib.rs index 836018e..fc4b7f6 100644 --- a/crates/docprims-ooxml/src/lib.rs +++ b/crates/docprims-ooxml/src/lib.rs @@ -22,6 +22,10 @@ //! println!("{}", text.content); //! ``` +#![cfg_attr( + not(any(feature = "docx", feature = "xlsx", feature = "pptx")), + allow(unused_imports) +)] use docprims_core::{DocprimsError, DocprimsExtract, ExtractLimits, ExtractedText, Result}; use std::fs::File; use std::io::{BufReader, Read}; @@ -61,9 +65,14 @@ pub enum OoxmlFormat { pub fn extract(path: impl AsRef) -> Result { let path = path.as_ref(); match detect_format(path) { + #[cfg(feature = "docx")] Some(OoxmlFormat::Docx) => extract_docx(path), + #[cfg(feature = "xlsx")] Some(OoxmlFormat::Xlsx) => extract_xlsx(path), + #[cfg(feature = "pptx")] Some(OoxmlFormat::Pptx) => extract_pptx(path), + #[allow(unreachable_patterns)] + Some(format) => Err(DocprimsError::UnsupportedFormat(format!("{format:?}"))), None => Err(DocprimsError::UnknownFormat(path.display().to_string())), } } @@ -195,6 +204,10 @@ pub fn extract_pptx_v0_reader( } #[cfg(test)] +#[cfg_attr( + not(all(feature = "docx", feature = "xlsx", feature = "pptx")), + allow(dead_code) +)] pub(crate) mod test_support { use docprims_core::DocprimsExtract; use std::io::{Cursor, Write}; diff --git a/crates/docprims-ooxml/tests/hostile_archives.rs b/crates/docprims-ooxml/tests/hostile_archives.rs index 681f9b0..95425e7 100644 --- a/crates/docprims-ooxml/tests/hostile_archives.rs +++ b/crates/docprims-ooxml/tests/hostile_archives.rs @@ -5,6 +5,8 @@ //! an error rather than a panic, resource use stays bounded, and text is //! neither lost nor double-decoded. +#![cfg(all(feature = "docx", feature = "xlsx", feature = "pptx"))] + use std::io::{Cursor, Write}; use std::time::{Duration, Instant}; diff --git a/crates/docprims-text/src/lib.rs b/crates/docprims-text/src/lib.rs index be4a641..81b669f 100644 --- a/crates/docprims-text/src/lib.rs +++ b/crates/docprims-text/src/lib.rs @@ -13,6 +13,10 @@ //! - `html` - Enable HTML extraction (default) //! - `xml` - Enable XML extraction (default) +#![cfg_attr( + not(any(feature = "markdown", feature = "html", feature = "xml")), + allow(unused_imports, dead_code) +)] use docprims_core::{DocprimsExtract, ExtractLimits, ExtractedText, Result}; use std::fs::File; use std::io::Read; diff --git a/crates/docprims/Cargo.toml b/crates/docprims/Cargo.toml new file mode 100644 index 0000000..01cae7a --- /dev/null +++ b/crates/docprims/Cargo.toml @@ -0,0 +1,72 @@ +[package] +name = "docprims" +description = "GPL-free text extraction with provenance for Markdown, HTML, XML, DOCX, XLSX and PPTX" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[features] +default = ["text", "ooxml"] +# Format groups +text = ["markdown", "html", "xml"] +ooxml = ["docx", "xlsx", "pptx"] +# Individual formats +markdown = ["dep:docprims-text", "docprims-text/markdown"] +html = ["dep:docprims-text", "docprims-text/html"] +xml = ["dep:docprims-text", "docprims-text/xml"] +docx = ["dep:docprims-ooxml", "docprims-ooxml/docx"] +xlsx = ["dep:docprims-ooxml", "docprims-ooxml/xlsx"] +pptx = ["dep:docprims-ooxml", "docprims-ooxml/pptx"] +# The `docprims` command-line tool: `cargo install docprims --features cli` +cli = [ + "text", + "ooxml", + "dep:clap", + "dep:rsfulmen", + "dep:serde", + "dep:serde_json", + "dep:tracing", + "dep:tracing-subscriber", +] + +[dependencies] +docprims-core = { workspace = true } +docprims-text = { workspace = true, optional = true } +docprims-ooxml = { workspace = true, optional = true } + +# CLI only +clap = { workspace = true, optional = true } +rsfulmen = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } +tracing-subscriber = { workspace = true, optional = true, features = ["env-filter", "json"] } + +[dev-dependencies] +assert_cmd = "2.0" +predicates = "3.1" +base64 = "0.23" +jsonschema = { version = "0.57", default-features = false } +serde_json = { workspace = true } +zip = { workspace = true } + +# Integration tests exercise every format. +[[test]] +name = "golden_v0" +required-features = ["text", "ooxml"] + +[[test]] +name = "char_refs" +required-features = ["text", "ooxml"] + +[[bin]] +name = "docprims" +path = "src/main.rs" +required-features = ["cli"] + +[package.metadata.docs.rs] +features = ["text", "ooxml"] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/crates/docprims/src/lib.rs b/crates/docprims/src/lib.rs new file mode 100644 index 0000000..b7b2887 --- /dev/null +++ b/crates/docprims/src/lib.rs @@ -0,0 +1,250 @@ +//! GPL-free text extraction with provenance. +//! +//! `docprims` extracts text from Markdown, HTML, XML, DOCX, XLSX and PPTX +//! documents into the `extract/v0` structure: the document text, blocks with +//! byte ranges into that text, and a locator for where each block came from. +//! Input is treated as untrusted; [`ExtractLimits`] bounds input size, output +//! size and block count. +//! +//! ```no_run +//! let extract = docprims::extract_file("report.docx", docprims::ExtractLimits::default())?; +//! for block in &extract.document.blocks { +//! println!("{}: {}", block.kind, block.text); +//! } +//! # Ok::<(), docprims::DocprimsError>(()) +//! ``` +//! +//! # Features +//! +//! Each format is a feature; `default` enables all of them. +//! +//! | Feature | Formats | +//! |---------|---------| +//! | `text` | `markdown`, `html`, `xml` | +//! | `ooxml` | `docx`, `xlsx`, `pptx` | +//! | `cli` | all formats, plus the `docprims` binary | +//! +//! Extracting a format whose feature is disabled returns +//! [`DocprimsError::UnsupportedFormat`]. +//! +//! This crate is the supported entry point. The `docprims-core`, +//! `docprims-text` and `docprims-ooxml` crates it builds on carry no stability +//! promise beyond what is re-exported here. + +#![cfg_attr(docsrs, feature(doc_cfg))] + +use std::path::Path; + +pub use docprims_core::{ + DocprimsBlock, DocprimsByteRange, DocprimsContainer, DocprimsDocument, DocprimsError, + DocprimsExtract, DocprimsFormat, DocprimsGenerator, DocprimsLocation, DocprimsQuality, + DocprimsSource, ExtractLimits, Result, +}; + +/// A document format docprims can extract. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Format { + /// Markdown (`.md`, `.markdown`) + Markdown, + /// HTML (`.html`, `.htm`) + Html, + /// XML (`.xml`) + Xml, + /// Word document (`.docx`) + Docx, + /// Excel workbook (`.xlsx`) + Xlsx, + /// PowerPoint presentation (`.pptx`) + Pptx, +} + +impl Format { + /// Every format, whether or not its feature is enabled. + pub const ALL: [Format; 6] = [ + Format::Markdown, + Format::Html, + Format::Xml, + Format::Docx, + Format::Xlsx, + Format::Pptx, + ]; + + /// The format for a file extension, ignoring ASCII case. + pub fn from_extension(extension: &str) -> Option { + match extension.to_ascii_lowercase().as_str() { + "md" | "markdown" => Some(Format::Markdown), + "html" | "htm" => Some(Format::Html), + "xml" => Some(Format::Xml), + "docx" => Some(Format::Docx), + "xlsx" => Some(Format::Xlsx), + "pptx" => Some(Format::Pptx), + _ => None, + } + } + + /// The format for a path or URI, from its extension. + pub fn from_path(path: impl AsRef) -> Option { + path.as_ref() + .extension() + .and_then(|e| e.to_str()) + .and_then(Format::from_extension) + } + + /// The format's name as used in `extract/v0` output (`source.format.kind`). + pub fn name(self) -> &'static str { + match self { + Format::Markdown => "markdown", + Format::Html => "html", + Format::Xml => "xml", + Format::Docx => "docx", + Format::Xlsx => "xlsx", + Format::Pptx => "pptx", + } + } + + /// Whether this build of docprims can extract the format. + pub fn is_enabled(self) -> bool { + match self { + Format::Markdown => cfg!(feature = "markdown"), + Format::Html => cfg!(feature = "html"), + Format::Xml => cfg!(feature = "xml"), + Format::Docx => cfg!(feature = "docx"), + Format::Xlsx => cfg!(feature = "xlsx"), + Format::Pptx => cfg!(feature = "pptx"), + } + } +} + +fn format_for(path: &Path) -> Result { + Format::from_path(path).ok_or_else(|| { + let ext = path + .extension() + .and_then(|e| e.to_str()) + .map(str::to_ascii_lowercase) + .unwrap_or_default(); + DocprimsError::UnknownFormat(ext) + }) +} + +#[allow(dead_code)] // unused when every format feature is enabled +fn disabled(format: Format) -> DocprimsError { + DocprimsError::UnsupportedFormat(format!( + "{} (docprims built without the `{}` feature)", + format.name(), + format.name() + )) +} + +/// Extract a file, choosing the format from its extension. +pub fn extract_file(path: impl AsRef, limits: ExtractLimits) -> Result { + let path = path.as_ref(); + extract_file_as(format_for(path)?, path, limits) +} + +/// Extract a file as the given format. +#[cfg_attr( + not(any( + feature = "markdown", + feature = "html", + feature = "xml", + feature = "docx", + feature = "xlsx", + feature = "pptx" + )), + allow(unused_variables) +)] +pub fn extract_file_as( + format: Format, + path: impl AsRef, + limits: ExtractLimits, +) -> Result { + let path = path.as_ref(); + match format { + #[cfg(feature = "markdown")] + Format::Markdown => docprims_text::extract_markdown_v0(path, limits), + #[cfg(feature = "html")] + Format::Html => docprims_text::extract_html_v0(path, limits), + #[cfg(feature = "xml")] + Format::Xml => docprims_text::extract_xml_v0(path, limits), + #[cfg(feature = "docx")] + Format::Docx => docprims_ooxml::extract_docx_v0(path, limits), + #[cfg(feature = "xlsx")] + Format::Xlsx => docprims_ooxml::extract_xlsx_v0(path, limits), + #[cfg(feature = "pptx")] + Format::Pptx => docprims_ooxml::extract_pptx_v0(path, limits), + #[allow(unreachable_patterns)] + other => Err(disabled(other)), + } +} + +/// Extract in-memory document bytes, choosing the format from the extension +/// of `source_uri`. `source_uri` is recorded in the output as the source. +pub fn extract_bytes( + source_uri: &str, + data: &[u8], + limits: ExtractLimits, +) -> Result { + extract_bytes_as(format_for(Path::new(source_uri))?, source_uri, data, limits) +} + +/// Extract in-memory document bytes as the given format. +/// +/// Markdown, HTML and XML input must be UTF-8. +#[cfg_attr( + not(any( + feature = "markdown", + feature = "html", + feature = "xml", + feature = "docx", + feature = "xlsx", + feature = "pptx" + )), + allow(unused_variables) +)] +pub fn extract_bytes_as( + format: Format, + source_uri: &str, + data: &[u8], + limits: ExtractLimits, +) -> Result { + if data.len() > limits.max_input_bytes { + return Err(DocprimsError::ResourceLimit(format!( + "input exceeds max_input_bytes ({} > {})", + data.len(), + limits.max_input_bytes + ))); + } + match format { + #[cfg(feature = "markdown")] + Format::Markdown => { + docprims_text::markdown::extract_v0_str(utf8(data, format)?, source_uri, limits) + } + #[cfg(feature = "html")] + Format::Html => { + docprims_text::html::extract_v0_str(utf8(data, format)?, source_uri, limits) + } + #[cfg(feature = "xml")] + Format::Xml => docprims_text::xml::extract_v0_str(utf8(data, format)?, source_uri, limits), + #[cfg(feature = "docx")] + Format::Docx => { + docprims_ooxml::extract_docx_v0_reader(std::io::Cursor::new(data), source_uri, limits) + } + #[cfg(feature = "xlsx")] + Format::Xlsx => { + docprims_ooxml::extract_xlsx_v0_reader(std::io::Cursor::new(data), source_uri, limits) + } + #[cfg(feature = "pptx")] + Format::Pptx => { + docprims_ooxml::extract_pptx_v0_reader(std::io::Cursor::new(data), source_uri, limits) + } + #[allow(unreachable_patterns)] + other => Err(disabled(other)), + } +} + +#[cfg(any(feature = "markdown", feature = "html", feature = "xml"))] +fn utf8(data: &[u8], format: Format) -> Result<&str> { + std::str::from_utf8(data) + .map_err(|_| DocprimsError::Malformed(format!("non-utf8 {} input", format.name()))) +} diff --git a/crates/docprims-cli/src/main.rs b/crates/docprims/src/main.rs similarity index 93% rename from crates/docprims-cli/src/main.rs rename to crates/docprims/src/main.rs index c39eb2c..c503a20 100644 --- a/crates/docprims-cli/src/main.rs +++ b/crates/docprims/src/main.rs @@ -291,40 +291,16 @@ fn extract_file_v0(path: &PathBuf, args: &ExtractArgs) -> Result docprims_ooxml::extract_docx_v0(path, limits).map_err(|e| CliError::Extraction { - path: path.clone(), - source: e.to_string(), - }), - "xlsx" => docprims_ooxml::extract_xlsx_v0(path, limits).map_err(|e| CliError::Extraction { - path: path.clone(), - source: e.to_string(), - }), - "pptx" => docprims_ooxml::extract_pptx_v0(path, limits).map_err(|e| CliError::Extraction { - path: path.clone(), - source: e.to_string(), - }), - "md" | "markdown" => { - docprims_text::extract_markdown_v0(path, limits).map_err(|e| CliError::Extraction { - path: path.clone(), - source: e.to_string(), - }) - } - "html" | "htm" => { - docprims_text::extract_html_v0(path, limits).map_err(|e| CliError::Extraction { - path: path.clone(), - source: e.to_string(), - }) - } - "xml" => docprims_text::extract_xml_v0(path, limits).map_err(|e| CliError::Extraction { - path: path.clone(), - source: e.to_string(), - }), - _ => Err(CliError::UnsupportedFormat { + let Some(format) = docprims::Format::from_extension(&ext) else { + return Err(CliError::UnsupportedFormat { path: path.clone(), extension: ext, - }), - } + }); + }; + docprims::extract_file_as(format, path, limits).map_err(|e| CliError::Extraction { + path: path.clone(), + source: e.to_string(), + }) } /// Extract text from a single file. diff --git a/crates/docprims-cli/tests/char_refs.rs b/crates/docprims/tests/char_refs.rs similarity index 87% rename from crates/docprims-cli/tests/char_refs.rs rename to crates/docprims/tests/char_refs.rs index fdb2118..504cf12 100644 --- a/crates/docprims-cli/tests/char_refs.rs +++ b/crates/docprims/tests/char_refs.rs @@ -9,13 +9,13 @@ use std::io::{Cursor, Write}; -use docprims_core::ExtractLimits; +use docprims::ExtractLimits; const W_NS: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main""#; fn via_xml(body: &str) -> String { let xml = format!("{body}"); - docprims_text::xml::extract_v0_str(&xml, "mem.xml", ExtractLimits::default()) + docprims::extract_bytes("mem.xml", xml.as_bytes(), ExtractLimits::default()) .unwrap() .document .text @@ -33,14 +33,14 @@ fn via_docx(body: &str) -> String { .unwrap(); w.write_all(doc.as_bytes()).unwrap(); let bytes = w.finish().unwrap().into_inner(); - docprims_ooxml::extract_docx_v0_reader(Cursor::new(bytes), "mem.docx", ExtractLimits::default()) + docprims::extract_bytes("mem.docx", &bytes, ExtractLimits::default()) .unwrap() .document .text } fn via_markdown(body: &str) -> String { - docprims_text::markdown::extract_v0_str(body, "mem.md", ExtractLimits::default()) + docprims::extract_bytes("mem.md", body.as_bytes(), ExtractLimits::default()) .unwrap() .document .text @@ -48,7 +48,7 @@ fn via_markdown(body: &str) -> String { fn via_html(body: &str) -> String { let html = format!("

{body}

"); - docprims_text::html::extract_v0_str(&html, "mem.html", ExtractLimits::default()) + docprims::extract_bytes("mem.html", html.as_bytes(), ExtractLimits::default()) .unwrap() .document .text @@ -161,7 +161,7 @@ fn zip_parts(parts: &[(&str, &str)]) -> Vec { /// Every block's byte range slices the document text to exactly the block's /// text, and no non-XML character survives anywhere. -fn assert_clean_and_consistent(label: &str, extract: &docprims_core::DocprimsExtract) { +fn assert_clean_and_consistent(label: &str, extract: &docprims::DocprimsExtract) { let doc = &extract.document.text; assert!( doc.chars().all(docprims_core::xml::is_output_char), @@ -184,19 +184,19 @@ fn block_ranges_slice_filtered_text_on_every_format() { // Each input has a clean block, a block with control bytes mid-text, and a // block made only of control bytes (which must vanish, not leave a gap). let md = "first\n\nmid\u{1}d\u{7f}l\u{85}\u{9b}e \u{1b}[0m\n\n\u{7}\u{8}\n\nlast\n"; - let e = docprims_text::markdown::extract_v0_str(md, "m.md", l()).unwrap(); + let e = docprims::extract_bytes("m.md", md.as_bytes(), l()).unwrap(); assert_clean_and_consistent("markdown", &e); assert_eq!(e.document.text, "first\nmiddle [0m\nlast"); let html = "

first

mid\u{1}d\u{7f}l\u{85}\u{9b}e \u{1b}bold

\u{7}

last

"; - let e = docprims_text::html::extract_v0_str(html, "h.html", l()).unwrap(); + let e = docprims::extract_bytes("h.html", html.as_bytes(), l()).unwrap(); assert_clean_and_consistent("html", &e); assert_eq!(e.document.text, "first\nmiddle bold\nlast"); let xml = "firstmid\u{1}d\u{7f}l\u{85}\u{9b}e\u{7}last"; - let e = docprims_text::xml::extract_v0_str(xml, "x.xml", l()).unwrap(); + let e = docprims::extract_bytes("x.xml", xml.as_bytes(), l()).unwrap(); assert_clean_and_consistent("xml", &e); assert_eq!(e.document.text, "first middle last"); @@ -211,7 +211,7 @@ fn block_ranges_slice_filtered_text_on_every_format() { p("last") ), )]); - let e = docprims_ooxml::extract_docx_v0_reader(Cursor::new(docx), "d.docx", l()).unwrap(); + let e = docprims::extract_bytes("d.docx", &docx, l()).unwrap(); assert_clean_and_consistent("docx", &e); assert_eq!(e.document.text, "first\nmiddle\nlast"); @@ -242,7 +242,7 @@ fn block_ranges_slice_filtered_text_on_every_format() { ), ), ]); - let e = docprims_ooxml::extract_xlsx_v0_reader(Cursor::new(xlsx), "s.xlsx", l()).unwrap(); + let e = docprims::extract_bytes("s.xlsx", &xlsx, l()).unwrap(); assert_clean_and_consistent("xlsx", &e); assert_eq!(e.document.text, "first\nmiddle\nlast"); @@ -253,15 +253,7 @@ fn block_ranges_slice_filtered_text_on_every_format() { ("ppt/_rels/presentation.xml.rels", r#""#), ("ppt/slides/slide1.xml", &format!("{}{}{}{}", ap("first"), ap("mid\u{1}d\u{7f}l\u{85}\u{9b}e"), ap("\u{7}"), ap("last"))), ]); - let e = docprims_ooxml::extract_pptx_v0_reader(Cursor::new(pptx), "p.pptx", l()).unwrap(); + let e = docprims::extract_bytes("p.pptx", &pptx, l()).unwrap(); assert_clean_and_consistent("pptx", &e); assert_eq!(e.document.text, "first\nmiddle\nlast"); } - -#[test] -fn legacy_extract_api_output_is_filtered_too() { - let out = docprims_text::xml::extract("a\u{1}b").unwrap(); - assert_eq!(out.content, "ab"); - let out = docprims_text::markdown::extract("a\u{1b}b\n").unwrap(); - assert!(!out.content.contains('\u{1b}'), "{:?}", out.content); -} diff --git a/crates/docprims-cli/tests/golden_v0.rs b/crates/docprims/tests/golden_v0.rs similarity index 86% rename from crates/docprims-cli/tests/golden_v0.rs rename to crates/docprims/tests/golden_v0.rs index d07a154..6bd4a72 100644 --- a/crates/docprims-cli/tests/golden_v0.rs +++ b/crates/docprims/tests/golden_v0.rs @@ -1,7 +1,6 @@ use base64::engine::general_purpose::STANDARD; use base64::Engine; -use docprims_core::{DocprimsExtract, ExtractLimits}; -use std::io::Cursor; +use docprims::{DocprimsExtract, ExtractLimits}; use std::path::{Path, PathBuf}; fn limits() -> ExtractLimits { @@ -96,7 +95,7 @@ fn assert_golden(name: &str, got: &DocprimsExtract) { fn golden_markdown_simple_v0() { let src = "testdata/fixtures/text/simple.md"; let s = read_text_fixture(src); - let got = docprims_text::markdown::extract_v0_str(&s, src, limits()).unwrap(); + let got = docprims::extract_bytes(src, s.as_bytes(), limits()).unwrap(); assert_golden("markdown-simple", &got); } @@ -104,7 +103,7 @@ fn golden_markdown_simple_v0() { fn golden_html_simple_v0() { let src = "testdata/fixtures/text/simple.html"; let s = read_text_fixture(src); - let got = docprims_text::html::extract_v0_str(&s, src, limits()).unwrap(); + let got = docprims::extract_bytes(src, s.as_bytes(), limits()).unwrap(); assert_golden("html-simple", &got); } @@ -112,7 +111,7 @@ fn golden_html_simple_v0() { fn golden_xml_simple_v0() { let src = "testdata/fixtures/text/simple.xml"; let s = read_text_fixture(src); - let got = docprims_text::xml::extract_v0_str(&s, src, limits()).unwrap(); + let got = docprims::extract_bytes(src, s.as_bytes(), limits()).unwrap(); assert_golden("xml-simple", &got); } @@ -120,8 +119,7 @@ fn golden_xml_simple_v0() { fn golden_docx_mini_v0() { let src = "testdata/fixtures/ooxml/mini.docx"; let zip_bytes = read_b64_fixture("testdata/fixtures/ooxml/mini.docx.b64"); - let got = - docprims_ooxml::extract_docx_v0_reader(Cursor::new(zip_bytes), src, limits()).unwrap(); + let got = docprims::extract_bytes(src, &zip_bytes, limits()).unwrap(); assert_golden("docx-mini", &got); } @@ -129,8 +127,7 @@ fn golden_docx_mini_v0() { fn golden_xlsx_mini_v0() { let src = "testdata/fixtures/ooxml/mini.xlsx"; let zip_bytes = read_b64_fixture("testdata/fixtures/ooxml/mini.xlsx.b64"); - let got = - docprims_ooxml::extract_xlsx_v0_reader(Cursor::new(zip_bytes), src, limits()).unwrap(); + let got = docprims::extract_bytes(src, &zip_bytes, limits()).unwrap(); assert_golden("xlsx-mini", &got); } @@ -138,7 +135,6 @@ fn golden_xlsx_mini_v0() { fn golden_pptx_mini_v0() { let src = "testdata/fixtures/ooxml/mini.pptx"; let zip_bytes = read_b64_fixture("testdata/fixtures/ooxml/mini.pptx.b64"); - let got = - docprims_ooxml::extract_pptx_v0_reader(Cursor::new(zip_bytes), src, limits()).unwrap(); + let got = docprims::extract_bytes(src, &zip_bytes, limits()).unwrap(); assert_golden("pptx-mini", &got); } diff --git a/ffi/docprims-ffi/Cargo.toml b/ffi/docprims-ffi/Cargo.toml index 3e2e3f0..6ff9019 100644 --- a/ffi/docprims-ffi/Cargo.toml +++ b/ffi/docprims-ffi/Cargo.toml @@ -13,8 +13,8 @@ crate-type = ["staticlib", "cdylib", "rlib"] [dependencies] docprims-core = { workspace = true } -docprims-text = { workspace = true } -docprims-ooxml = { workspace = true } +docprims-text = { workspace = true, features = ["markdown", "html", "xml"] } +docprims-ooxml = { workspace = true, features = ["docx", "xlsx", "pptx"] } serde_json = { workspace = true } serde = { workspace = true } diff --git a/scripts/check-lean-lib.sh b/scripts/check-lean-lib.sh new file mode 100755 index 0000000..15d2e48 --- /dev/null +++ b/scripts/check-lean-lib.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Verify that library builds of the `docprims` crate pull only what their +# features need: no CLI dependencies without `cli`, and no dependencies of +# formats that are not enabled. +set -euo pipefail + +cd "$(dirname "$0")/.." + +fail=0 + +# check ").into_bytes(), + ' ', + ), + ("d.docx", docx, '\n'), + ("s.xlsx", xlsx, '\n'), + ("p.pptx", pptx, '\n'), + ] +} + +fn check(label: &str, out: &DocprimsExtract, limit: usize, full_len: usize, sep: char) { + let doc = &out.document; + assert!( + doc.text.len() <= limit, + "{label}: {} bytes > limit", + doc.text.len() + ); + + // The text is exactly the blocks joined by single separators. + let mut rebuilt = String::new(); + for (i, block) in doc.blocks.iter().enumerate() { + assert!(!block.text.is_empty(), "{label}: empty block {}", block.id); + if i > 0 { + rebuilt.push(sep); + } + let r = &block.doc_text_range; + assert_eq!( + r.start_byte, + rebuilt.len(), + "{label}: block {} start", + block.id + ); + rebuilt.push_str(&block.text); + assert_eq!(r.end_byte, rebuilt.len(), "{label}: block {} end", block.id); + } + assert_eq!(doc.text, rebuilt, "{label}: text is not the joined blocks"); + + let cut = doc.text.len() < full_len; + assert_eq!( + doc.quality.status == DocprimsQualityStatus::Partial, + cut, + "{label}: quality {:?} with {} of {full_len} bytes", + doc.quality, + doc.text.len() + ); +} + +#[test] +fn every_output_limit_yields_well_formed_output() { + for (uri, bytes, sep) in inputs() { + let full = docprims::extract_bytes(uri, &bytes, ExtractLimits::default()).unwrap(); + let full_len = full.document.text.len(); + assert_eq!(full.document.blocks.len(), 3, "{uri}: fixture blocks"); + for limit in 0..=full_len + 2 { + let limits = ExtractLimits { + max_output_bytes: limit, + ..ExtractLimits::default() + }; + let out = docprims::extract_bytes(uri, &bytes, limits).unwrap(); + check(&format!("{uri} limit={limit}"), &out, limit, full_len, sep); + } + } +} + +#[test] +fn zero_output_limit_yields_no_blocks() { + let limits = ExtractLimits { + max_output_bytes: 0, + ..ExtractLimits::default() + }; + for (uri, bytes, _) in inputs() { + let out = docprims::extract_bytes(uri, &bytes, limits).unwrap(); + assert!(out.document.blocks.is_empty(), "{uri}"); + assert_eq!(out.document.text, "", "{uri}"); + assert_eq!( + out.document.quality.status, + DocprimsQualityStatus::Partial, + "{uri}" + ); + } +} From 7d0114f982192ea1624235acb28e23778a62ec85 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Thu, 24 Sep 2026 19:56:31 -0400 Subject: [PATCH 14/33] fix: mark public error and schema enums non_exhaustive `DocprimsError`, `DocprimsQualityStatus` and `DocprimsContainerKind` can gain variants, so matches on them outside docprims-core need a wildcard arm. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- crates/docprims-core/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/docprims-core/src/lib.rs b/crates/docprims-core/src/lib.rs index 3f45ab7..adc1259 100644 --- a/crates/docprims-core/src/lib.rs +++ b/crates/docprims-core/src/lib.rs @@ -49,6 +49,7 @@ pub fn truncate_to_utf8_boundary(s: &str, max_bytes: usize) -> &str { /// Errors that can occur during document extraction. #[derive(Error, Debug)] +#[non_exhaustive] pub enum DocprimsError { /// I/O error reading the document #[error("I/O error: {0}")] @@ -283,6 +284,7 @@ impl DocprimsQuality { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] +#[non_exhaustive] pub enum DocprimsQualityStatus { Complete, Partial, @@ -369,6 +371,7 @@ pub struct DocprimsContainer { #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] +#[non_exhaustive] pub enum DocprimsContainerKind { File, Archive, From dbc90b2c55efef4b44ee30290ec85a678d2e8d45 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:04:54 -0400 Subject: [PATCH 15/33] build(ts): update TypeScript to 7.0.2 The TypeScript compiler binary is an optional dependency of the typescript package, so workflows install Node dependencies with `npm install` instead of `npm install --omit=optional`. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .github/workflows/typescript-bindings.yml | 4 +- .../workflows/typescript-napi-prebuilds.yml | 10 +- .github/workflows/typescript-npm-publish.yml | 11 +- .github/workflows/validate-release.yml | 4 +- .../typescript/docprims/package-lock.json | 389 +++++++++++++++++- bindings/typescript/docprims/package.json | 2 +- 6 files changed, 398 insertions(+), 22 deletions(-) diff --git a/.github/workflows/typescript-bindings.yml b/.github/workflows/typescript-bindings.yml index 34366d9..eb202dd 100644 --- a/.github/workflows/typescript-bindings.yml +++ b/.github/workflows/typescript-bindings.yml @@ -20,7 +20,7 @@ jobs: - name: Install deps and run tests run: | cd bindings/typescript/docprims - npm install --omit=optional + npm install npm run test:ci test-typescript-alpine: name: Test TypeScript Bindings (Alpine/musl) @@ -38,6 +38,6 @@ jobs: apk add --no-cache bash curl build-base python3 ca-certificates curl -fsSL https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.94.1 . "$HOME/.cargo/env" - npm install --omit=optional + npm install npm run test:ci ' diff --git a/.github/workflows/typescript-napi-prebuilds.yml b/.github/workflows/typescript-napi-prebuilds.yml index e3183a8..7dfc5bb 100644 --- a/.github/workflows/typescript-napi-prebuilds.yml +++ b/.github/workflows/typescript-napi-prebuilds.yml @@ -31,7 +31,7 @@ jobs: run: cargo install cargo-zigbuild --locked --version 0.23.4 - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install --omit=optional + run: npm install - name: Build .node files (zig) working-directory: bindings/typescript/docprims shell: bash @@ -72,7 +72,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install --omit=optional + run: npm install - name: Build .node file working-directory: bindings/typescript/docprims shell: bash @@ -100,7 +100,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install --omit=optional + run: npm install - name: Build .node file working-directory: bindings/typescript/docprims shell: bash @@ -128,7 +128,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install --omit=optional + run: npm install - name: Build .node file working-directory: bindings/typescript/docprims shell: pwsh @@ -157,7 +157,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install --omit=optional + run: npm install - name: Build JS dist (root package) working-directory: bindings/typescript/docprims run: npm run clean && npm run build && npm run build:native diff --git a/.github/workflows/typescript-npm-publish.yml b/.github/workflows/typescript-npm-publish.yml index 8ef0ee7..38c1ec8 100644 --- a/.github/workflows/typescript-npm-publish.yml +++ b/.github/workflows/typescript-npm-publish.yml @@ -234,11 +234,12 @@ jobs: working-directory: bindings/typescript/docprims run: | set -euo pipefail - # Use npm install instead of npm ci because: - # 1. Optional dependencies (platform packages) are being published in this workflow - # 2. They don't exist yet when we build the root package - # 3. We only need devDependencies for TypeScript compilation - npm install --omit=optional + # Use npm install instead of npm ci: the platform packages listed in + # optionalDependencies are published by this workflow, so they are not + # in the lock file and do not exist yet. npm install skips them. Do not + # pass --omit=optional: the TypeScript compiler binary is an optional + # dependency of typescript. + npm install npm run build - name: Publish (dry run) if: ${{ inputs.dry_run }} diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index bb01c8b..2c44632 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -50,7 +50,7 @@ jobs: - name: Build and test TypeScript bindings run: | cd bindings/typescript/docprims - npm install --omit=optional + npm install npm run test:ci validate-typescript-musl: name: Validate TypeScript Bindings (Alpine/musl) @@ -80,5 +80,5 @@ jobs: run: | set -euo pipefail . "$HOME/.cargo/env" - npm install --omit=optional + npm install npm run test:ci diff --git a/bindings/typescript/docprims/package-lock.json b/bindings/typescript/docprims/package-lock.json index 5021dc1..8904b41 100644 --- a/bindings/typescript/docprims/package-lock.json +++ b/bindings/typescript/docprims/package-lock.json @@ -11,7 +11,7 @@ "devDependencies": { "@napi-rs/cli": "^3.10.5", "@types/node": "^22.20.4", - "typescript": "^6.0.3" + "typescript": "^7.0.2" }, "engines": { "node": ">=22" @@ -454,6 +454,20 @@ } } }, + "node_modules/@napi-rs/cli/node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/@napi-rs/cross-toolchain": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@napi-rs/cross-toolchain/-/cross-toolchain-1.0.3.tgz", @@ -1750,6 +1764,346 @@ "undici-types": "~6.21.0" } }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -2006,17 +2360,38 @@ ] }, "node_modules/typescript": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", - "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", "dev": true, "license": "Apache-2.0", "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" + "tsc": "bin/tsc" }, "engines": { - "node": ">=14.17" + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" } }, "node_modules/undici-types": { diff --git a/bindings/typescript/docprims/package.json b/bindings/typescript/docprims/package.json index 1ecf65e..02edef0 100644 --- a/bindings/typescript/docprims/package.json +++ b/bindings/typescript/docprims/package.json @@ -51,6 +51,6 @@ "devDependencies": { "@napi-rs/cli": "^3.10.5", "@types/node": "^22.20.4", - "typescript": "^6.0.3" + "typescript": "^7.0.2" } } From 588d9c14ef59609da83e79888fab20f03e95f3d2 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:04:54 -0400 Subject: [PATCH 16/33] docs(changelog): note decompression and output limit fixes Also record the non_exhaustive public enums. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4447e7..c685110 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **BREAKING: `DocprimsError`, `DocprimsQualityStatus` and `DocprimsContainerKind` are `#[non_exhaustive]`**: Matches on them outside docprims need a wildcard arm. `DocprimsQualityStatus` implements `PartialEq` and `Eq`. - **CLI**: The `docprims` command-line tool is now built from the `docprims` crate with the `cli` feature (`cargo install docprims --features cli`). The binary name, commands and output are unchanged. - **C, Go and TypeScript bindings** extract through the `docprims` crate; results and error codes are unchanged. @@ -29,6 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Extracted text character set**: Extracted text from every format (Markdown, HTML, XML, DOCX, XLSX, PPTX) contains only XML 1.0 characters, excluding DEL and C1 controls (U+007F–U+009F); other characters are dropped before block byte ranges are computed. Markdown and HTML keep their parsers' U+FFFD replacement for NUL. Numeric character references (`é`, `😀`) resolve under the same rule, including in DOCX, XLSX and PPTX, which previously dropped all numeric references. Extracted text changes for documents that contain such characters or references. +- **OOXML decompression limits**: The decompressed size of DOCX, XLSX and PPTX parts is now bounded by the bytes actually read, not the size declared in the archive: 100 MiB per part and 400 MiB across all parts of one archive. Exceeding either limit is a `ResourceLimit` error. +- **Output limits**: When `max_output_bytes` truncates extraction, the result no longer contains an empty block or a trailing separator. ## [0.1.5] - 2026-07-07 From 667c93f22fc286bb72ccd06149d1cc67d0fe0b15 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:53:32 -0400 Subject: [PATCH 17/33] ci(napi): write prebuilds to the prebuilds directory napi-rs 3 writes .node files next to the addon's Cargo.toml unless --output-dir is given. Build straight into prebuilds/ and fail if the expected file is missing. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .../workflows/typescript-napi-prebuilds.yml | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/typescript-napi-prebuilds.yml b/.github/workflows/typescript-napi-prebuilds.yml index 7dfc5bb..9b23488 100644 --- a/.github/workflows/typescript-napi-prebuilds.yml +++ b/.github/workflows/typescript-napi-prebuilds.yml @@ -37,23 +37,23 @@ jobs: shell: bash run: | set -euo pipefail - rm -f docprims.*.node + rm -rf prebuilds mkdir -p prebuilds # glibc baseline (2.17) for broad compatibility - npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip \ + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds \ --target x86_64-unknown-linux-gnu --use-napi-cross - mv docprims.linux-x64-gnu.node prebuilds/ + test -f prebuilds/docprims.linux-x64-gnu.node - npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip \ + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds \ --target x86_64-unknown-linux-musl --cross-compile - mv docprims.linux-x64-musl.node prebuilds/ + test -f prebuilds/docprims.linux-x64-musl.node # arm64-gnu built natively on ubuntu-latest-arm64-s (build-linux-arm64 job) - npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip \ + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds \ --target aarch64-unknown-linux-musl --cross-compile - mv docprims.linux-arm64-musl.node prebuilds/ + test -f prebuilds/docprims.linux-arm64-musl.node - name: Upload prebuilds uses: actions/upload-artifact@v7 with: @@ -78,10 +78,10 @@ jobs: shell: bash run: | set -euo pipefail - rm -f docprims.*.node + rm -rf prebuilds mkdir -p prebuilds - npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip - mv docprims.linux-arm64-gnu.node prebuilds/ + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds + test -f prebuilds/docprims.linux-arm64-gnu.node - name: Upload prebuild uses: actions/upload-artifact@v7 with: @@ -106,10 +106,10 @@ jobs: shell: bash run: | set -euo pipefail - rm -f docprims.*.node + rm -rf prebuilds mkdir -p prebuilds - npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip - mv docprims.darwin-arm64.node prebuilds/ + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds + test -f prebuilds/docprims.darwin-arm64.node - name: Upload prebuild uses: actions/upload-artifact@v7 with: @@ -134,10 +134,11 @@ jobs: shell: pwsh run: | $ErrorActionPreference = 'Stop' - Remove-Item -Force docprims.*.node -ErrorAction SilentlyContinue + Remove-Item -Recurse -Force prebuilds -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path prebuilds | Out-Null - npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip - Move-Item docprims.win32-x64-msvc.node prebuilds/ + npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds + $node = 'prebuilds/docprims.win32-x64-msvc.node' + if (-not (Test-Path $node)) { throw "missing $node" } - name: Upload prebuild uses: actions/upload-artifact@v7 with: From a9afca0447960174f71198a4fd77564c5203f721 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:42:20 -0400 Subject: [PATCH 18/33] build: declare crates.io publication per crate The workspace defaults to publish = false. docprims, docprims-core, docprims-text and docprims-ooxml opt in and carry docs.rs, README and license metadata; docprims-ffi and docprims-ts-napi stay unpublished. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- Cargo.toml | 2 ++ bindings/typescript/docprims/native/Cargo.toml | 1 + crates/docprims-core/Cargo.toml | 3 +++ crates/docprims-core/LICENSE-APACHE | 1 + crates/docprims-core/LICENSE-MIT | 1 + crates/docprims-core/README.md | 9 +++++++++ crates/docprims-ooxml/Cargo.toml | 3 +++ crates/docprims-ooxml/LICENSE-APACHE | 1 + crates/docprims-ooxml/LICENSE-MIT | 1 + crates/docprims-ooxml/README.md | 9 +++++++++ crates/docprims-text/Cargo.toml | 3 +++ crates/docprims-text/LICENSE-APACHE | 1 + crates/docprims-text/LICENSE-MIT | 1 + crates/docprims-text/README.md | 9 +++++++++ crates/docprims/Cargo.toml | 5 +++++ crates/docprims/LICENSE-APACHE | 1 + crates/docprims/LICENSE-MIT | 1 + ffi/docprims-ffi/Cargo.toml | 1 + 18 files changed, 53 insertions(+) create mode 120000 crates/docprims-core/LICENSE-APACHE create mode 120000 crates/docprims-core/LICENSE-MIT create mode 100644 crates/docprims-core/README.md create mode 120000 crates/docprims-ooxml/LICENSE-APACHE create mode 120000 crates/docprims-ooxml/LICENSE-MIT create mode 100644 crates/docprims-ooxml/README.md create mode 120000 crates/docprims-text/LICENSE-APACHE create mode 120000 crates/docprims-text/LICENSE-MIT create mode 100644 crates/docprims-text/README.md create mode 120000 crates/docprims/LICENSE-APACHE create mode 120000 crates/docprims/LICENSE-MIT diff --git a/Cargo.toml b/Cargo.toml index a0619a0..c644820 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,8 @@ rust-version = "1.88.0" license = "MIT OR Apache-2.0" repository = "https://github.com/3leaps/docprims" authors = ["3leaps Architecture Council"] +# Crates opt in to crates.io with `publish = true`; config/release/publishable-crates.txt lists them. +publish = false [workspace.dependencies] docprims = { path = "crates/docprims" } diff --git a/bindings/typescript/docprims/native/Cargo.toml b/bindings/typescript/docprims/native/Cargo.toml index 8dc8c8f..2f0c9a3 100644 --- a/bindings/typescript/docprims/native/Cargo.toml +++ b/bindings/typescript/docprims/native/Cargo.toml @@ -5,6 +5,7 @@ edition.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +publish.workspace = true rust-version.workspace = true description = "Node-API addon for @3leaps/docprims (TypeScript bindings)" diff --git a/crates/docprims-core/Cargo.toml b/crates/docprims-core/Cargo.toml index 6daf54f..52f2a85 100644 --- a/crates/docprims-core/Cargo.toml +++ b/crates/docprims-core/Cargo.toml @@ -7,6 +7,9 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +publish = true +documentation = "https://docs.rs/docprims-core" +readme = "README.md" [dependencies] serde = { workspace = true } diff --git a/crates/docprims-core/LICENSE-APACHE b/crates/docprims-core/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/docprims-core/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/docprims-core/LICENSE-MIT b/crates/docprims-core/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/docprims-core/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/docprims-core/README.md b/crates/docprims-core/README.md new file mode 100644 index 0000000..ec9991d --- /dev/null +++ b/crates/docprims-core/README.md @@ -0,0 +1,9 @@ +# docprims-core + +Core types, errors and limits for [docprims](https://crates.io/crates/docprims). + +This crate is an internal component of docprims and carries no stability +promise beyond what the `docprims` crate re-exports. Depend on +[`docprims`](https://docs.rs/docprims) instead. + +Licensed under either of MIT or Apache-2.0 at your option. diff --git a/crates/docprims-ooxml/Cargo.toml b/crates/docprims-ooxml/Cargo.toml index 17c2c92..898ca8d 100644 --- a/crates/docprims-ooxml/Cargo.toml +++ b/crates/docprims-ooxml/Cargo.toml @@ -7,6 +7,9 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +publish = true +documentation = "https://docs.rs/docprims-ooxml" +readme = "README.md" [features] default = ["docx", "xlsx", "pptx"] diff --git a/crates/docprims-ooxml/LICENSE-APACHE b/crates/docprims-ooxml/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/docprims-ooxml/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/docprims-ooxml/LICENSE-MIT b/crates/docprims-ooxml/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/docprims-ooxml/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/docprims-ooxml/README.md b/crates/docprims-ooxml/README.md new file mode 100644 index 0000000..1cb4a14 --- /dev/null +++ b/crates/docprims-ooxml/README.md @@ -0,0 +1,9 @@ +# docprims-ooxml + +DOCX, XLSX and PPTX extraction for [docprims](https://crates.io/crates/docprims). + +This crate is an internal component of docprims and carries no stability +promise beyond what the `docprims` crate re-exports. Depend on +[`docprims`](https://docs.rs/docprims) instead. + +Licensed under either of MIT or Apache-2.0 at your option. diff --git a/crates/docprims-text/Cargo.toml b/crates/docprims-text/Cargo.toml index 2663754..d93834a 100644 --- a/crates/docprims-text/Cargo.toml +++ b/crates/docprims-text/Cargo.toml @@ -7,6 +7,9 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +publish = true +documentation = "https://docs.rs/docprims-text" +readme = "README.md" [features] default = ["markdown", "html", "xml"] diff --git a/crates/docprims-text/LICENSE-APACHE b/crates/docprims-text/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/docprims-text/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/docprims-text/LICENSE-MIT b/crates/docprims-text/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/docprims-text/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/crates/docprims-text/README.md b/crates/docprims-text/README.md new file mode 100644 index 0000000..a0022c9 --- /dev/null +++ b/crates/docprims-text/README.md @@ -0,0 +1,9 @@ +# docprims-text + +Markdown, HTML and XML extraction for [docprims](https://crates.io/crates/docprims). + +This crate is an internal component of docprims and carries no stability +promise beyond what the `docprims` crate re-exports. Depend on +[`docprims`](https://docs.rs/docprims) instead. + +Licensed under either of MIT or Apache-2.0 at your option. diff --git a/crates/docprims/Cargo.toml b/crates/docprims/Cargo.toml index 277aceb..4ea902b 100644 --- a/crates/docprims/Cargo.toml +++ b/crates/docprims/Cargo.toml @@ -7,6 +7,11 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +publish = true +documentation = "https://docs.rs/docprims" +readme = "../../README.md" +keywords = ["text-extraction", "docx", "xlsx", "markdown", "html"] +categories = ["text-processing", "parser-implementations"] [features] default = ["text", "ooxml"] diff --git a/crates/docprims/LICENSE-APACHE b/crates/docprims/LICENSE-APACHE new file mode 120000 index 0000000..1cd601d --- /dev/null +++ b/crates/docprims/LICENSE-APACHE @@ -0,0 +1 @@ +../../LICENSE-APACHE \ No newline at end of file diff --git a/crates/docprims/LICENSE-MIT b/crates/docprims/LICENSE-MIT new file mode 120000 index 0000000..b2cfbdc --- /dev/null +++ b/crates/docprims/LICENSE-MIT @@ -0,0 +1 @@ +../../LICENSE-MIT \ No newline at end of file diff --git a/ffi/docprims-ffi/Cargo.toml b/ffi/docprims-ffi/Cargo.toml index a4ea180..ebcf222 100644 --- a/ffi/docprims-ffi/Cargo.toml +++ b/ffi/docprims-ffi/Cargo.toml @@ -7,6 +7,7 @@ rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true +publish.workspace = true [lib] crate-type = ["staticlib", "cdylib", "rlib"] From e0bb730097181bdf0aebc9829269a4fc3831aaaf Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:42:20 -0400 Subject: [PATCH 19/33] chore(release): prepare v0.2.0 Set the version to 0.2.0 across the workspace, npm manifest and lock files, and move the unreleased notes under 0.2.0. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- CHANGELOG.md | 25 ++--- Cargo.lock | 12 +-- Cargo.toml | 10 +- RELEASE_NOTES.md | 48 +++------- VERSION | 2 +- .../typescript/docprims/package-lock.json | 16 ++-- bindings/typescript/docprims/package.json | 14 +-- docs/releases/v0.1.5.md | 93 ------------------- docs/releases/v0.2.0.md | 83 +++++++++++++++++ 9 files changed, 132 insertions(+), 171 deletions(-) delete mode 100644 docs/releases/v0.1.5.md create mode 100644 docs/releases/v0.2.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c685110..88e8413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,15 +10,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.2.0] - 2026-09-25 + ### Added - **`docprims` crate**: The library's entry point. `extract_file` and `extract_bytes` return the `extract/v0` structure and choose the parser from the path or source URI extension, or from an explicit `Format` (`extract_file_as`, `extract_bytes_as`); content is never inspected to choose a parser. Each format is a feature (`markdown`, `html`, `xml`, `docx`, `xlsx`, `pptx`, grouped as `text` and `ooxml`; all enabled by default). `docprims-core`, `docprims-text` and `docprims-ooxml` carry no stability promise beyond what `docprims` re-exports. +- **npm trusted publishing runtime guard**: `scripts/check-npm-trusted-publish-runtime.sh`, run by `make lint` and `make npm-publish-prereqs-check`. ### Changed - **BREAKING: `DocprimsError`, `DocprimsQualityStatus` and `DocprimsContainerKind` are `#[non_exhaustive]`**: Matches on them outside docprims need a wildcard arm. `DocprimsQualityStatus` implements `PartialEq` and `Eq`. - **CLI**: The `docprims` command-line tool is now built from the `docprims` crate with the `cli` feature (`cargo install docprims --features cli`). The binary name, commands and output are unchanged. - **C, Go and TypeScript bindings** extract through the `docprims` crate; results and error codes are unchanged. +- **Dependencies**: quick-xml 0.42, zip 8, scraper 0.27 and rsfulmen 0.2. The TypeScript binding builds with napi-rs 3 and TypeScript 7. +- **Rust MSRV**: The workspace minimum supported Rust version is 1.88.0. +- **TypeScript runtime floor**: `@3leaps/docprims` requires Node.js 22 or newer. npm publishing checks for Node >=22.14.0 and npm >=11.5.1. ### Removed @@ -32,24 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Extracted text character set**: Extracted text from every format (Markdown, HTML, XML, DOCX, XLSX, PPTX) contains only XML 1.0 characters, excluding DEL and C1 controls (U+007F–U+009F); other characters are dropped before block byte ranges are computed. Markdown and HTML keep their parsers' U+FFFD replacement for NUL. Numeric character references (`é`, `😀`) resolve under the same rule, including in DOCX, XLSX and PPTX, which previously dropped all numeric references. Extracted text changes for documents that contain such characters or references. - **OOXML decompression limits**: The decompressed size of DOCX, XLSX and PPTX parts is now bounded by the bytes actually read, not the size declared in the archive: 100 MiB per part and 400 MiB across all parts of one archive. Exceeding either limit is a `ResourceLimit` error. - **Output limits**: When `max_output_bytes` truncates extraction, the result no longer contains an empty block or a trailing separator. - -## [0.1.5] - 2026-07-07 - -### Added - -- **npm trusted publishing runtime guard**: Added `scripts/check-npm-trusted-publish-runtime.sh` and wired it into `make lint` plus `npm-publish-prereqs-check`. - -### Changed - -- **Rust MSRV**: Raised the workspace minimum supported Rust version to 1.88.0. -- **TypeScript runtime floor**: Raised `@3leaps/docprims` and repo tooling Node.js engine requirements to Node 22 or newer. -- **TypeScript workflows**: Moved TypeScript binding CI, release validation, and N-API prebuild workflows to Node 22 validation; npm publish remains on Node 24 with explicit Node >=22.14.0 and npm >=11.5.1 trusted-publishing checks. -- **Repository guidance**: Refreshed agent guidance, canonical documentation pointers, YAML linting configuration, and local-only planning artifact conventions. -- **README install guidance**: Replaced the unpublished CLI crate install command with repo-root `cargo install --path crates/docprims-cli` guidance. - -### Fixed - -- **Current stable clippy compatibility**: Updated parser match handling flagged by newer clippy lints. +- **Current stable clippy compatibility**: Parser match handling updated for newer clippy lints. ## [0.1.4] - 2026-01-31 diff --git a/Cargo.lock b/Cargo.lock index 749d6fe..52e1c52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,7 +345,7 @@ dependencies = [ [[package]] name = "docprims" -version = "0.1.5" +version = "0.2.0" dependencies = [ "assert_cmd", "base64", @@ -365,7 +365,7 @@ dependencies = [ [[package]] name = "docprims-core" -version = "0.1.5" +version = "0.2.0" dependencies = [ "serde", "serde_json", @@ -375,7 +375,7 @@ dependencies = [ [[package]] name = "docprims-ffi" -version = "0.1.5" +version = "0.2.0" dependencies = [ "base64", "docprims", @@ -386,7 +386,7 @@ dependencies = [ [[package]] name = "docprims-ooxml" -version = "0.1.5" +version = "0.2.0" dependencies = [ "docprims-core", "encoding_rs", @@ -398,7 +398,7 @@ dependencies = [ [[package]] name = "docprims-text" -version = "0.1.5" +version = "0.2.0" dependencies = [ "docprims-core", "jsonschema", @@ -410,7 +410,7 @@ dependencies = [ [[package]] name = "docprims-ts-napi" -version = "0.1.5" +version = "0.2.0" dependencies = [ "docprims", "docprims-core", diff --git a/Cargo.toml b/Cargo.toml index c644820..7f1ba96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ ] [workspace.package] -version = "0.1.5" +version = "0.2.0" edition = "2021" rust-version = "1.88.0" license = "MIT OR Apache-2.0" @@ -20,10 +20,10 @@ authors = ["3leaps Architecture Council"] publish = false [workspace.dependencies] -docprims = { path = "crates/docprims" } -docprims-core = { path = "crates/docprims-core" } -docprims-text = { path = "crates/docprims-text", default-features = false } -docprims-ooxml = { path = "crates/docprims-ooxml", default-features = false } +docprims = { version = "0.2.0", path = "crates/docprims" } +docprims-core = { version = "0.2.0", path = "crates/docprims-core" } +docprims-text = { version = "0.2.0", path = "crates/docprims-text", default-features = false } +docprims-ooxml = { version = "0.2.0", path = "crates/docprims-ooxml", default-features = false } # Fulmen Ecosystem rsfulmen = { version = "=0.2.0", default-features = false, features = ["foundry-core"] } diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 052646c..a298ad7 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,47 +6,29 @@ --- -## v0.1.5 - 2026-07-07 +## v0.2.0 — 2026-09-25 -**Status:** Patch Release (runtime floors + repository guidance) +**Status:** Minor Release (library entry point, extraction fixes, dependency refresh) -Refreshes package runtime floors and repository guidance before the next coordinated dependency modernization pass. +Adds the `docprims` crate as the library's entry point, fixes extraction edge cases, and refreshes dependencies and toolchains. Breaking for Rust users of the subcrates; bindings keep their results and error codes. ### Highlights -- **Rust MSRV**: Raised the workspace MSRV to Rust 1.88.0. -- **TypeScript Node floor**: `@3leaps/docprims` now requires Node.js 22 or newer. -- **Trusted publishing guard**: npm publishing now hard-checks Node >=22.14.0 and npm >=11.5.1. -- **TypeScript workflow alignment**: Binding CI, release validation, and N-API prebuild workflows validate on Node 22. -- **Repository guidance cleanup**: Agent guidance, local-only planning conventions, and YAML linting configuration were refreshed. +- **`docprims` crate**: One dependency for all formats, a feature per format; the parser is chosen by extension or an explicit `Format`, never by content. +- **CLI**: `cargo install docprims --features cli`; binary name, commands and output are unchanged. +- **Extracted text character set**: XML 1.0 characters only, excluding DEL and C1 controls, on every format; numeric character references resolve in DOCX, XLSX and PPTX. +- **OOXML decompression limits**: Enforced on bytes actually read, per part (100 MiB) and per archive (400 MiB). +- **Output limits**: No empty block or trailing separator when `max_output_bytes` truncates. +- **Toolchains**: MSRV 1.88.0; Node.js 22+; the TypeScript binding builds with napi-rs 3 and TypeScript 7. -### Runtime Floors +### Breaking Changes -v0.1.5 aligns the project with currently supported runtime floors: +- `docprims-cli` crate removed (use the `docprims` crate's `cli` feature). +- `--timeout-ms` / `ExtractOptions::timeout_ms` and `ExtractOptions` removed. +- `docprims_ooxml::common` is private. +- `DocprimsError`, `DocprimsQualityStatus` and `DocprimsContainerKind` are `#[non_exhaustive]`. -| Component | Floor | -|-----------|-------| -| Rust | 1.88.0 | -| TypeScript package | Node.js 22+ | -| npm trusted publishing | Node.js >=22.14.0 and npm >=11.5.1 | - -The TypeScript package remains on TypeScript 5.x and napi-rs 2.x for this release. TypeScript 6.x and napi-rs 3.x are reserved for a later coordinated modernization pass. - -### Release Workflow Hardening - -The npm publish workflow now uses a dedicated runtime guard that fails fast if the publish runner is below the trusted-publishing floor. The same guard is available locally through: - -```bash -make npm-publish-prereqs-check -``` - -`make lint` also runs this guard so workflow drift is caught during normal quality checks. - -### Documentation Updates - -- README CLI install guidance now uses `cargo install --path crates/docprims-cli` from a repo checkout. -- Repository guidance now treats planning artifacts as local-only and non-canonical. -- YAML linting configuration is explicit at repo root. +See `docs/releases/v0.2.0.md` for usage and migration. --- diff --git a/VERSION b/VERSION index 9faa1b7..0ea3a94 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.5 +0.2.0 diff --git a/bindings/typescript/docprims/package-lock.json b/bindings/typescript/docprims/package-lock.json index 8904b41..6b4150b 100644 --- a/bindings/typescript/docprims/package-lock.json +++ b/bindings/typescript/docprims/package-lock.json @@ -1,12 +1,12 @@ { "name": "@3leaps/docprims", - "version": "0.1.5", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@3leaps/docprims", - "version": "0.1.5", + "version": "0.2.0", "license": "MIT OR Apache-2.0", "devDependencies": { "@napi-rs/cli": "^3.10.5", @@ -17,12 +17,12 @@ "node": ">=22" }, "optionalDependencies": { - "@3leaps/docprims-darwin-arm64": "0.1.5", - "@3leaps/docprims-linux-arm64-gnu": "0.1.5", - "@3leaps/docprims-linux-arm64-musl": "0.1.5", - "@3leaps/docprims-linux-x64-gnu": "0.1.5", - "@3leaps/docprims-linux-x64-musl": "0.1.5", - "@3leaps/docprims-win32-x64-msvc": "0.1.5" + "@3leaps/docprims-darwin-arm64": "0.2.0", + "@3leaps/docprims-linux-arm64-gnu": "0.2.0", + "@3leaps/docprims-linux-arm64-musl": "0.2.0", + "@3leaps/docprims-linux-x64-gnu": "0.2.0", + "@3leaps/docprims-linux-x64-musl": "0.2.0", + "@3leaps/docprims-win32-x64-msvc": "0.2.0" } }, "node_modules/@emnapi/core": { diff --git a/bindings/typescript/docprims/package.json b/bindings/typescript/docprims/package.json index 02edef0..5f38f7b 100644 --- a/bindings/typescript/docprims/package.json +++ b/bindings/typescript/docprims/package.json @@ -1,6 +1,6 @@ { "name": "@3leaps/docprims", - "version": "0.1.5", + "version": "0.2.0", "description": "GPL-free document text extraction (TypeScript bindings via Node-API native addon)", "license": "MIT OR Apache-2.0", "repository": { @@ -41,12 +41,12 @@ }, "dependencies": {}, "optionalDependencies": { - "@3leaps/docprims-linux-x64-gnu": "0.1.5", - "@3leaps/docprims-linux-x64-musl": "0.1.5", - "@3leaps/docprims-linux-arm64-gnu": "0.1.5", - "@3leaps/docprims-linux-arm64-musl": "0.1.5", - "@3leaps/docprims-darwin-arm64": "0.1.5", - "@3leaps/docprims-win32-x64-msvc": "0.1.5" + "@3leaps/docprims-linux-x64-gnu": "0.2.0", + "@3leaps/docprims-linux-x64-musl": "0.2.0", + "@3leaps/docprims-linux-arm64-gnu": "0.2.0", + "@3leaps/docprims-linux-arm64-musl": "0.2.0", + "@3leaps/docprims-darwin-arm64": "0.2.0", + "@3leaps/docprims-win32-x64-msvc": "0.2.0" }, "devDependencies": { "@napi-rs/cli": "^3.10.5", diff --git a/docs/releases/v0.1.5.md b/docs/releases/v0.1.5.md deleted file mode 100644 index ad7f3ef..0000000 --- a/docs/releases/v0.1.5.md +++ /dev/null @@ -1,93 +0,0 @@ -# docprims v0.1.5 - -**Release Date:** 2026-07-07 -**Status:** Patch Release (runtime floors + repository guidance) - -## Summary - -This release refreshes package runtime floors and repository guidance after the v0.1.4 publishing fixes. It keeps the TypeScript 6.x and napi-rs 3.x upgrade out of scope for the later coordinated modernization pass. - -## Highlights - -- **Rust MSRV**: Raised the workspace minimum supported Rust version to 1.88.0. -- **TypeScript Node floor**: `@3leaps/docprims` now requires Node.js 22 or newer. -- **npm trusted publishing hardening**: Publish workflow checks Node >=22.14.0 and npm >=11.5.1 before publishing. -- **TypeScript workflow alignment**: Binding tests, release validation, and N-API prebuilds now validate on Node 22. -- **Repository guidance cleanup**: Agent guidance, local-only planning conventions, and YAML linting configuration were refreshed. - -## Runtime Floors - -| Component | Floor | -|-----------|-------| -| Rust workspace | 1.88.0 | -| TypeScript package | Node.js 22+ | -| TypeScript CI/prebuild validation | Node.js 22 | -| npm trusted publishing | Node.js >=22.14.0 and npm >=11.5.1 | - -## TypeScript Package Changes - -The TypeScript package metadata now declares: - -```json -{ - "engines": { - "node": ">=22" - } -} -``` - -Platform optional dependencies were advanced to the v0.1.5 package set: - -| Platform | Package | -|----------|---------| -| Linux x64 (glibc) | `@3leaps/docprims-linux-x64-gnu` | -| Linux x64 (musl) | `@3leaps/docprims-linux-x64-musl` | -| Linux arm64 (glibc) | `@3leaps/docprims-linux-arm64-gnu` | -| Linux arm64 (musl) | `@3leaps/docprims-linux-arm64-musl` | -| macOS arm64 | `@3leaps/docprims-darwin-arm64` | -| Windows x64 | `@3leaps/docprims-win32-x64-msvc` | - -## Trusted Publishing Guard - -The npm publish workflow runs on Node 24 but explicitly enforces the trusted-publishing minimums before publishing: - -- Node.js >=22.14.0 -- npm >=11.5.1 - -The same static guard runs locally through: - -```bash -make npm-publish-prereqs-check -``` - -It is also wired into `make lint`, so the normal quality gate catches workflow drift. - -## Repository Guidance - -v0.1.5 refreshes public repository guidance: - -- Planning artifacts are local-only and non-canonical. -- `docs/orientation/sources-of-truth.md` remains the canonical guide for source-of-truth boundaries. -- YAML linting is configured explicitly at repo root. -- Commit attribution guidance uses bare role slugs. - -## Installation Notes - -### CLI from a checkout - -```bash -cargo install --path crates/docprims-cli -``` - -### TypeScript from a checkout - -```bash -cd bindings/typescript/docprims -npm install -npm run build -npm run build:native -``` - -## Changelog - -See `CHANGELOG.md` for detailed changes. diff --git a/docs/releases/v0.2.0.md b/docs/releases/v0.2.0.md new file mode 100644 index 0000000..af518ed --- /dev/null +++ b/docs/releases/v0.2.0.md @@ -0,0 +1,83 @@ +# docprims v0.2.0 + +**Release Date:** 2026-09-25 +**Status:** Minor Release (library entry point, extraction fixes, dependency refresh) + +## Summary + +v0.2.0 adds the `docprims` crate as the library's entry point, fixes several +extraction edge cases, and updates dependencies and toolchains across the +Rust crates and the C, Go and TypeScript bindings. It contains breaking +changes for Rust users of the subcrates; bindings keep their results and +error codes. + +## Highlights + +- **`docprims` crate**: One dependency for all formats, with a feature per format. +- **Extracted text character set**: Every format emits only XML 1.0 characters, excluding DEL and C1 controls. +- **Numeric character references** resolve in DOCX, XLSX and PPTX. +- **OOXML decompression limits** are enforced on bytes actually read, per part and per archive. +- **Output limits** never produce an empty block or a trailing separator. +- **CLI** is installed with `cargo install docprims --features cli`. + +## Using the `docprims` crate + +```toml +[dependencies] +docprims = "0.2" +``` + +```rust +use docprims::{extract_file, ExtractLimits}; + +let result = extract_file("report.docx", ExtractLimits::default())?; +println!("{}", result.document.text); +``` + +The parser is chosen from the file extension, or explicitly with +`extract_file_as` / `extract_bytes_as` and a `Format`. Content is never +inspected to choose a parser. + +Formats are features: `markdown`, `html`, `xml`, `docx`, `xlsx`, `pptx`, +grouped as `text` and `ooxml`, all on by default. A Markdown-only build: + +```toml +docprims = { version = "0.2", default-features = false, features = ["markdown"] } +``` + +`docprims-core`, `docprims-text` and `docprims-ooxml` are published as +components of `docprims` and carry no stability promise beyond what +`docprims` re-exports. + +## Breaking Changes + +| Change | Migration | +|--------|-----------| +| `docprims-cli` crate removed | `cargo install docprims --features cli` (same binary name, commands and output) | +| `--timeout-ms` / `ExtractOptions::timeout_ms` removed (never applied) | Remove the flag or field | +| `ExtractOptions` removed from `docprims-core` | Use `ExtractLimits` | +| `docprims_ooxml::common` no longer public | Use `docprims::extract_*` | +| `DocprimsError`, `DocprimsQualityStatus`, `DocprimsContainerKind` are `#[non_exhaustive]` | Add a wildcard arm to matches | + +## Extraction Changes + +Extracted text may differ from v0.1.x for documents that contain: + +- control characters, DEL or C1 controls (now dropped); +- numeric character references in DOCX, XLSX or PPTX (now resolved); +- output truncated by `max_output_bytes` (no empty final block). + +Block byte ranges are computed after characters are dropped, so they always +index the emitted text. + +## Runtime Floors + +| Component | Floor | +|-----------|-------| +| Rust (MSRV) | 1.88.0 | +| TypeScript package | Node.js 22+ | +| npm trusted publishing | Node.js >=22.14.0 and npm >=11.5.1 | + +## Changelog + +See `CHANGELOG.md` for the full list of changes. From 1f538bcdd3e37aeef3a27932be17962d67027e8e Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:50:34 -0400 Subject: [PATCH 20/33] build(release): add version, crate and release-notes gates Adapted from the storageprims release tooling. - config/release/publishable-crates.txt is the ordered crates.io list; release-crates.py checks it against Cargo metadata, and docprims-ffi and docprims-ts-napi must stay unpublished - check-packages.sh packages and verifies the listed crates without publishing; release-crates-dry-run.sh and release-crates-verify.sh cover the ordered publish from a guarded tag - version-sync.py writes VERSION to Cargo.toml, Cargo.lock and the npm manifests; check-version.sh checks every one of those sites - release-guard-tag-version.sh binds the release tag to VERSION and, in strict mode, to origin/main - check-release-notes.sh requires docs/releases/vX.Y.Z.md to match its RELEASE_NOTES.md section - make release-preflight runs these before tagging; prepush runs the tooling tests; help is generated from target comments - RELEASE_CHECKLIST.md gains the crates.io steps Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- Makefile | 159 ++++++++++------------ RELEASE_CHECKLIST.md | 50 ++++++- config/release/publishable-crates.txt | 4 + scripts/check-packages.sh | 77 +++++++++++ scripts/check-release-notes.sh | 32 +++++ scripts/check-release-notes.test.sh | 40 ++++++ scripts/check-version.sh | 97 +++++++++++++ scripts/check-version.test.sh | 60 ++++++++ scripts/release-checklist.test.sh | 14 ++ scripts/release-crates-dry-run.sh | 17 +++ scripts/release-crates-registry.sh | 25 ++++ scripts/release-crates-verify.sh | 26 ++++ scripts/release-crates-verify.test.sh | 31 +++++ scripts/release-crates.py | 75 ++++++++++ scripts/release-crates.test.sh | 72 ++++++++++ scripts/release-guard-tag-version.sh | 97 +++++++++++++ scripts/release-guard-tag-version.test.sh | 76 +++++++++++ scripts/version-sync.py | 63 +++++++++ 18 files changed, 926 insertions(+), 89 deletions(-) create mode 100644 config/release/publishable-crates.txt create mode 100755 scripts/check-packages.sh create mode 100755 scripts/check-release-notes.sh create mode 100755 scripts/check-release-notes.test.sh create mode 100755 scripts/check-version.sh create mode 100755 scripts/check-version.test.sh create mode 100755 scripts/release-checklist.test.sh create mode 100755 scripts/release-crates-dry-run.sh create mode 100755 scripts/release-crates-registry.sh create mode 100755 scripts/release-crates-verify.sh create mode 100755 scripts/release-crates-verify.test.sh create mode 100755 scripts/release-crates.py create mode 100755 scripts/release-crates.test.sh create mode 100755 scripts/release-guard-tag-version.sh create mode 100755 scripts/release-guard-tag-version.test.sh create mode 100755 scripts/version-sync.py diff --git a/Makefile b/Makefile index 41d1024..0bdee83 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,9 @@ .PHONY: release-export-keys release-verify-checksums release-verify-signatures .PHONY: release-verify-keys release-notes release-upload release .PHONY: build-local-go go-test go-test-shared -.PHONY: version-patch version-minor version-major version-set version-sync +.PHONY: version-patch version-minor version-major version-set version-sync version-check +.PHONY: release-check release-crates-list release-crates-dry-run release-crates-verify +.PHONY: release-tooling-test release-preflight release-guard-tag-version .PHONY: check-windows check-windows-msvc check-windows-gnu # ----------------------------------------------------------------------------- @@ -53,52 +55,8 @@ all: check help: ## Show available targets @echo "docprims - GPL-free Document Text Extraction" - @echo "Extract text from documents without license contamination." @echo "" - @echo "Development:" - @echo " help Show this help message" - @echo " bootstrap Install tools (sfetch -> goneat)" - @echo " build Build all crates (debug)" - @echo " build-release Build all crates (release)" - @echo " build-ffi Build FFI library with C header" - @echo " install Install docprims binary to ~/.local/bin" - @echo " clean Remove build artifacts" - @echo "" - @echo "Go bindings:" - @echo " build-local-go Build FFI for local Go development" - @echo " go-test Run Go binding tests" - @echo " go-test-shared Run Go binding tests (shared library)" - @echo "" - @echo "Release (manual signing):" - @echo " release-clean Remove dist/release contents" - @echo " release-download Download draft release assets" - @echo " release-checksums Generate SHA256SUMS and SHA512SUMS" - @echo " release-sign Sign checksum manifests (minisign/optional PGP)" - @echo " release-export-keys Export public signing keys" - @echo " release-verify Verify checksums/signatures/keys" - @echo " release-upload Upload signed assets and publish" - @echo "" - @echo "Quality gates:" - @echo " check Run all quality checks (fmt, lint, test, deny)" - @echo " test Run test suite" - @echo " fmt Format code (cargo fmt)" - @echo " lint Run linting (cargo clippy)" - @echo " precommit Pre-commit checks (fast: fmt, clippy)" - @echo " prepush Pre-push checks (thorough: fmt, clippy, test, deny)" - @echo " deny Run cargo-deny license and advisory checks" - @echo " audit Run cargo-audit security scan" - @echo " miri Run Miri UB detection on unsafe code (nightly)" - @echo " msrv Verify build and tests with MSRV (Rust $(MSRV))" - @echo " npm-publish-prereqs-check Verify npm trusted publishing runtime guard" - @echo " check-windows Cross-check Windows targets (no SDK required)" - @echo "" - @echo "Version management:" - @echo " version Print current version" - @echo " version-patch Bump patch version (0.1.0 -> 0.1.1)" - @echo " version-minor Bump minor version (0.1.0 -> 0.2.0)" - @echo " version-major Bump major version (0.1.0 -> 1.0.0)" - @echo " version-set Set explicit version (V=X.Y.Z)" - @echo " version-sync Sync VERSION to Cargo.toml" + @awk 'BEGIN { FS = ":.*## " } /^[a-zA-Z0-9_-]+:.*## / { printf " %-26s %s\n", $$1, $$2 }' $(MAKEFILE_LIST) @echo "" @echo "Current version: $(VERSION)" @@ -456,9 +414,59 @@ install: build-release ## Install docprims binary to INSTALL_BINDIR precommit: fmt-check lint ## Run pre-commit checks (fast) @echo "[ok] Pre-commit checks passed" -prepush: check ## Run pre-push checks (thorough) +prepush: check version-check release-tooling-test ## Run pre-push checks (thorough) @echo "[ok] Pre-push checks passed" +# ----------------------------------------------------------------------------- +# Release Governance (crates.io, version, tag guard) +# ----------------------------------------------------------------------------- + +release-check: version-check ## Version consistency + crate package check (does not publish) + @./scripts/check-packages.sh + @echo "[ok] Package check passed; cargo publish was not run" + +release-crates-list: ## Print the validated crates.io publication order + @./scripts/release-crates.py list + +release-crates-dry-run: ## Dry-run crates.io publishing from the guarded tag + @./scripts/release-crates-dry-run.sh + +release-crates-verify: ## Wait for the released version on crates.io and verify it (CRATE= for one) + @CRATE="$(CRATE)" ./scripts/release-crates-verify.sh + +release-guard-tag-version: ## Validate the canonical release tag + @./scripts/release-guard-tag-version.sh + +release-tooling-test: ## Run release guard, crate, version and notes tests + @./scripts/release-crates.test.sh + @./scripts/release-crates-verify.test.sh + @./scripts/release-guard-tag-version.test.sh + @./scripts/check-version.test.sh + @./scripts/check-release-notes.test.sh + @./scripts/release-checklist.test.sh + @echo "[ok] Release tooling tests passed" + +release-preflight: ## Verify clean-tree pre-tag requirements + @echo "Running release preflight checks..." + @if [ -n "$$(git status --porcelain 2>/dev/null)" ]; then \ + echo "[!!] Working tree not clean - commit or stash changes first"; \ + git status --short; \ + exit 1; \ + fi + @$(MAKE) prepush --silent + @$(MAKE) release-check --silent + @grep -Eq "^## v$(VERSION) — [0-9]{4}-[0-9]{2}-[0-9]{2}$$" RELEASE_NOTES.md || \ + { echo "[!!] RELEASE_NOTES.md lacks the exact v$(VERSION) heading"; exit 1; } + @grep -Eq "^## \[$(VERSION)\] - [0-9]{4}-[0-9]{2}-[0-9]{2}$$" CHANGELOG.md || \ + { echo "[!!] CHANGELOG.md lacks the [$(VERSION)] heading"; exit 1; } + @./scripts/check-release-notes.sh "v$(VERSION)" + @git fetch origin main + @test "$$(git rev-parse HEAD)" = "$$(git rev-parse origin/main)" || \ + { echo "[!!] HEAD must equal fetched origin/main"; exit 1; } + @test -z "$$(git status --porcelain)" || \ + { echo "[!!] Preflight gates changed the working tree"; exit 1; } + @echo "[ok] All preflight checks passed - ready to tag v$(VERSION)" + # ----------------------------------------------------------------------------- # Release Workflow # ----------------------------------------------------------------------------- @@ -556,49 +564,30 @@ version: ## Print current version @echo "$(VERSION)" version-patch: ## Bump patch version (0.1.0 -> 0.1.1) - @current=$$(cat $(VERSION_FILE)); \ - major=$$(echo $$current | cut -d. -f1); \ - minor=$$(echo $$current | cut -d. -f2); \ - patch=$$(echo $$current | cut -d. -f3); \ - new_patch=$$((patch + 1)); \ - new_version="$$major.$$minor.$$new_patch"; \ - echo "$$new_version" > $(VERSION_FILE); \ - sed -i '' "s/^version = \"$$current\"/version = \"$$new_version\"/" Cargo.toml; \ - echo "Version bumped: $$current -> $$new_version" + @current=$$(tr -d ' \t\r\n' < $(VERSION_FILE)); \ + IFS=. read -r major minor patch <<< "$$current"; \ + echo "$$major.$$minor.$$((patch + 1))" > $(VERSION_FILE); \ + ./scripts/version-sync.py version-minor: ## Bump minor version (0.1.0 -> 0.2.0) - @current=$$(cat $(VERSION_FILE)); \ - major=$$(echo $$current | cut -d. -f1); \ - minor=$$(echo $$current | cut -d. -f2); \ - new_minor=$$((minor + 1)); \ - new_version="$$major.$$new_minor.0"; \ - echo "$$new_version" > $(VERSION_FILE); \ - sed -i '' "s/^version = \"$$current\"/version = \"$$new_version\"/" Cargo.toml; \ - echo "Version bumped: $$current -> $$new_version" + @current=$$(tr -d ' \t\r\n' < $(VERSION_FILE)); \ + IFS=. read -r major minor patch <<< "$$current"; \ + echo "$$major.$$((minor + 1)).0" > $(VERSION_FILE); \ + ./scripts/version-sync.py version-major: ## Bump major version (0.1.0 -> 1.0.0) - @current=$$(cat $(VERSION_FILE)); \ - major=$$(echo $$current | cut -d. -f1); \ - new_major=$$((major + 1)); \ - new_version="$$new_major.0.0"; \ - echo "$$new_version" > $(VERSION_FILE); \ - sed -i '' "s/^version = \"$$current\"/version = \"$$new_version\"/" Cargo.toml; \ - echo "Version bumped: $$current -> $$new_version" + @current=$$(tr -d ' \t\r\n' < $(VERSION_FILE)); \ + IFS=. read -r major minor patch <<< "$$current"; \ + echo "$$((major + 1)).0.0" > $(VERSION_FILE); \ + ./scripts/version-sync.py version-set: ## Set explicit version (V=X.Y.Z) - @if [ -z "$(V)" ]; then \ - echo "Usage: make version-set V=1.2.3"; \ - exit 1; \ - fi + @if [ -z "$(V)" ]; then echo "Usage: make version-set V=1.2.3"; exit 1; fi @echo "$(V)" > $(VERSION_FILE) - @echo "Version set to $(V)" + @./scripts/version-sync.py -version-sync: ## Sync VERSION file to Cargo.toml (requires cargo-edit) - @ver=$$(cat $(VERSION_FILE)); \ - if command -v cargo-set-version >/dev/null 2>&1; then \ - cargo set-version --workspace "$$ver"; \ - echo "[ok] Synced Cargo.toml to $$ver"; \ - else \ - echo "[!!] cargo-edit not installed (cargo install cargo-edit)"; \ - echo "Manual update required: set version = \"$$ver\" in Cargo.toml"; \ - fi +version-sync: ## Write VERSION to Cargo.toml, Cargo.lock and the npm manifests + @./scripts/version-sync.py + +version-check: ## Validate version consistency across Cargo and npm manifests + @./scripts/check-version.sh diff --git a/RELEASE_CHECKLIST.md b/RELEASE_CHECKLIST.md index f2ec6db..1147723 100644 --- a/RELEASE_CHECKLIST.md +++ b/RELEASE_CHECKLIST.md @@ -25,9 +25,12 @@ docprims uses the same high-level release posture as sysprims: ### Version & Documentation -- [ ] Update `VERSION` file with new semver (e.g. `0.1.1`) -- [ ] Ensure workspace version matches `VERSION` (if applicable) -- [ ] Update `CHANGELOG.md` (if present) +- [ ] Set the version everywhere: `make version-set V=X.Y.Z` (writes `VERSION`, + `Cargo.toml`, `Cargo.lock` and the npm manifests), then `make version-check` +- [ ] Move `CHANGELOG.md` `[Unreleased]` entries under `## [X.Y.Z] - YYYY-MM-DD` +- [ ] Add `## vX.Y.Z — YYYY-MM-DD` to `RELEASE_NOTES.md` and copy that exact + section to `docs/releases/vX.Y.Z.md` (`scripts/check-release-notes.sh vX.Y.Z`) +- [ ] Package the publishable crates without publishing: `make release-check` ### Commit & Push @@ -49,6 +52,8 @@ docprims uses the same high-level release posture as sysprims: ``` Both commands must show no output. +- [ ] Run the pre-tag gate on the merged `main`: `make release-preflight` + ### Go Bindings Prep (Required) Go bindings must be fetchable via `go get` without requiring Rust. @@ -229,9 +234,46 @@ npm publish --access public This publishes with local platform binary only. Run the workflow publish afterward for cross-platform support. -## 4. Post-Release Verification +## 4. crates.io (maintainer action after the tag) + +CI has no registry token and never uploads crates. A registry version cannot +be overwritten: a correction takes a new patch version. + +The only ordered list is `config/release/publishable-crates.txt` +(`docprims-core`, `docprims-text`, `docprims-ooxml`, `docprims`); print and +validate it with `make release-crates-list`. `docprims-ffi` and +`docprims-ts-napi` are unpublished (`publish = false`), which +`make release-tooling-test` proves. + +- [ ] In a clean detached checkout of the pushed tag: + ```bash + git checkout --detach "v$(cat VERSION)" + DOCPRIMS_RELEASE_TAG="v$(cat VERSION)" DOCPRIMS_REQUIRE_TAG=1 make release-guard-tag-version + DOCPRIMS_RELEASE_TAG="v$(cat VERSION)" make release-crates-dry-run + ``` + The dry run patches earlier workspace crates in by path; it uploads nothing. +- [ ] Use a crates.io token scoped to the publishable names, kept outside the + repository and CI, with an expiry of 30–90 days. The first upload needs + `publish-new` and `publish-update`; later versions need only + `publish-update`. Do not grant `yank`. Load it only into the environment + of each publish command; do not use `cargo login`. +- [ ] Publish **one crate at a time** in list order, verifying each before the next: + ```bash + make release-crates-list + export DOCPRIMS_RELEASE_TAG="v$(cat VERSION)" + cargo publish --locked -p "${crate:?set the next ordered crate}" + make release-crates-verify CRATE="$crate" + # Repeat for each remaining entry; after the last: + make release-crates-verify + ``` + Each verification waits for `cargo info --registry crates-io @` + and checks the exact version on the crates.io API. +- [ ] Review the crates.io pages and the docs.rs builds. + +## 5. Post-Release Verification - [ ] Verify release is public: `gh release view v$(cat VERSION)` - [ ] Download and verify checksums/signatures using the published public keys - [ ] Verify npm package: `npm view @3leaps/docprims` +- [ ] Verify crates: `cargo info --registry crates-io docprims@$(cat VERSION)` - [ ] Test Go module: `go get github.com/3leaps/docprims/bindings/go/docprims@v$(cat VERSION)` diff --git a/config/release/publishable-crates.txt b/config/release/publishable-crates.txt new file mode 100644 index 0000000..16dd0e6 --- /dev/null +++ b/config/release/publishable-crates.txt @@ -0,0 +1,4 @@ +docprims-core +docprims-text +docprims-ooxml +docprims diff --git a/scripts/check-packages.sh b/scripts/check-packages.sh new file mode 100755 index 0000000..832ab0a --- /dev/null +++ b/scripts/check-packages.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Verify publishable crate archives without publishing. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VERSION_FILE="$PROJECT_ROOT/VERSION" + +version=$(tr -d '[:space:]' <"$VERSION_FILE") +"$SCRIPT_DIR/release-crates.py" check +mapfile_crates() { + while IFS= read -r crate; do crates+=("$crate"); done < <("$SCRIPT_DIR/release-crates.py" list) +} +crates=() +mapfile_crates +temp_root="${RUNNER_TEMP:-${TMPDIR:-/tmp}}" +package_target=$(mktemp -d "$temp_root/docprims-package.XXXXXX") +trap 'rm -rf "$package_target"' EXIT + +export CARGO_TARGET_DIR="$package_target" + +cd "$PROJECT_ROOT" +package_args=() +expected=() +for crate in "${crates[@]}"; do + package_args+=(-p "$crate") + expected+=("$crate-$version.crate") +done +# Packaging the crates together resolves their dependencies on each other +# from the packaged sources, as crates.io will after an ordered publish. +cargo package "${package_args[@]}" --locked + +package_dir="$CARGO_TARGET_DIR/package" +actual=() +while IFS= read -r archive; do + actual+=("$(basename "$archive")") +done < <(find "$package_dir" -maxdepth 1 -type f -name 'docprims-*.crate' | sort) + +sorted_expected=() +while IFS= read -r archive; do + sorted_expected+=("$archive") +done < <(printf '%s\n' "${expected[@]}" | LC_ALL=C sort) +if [[ "${actual[*]}" != "${sorted_expected[*]}" ]]; then + printf '[ERROR] package artifacts differ from the expected publishable crates\n' >&2 + printf 'expected: %s\n' "${sorted_expected[*]}" >&2 + printf 'actual: %s\n' "${actual[*]}" >&2 + exit 1 +fi + +for crate in "${crates[@]}"; do + archive="$package_dir/$crate-$version.crate" + manifest_path="$crate-$version/Cargo.toml" + python3 - "$archive" "$manifest_path" "$version" <<'PY' +import sys +import tarfile +if sys.version_info < (3, 11): + sys.exit("error: package inspection requires Python 3.11 or newer") +import tomllib + +archive, manifest_path, version = sys.argv[1:] +with tarfile.open(archive) as packaged: + manifest = tomllib.loads(packaged.extractfile(manifest_path).read().decode()) + +def check_dependencies(sections): + for section in ("dependencies", "dev-dependencies", "build-dependencies"): + for name, spec in sections.get(section, {}).items(): + if name == "docprims" or name.startswith("docprims-"): + assert isinstance(spec, dict) and spec.get("version") == version and "path" not in spec, (name, spec) + +check_dependencies(manifest) +for target in manifest.get("target", {}).values(): + check_dependencies(target) +PY +done + +printf '[ok] Verified package archives: %s\n' "${expected[*]}" diff --git a/scripts/check-release-notes.sh b/scripts/check-release-notes.sh new file mode 100755 index 0000000..1e129e1 --- /dev/null +++ b/scripts/check-release-notes.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Prove the per-cut file is exactly the matching RELEASE_NOTES section. + +set -euo pipefail + +tag="${1:?usage: check-release-notes.sh vX.Y.Z}" +[[ "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || { + echo "error: release tag must be canonical vX.Y.Z" >&2 + exit 1 +} +cut_file="docs/releases/${tag}.md" +[[ -f "$cut_file" ]] || { + echo "error: per-cut release notes are missing" >&2 + exit 1 +} + +extracted="$(mktemp "${TMPDIR:-/tmp}/docprims-release-notes.XXXXXX")" +trap 'rm -f "$extracted"' EXIT +# The section runs from its heading to the next `---`, without trailing blank lines. +awk -v heading="## ${tag} " ' + index($0, heading) == 1 { found = 1 } + found && /^---$/ { exit } + found && /^$/ { blanks = blanks "\n"; next } + found { printf "%s%s\n", blanks, $0; blanks = "" } + END { if (!found) exit 1 } +' RELEASE_NOTES.md >"$extracted" + +cmp -s "$extracted" "$cut_file" || { + echo "error: per-cut notes must exactly match their RELEASE_NOTES section" >&2 + exit 1 +} +echo "[ok] per-cut release notes match the landing page" diff --git a/scripts/check-release-notes.test.sh b/scripts/check-release-notes.test.sh new file mode 100755 index 0000000..5bc2a40 --- /dev/null +++ b/scripts/check-release-notes.test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Prove the per-cut notes check accepts only an exact RELEASE_NOTES section. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +fixture="$(mktemp -d "${TMPDIR:-/tmp}/docprims-notes-test.XXXXXX")" +trap 'rm -rf "$fixture"' EXIT +mkdir -p "$fixture/docs/releases" + +section=$'## v1.2.0 — 2026-01-02\n\nBody line.\n\n### Detail\n\n- item' +write_fixture() { + printf '# Release Notes\n\n---\n\n%s\n\n---\n\n## v1.1.0 — 2026-01-01\n\nOld.\n' "$1" >"$fixture/RELEASE_NOTES.md" + printf '%s\n' "$2" >"$fixture/docs/releases/v1.2.0.md" +} +check() { (cd "$fixture" && "$SCRIPT_DIR/check-release-notes.sh" "$@") >/dev/null 2>&1; } +expect_fail() { + if check "$@"; then + echo "error: notes check passed unexpectedly: $*" >&2 + exit 1 + fi +} + +write_fixture "$section" "$section" +check v1.2.0 || { + echo "error: matching section was rejected" >&2 + exit 1 +} +write_fixture "$section" "${section} changed" +expect_fail v1.2.0 +write_fixture "$section" "${section%$'\n- item'}" +expect_fail v1.2.0 +write_fixture "${section/2026-01-02/2026-01-03}" "$section" +expect_fail v1.2.0 +write_fixture "$section" "$section" +expect_fail v1.1.0 +expect_fail v1.3.0 +expect_fail 1.2.0 +rm "$fixture/docs/releases/v1.2.0.md" +expect_fail v1.2.0 +echo '[ok] per-cut notes check accepts only the exact section' diff --git a/scripts/check-version.sh b/scripts/check-version.sh new file mode 100755 index 0000000..f348925 --- /dev/null +++ b/scripts/check-version.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Validate the docprims VERSION, workspace, members, path dependencies and npm manifests. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +VERSION_FILE="$PROJECT_ROOT/VERSION" +CARGO_TOML="$PROJECT_ROOT/Cargo.toml" + +error() { + printf '[ERROR] %s\n' "$*" >&2 +} + +ok() { + printf '[OK] %s\n' "$*" +} + +if [[ ! -f "$VERSION_FILE" ]]; then + error "VERSION file not found: $VERSION_FILE" + exit 1 +fi + +version=$(tr -d '[:space:]' <"$VERSION_FILE") +if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.-]+)?(\+[a-zA-Z0-9.-]+)?$ ]]; then + error "VERSION contains invalid semver: $version" + exit 1 +fi +ok "VERSION file: $version" + +workspace_version=$( + awk ' + /^\[workspace\.package\]$/ { in_package = 1; next } + /^\[/ { in_package = 0 } + in_package && /^version[[:space:]]*=/ { + gsub(/^[^"]*"|"[[:space:]]*$/, "") + print + exit + } + ' "$CARGO_TOML" +) +if [[ "$workspace_version" != "$version" ]]; then + error "workspace version $workspace_version does not match VERSION $version" + exit 1 +fi +ok "workspace version matches" + +failed=0 +while IFS= read -r manifest; do + crate_dir=$(dirname "$manifest") + if grep -A 12 '^\[package\]' "$manifest" | grep -Eq '^version\.workspace[[:space:]]*=[[:space:]]*true'; then + ok "${crate_dir#"$PROJECT_ROOT/"} uses workspace version" + else + error "${crate_dir#"$PROJECT_ROOT/"} must set version.workspace = true" + failed=1 + fi +done < <( + cargo metadata --no-deps --format-version 1 --locked --manifest-path "$CARGO_TOML" | + jq -r '.packages[].manifest_path' | sort +) + +while IFS= read -r line; do + dependency_version=$(sed -E 's/.*version = "([^"]*)".*/\1/' <<<"$line") + if [[ "$dependency_version" != "$version" ]]; then + error "workspace dependency version $dependency_version does not match VERSION $version" + error "$line" + failed=1 + fi +done < <( + grep -E '^docprims(-core|-text|-ooxml)? = \{ version = "' "$CARGO_TOML" || true +) + +if [[ "$failed" -ne 0 ]]; then + error "version consistency check failed" + exit 1 +fi + +ok "workspace path-dependency versions match" + +npm_dir="$PROJECT_ROOT/bindings/typescript/docprims" +npm_versions=$( + jq -r '.version, (.optionalDependencies // {} | .[])' "$npm_dir/package.json" + jq -r '.version, .packages[""].version, (.packages[""].optionalDependencies // {} | .[])' \ + "$npm_dir/package-lock.json" +) +while IFS= read -r npm_version; do + if [[ "$npm_version" != "$version" ]]; then + error "npm manifest version $npm_version does not match VERSION $version" + failed=1 + fi +done <<<"$npm_versions" +if [[ "$failed" -ne 0 ]]; then + error "version consistency check failed" + exit 1 +fi +ok "npm package and platform package versions match" +ok "version consistency check passed" diff --git a/scripts/check-version.test.sh b/scripts/check-version.test.sh new file mode 100755 index 0000000..4c55990 --- /dev/null +++ b/scripts/check-version.test.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Prove check-version.sh rejects drift at every version site it owns. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(dirname "$SCRIPT_DIR")" +work="$(mktemp -d "${TMPDIR:-/tmp}/docprims-version-test.XXXXXX")" +trap 'rm -rf "$work"' EXIT + +fresh_copy() { + rm -rf "$work/repo" + mkdir "$work/repo" + (cd "$root" && git ls-files -z --cached --others --exclude-standard) | + (cd "$root" && xargs -0 tar -cf - --) | tar -xf - -C "$work/repo" +} + +fresh_copy +"$work/repo/scripts/check-version.sh" >/dev/null + +npm="bindings/typescript/docprims" +for mutation in \ + "VERSION|s/^.*$/9.9.9/" \ + "Cargo.toml|s/^docprims-text = { version = \"[^\"]*\"/docprims-text = { version = \"9.9.9\"/" \ + "Cargo.toml|s/^docprims = { version = \"[^\"]*\"/docprims = { version = \"9.9.9\"/" \ + "crates/docprims-ooxml/Cargo.toml|s/^version.workspace = true/version = \"9.9.9\"/" \ + "$npm/native/Cargo.toml|s/^version.workspace = true/version = \"9.9.9\"/" \ + "$npm/package.json|1,/\"version\": \"[^\"]*\"/s/\"version\": \"[^\"]*\"/\"version\": \"9.9.9\"/" \ + "$npm/package.json|s/\"@3leaps\/docprims-win32-x64-msvc\": \"[^\"]*\"/\"@3leaps\/docprims-win32-x64-msvc\": \"9.9.9\"/" \ + "$npm/package-lock.json|1,/\"version\": \"[^\"]*\"/s/\"version\": \"[^\"]*\"/\"version\": \"9.9.9\"/" \ + "$npm/package-lock.json|s/\"@3leaps\/docprims-darwin-arm64\": \"[^\"]*\"/\"@3leaps\/docprims-darwin-arm64\": \"9.9.9\"/"; do + file="${mutation%%|*}" + expr="${mutation#*|}" + fresh_copy + before="$(cat "$work/repo/$file")" + sed -i.bak -e "$expr" "$work/repo/$file" + rm -f "$work/repo/$file.bak" + if [[ "$before" == "$(cat "$work/repo/$file")" ]]; then + echo "error: mutation did not apply: $mutation" >&2 + exit 1 + fi + if "$work/repo/scripts/check-version.sh" >/dev/null 2>&1; then + echo "error: version drift passed the check: $mutation" >&2 + exit 1 + fi +done + +# version-sync.py writes every site the check reads. +fresh_copy +printf '9.9.9\n' >"$work/repo/VERSION" +if "$work/repo/scripts/check-version.sh" >/dev/null 2>&1; then + echo 'error: VERSION-only bump passed the check' >&2 + exit 1 +fi +"$work/repo/scripts/version-sync.py" >/dev/null +"$work/repo/scripts/check-version.sh" >/dev/null +grep -q '^version = "9.9.9"' "$work/repo/Cargo.lock" || { + echo 'error: version-sync did not update Cargo.lock' >&2 + exit 1 +} +echo '[ok] version check rejects drift at every version site; version-sync fixes it' diff --git a/scripts/release-checklist.test.sh b/scripts/release-checklist.test.sh new file mode 100755 index 0000000..df7f7cc --- /dev/null +++ b/scripts/release-checklist.test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Every make target the release checklist names must exist in the Makefile. +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +missing=0 +while IFS= read -r target; do + grep -Eq "^${target}:" "$root/Makefile" || { + echo "error: RELEASE_CHECKLIST.md names missing target: make $target" >&2 + missing=1 + } +done < <(grep -oE 'make [a-z][a-z0-9-]*' "$root/RELEASE_CHECKLIST.md" | awk '{ print $2 }' | sort -u) +[[ "$missing" -eq 0 ]] +echo '[ok] release checklist names only existing make targets' diff --git a/scripts/release-crates-dry-run.sh b/scripts/release-crates-dry-run.sh new file mode 100755 index 0000000..acd9dcb --- /dev/null +++ b/scripts/release-crates-dry-run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Run per-crate publish dry runs in registry dependency order, never upload. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(dirname "$SCRIPT_DIR")" +cd "$root" +DOCPRIMS_REQUIRE_TAG=1 "$SCRIPT_DIR/release-guard-tag-version.sh" >/dev/null +"$SCRIPT_DIR/release-crates.py" check +version="$(cat VERSION)" +patches=() +while IFS= read -r crate; do + printf 'Dry-run crates.io: %s@%s\n' "$crate" "$version" + cargo publish --dry-run --locked -p "$crate" ${patches[@]+"${patches[@]}"} + patches+=(--config "patch.crates-io.${crate}.path=\"crates/${crate}\"") +done <"$root/config/release/publishable-crates.txt" +echo '[ok] Per-crate registry dry runs passed; nothing was published' diff --git a/scripts/release-crates-registry.sh b/scripts/release-crates-registry.sh new file mode 100755 index 0000000..4920561 --- /dev/null +++ b/scripts/release-crates-registry.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Read-only index and crates.io API checks, also used between manual publishes. +registry_wait() { + local crate="$1" version="$2" attempt + for attempt in {1..30}; do + if cargo info --registry crates-io "${crate}@${version}" >/dev/null 2>&1; then + return 0 + fi + sleep 10 + done + echo "error: crates.io index did not expose ${crate}@${version} after ${attempt} attempts" >&2 + return 1 +} + +registry_api_check() { + local crate="$1" version="$2" response + response="$(curl --fail --silent --show-error --retry 3 \ + "https://crates.io/api/v1/crates/${crate}/${version}")" + jq -e --arg name "$crate" --arg version "$version" \ + '.version.num == $version and .version.crate == $name and .version.yanked == false' \ + <<<"$response" >/dev/null || { + echo "error: crates.io API did not confirm ${crate}@${version}" >&2 + return 1 + } +} diff --git a/scripts/release-crates-verify.sh b/scripts/release-crates-verify.sh new file mode 100755 index 0000000..7e8fa10 --- /dev/null +++ b/scripts/release-crates-verify.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(dirname "$SCRIPT_DIR")" +# shellcheck source=release-crates-registry.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-crates-registry.sh" +cd "$root" +DOCPRIMS_REQUIRE_TAG=1 "$SCRIPT_DIR/release-guard-tag-version.sh" >/dev/null +"$SCRIPT_DIR/release-crates.py" check +version="$(cat VERSION)" +if [[ -n "${CRATE:-}" ]]; then + grep -Fxq -- "$CRATE" config/release/publishable-crates.txt || { + echo 'error: CRATE is not in the publishable crate list' >&2 + exit 1 + } + registry_wait "$CRATE" "$version" + registry_api_check "$CRATE" "$version" +else + while IFS= read -r crate; do + registry_wait "$crate" "$version" + registry_api_check "$crate" "$version" + done /dev/null 2>&1; then + echo 'error: mismatched version passed API check' >&2 + exit 1 +fi +if registry_api_check docprims-core 0.2.0 >/dev/null 2>&1; then + echo 'error: mismatched crate passed API check' >&2 + exit 1 +fi +curl() { printf '%s\n' '{"version":{"num":"0.2.0","crate":"docprims","yanked":true}}'; } +if registry_api_check docprims 0.2.0 >/dev/null 2>&1; then + echo 'error: yanked version passed API check' >&2 + exit 1 +fi +cargo() { return 1; } +sleep() { :; } +if registry_wait docprims 0.2.0 >/dev/null 2>&1; then + echo 'error: missing registry version passed index wait' >&2 + exit 1 +fi +echo '[ok] registry index and API checks reject mismatched releases' diff --git a/scripts/release-crates.py b/scripts/release-crates.py new file mode 100755 index 0000000..af25e02 --- /dev/null +++ b/scripts/release-crates.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Validate the ordered crates.io publication plan against Cargo metadata.""" + +import json +import pathlib +import subprocess +import sys + +if sys.version_info < (3, 11): + sys.exit("error: release crate checks require Python 3.11 or newer") + +import tomllib + +ROOT = pathlib.Path(__file__).resolve().parent.parent +PLAN = ROOT / "config/release/publishable-crates.txt" +# Workspace crates that must never reach crates.io. +UNPUBLISHED = ("docprims-ffi", "docprims-ts-napi") + + +def validate(packages, names, manifests): + if ( + not names + or len(names) != len(set(names)) + or any(n != "docprims" and not n.startswith("docprims-") for n in names) + ): + raise ValueError("invalid or duplicate publishable crate name") + by_name = {p["name"]: p for p in packages} + if len(by_name) != len(packages): + raise ValueError("duplicate workspace package") + publishable = {p["name"] for p in packages if p.get("publish") != []} + if set(names) != publishable: + raise ValueError( + f"crate list differs from publishable workspace crates: {sorted(publishable ^ set(names))}" + ) + for name in names: + if manifests[name]["package"].get("publish") is not True: + raise ValueError(f"{name} must explicitly set publish = true") + for dependency in by_name[name]["dependencies"]: + dep_name = dependency["name"] + if dep_name in names and names.index(dep_name) >= names.index(name): + raise ValueError( + f"{name} must follow {dep_name} (including dev dependencies)" + ) + for name in UNPUBLISHED: + if name in by_name and by_name[name].get("publish") != []: + raise ValueError(f"{name} must remain unpublished") + + +def main(): + names = PLAN.read_text().splitlines() + if any(not n or n != n.strip() or n.startswith("#") for n in names): + raise ValueError("crate list must contain one bare name per line") + metadata = json.loads( + subprocess.check_output( + ["cargo", "metadata", "--no-deps", "--format-version", "1", "--locked"], + cwd=ROOT, + ) + ) + packages = metadata["packages"] + manifests = { + p["name"]: tomllib.loads(pathlib.Path(p["manifest_path"]).read_text()) + for p in packages + } + validate(packages, names, manifests) + if sys.argv[1:] == ["list"]: + print("\n".join(names)) + elif sys.argv[1:] != ["check"]: + raise ValueError("usage: release-crates.py check|list") + + +if __name__ == "__main__": + try: + main() + except (ValueError, KeyError) as error: + sys.exit(f"error: {error}") diff --git a/scripts/release-crates.test.sh b/scripts/release-crates.test.sh new file mode 100755 index 0000000..467cb52 --- /dev/null +++ b/scripts/release-crates.test.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"$SCRIPT_DIR/release-crates.py" check +# Every unpublished workspace crate must be refused by cargo itself. +for crate in docprims-ffi docprims-ts-napi; do + cargo metadata --no-deps --format-version 1 --locked | + jq -e --arg name "$crate" 'any(.packages[]; .name == $name)' >/dev/null || { + echo "error: unpublished crate $crate is not a workspace member" >&2 + exit 1 + } + result="$(cargo publish --dry-run --locked --allow-dirty -p "$crate" 2>&1)" && { + echo "error: unpublished crate $crate passed publish dry run" >&2 + exit 1 + } + grep -q 'cannot be published' <<<"$result" || { + echo "error: $crate failed for a reason other than publish = false" >&2 + exit 1 + } +done +python3 -B - "$SCRIPT_DIR" <<'PY' +import importlib.util +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) / "release-crates.py" +spec = importlib.util.spec_from_file_location("release_crates", path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +names = ["docprims-core", "docprims-text", "docprims-ooxml", "docprims"] +core = {"name": "docprims-core"} +packages = [ + {"name": "docprims-core", "publish": None, "dependencies": []}, + {"name": "docprims-text", "publish": None, "dependencies": [core]}, + {"name": "docprims-ooxml", "publish": None, "dependencies": [core]}, + { + "name": "docprims", + "publish": None, + "dependencies": [core, {"name": "docprims-text"}, {"name": "docprims-ooxml", "kind": "dev"}], + }, + {"name": "docprims-ffi", "publish": [], "dependencies": [{"name": "docprims"}]}, + {"name": "docprims-ts-napi", "publish": [], "dependencies": [{"name": "docprims"}]}, +] +manifests = {name: {"package": {"publish": True}} for name in names} +module.validate(packages, names, manifests) + + +def publishable(name): + return [{**p, "publish": None} if p["name"] == name else p for p in packages] + + +for bad_names, bad_packages, bad_manifests in ( + (names[::-1], packages, manifests), + (["docprims-core", "docprims-text", "docprims", "docprims-ooxml"], packages, manifests), + (names[:-1], packages, manifests), + (names + names[:1], packages, manifests), + (names + ["docprims-ffi"], publishable("docprims-ffi"), {**manifests, "docprims-ffi": {"package": {"publish": True}}}), + (names, publishable("docprims-ffi"), manifests), + (names, publishable("docprims-ts-napi"), manifests), + (names, packages, {**manifests, "docprims-core": {"package": {"publish": False}}}), + (names, packages, {**manifests, "docprims": {"package": {}}}), + (["docprimsx"] + names[1:], packages, manifests), + ([], packages, manifests), +): + try: + module.validate(bad_packages, bad_names, bad_manifests) + except ValueError: + continue + raise AssertionError(f"negative control unexpectedly passed: {bad_names}") +print("[ok] publishable crate list and negative controls") +PY diff --git a/scripts/release-guard-tag-version.sh b/scripts/release-guard-tag-version.sh new file mode 100755 index 0000000..6cdee76 --- /dev/null +++ b/scripts/release-guard-tag-version.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Validate the sole release tag input and, in strict mode, bind it to origin/main. + +set -euo pipefail + +main() { + local root version expected tag require_tag + root="$(git rev-parse --show-toplevel)" + cd "$root" + + if [[ -n "${RELEASE_TAG+x}" || -n "${DOCPRIMS_RELEASE_KEY+x}" ]]; then + echo "error: generic or legacy release tag aliases are forbidden" >&2 + exit 1 + fi + + version="$(cat VERSION)" + if [[ ! "$version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "error: VERSION must contain one stable semantic version" >&2 + exit 1 + fi + expected="v${version}" + tag="${DOCPRIMS_RELEASE_TAG:-$expected}" + require_tag="${DOCPRIMS_REQUIRE_TAG:-0}" + + if [[ ! "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "error: release tag must be canonical vX.Y.Z" >&2 + exit 1 + fi + if [[ "$tag" != "$expected" ]]; then + echo "error: release tag does not match VERSION" >&2 + exit 1 + fi + + if [[ "$require_tag" == "1" ]]; then + if [[ -z "${DOCPRIMS_RELEASE_TAG:-}" ]]; then + echo "error: strict release guard requires DOCPRIMS_RELEASE_TAG" >&2 + exit 1 + fi + if git symbolic-ref -q HEAD >/dev/null; then + echo "error: strict release work must use a detached checkout" >&2 + exit 1 + fi + if [[ -n "$(git status --porcelain --untracked-files=normal)" ]]; then + echo "error: strict release work requires a clean checkout" >&2 + exit 1 + fi + local origin_url + origin_url="$(git config --get remote.origin.url || true)" + case "$origin_url" in + https://github.com/3leaps/docprims | \ + https://github.com/3leaps/docprims.git) ;; + git@*:3leaps/docprims | git@*:3leaps/docprims.git) + local ssh_host resolved_host + ssh_host="${origin_url#git@}" + ssh_host="${ssh_host%%:*}" + resolved_host="$(ssh -G "$ssh_host" 2>/dev/null | + awk '$1 == "hostname" { print $2; exit }')" + [[ "$resolved_host" == "github.com" ]] || { + echo "error: origin SSH host must resolve to github.com" >&2 + exit 1 + } + ;; + *) + echo "error: origin must identify github.com/3leaps/docprims" >&2 + exit 1 + ;; + esac + git fetch --quiet --no-tags origin "+refs/heads/main:refs/remotes/origin/main" + if [[ "$(git cat-file -t "refs/tags/$tag" 2>/dev/null || true)" != "tag" ]]; then + echo "error: strict release tag must be annotated" >&2 + exit 1 + fi + local remote_object local_object + remote_object="$(git ls-remote origin "refs/tags/${tag}" | awk '{print $1}')" + local_object="$(git rev-parse "refs/tags/${tag}")" + if [[ -z "$remote_object" || "$remote_object" != "$local_object" ]]; then + echo "error: remote and local annotated tag objects must match" >&2 + exit 1 + fi + local tag_commit head_commit main_commit + tag_commit="$(git rev-parse "refs/tags/${tag}^{}")" + head_commit="$(git rev-parse HEAD)" + main_commit="$(git rev-parse refs/remotes/origin/main)" + if [[ "$tag_commit" != "$head_commit" ]]; then + echo "error: release tag and HEAD must be identical" >&2 + exit 1 + fi + if [[ "$tag_commit" != "$main_commit" ]]; then + echo "error: release tag and origin/main must be identical" >&2 + exit 1 + fi + fi + + echo "[ok] release guard passed for $tag" +} + +main "$@" diff --git a/scripts/release-guard-tag-version.test.sh b/scripts/release-guard-tag-version.test.sh new file mode 100755 index 0000000..8a323c6 --- /dev/null +++ b/scripts/release-guard-tag-version.test.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +fixture="$(mktemp -d "${TMPDIR:-/tmp}/docprims-release-guard.XXXXXX")" +remote="$(mktemp -d "${TMPDIR:-/tmp}/docprims-release-remote.XXXXXX")" +trap 'rm -rf "$fixture" "$remote"' EXIT + +cp "$SCRIPT_DIR/release-guard-tag-version.sh" "$fixture/guard.sh" +printf '1.2.3\n' >"$fixture/VERSION" +git init -q -b main "$fixture" +git init -q --bare "$remote" +git -C "$fixture" config user.name "release guard test" +git -C "$fixture" config user.email "noreply@example.invalid" +git -C "$fixture" add VERSION guard.sh +git -C "$fixture" commit -qm "fixture" +git -C "$fixture" tag -a v1.2.3 -m "fixture" +git -C "$fixture" remote add origin https://github.com/3leaps/docprims.git +git -C "$fixture" config "url.$remote.insteadOf" \ + https://github.com/3leaps/docprims.git +git -C "$fixture" push -q origin main refs/tags/v1.2.3 + +run_guard() { + ( + cd "$fixture" + env -u DOCPRIMS_RELEASE_TAG \ + -u DOCPRIMS_REQUIRE_TAG \ + -u DOCPRIMS_RELEASE_KEY \ + -u RELEASE_TAG \ + "$@" + ) +} + +expect_fail() { + if "$@" >/dev/null 2>&1; then + echo "expected failure: $*" >&2 + exit 1 + fi +} + +run_guard ./guard.sh >/dev/null +run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 ./guard.sh >/dev/null +for invalid in 1.2.3 vv1.2.3 "v1.2.3 " v1.2.3-rc.1 v1.2.3+build; do + expect_fail run_guard DOCPRIMS_RELEASE_TAG="$invalid" ./guard.sh +done +expect_fail run_guard DOCPRIMS_RELEASE_TAG=v1.2.4 ./guard.sh +expect_fail run_guard RELEASE_TAG=v1.2.3 ./guard.sh +expect_fail run_guard DOCPRIMS_RELEASE_KEY=v1.2.3 ./guard.sh +expect_fail run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ + DOCPRIMS_REQUIRE_TAG=1 ./guard.sh + +git -C "$fixture" checkout -q --detach v1.2.3 +run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ + DOCPRIMS_REQUIRE_TAG=1 ./guard.sh >/dev/null +original_tag="$(git -C "$fixture" rev-parse refs/tags/v1.2.3)" +git -C "$fixture" tag -d v1.2.3 >/dev/null +expect_fail run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ + DOCPRIMS_REQUIRE_TAG=1 ./guard.sh +git -C "$fixture" update-ref refs/tags/v1.2.3 "$original_tag" +git -C "$fixture" tag -fa v1.2.3 -m 'different object' HEAD +git -C "$fixture" push -q --force origin refs/tags/v1.2.3 +git -C "$fixture" update-ref refs/tags/v1.2.3 "$original_tag" +expect_fail run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ + DOCPRIMS_REQUIRE_TAG=1 ./guard.sh + +git -C "$fixture" checkout -q main +printf 'next\n' >"$fixture/next" +git -C "$fixture" add next +git -C "$fixture" commit -qm "move main" +git -C "$fixture" push -q origin main +git -C "$fixture" checkout -q --detach v1.2.3 +expect_fail run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ + DOCPRIMS_REQUIRE_TAG=1 ./guard.sh + +echo "[ok] release tag guard tests passed" diff --git a/scripts/version-sync.py b/scripts/version-sync.py new file mode 100755 index 0000000..ac1a1e9 --- /dev/null +++ b/scripts/version-sync.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Write VERSION to every version site that scripts/check-version.sh checks.""" + +import json +import pathlib +import re +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent +NPM = ROOT / "bindings/typescript/docprims" + + +def sync(version): + cargo = ROOT / "Cargo.toml" + text = cargo.read_text() + text, count = re.subn( + r'(?ms)^(\[workspace\.package\]\n(?:(?!^\[).)*?^version = )"[^"]*"', + rf'\1"{version}"', + text, + count=1, + ) + if count != 1: + raise ValueError("Cargo.toml has no [workspace.package] version") + text, count = re.subn( + r'(?m)^(docprims(?:-core|-text|-ooxml)? = \{ version = )"[^"]*"', + rf'\1"{version}"', + text, + ) + if count != 4: + raise ValueError(f"expected 4 docprims workspace dependencies, found {count}") + cargo.write_text(text) + + package = json.loads((NPM / "package.json").read_text()) + package["version"] = version + for name in package.get("optionalDependencies", {}): + package["optionalDependencies"][name] = version + (NPM / "package.json").write_text(json.dumps(package, indent=2) + "\n") + + lock = json.loads((NPM / "package-lock.json").read_text()) + lock["version"] = version + root = lock["packages"][""] + root["version"] = version + for name in root.get("optionalDependencies", {}): + root["optionalDependencies"][name] = version + (NPM / "package-lock.json").write_text(json.dumps(lock, indent=2) + "\n") + + subprocess.run(["cargo", "update", "--workspace", "--offline", "--quiet"], cwd=ROOT, check=True) + + +def main(): + version = (ROOT / "VERSION").read_text().strip() + if not re.fullmatch(r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)", version): + raise ValueError("VERSION must contain one stable semantic version") + sync(version) + print(f"[ok] Synced version sites to {version}") + + +if __name__ == "__main__": + try: + main() + except (ValueError, KeyError, subprocess.CalledProcessError) as error: + sys.exit(f"error: {error}") From 4168f829fc9d10d781791bcfa6b95ebb2fb94c8f Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:50:34 -0400 Subject: [PATCH 21/33] docs(release): match v0.2.0 notes to the release notes section Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- RELEASE_NOTES.md | 83 +++++++++++++++++++++++++++++++++++------ docs/releases/v0.2.0.md | 17 ++++----- 2 files changed, 79 insertions(+), 21 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index a298ad7..cac22b3 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -10,25 +10,84 @@ **Status:** Minor Release (library entry point, extraction fixes, dependency refresh) -Adds the `docprims` crate as the library's entry point, fixes extraction edge cases, and refreshes dependencies and toolchains. Breaking for Rust users of the subcrates; bindings keep their results and error codes. +### Summary + +v0.2.0 adds the `docprims` crate as the library's entry point, fixes several +extraction edge cases, and updates dependencies and toolchains across the +Rust crates and the C, Go and TypeScript bindings. It contains breaking +changes for Rust users of the subcrates; bindings keep their results and +error codes. ### Highlights -- **`docprims` crate**: One dependency for all formats, a feature per format; the parser is chosen by extension or an explicit `Format`, never by content. -- **CLI**: `cargo install docprims --features cli`; binary name, commands and output are unchanged. -- **Extracted text character set**: XML 1.0 characters only, excluding DEL and C1 controls, on every format; numeric character references resolve in DOCX, XLSX and PPTX. -- **OOXML decompression limits**: Enforced on bytes actually read, per part (100 MiB) and per archive (400 MiB). -- **Output limits**: No empty block or trailing separator when `max_output_bytes` truncates. -- **Toolchains**: MSRV 1.88.0; Node.js 22+; the TypeScript binding builds with napi-rs 3 and TypeScript 7. +- **`docprims` crate**: One dependency for all formats, with a feature per format. +- **Extracted text character set**: Every format emits only XML 1.0 characters, excluding DEL and C1 controls. +- **Numeric character references** resolve in DOCX, XLSX and PPTX. +- **OOXML decompression limits** are enforced on bytes actually read, per part and per archive. +- **Output limits** never produce an empty block or a trailing separator. +- **CLI** is installed with `cargo install docprims --features cli`. + +### Using the `docprims` crate + +```toml +[dependencies] +docprims = "0.2" +``` + +```rust +use docprims::{extract_file, ExtractLimits}; + +let result = extract_file("report.docx", ExtractLimits::default())?; +println!("{}", result.document.text); +``` + +The parser is chosen from the file extension, or explicitly with +`extract_file_as` / `extract_bytes_as` and a `Format`. Content is never +inspected to choose a parser. + +Formats are features: `markdown`, `html`, `xml`, `docx`, `xlsx`, `pptx`, +grouped as `text` and `ooxml`, all on by default. A Markdown-only build: + +```toml +docprims = { version = "0.2", default-features = false, features = ["markdown"] } +``` + +`docprims-core`, `docprims-text` and `docprims-ooxml` are published as +components of `docprims` and carry no stability promise beyond what +`docprims` re-exports. ### Breaking Changes -- `docprims-cli` crate removed (use the `docprims` crate's `cli` feature). -- `--timeout-ms` / `ExtractOptions::timeout_ms` and `ExtractOptions` removed. -- `docprims_ooxml::common` is private. -- `DocprimsError`, `DocprimsQualityStatus` and `DocprimsContainerKind` are `#[non_exhaustive]`. +| Change | Migration | +|--------|-----------| +| `docprims-cli` crate removed | `cargo install docprims --features cli` (same binary name, commands and output) | +| `--timeout-ms` / `ExtractOptions::timeout_ms` removed (never applied) | Remove the flag or field | +| `ExtractOptions` removed from `docprims-core` | Use `ExtractLimits` | +| `docprims_ooxml::common` no longer public | Use `docprims::extract_*` | +| `DocprimsError`, `DocprimsQualityStatus`, `DocprimsContainerKind` are `#[non_exhaustive]` | Add a wildcard arm to matches | + +### Extraction Changes + +Extracted text may differ from v0.1.x for documents that contain: + +- control characters, DEL or C1 controls (now dropped); +- numeric character references in DOCX, XLSX or PPTX (now resolved); +- output truncated by `max_output_bytes` (no empty final block). + +Block byte ranges are computed after characters are dropped, so they always +index the emitted text. + +### Runtime Floors + +| Component | Floor | +|-----------|-------| +| Rust (MSRV) | 1.88.0 | +| TypeScript package | Node.js 22+ | +| npm trusted publishing | Node.js >=22.14.0 and npm >=11.5.1 | + +### Changelog -See `docs/releases/v0.2.0.md` for usage and migration. +See `CHANGELOG.md` for the full list of changes. --- diff --git a/docs/releases/v0.2.0.md b/docs/releases/v0.2.0.md index af518ed..7caf59a 100644 --- a/docs/releases/v0.2.0.md +++ b/docs/releases/v0.2.0.md @@ -1,9 +1,8 @@ -# docprims v0.2.0 +## v0.2.0 — 2026-09-25 -**Release Date:** 2026-09-25 **Status:** Minor Release (library entry point, extraction fixes, dependency refresh) -## Summary +### Summary v0.2.0 adds the `docprims` crate as the library's entry point, fixes several extraction edge cases, and updates dependencies and toolchains across the @@ -11,7 +10,7 @@ Rust crates and the C, Go and TypeScript bindings. It contains breaking changes for Rust users of the subcrates; bindings keep their results and error codes. -## Highlights +### Highlights - **`docprims` crate**: One dependency for all formats, with a feature per format. - **Extracted text character set**: Every format emits only XML 1.0 characters, excluding DEL and C1 controls. @@ -20,7 +19,7 @@ error codes. - **Output limits** never produce an empty block or a trailing separator. - **CLI** is installed with `cargo install docprims --features cli`. -## Using the `docprims` crate +### Using the `docprims` crate ```toml [dependencies] @@ -49,7 +48,7 @@ docprims = { version = "0.2", default-features = false, features = ["markdown"] components of `docprims` and carry no stability promise beyond what `docprims` re-exports. -## Breaking Changes +### Breaking Changes | Change | Migration | |--------|-----------| @@ -59,7 +58,7 @@ components of `docprims` and carry no stability promise beyond what | `docprims_ooxml::common` no longer public | Use `docprims::extract_*` | | `DocprimsError`, `DocprimsQualityStatus`, `DocprimsContainerKind` are `#[non_exhaustive]` | Add a wildcard arm to matches | -## Extraction Changes +### Extraction Changes Extracted text may differ from v0.1.x for documents that contain: @@ -70,7 +69,7 @@ Extracted text may differ from v0.1.x for documents that contain: Block byte ranges are computed after characters are dropped, so they always index the emitted text. -## Runtime Floors +### Runtime Floors | Component | Floor | |-----------|-------| @@ -78,6 +77,6 @@ index the emitted text. | TypeScript package | Node.js 22+ | | npm trusted publishing | Node.js >=22.14.0 and npm >=11.5.1 | -## Changelog +### Changelog See `CHANGELOG.md` for the full list of changes. From add0b4af308efba9ae4564a016ad2b1f177732e8 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:51:05 -0400 Subject: [PATCH 22/33] test(release): isolate crate name and empty plan controls Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- scripts/release-crates.test.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/release-crates.test.sh b/scripts/release-crates.test.sh index 467cb52..db37afe 100755 --- a/scripts/release-crates.test.sh +++ b/scripts/release-crates.test.sh @@ -60,8 +60,14 @@ for bad_names, bad_packages, bad_manifests in ( (names, publishable("docprims-ts-napi"), manifests), (names, packages, {**manifests, "docprims-core": {"package": {"publish": False}}}), (names, packages, {**manifests, "docprims": {"package": {}}}), - (["docprimsx"] + names[1:], packages, manifests), - ([], packages, manifests), + # A foreign crate, even if consistently listed and publishable. + ( + names + ["otherlib"], + packages + [{"name": "otherlib", "publish": None, "dependencies": []}], + {**manifests, "otherlib": {"package": {"publish": True}}}, + ), + # An empty plan, even when no crate is publishable. + ([], [{**p, "publish": []} for p in packages], manifests), ): try: module.validate(bad_packages, bad_names, bad_manifests) From 39ba5aa9308bd6315e3b2177291298b362cd89f1 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:51:34 -0400 Subject: [PATCH 23/33] test(release): isolate the moved-main and missing-section cases Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- scripts/check-release-notes.test.sh | 4 ++++ scripts/release-guard-tag-version.test.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/scripts/check-release-notes.test.sh b/scripts/check-release-notes.test.sh index 5bc2a40..3d8b11f 100755 --- a/scripts/check-release-notes.test.sh +++ b/scripts/check-release-notes.test.sh @@ -35,6 +35,10 @@ write_fixture "$section" "$section" expect_fail v1.1.0 expect_fail v1.3.0 expect_fail 1.2.0 +# A missing section must fail even against an empty per-cut file. +: >"$fixture/docs/releases/v1.2.0.md" +printf '# Release Notes\n\n---\n' >"$fixture/RELEASE_NOTES.md" +expect_fail v1.2.0 rm "$fixture/docs/releases/v1.2.0.md" expect_fail v1.2.0 echo '[ok] per-cut notes check accepts only the exact section' diff --git a/scripts/release-guard-tag-version.test.sh b/scripts/release-guard-tag-version.test.sh index 8a323c6..f3df586 100755 --- a/scripts/release-guard-tag-version.test.sh +++ b/scripts/release-guard-tag-version.test.sh @@ -63,6 +63,10 @@ git -C "$fixture" push -q --force origin refs/tags/v1.2.3 git -C "$fixture" update-ref refs/tags/v1.2.3 "$original_tag" expect_fail run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ DOCPRIMS_REQUIRE_TAG=1 ./guard.sh +# Restore the remote tag so the next case fails only because main moved. +git -C "$fixture" push -q --force origin refs/tags/v1.2.3 +run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ + DOCPRIMS_REQUIRE_TAG=1 ./guard.sh >/dev/null git -C "$fixture" checkout -q main printf 'next\n' >"$fixture/next" From 69f677df999cd66de2343e1b98709f1867648dee Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 07:51:55 -0400 Subject: [PATCH 24/33] test(release): cover a detached commit past the release tag Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- scripts/release-guard-tag-version.test.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/release-guard-tag-version.test.sh b/scripts/release-guard-tag-version.test.sh index f3df586..fbe5786 100755 --- a/scripts/release-guard-tag-version.test.sh +++ b/scripts/release-guard-tag-version.test.sh @@ -68,6 +68,13 @@ git -C "$fixture" push -q --force origin refs/tags/v1.2.3 run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ DOCPRIMS_REQUIRE_TAG=1 ./guard.sh >/dev/null +# A detached commit past the tag fails even though the tag still equals main. +printf 'detached\n' >"$fixture/detached" +git -C "$fixture" add detached +git -C "$fixture" commit -qm "detached" +expect_fail run_guard DOCPRIMS_RELEASE_TAG=v1.2.3 \ + DOCPRIMS_REQUIRE_TAG=1 ./guard.sh + git -C "$fixture" checkout -q main printf 'next\n' >"$fixture/next" git -C "$fixture" add next From 2486b958268a7e89811738f0e3cbe2fe065ab488 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:05:12 -0400 Subject: [PATCH 25/33] build(release): add signed tag, anchor and asset ceremony tooling Adapted from the storageprims release tooling. - Release tags are signed by a subkey of the committed public pin, bound to VERSION and origin/main, and checked against the tag ruleset; public fingerprint anchors are generated and verified with Decernor - The release asset set is defined by config/release/cli-platforms.txt; CLI archives must hold exactly the binary and both licenses as top-level regular files - make release runs the serialized signing ceremony against the exact unsigned draft; CI never holds signing keys - The release workflow can run manually to build and validate the full asset set without releasing; on a tag push it verifies the signed tag before opening the draft, and adds a CycloneDX SBOM - RELEASE_CHECKLIST.md and docs/cicd.md describe the flow Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .github/workflows/release.yml | 399 ++++++++++++------ Makefile | 164 +++---- RELEASE_CHECKLIST.md | 386 +++++++++++------ config/release/cli-platforms.txt | 6 + config/release/decernor-ci-sha256.txt | 1 + config/release/tagger-identity.txt | 1 + docs/cicd.md | 56 +++ .../release/fingerprint-record.v0.schema.json | 246 +++++++++++ scripts/check-draft-release.sh | 45 ++ scripts/download-release-assets.sh | 49 ++- scripts/export-release-keys.sh | 73 ++-- scripts/generate-checksums.sh | 81 ++-- scripts/install-release-anchors.sh | 43 ++ scripts/release-assets.test.sh | 174 ++++++++ scripts/release-checklist.test.sh | 14 - scripts/release-clean.sh | 28 ++ scripts/release-common.sh | 243 +++++++++++ scripts/release-decernor.sh | 50 +++ scripts/release-decernor.test.sh | 142 +++++++ scripts/release-github-state.test.sh | 109 +++++ scripts/release-guard-tag-ruleset.sh | 181 ++++++++ scripts/release-insert-anchors.sh | 50 +++ scripts/release-push-tag.sh | 10 + scripts/release-safety.test.sh | 27 ++ scripts/release-tag-body.sh | 23 + scripts/release-tag-common.sh | 187 ++++++++ scripts/release-tag-controls.test.sh | 138 ++++++ scripts/release-tag.sh | 19 + scripts/release-verify-remote-tag.sh | 16 + scripts/release-verify-tag.sh | 17 + scripts/sign-release-assets.sh | 109 +++-- scripts/stage-release-anchors.sh | 13 + scripts/upload-release-assets.sh | 86 ++-- scripts/validate-release-anchors.sh | 48 +++ scripts/validate-release-assets.sh | 39 ++ scripts/verify-checksums.sh | 50 +++ scripts/verify-pinned-tag.sh | 44 ++ scripts/verify-pinned-tag.test.sh | 135 ++++++ scripts/verify-public-keys.sh | 82 ++-- scripts/verify-signatures.sh | 95 ++--- 40 files changed, 3005 insertions(+), 674 deletions(-) create mode 100644 config/release/cli-platforms.txt create mode 100644 config/release/decernor-ci-sha256.txt create mode 100644 config/release/tagger-identity.txt create mode 100644 docs/cicd.md create mode 100644 schemas/release/fingerprint-record.v0.schema.json create mode 100755 scripts/check-draft-release.sh create mode 100755 scripts/install-release-anchors.sh create mode 100755 scripts/release-assets.test.sh delete mode 100755 scripts/release-checklist.test.sh create mode 100755 scripts/release-clean.sh create mode 100755 scripts/release-common.sh create mode 100644 scripts/release-decernor.sh create mode 100755 scripts/release-decernor.test.sh create mode 100755 scripts/release-github-state.test.sh create mode 100755 scripts/release-guard-tag-ruleset.sh create mode 100755 scripts/release-insert-anchors.sh create mode 100755 scripts/release-push-tag.sh create mode 100755 scripts/release-safety.test.sh create mode 100755 scripts/release-tag-body.sh create mode 100755 scripts/release-tag-common.sh create mode 100755 scripts/release-tag-controls.test.sh create mode 100755 scripts/release-tag.sh create mode 100755 scripts/release-verify-remote-tag.sh create mode 100755 scripts/release-verify-tag.sh create mode 100755 scripts/stage-release-anchors.sh create mode 100755 scripts/validate-release-anchors.sh create mode 100755 scripts/validate-release-assets.sh create mode 100755 scripts/verify-checksums.sh create mode 100755 scripts/verify-pinned-tag.sh create mode 100755 scripts/verify-pinned-tag.test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32e36b2..94eec0f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,35 +1,92 @@ +# Build the exact release asset set. A tag push opens an unsigned draft +# release; a manual run builds and validates the same set without releasing. +# Signing keys remain local to the maintainer ceremony. name: Release on: push: tags: ['v*'] + workflow_dispatch: +permissions: + contents: read +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false env: CARGO_TERM_COLOR: always MACOSX_DEPLOYMENT_TARGET: "11.0" jobs: - validate-version: - name: Validate Version + validate: + name: Validate version and package set runs-on: ubuntu-latest outputs: - version: ${{ steps.version.outputs.version }} + commit: ${{ steps.release.outputs.commit }} + version: ${{ steps.release.outputs.version }} + defaults: + run: + shell: bash steps: - - uses: actions/checkout@v7 - - name: Validate VERSION matches tag - id: version + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + # The strict tag guard fetches origin/main in this read-only job. + persist-credentials: true + - name: Install Rust 1.94.1 + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 + - name: Bind annotated tag to main + if: github.event_name == 'push' + env: + DOCPRIMS_RELEASE_TAG: ${{ github.ref_name }} + DOCPRIMS_REQUIRE_TAG: '1' + run: ./scripts/release-guard-tag-version.sh + - name: Record commit and version + id: release run: | - FILE_VERSION=$(cat VERSION) - TAG_VERSION=${GITHUB_REF_NAME#v} - if [ "$FILE_VERSION" != "$TAG_VERSION" ]; then - echo "VERSION mismatch: file=$FILE_VERSION tag=$TAG_VERSION" - exit 1 - fi - echo "version=$FILE_VERSION" >> "$GITHUB_OUTPUT" - echo "Version validated: $FILE_VERSION" + set -euo pipefail + ./scripts/release-guard-tag-version.sh + printf 'commit=%s\n' "$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT" + printf 'version=%s\n' "$(cat VERSION)" >>"$GITHUB_OUTPUT" + - name: Verify package archives without publishing + run: make release-check + verify-signature: + name: Verify pinned tag signature and publication policy + if: github.event_name == 'push' + needs: validate + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Checkout exact signed tag and committed pin + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Verify committed pin and GitHub signature + env: + GH_TOKEN: ${{ github.token }} + DOCPRIMS_RELEASE_TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + ./scripts/verify-pinned-tag.sh + EXPECTED_OBJECT="$(git rev-parse "refs/tags/$DOCPRIMS_RELEASE_TAG")" + tag_json="$(gh api "repos/3leaps/docprims/git/tags/$EXPECTED_OBJECT")" + jq -e --arg tag "$DOCPRIMS_RELEASE_TAG" --arg commit "$(git rev-parse HEAD)" \ + '.tag == $tag and .object.type == "commit" and .object.sha == $commit and + .verification.verified == true and .verification.reason == "valid"' \ + <<<"$tag_json" >/dev/null + ./scripts/release-guard-tag-ruleset.sh --read-only --verify-tag-attestation "$EXPECTED_OBJECT" verify-go-bindings-assets: name: Verify Go Binding Assets + if: github.event_name == 'push' runs-on: ubuntu-latest - needs: validate-version + needs: validate steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Verify Go module contains prebuilt libs run: | set -euo pipefail @@ -88,37 +145,40 @@ jobs: echo "Go binding assets verified." build-linux: - name: Build Linux + name: CLI (${{ matrix.platform }}) runs-on: ubuntu-latest container: # goneat-tools-runner-glibc v0.5.6 image: ghcr.io/fulmenhq/goneat-tools-runner-glibc@sha256:afcffa00283c765928dfef40966c282f2ca190653cf4443f634cb8f67c1d51ab options: --user root - needs: validate-version + needs: validate strategy: + fail-fast: false matrix: include: - target: x86_64-unknown-linux-gnu glibc: "2.17" - artifact_suffix: linux-amd64 + platform: linux-amd64 - target: x86_64-unknown-linux-musl glibc: "" - artifact_suffix: linux-amd64-musl + platform: linux-amd64-musl - target: aarch64-unknown-linux-gnu glibc: "2.17" - artifact_suffix: linux-arm64 + platform: linux-arm64 - target: aarch64-unknown-linux-musl glibc: "" - artifact_suffix: linux-arm64-musl + platform: linux-arm64-musl steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Build CLI run: | + target="${{ matrix.target }}" if [ -n "${{ matrix.glibc }}" ]; then - cargo zigbuild --release --target ${{ matrix.target }}.${{ matrix.glibc }} -p docprims --features cli - else - cargo zigbuild --release --target ${{ matrix.target }} -p docprims --features cli + target="${target}.${{ matrix.glibc }}" fi + cargo zigbuild --release --locked --target "$target" -p docprims --features cli - name: Package CLI shell: bash run: | @@ -146,48 +206,71 @@ jobs: mkdir -p dist/cli cp "target/${TARGET_DIR}/release/docprims" dist/cli/docprims cp LICENSE-MIT LICENSE-APACHE dist/cli/ - VERSION=${{ needs.validate-version.outputs.version }} - tar -czf docprims-${VERSION}-${{ matrix.artifact_suffix }}.tar.gz -C dist/cli . + VERSION=${{ needs.validate.outputs.version }} + tar -czf "docprims-${VERSION}-${{ matrix.platform }}.tar.gz" -C dist/cli \ + docprims LICENSE-APACHE LICENSE-MIT - name: Upload CLI artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-${{ matrix.artifact_suffix }} - path: docprims-${{ needs.validate-version.outputs.version }}-${{ matrix.artifact_suffix }}.tar.gz + name: cli-${{ matrix.platform }} + path: docprims-${{ needs.validate.outputs.version }}-${{ matrix.platform }}.tar.gz + if-no-files-found: error + retention-days: 7 build-macos: - name: Build macOS + name: CLI (${{ matrix.platform }}) runs-on: macos-14 - needs: validate-version - env: - MACOSX_DEPLOYMENT_TARGET: "11.0" + needs: validate + strategy: + matrix: + include: + - target: aarch64-apple-darwin + platform: darwin-arm64 steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: - targets: aarch64-apple-darwin + toolchain: 1.94.1 + targets: ${{ matrix.target }} - name: Build CLI - run: cargo build --release --target aarch64-apple-darwin -p docprims --features cli --locked + run: cargo build --release --target ${{ matrix.target }} -p docprims --features cli --locked - name: Package CLI run: | + set -euo pipefail mkdir -p dist/cli - cp target/aarch64-apple-darwin/release/docprims dist/cli/docprims + cp target/${{ matrix.target }}/release/docprims dist/cli/docprims cp LICENSE-MIT LICENSE-APACHE dist/cli/ - tar -czf docprims-${{ needs.validate-version.outputs.version }}-darwin-arm64.tar.gz -C dist/cli . + # No AppleDouble or extended-attribute entries in the archive. + COPYFILE_DISABLE=1 tar --no-mac-metadata -czf \ + "docprims-${{ needs.validate.outputs.version }}-${{ matrix.platform }}.tar.gz" -C dist/cli \ + docprims LICENSE-APACHE LICENSE-MIT - name: Upload CLI artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-darwin-arm64 - path: docprims-${{ needs.validate-version.outputs.version }}-darwin-arm64.tar.gz + name: cli-${{ matrix.platform }} + path: docprims-${{ needs.validate.outputs.version }}-${{ matrix.platform }}.tar.gz + if-no-files-found: error + retention-days: 7 build-windows: - name: Build Windows + name: CLI (${{ matrix.platform }}) runs-on: windows-latest - needs: validate-version + needs: validate + strategy: + matrix: + include: + - target: x86_64-pc-windows-gnu + platform: windows-amd64 steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - targets: x86_64-pc-windows-gnu + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 + targets: ${{ matrix.target }} - name: Install MinGW - uses: msys2/setup-msys2@v2 + uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 with: msystem: MINGW64 update: true @@ -195,19 +278,23 @@ jobs: path-type: inherit - name: Build CLI (GNU target) shell: msys2 {0} - run: cargo build --release --target x86_64-pc-windows-gnu -p docprims --features cli --locked + run: cargo build --release --target ${{ matrix.target }} -p docprims --features cli --locked - name: Package CLI shell: bash run: | + set -euo pipefail mkdir -p dist/cli - cp target/x86_64-pc-windows-gnu/release/docprims.exe dist/cli/docprims.exe + cp target/${{ matrix.target }}/release/docprims.exe dist/cli/docprims.exe cp LICENSE-MIT LICENSE-APACHE dist/cli/ - cd dist/cli && 7z a -tzip ../../docprims-${{ needs.validate-version.outputs.version }}-windows-amd64.zip . + cd dist/cli && 7z a -tzip "../../docprims-${{ needs.validate.outputs.version }}-${{ matrix.platform }}.zip" \ + docprims.exe LICENSE-APACHE LICENSE-MIT - name: Upload CLI artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: cli-windows-amd64 - path: docprims-${{ needs.validate-version.outputs.version }}-windows-amd64.zip + name: cli-${{ matrix.platform }} + path: docprims-${{ needs.validate.outputs.version }}-${{ matrix.platform }}.zip + if-no-files-found: error + retention-days: 7 cbindgen: name: Generate C Header runs-on: ubuntu-latest @@ -215,95 +302,145 @@ jobs: # goneat-tools-runner-glibc v0.5.6 image: ghcr.io/fulmenhq/goneat-tools-runner-glibc@sha256:afcffa00283c765928dfef40966c282f2ca190653cf4443f634cb8f67c1d51ab options: --user root - needs: validate-version + needs: validate steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Generate C header run: cbindgen --config cbindgen.toml --crate docprims-ffi --output docprims.h - name: Upload header - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: c-header path: docprims.h - publish: - name: Publish Release + if-no-files-found: error + retention-days: 7 + sbom: + name: CycloneDX SBOM + needs: validate runs-on: ubuntu-latest - needs: - - validate-version - - verify-go-bindings-assets - - build-linux - - build-macos - - build-windows - - cbindgen - permissions: - contents: write + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Generate SBOM with pinned Syft image + run: | + set -euo pipefail + docker run --rm \ + --volume "$PWD:/src" \ + --workdir /src \ + anchore/syft@sha256:f94e5d9fce1f2278491a8e3a63bd5f6ddb81fdfdbb8bf7a1637565c1d5344357 \ + dir:. \ + --output "cyclonedx-json=sbom-${{ needs.validate.outputs.version }}.cdx.json" + test -s "sbom-${{ needs.validate.outputs.version }}.cdx.json" + - name: Upload SBOM + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sbom + path: sbom-${{ needs.validate.outputs.version }}.cdx.json + if-no-files-found: error + retention-days: 7 + assemble: + name: Assemble and validate the exact asset set + needs: [validate, build-linux, build-macos, build-windows, cbindgen, sbom] + runs-on: ubuntu-latest + defaults: + run: + shell: bash steps: - - uses: actions/checkout@v7 - - name: Download all CLI artifacts - uses: actions/download-artifact@v8 + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Download CLI archives + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: cli-* - path: release-cli + path: dist/release + merge-multiple: true - name: Download C header - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: c-header - path: release-header - - name: Organize release files + path: dist/release + - name: Download SBOM + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: sbom + path: dist/release + - name: Assert exact unsigned inventory + env: + DOCPRIMS_RELEASE_TAG: v${{ needs.validate.outputs.version }} run: | - mkdir -p release - - # CLI archives - find release-cli -type f \( -name '*.tar.gz' -o -name '*.zip' \) -exec mv {} release/ \; - - # Header - mv release-header/docprims.h release/ - - # Licenses - cp LICENSE-MIT LICENSE-APACHE release/ - - ls -la release/ - - name: Create Release - uses: softprops/action-gh-release@v3 + set -euo pipefail + cp LICENSE-MIT LICENSE-APACHE dist/release/ + ./scripts/validate-release-assets.sh dist/release base + - name: Upload release asset set + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-assets + path: dist/release + if-no-files-found: error + retention-days: 7 + draft: + name: Create exact unsigned draft + if: github.event_name == 'push' + needs: [validate, verify-signature, verify-go-bindings-assets, assemble] + runs-on: ubuntu-latest + permissions: + contents: write + defaults: + run: + shell: bash + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Download release asset set + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-assets + path: dist/release + - name: Assert exact unsigned inventory + env: + GITHUB_TOKEN: ${{ github.token }} + DOCPRIMS_RELEASE_TAG: ${{ github.ref_name }} + DOCPRIMS_REQUIRE_TAG: '1' + run: | + set -euo pipefail + auth_header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GITHUB_TOKEN" | base64 | tr -d '\n')" + git config --local http.https://github.com/.extraheader "$auth_header" + cleanup_credentials() { + git config --local --unset-all http.https://github.com/.extraheader || true + } + trap cleanup_credentials EXIT + ./scripts/release-guard-tag-version.sh + cleanup_credentials + trap - EXIT + ./scripts/validate-release-assets.sh dist/release base + - name: Refuse unsafe existing release state + env: + GH_TOKEN: ${{ github.token }} + DOCPRIMS_RELEASE_TAG: ${{ github.ref_name }} + run: ./scripts/check-draft-release.sh before + - name: Create draft GitHub release + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: - files: release/* - generate_release_notes: true draft: true - body: | - ## docprims v${{ needs.validate-version.outputs.version }} - - GPL-free document text extraction primitives for Rust with Go bindings. - - ### Installation - - Download the appropriate archive for your platform and extract. - - ### Verification - - **Note:** This release is unsigned. Signed checksums will be uploaded after manual signing. - - Once signed artifacts are available: - ```bash - # Verify checksum signature - minisign -Vm SHA256SUMS -p docprims-minisign.pub - - # Verify file checksums - shasum -a 256 -c SHA256SUMS - ``` - - ### Platform Matrix - - | Platform | CLI | Go Bindings | - |----------|-----|-------------| - | Linux x64 (glibc 2.17+) | `docprims-*-linux-amd64.tar.gz` | Supported | - | Linux x64 (musl) | `docprims-*-linux-amd64-musl.tar.gz` | Supported | - | Linux arm64 (glibc 2.17+) | `docprims-*-linux-arm64.tar.gz` | Supported | - | Linux arm64 (musl) | `docprims-*-linux-arm64-musl.tar.gz` | Supported | - | macOS arm64 | `docprims-*-darwin-arm64.tar.gz` | Supported | - | Windows x64 | `docprims-*-windows-amd64.zip` | Supported | - - ### Go Bindings - - ```bash - go get github.com/3leaps/docprims/bindings/go/docprims@v${{ needs.validate-version.outputs.version }} - ``` + prerelease: false + target_commitish: ${{ needs.validate.outputs.commit }} + body_path: docs/releases/${{ github.ref_name }}.md + files: | + dist/release/* + - name: Verify exact draft state and remote inventory + env: + GH_TOKEN: ${{ github.token }} + DOCPRIMS_RELEASE_TAG: ${{ github.ref_name }} + run: ./scripts/check-draft-release.sh after diff --git a/Makefile b/Makefile index 0bdee83..1432c51 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ .PHONY: all help bootstrap bootstrap-force tools check test fmt lint build clean version install .PHONY: precommit prepush deps-check audit deny miri msrv fmt-check npm-publish-prereqs-check .PHONY: build-release build-ffi cbindgen -.PHONY: release-clean release-download release-checksums release-sign +.PHONY: release-clean release-download release-checksums release-sign release-tag release-push-tag +.PHONY: release-verify-tag release-verify-remote-tag release-insert-anchors .PHONY: release-export-keys release-verify-checksums release-verify-signatures .PHONY: release-verify-keys release-notes release-upload release .PHONY: build-local-go go-test go-test-shared @@ -24,6 +25,8 @@ # Configuration # ----------------------------------------------------------------------------- +.NOTPARALLEL: release + # Version from VERSION file (SSOT) VERSION := $(shell cat VERSION 2>/dev/null || echo "dev") @@ -437,16 +440,22 @@ release-crates-verify: ## Wait for the released version on crates.io and verify release-guard-tag-version: ## Validate the canonical release tag @./scripts/release-guard-tag-version.sh -release-tooling-test: ## Run release guard, crate, version and notes tests +release-tooling-test: ## Run release guard, signing, asset, crate, version and notes tests + @./scripts/release-decernor.test.sh + @./scripts/release-tag-controls.test.sh + @./scripts/verify-pinned-tag.test.sh @./scripts/release-crates.test.sh @./scripts/release-crates-verify.test.sh @./scripts/release-guard-tag-version.test.sh + @./scripts/release-assets.test.sh + @./scripts/release-github-state.test.sh + @./scripts/release-safety.test.sh @./scripts/check-version.test.sh @./scripts/check-release-notes.test.sh - @./scripts/release-checklist.test.sh @echo "[ok] Release tooling tests passed" release-preflight: ## Verify clean-tree pre-tag requirements + @./scripts/validate-release-anchors.sh @echo "Running release preflight checks..." @if [ -n "$$(git status --porcelain 2>/dev/null)" ]; then \ echo "[!!] Working tree not clean - commit or stash changes first"; \ @@ -468,91 +477,82 @@ release-preflight: ## Verify clean-tree pre-tag requirements @echo "[ok] All preflight checks passed - ready to tag v$(VERSION)" # ----------------------------------------------------------------------------- -# Release Workflow +# Release Ceremony (maintainer, local keys; see RELEASE_CHECKLIST.md) # ----------------------------------------------------------------------------- # -# Manual signing workflow (CI builds unsigned, maintainer signs locally): -# -# 1. CI creates draft release on tag push (release workflow) -# 2. Download artifacts: make release-download -# 3. Generate checksums: make release-checksums -# 4. Sign checksums: make release-sign (requires DOCPRIMS_MINISIGN_KEY) -# 5. Export public keys: make release-export-keys -# 6. Verify everything: make release-verify -# 7. Upload signed artifacts: make release-upload - -DIST_RELEASE := dist/release -DOCPRIMS_RELEASE_TAG ?= $(shell git describe --tags --abbrev=0 2>/dev/null || echo v$(shell cat VERSION)) - -# Signing keys (set these environment variables) -DOCPRIMS_MINISIGN_KEY ?= -DOCPRIMS_MINISIGN_PUB ?= -DOCPRIMS_PGP_KEY_ID ?= -DOCPRIMS_GPG_HOMEDIR ?= - -release-clean: ## Remove dist/release contents - @echo "Cleaning release directory..." - rm -rf $(DIST_RELEASE) - @echo "[ok] Release directory cleaned" - -release-download: ## Download release assets from GitHub - @if [ -z "$(DOCPRIMS_RELEASE_TAG)" ] || [ "$(DOCPRIMS_RELEASE_TAG)" = "v" ]; then \ - echo "Error: No release tag found. Set DOCPRIMS_RELEASE_TAG=vX.Y.Z"; \ - exit 1; \ - fi - ./scripts/download-release-assets.sh $(DOCPRIMS_RELEASE_TAG) $(DIST_RELEASE) +# CI builds the exact unsigned asset set and opens a draft release on a signed +# tag push. The maintainer signs checksum manifests locally and uploads them. +# Every target takes the tag from DOCPRIMS_RELEASE_TAG only. -release-checksums: ## Generate SHA256SUMS and SHA512SUMS - ./scripts/generate-checksums.sh $(DIST_RELEASE) +RELEASE_DIR := $(CURDIR)/dist/release -release-sign: ## Sign checksum manifests (requires DOCPRIMS_MINISIGN_KEY) - @if [ -z "$(DOCPRIMS_MINISIGN_KEY)" ]; then \ - echo "Error: DOCPRIMS_MINISIGN_KEY not set"; \ - echo ""; \ - echo "Set the path to your minisign secret key:"; \ - echo " export DOCPRIMS_MINISIGN_KEY=/path/to/docprims.key"; \ - exit 1; \ - fi - DOCPRIMS_MINISIGN_KEY=$(DOCPRIMS_MINISIGN_KEY) \ - DOCPRIMS_PGP_KEY_ID=$(DOCPRIMS_PGP_KEY_ID) \ - DOCPRIMS_GPG_HOMEDIR=$(DOCPRIMS_GPG_HOMEDIR) \ - ./scripts/sign-release-assets.sh $(DOCPRIMS_RELEASE_TAG) $(DIST_RELEASE) - -release-export-keys: ## Export public signing keys - DOCPRIMS_MINISIGN_KEY=$(DOCPRIMS_MINISIGN_KEY) \ - DOCPRIMS_MINISIGN_PUB=$(DOCPRIMS_MINISIGN_PUB) \ - DOCPRIMS_PGP_KEY_ID=$(DOCPRIMS_PGP_KEY_ID) \ - DOCPRIMS_GPG_HOMEDIR=$(DOCPRIMS_GPG_HOMEDIR) \ - ./scripts/export-release-keys.sh $(DIST_RELEASE) - -release-verify-checksums: ## Verify checksums match artifacts - @echo "Verifying checksums..." - cd $(DIST_RELEASE) && shasum -a 256 -c SHA256SUMS - @echo "[ok] Checksums verified" - -release-verify-signatures: ## Verify minisign/PGP signatures - ./scripts/verify-signatures.sh $(DIST_RELEASE) - -release-verify-keys: ## Verify exported keys are public-only - ./scripts/verify-public-keys.sh $(DIST_RELEASE) - -release-verify: release-verify-checksums release-verify-signatures release-verify-keys ## Run all release verification - @echo "[ok] All release verifications passed" - -release-notes: ## Copy release notes to dist - @src="docs/releases/$(DOCPRIMS_RELEASE_TAG).md"; \ - if [ -f "$$src" ]; then \ - cp "$$src" "$(DIST_RELEASE)/release-notes-$(DOCPRIMS_RELEASE_TAG).md"; \ - echo "[ok] Copied release notes"; \ - else \ - echo "[--] No release notes found at $$src"; \ - fi +release-tag: ## Create and verify a local signed version tag + @./scripts/release-tag.sh + +release-push-tag: ## Publish and verify the signed version tag + @./scripts/release-push-tag.sh + +release-verify-tag: ## Verify the tag using only the committed public pin + @./scripts/release-verify-tag.sh + +release-verify-remote-tag: ## Compare local and remote tag objects and GitHub verification + @./scripts/release-verify-remote-tag.sh + +release-insert-anchors: ## Maintainer-only: generate and review public fingerprint anchors + @./scripts/release-insert-anchors.sh + +release-clean: ## Safely empty the repository release staging directory + @./scripts/release-clean.sh "$(RELEASE_DIR)" + +release-download: ## Download exact unsigned assets from the trusted draft + @./scripts/download-release-assets.sh "$(RELEASE_DIR)" + +release-notes: ## Add the exact per-cut notes to the signable asset set + @DOCPRIMS_REQUIRE_TAG=1 ./scripts/release-guard-tag-version.sh >/dev/null + @test -f "docs/releases/$${DOCPRIMS_RELEASE_TAG}.md" || \ + { echo "[!!] Exact per-cut release notes are missing"; exit 1; } + @./scripts/validate-release-assets.sh "$(RELEASE_DIR)" base >/dev/null + @cp "docs/releases/$${DOCPRIMS_RELEASE_TAG}.md" \ + "$(RELEASE_DIR)/release-notes-$${DOCPRIMS_RELEASE_TAG}.md" + @./scripts/stage-release-anchors.sh "$(RELEASE_DIR)" + @./scripts/validate-release-assets.sh "$(RELEASE_DIR)" signable >/dev/null + @echo "[ok] Per-cut release notes added to the signed set" + +release-checksums: ## Generate exact SHA256 and SHA512 manifests + @./scripts/generate-checksums.sh "$(RELEASE_DIR)" + +release-sign: ## Sign checksum manifests with local MFA-held keys + @./scripts/sign-release-assets.sh "$(RELEASE_DIR)" + +release-export-keys: ## Export and prove public verification material + @./scripts/export-release-keys.sh "$(RELEASE_DIR)" + +release-verify-checksums: ## Verify exact dual checksum manifests + @./scripts/verify-checksums.sh "$(RELEASE_DIR)" + +release-verify-signatures: ## Verify every configured signature + @./scripts/verify-signatures.sh "$(RELEASE_DIR)" + +release-verify-keys: ## Verify exported public keys against staged anchors + @./scripts/verify-public-keys.sh "$(RELEASE_DIR)" + +release-verify: release-verify-checksums release-verify-signatures release-verify-keys ## Verify signed release set + @./scripts/validate-release-assets.sh "$(RELEASE_DIR)" signed >/dev/null + @echo "[ok] Signed release set verified" -release-upload: release-verify release-notes ## Upload signed artifacts to GitHub release - ./scripts/upload-release-assets.sh $(DOCPRIMS_RELEASE_TAG) $(DIST_RELEASE) +release-upload: ## Verify once and update the exact trusted draft release + @./scripts/upload-release-assets.sh "$(RELEASE_DIR)" -release: release-clean release-download release-checksums release-sign release-export-keys release-upload ## Full release workflow (after CI build) - @echo "[ok] Release $(DOCPRIMS_RELEASE_TAG) complete" +release: release-guard-tag-version ## Run the serialized local signing ceremony + @DOCPRIMS_REQUIRE_TAG=1 ./scripts/release-guard-tag-version.sh >/dev/null + @$(MAKE) release-clean + @$(MAKE) release-download + @$(MAKE) release-notes + @$(MAKE) release-checksums + @$(MAKE) release-sign + @$(MAKE) release-export-keys + @$(MAKE) release-upload + @echo "[ok] Release assets signed and uploaded; GitHub release remains draft" # ----------------------------------------------------------------------------- # Version Management diff --git a/RELEASE_CHECKLIST.md b/RELEASE_CHECKLIST.md index 1147723..34c0c4c 100644 --- a/RELEASE_CHECKLIST.md +++ b/RELEASE_CHECKLIST.md @@ -1,58 +1,72 @@ # Release Checklist -This document walks maintainers through the build/sign/upload flow for each docprims release. - -docprims uses the same high-level release posture as sysprims: - -- CI builds unsigned artifacts. -- Maintainers sign checksum manifests locally and upload signatures + public keys. +This checklist covers release preparation, the signed tag and unsigned +draft, the maintainer MFA signing ceremony, and npm and crates.io +publication. CI never receives signing keys and never publishes a GitHub +release. Earlier unsigned annotated tags remain historical; signing begins +with v0.2.0. ## Prerequisites -- `gh` CLI authenticated with push access -- `minisign` installed (required) -- `gpg` installed (optional; only if you also PGP sign) -- Signing keys configured and available on your machine - -## 1. Pre-Release Preparation - -### Code Quality Gates - -- [ ] Ensure `main` is clean: `git status` shows no uncommitted changes -- [ ] Run quality gates: `make check` -- [ ] Run full tests: `cargo test --workspace` -- [ ] Verify cargo-deny passes: `cargo deny check` - -### Version & Documentation +- `make bootstrap` and `make tools` are green +- `gh`, minisign, and optional GPG tooling are available +- `gh` is authenticated for `3leaps/docprims` +- Approved signing material is loaded from an external secret store without + printing values or paths + +Required ceremony variables: + +- `DOCPRIMS_RELEASE_TAG` — the sole canonical tag input, for example + `v0.2.0` when `VERSION` contains `0.2.0` +- `DOCPRIMS_MINISIGN_KEY` — minisign secret-key file outside the repository +- `DOCPRIMS_MINISIGN_PUB` — explicit minisign public-key file +- `DOCPRIMS_DECERNOR_BIN` — absolute executable Decernor v0.1.8+ binary + selected by the maintainer; self-reported identity does not authenticate + the file, so bind it to a trusted tag-built regular file (not a symlink) +- `DOCPRIMS_TAG_MESSAGE_DIR` — external per-cut directory ending in the + canonical tag; its only message input is `message.txt` +- `DOCPRIMS_TAGGER_NAME` / `DOCPRIMS_TAGGER_EMAIL` — fixed infosec + tagger identity (`3 Leaps Infosec Team `) +- `DOCPRIMS_GPG_SIGNING_FINGERPRINT` — full 40-hex authorized primary + fingerprint; `DOCPRIMS_PGP_KEY_ID` selects an exact 40-hex signing + subkey with a trailing `!` + +GPG tag signing requires `DOCPRIMS_PGP_KEY_ID` and +`DOCPRIMS_GPG_HOMEDIR`. Both manifest signatures are made during the +subsequent maintainer ceremony. Partial configuration fails closed. + +## 1. Write and prepare - [ ] Set the version everywhere: `make version-set V=X.Y.Z` (writes `VERSION`, `Cargo.toml`, `Cargo.lock` and the npm manifests), then `make version-check` +- [ ] Check release-tooling fixtures for hard-coded prior-version values; + run `make release-tooling-test` and confirm mismatched tags fail - [ ] Move `CHANGELOG.md` `[Unreleased]` entries under `## [X.Y.Z] - YYYY-MM-DD` -- [ ] Add `## vX.Y.Z — YYYY-MM-DD` to `RELEASE_NOTES.md` and copy that exact - section to `docs/releases/vX.Y.Z.md` (`scripts/check-release-notes.sh vX.Y.Z`) -- [ ] Package the publishable crates without publishing: `make release-check` - -### Commit & Push - -- [ ] Commit changes: - ```bash - git add -A - git commit -m "release: prepare vX.Y.Z" - ``` -- [ ] Push to main: - ```bash - git push origin main - ``` - -- [ ] Verify local/remote sync (required before running workflows): - ```bash - git fetch origin - git log --oneline origin/main..HEAD - git log --oneline HEAD..origin/main - ``` - Both commands must show no output. - -- [ ] Run the pre-tag gate on the merged `main`: `make release-preflight` +- [ ] Add `## vX.Y.Z — YYYY-MM-DD` to `RELEASE_NOTES.md` (newest first, at most + three cuts) and copy that exact section to `docs/releases/vX.Y.Z.md` +- [ ] Confirm README MSRV prose matches `[workspace.package].rust-version` +- [ ] Run `make prepush` and `make release-check` +- [ ] Commit the release preparation and merge it through the normal review + process; confirm required CI on `main` is green +- [ ] Prove the release build matrix without releasing: run the Release + workflow manually on `main` (`gh workflow run release.yml --ref main`) + and confirm its `assemble` job validates the exact asset set +- [ ] Merge the release-tag and anchor tooling through a reviewed PR **before** + adding its public pin and anchors. A disposable worktree of the local + tooling branch may be used to rehearse the steps below; perform the real + public-pin/anchor change on a separate branch from updated `main` and + review it in a separate PR before tagging +- [ ] Confirm the reviewed public pin and decernor-generated anchor pair are + committed; Dave generates them with `make release-insert-anchors` using + `DOCPRIMS_DECERNOR_BIN` set to an absolute executable Decernor + v0.1.8 or newer. The ceremony never falls back to `DECERNOR_BIN` or PATH; + generation verifies the installed pair against both public exports +- [ ] Complete the Go bindings prep below and merge its PR +- [ ] From a clean, freshly fetched `main`, run `make release-preflight` + +The preflight requires a clean tree, the full `make prepush` gate, exact +release-note extraction, a successful fetch, and exact equality between +`HEAD` and fetched `origin/main`. ### Go Bindings Prep (Required) @@ -82,31 +96,154 @@ Go bindings must be fetchable via `go get` without requiring Rust. make go-test-shared # shared (from repo root) ``` -### Create and Push Tags +### Initial public pin and anchors (maintainer only) + +Export the **existing approved key's public portion** into the repository for +review; this does not generate a new key or change the key already registered +on GitHub. GitHub's Verified badge is a secondary check, while the committed +pin authorizes the signing key used by CI. Run the commands from the repository +root on the separate public-pin/anchor branch in a fresh shell. Set the intended +release tag by hand +**before** loading the approved external environment so its per-cut message +directory is derived from the intended tag. +Confirm the external message directory ends in that tag. Do not use an ambient +GPG home: + +```bash +export DOCPRIMS_RELEASE_TAG=v0.2.0 # example: first signed cut; set each cut explicitly +# Load the approved external environment for this repository before continuing. +( + set -euo pipefail # a failed guard stops this entire block, even in an interactive shell + : "${DOCPRIMS_RELEASE_TAG:?set the approved tag for this cut}" + : "${DOCPRIMS_GPG_HOMEDIR:?load the approved GPG home}" + : "${DOCPRIMS_PGP_KEY_ID:?load the exact signing-subkey selector}" + : "${DOCPRIMS_GPG_SIGNING_FINGERPRINT:?load the approved primary fingerprint}" + : "${DOCPRIMS_DECERNOR_BIN:?load the trusted Decernor executable}" + : "${DOCPRIMS_TAG_MESSAGE_DIR:?load the per-cut message directory}" + : "${DOCPRIMS_MINISIGN_PUB:?load the approved minisign public export}" + [[ "$DOCPRIMS_GPG_HOMEDIR" == /* && -d "$DOCPRIMS_GPG_HOMEDIR" ]] + [[ "$DOCPRIMS_DECERNOR_BIN" == /* && -f "$DOCPRIMS_DECERNOR_BIN" && + ! -L "$DOCPRIMS_DECERNOR_BIN" && -x "$DOCPRIMS_DECERNOR_BIN" ]] + python3 - "$DOCPRIMS_GPG_HOMEDIR" "$HOME/.gnupg" <<'PY' +from pathlib import Path +import sys +if Path(sys.argv[1]).resolve() == Path(sys.argv[2]).resolve(): + raise SystemExit('STOP: default GPG home is not approved for this ceremony') +PY + [[ "$DOCPRIMS_PGP_KEY_ID" == *'!' ]] + [[ "${DOCPRIMS_TAG_MESSAGE_DIR%/}" == */"$DOCPRIMS_RELEASE_TAG" ]] + bash -c 'source scripts/release-decernor.sh; resolve_release_decernor ceremony' + pin=docs/security/release-signing-keys.asc + [[ ! -e "$pin" && ! -L "$pin" ]] || { echo 'STOP: public pin already exists' >&2; exit 1; } + set -C # never overwrite a public pin + gpg --homedir "$DOCPRIMS_GPG_HOMEDIR" --batch --armor \ + --export "$DOCPRIMS_PGP_KEY_ID" > "$pin" + require_public_only() { + if "$DOCPRIMS_DECERNOR_BIN" fingerprint "$1" --kind "$2" \ + --class private --fail-on-empty --path-mode none >/dev/null; then + echo 'STOP: private material in public export' >&2; exit 1 + else + decernor_rc=$? + [[ "$decernor_rc" -eq 3 ]] || { echo 'STOP: private-material check failed' >&2; exit 1; } + fi + } + require_public_only "$pin" gpg + [[ -s "$DOCPRIMS_MINISIGN_PUB" && ! -L "$DOCPRIMS_MINISIGN_PUB" ]] + require_public_only "$DOCPRIMS_MINISIGN_PUB" minisign + grep -q '^untrusted comment:' "$DOCPRIMS_MINISIGN_PUB" + verify_tmp="$(mktemp -d)" + trap 'rm -rf "$verify_tmp"' EXIT + "$DOCPRIMS_DECERNOR_BIN" fingerprint "$pin" --kind gpg \ + --class public --fail-on-empty --path-mode none >"$verify_tmp/gpg.ndjson" + python3 - "$verify_tmp/gpg.ndjson" "$DOCPRIMS_GPG_SIGNING_FINGERPRINT" \ + "$DOCPRIMS_PGP_KEY_ID" <<'PY' +import json +import pathlib +import re +import sys +records = [json.loads(line) for line in pathlib.Path(sys.argv[1]).read_text().splitlines()] +primary, selector = sys.argv[2:] +if not re.fullmatch('[0-9A-F]{40}', primary) or not re.fullmatch('[0-9A-F]{40}!', selector): + raise SystemExit('STOP: invalid configured fingerprints') +if len(records) != 2 or any(r.get('kind') != 'gpg' or r.get('class') != 'public' for r in records): + raise SystemExit('STOP: expected exactly one public primary and signing subkey') +by_role = {r.get('key_role'): r.get('fingerprint') for r in records} +if by_role != {'primary': primary, 'subkey': selector[:-1]}: + raise SystemExit('STOP: exported public key differs from approved selector') +PY + gpg --homedir "$verify_tmp" --batch --show-keys \ + --fingerprint --with-subkey-fingerprint "$pin" # inspect readable expiry/revocation +) +``` -Create and push tags that point to the SAME commit (the commit that includes the merged Go bindings PR). +The `!` selects one signing subkey; the export includes its primary public key. +Decernor must find **no private record**. With `--fail-on-empty`, exit 3 is the +expected no-match result; exit 0 means private material was found and every +other result is a stop. The block asserts exactly one primary matching the +approved fingerprint and one subkey matching the `!` selector. Before +proceeding, inspect the human-readable GPG output for unexpired, non-revoked +primary and signing subkey; it uses a throwaway home, not the ceremony keyring. +Only after those checks pass, run: ```bash -VERSION=$(cat VERSION) +( + set -euo pipefail # stop before later checks if generation or verification fails + make release-insert-anchors + ./scripts/validate-release-anchors.sh + make release-tooling-test + make prepush + git status --short +) +``` + +The minisign export starts with `untrusted comment:`. After insertion, Decernor reports +`fingerprint verify: records=2 findings=0` and the script reports +`[ok] generated public fingerprint anchors for review`. The two generated +files contain exactly `gpg <40 uppercase hex>` and `minisign <64 lowercase hex>` +in TXT, with corresponding primary and public-blob NDJSON records. The GPG +anchor equals the approved primary. Only the public pin and those two anchors +should appear in the working-tree diff; reviewers re-derive the fingerprints +from the public exports before approving the PR. Stop on any unexpected file, +private marker, missing or extra key, expiry/revocation, mismatch, invalid +Decernor version, or nonzero generation/verification result. Fix the cause and +regenerate; never hand-edit an anchor. If a post-export check fails, Dave must +confirm that the new untracked public pin is his intended export, remove that +failed export, then rerun from the beginning. Never overwrite an existing pin. + +## 2. Create the signed tag and unsigned draft -# Canonical repo tag -git tag -a "v${VERSION}" -m "v${VERSION}: docprims release" +Only after an explicit tag cue, from clean `main` at the preflighted commit: + +```bash +: "${DOCPRIMS_RELEASE_TAG:?load the approved cut}" +: "${DOCPRIMS_TAG_MESSAGE_DIR:?load the external per-cut message dir}" +test -s "${DOCPRIMS_TAG_MESSAGE_DIR}/message.txt" +make release-tag # creates and verifies locally, no push +make release-push-tag # explicit maintainer action; verifies remote object +``` -# Go submodule tag (required for subdir Go module) -git tag -a "bindings/go/docprims/v${VERSION}" -m "bindings/go/docprims/v${VERSION}" +Then tag the Go submodule at the same commit (required for the subdirectory +Go module): -git push origin "v${VERSION}" "bindings/go/docprims/v${VERSION}" +```bash +git tag -a "bindings/go/docprims/${DOCPRIMS_RELEASE_TAG}" -m "bindings/go/docprims/${DOCPRIMS_RELEASE_TAG}" +git push origin "bindings/go/docprims/${DOCPRIMS_RELEASE_TAG}" ``` -### CI Verification +- [ ] Confirm the tag workflow is green +- [ ] Confirm the tag has GitHub **Verified** status and the + `verify-signature` gate passed before a draft is created +- [ ] Confirm the GitHub release is still a draft +- [ ] Confirm it contains the CLI archives from + `config/release/cli-platforms.txt`, `docprims.h`, the CycloneDX SBOM, + and both licenses +- [ ] Do not sign until the draft inventory is complete + +### Release validation -- [ ] Wait for GitHub Actions workflows on the tag (release build, if configured) -- [ ] (Recommended) Run Go bindings staged validation: - - `.github/workflows/go-bindings.yml` (glibc on PR; musl on manual) -- [ ] (Recommended) Validate the tag (includes Go + TypeScript): +- [ ] Validate the tag (Go and TypeScript): ```bash - VERSION=$(cat VERSION) - gh workflow run "Validate Release" -f tag="v${VERSION}" + gh workflow run "Validate Release" -f tag="${DOCPRIMS_RELEASE_TAG}" ``` ### TypeScript N-API Prebuilds (Required for npm) @@ -124,84 +261,57 @@ Run the prebuilds workflow on the release tag to build cross-platform native bin ``` - [ ] Verify all platforms built successfully -The prebuilds must complete before running the npm publish workflow in Section 3. - -## 2. Manual Signing (Local Machine) +The prebuilds must complete before running the npm publish workflow in Section 4. -docprims signs checksum manifests for release artifacts. +## 3. Maintainer MFA sign and upload -### Set Environment Variables +Use a clean detached checkout. Fetch both the exact tag and `main`; the strict +guard peels the annotated tag and requires the tag, `HEAD`, and fetched +`origin/main` to be the same commit. ```bash -export DOCPRIMS_RELEASE_TAG=v$(cat VERSION) +: "${DOCPRIMS_RELEASE_TAG:?load the approved release tag}" +: "${DOCPRIMS_MINISIGN_KEY:?load the approved secret key}" +: "${DOCPRIMS_MINISIGN_PUB:?load the approved public key}" +test -z "$(git status --porcelain)" +git fetch origin \ + "+refs/heads/main:refs/remotes/origin/main" \ + "+refs/tags/${DOCPRIMS_RELEASE_TAG}:refs/tags/${DOCPRIMS_RELEASE_TAG}" +git checkout --detach "$DOCPRIMS_RELEASE_TAG" +DOCPRIMS_REQUIRE_TAG=1 make release-guard-tag-version +make release-verify-tag +make release-verify-remote-tag +make release +``` -export DOCPRIMS_MINISIGN_KEY=/path/to/docprims.key -export DOCPRIMS_MINISIGN_PUB=/path/to/docprims.pub +`make release` performs the only serialized walk: -# Optional PGP signing -export DOCPRIMS_PGP_KEY_ID="keyid!" -export DOCPRIMS_GPG_HOMEDIR=/path/to/gpg/homedir -``` +Keep `DOCPRIMS_DECERNOR_BIN` bound to the reviewed absolute v0.1.8+ +executable throughout the ceremony; `release-verify-keys` re-derives both +exported publics against the anchors staged into the signed set. + +1. Safely empty the repository-owned `dist/release` +2. Download and structurally validate the exact unsigned draft assets +3. Add the exact per-cut release notes and staged public fingerprint anchors +4. Generate exact SHA-256 and SHA-512 manifests +5. Sign both manifests +6. Export and prove public verification keys +7. Verify once, upload the explicit provenance set, and recheck the exact + remote inventory -### Signing Steps - -1. Clean previous release artifacts: - ```bash - make release-clean - ``` - -2. Download artifacts from the GitHub draft release: - ```bash - make release-download - ``` - -3. Generate checksum manifests: - ```bash - make release-checksums - ``` - Produces: `SHA256SUMS`, `SHA512SUMS` - -4. Sign checksum manifests: - ```bash - make release-sign - ``` - Produces: `.minisig` and optional `.asc` signatures - - > **Note (Ghostty users):** If GPG fails with "Screen or window too small", prefix with: - > ```bash - > TERM=xterm-256color make release-sign - > ``` - -5. Export public keys into the release directory: - ```bash - make release-export-keys - ``` - Produces: - - `docprims-minisign.pub` - - `docprims-release-signing-key.asc` (optional) - -6. Verify everything before upload: - ```bash - make release-verify - ``` - -7. Copy release notes (optional): - ```bash - make release-notes - ``` - -8. Upload signed assets to GitHub and publish the release: - ```bash - make release-upload - ``` - -## 3. TypeScript npm Publishing +- [ ] Confirm the release remains a draft after upload +- [ ] Independently download into an empty directory and verify both checksum + manifests and minisign signatures +- [ ] Review the final release notes and asset list +- [ ] Publish the GitHub release only as a separate, explicit maintainer action + +## 4. TypeScript npm Publishing After signing and undrafting the GitHub release, publish to npm. ### Prerequisites -- Prebuilds workflow completed successfully (Section 1) +- Prebuilds workflow completed successfully (Section 2) - GitHub release is public (not draft) - `publish-npm` environment configured on GitHub (for OIDC trusted publishing) @@ -234,7 +344,7 @@ npm publish --access public This publishes with local platform binary only. Run the workflow publish afterward for cross-platform support. -## 4. crates.io (maintainer action after the tag) +## 5. crates.io (maintainer action after the tag) CI has no registry token and never uploads crates. A registry version cannot be overwritten: a correction takes a new patch version. @@ -249,6 +359,7 @@ validate it with `make release-crates-list`. `docprims-ffi` and ```bash git checkout --detach "v$(cat VERSION)" DOCPRIMS_RELEASE_TAG="v$(cat VERSION)" DOCPRIMS_REQUIRE_TAG=1 make release-guard-tag-version + DOCPRIMS_RELEASE_TAG="v$(cat VERSION)" make release-verify-remote-tag DOCPRIMS_RELEASE_TAG="v$(cat VERSION)" make release-crates-dry-run ``` The dry run patches earlier workspace crates in by path; it uploads nothing. @@ -270,10 +381,27 @@ validate it with `make release-crates-list`. `docprims-ffi` and and checks the exact version on the crates.io API. - [ ] Review the crates.io pages and the docs.rs builds. -## 5. Post-Release Verification +## 6. Post-Release Verification - [ ] Verify release is public: `gh release view v$(cat VERSION)` - [ ] Download and verify checksums/signatures using the published public keys - [ ] Verify npm package: `npm view @3leaps/docprims` - [ ] Verify crates: `cargo info --registry crates-io docprims@$(cat VERSION)` - [ ] Test Go module: `go get github.com/3leaps/docprims/bindings/go/docprims@v$(cat VERSION)` + +## Rotate release signing keys + +Before the first cut with a new key, generate a new public export and anchors +under maintainer identity and land both through a reviewed PR. CI must see the +new pinned public key _in the tagged commit_; an account-level GitHub Verified +indicator alone cannot authorize a release. Re-derive both fingerprint records +with `decernor fingerprint`, review the primary/subkey relationship, and confirm +the tagger account has the matching public key and verified email. Check the +primary and signing-subkey expiration with: + +```bash +gpg --homedir "$DOCPRIMS_GPG_HOMEDIR" --list-keys --with-subkey-fingerprint --with-colons "$DOCPRIMS_GPG_SIGNING_FINGERPRINT" +``` + +An expired key is a rotation nobody scheduled. Do not sign or publish until a +reviewed replacement pin is on `main` and the tag ruleset is active. diff --git a/config/release/cli-platforms.txt b/config/release/cli-platforms.txt new file mode 100644 index 0000000..0a053db --- /dev/null +++ b/config/release/cli-platforms.txt @@ -0,0 +1,6 @@ +linux-amd64 tar.gz docprims +linux-amd64-musl tar.gz docprims +linux-arm64 tar.gz docprims +linux-arm64-musl tar.gz docprims +darwin-arm64 tar.gz docprims +windows-amd64 zip docprims.exe diff --git a/config/release/decernor-ci-sha256.txt b/config/release/decernor-ci-sha256.txt new file mode 100644 index 0000000..9d308ea --- /dev/null +++ b/config/release/decernor-ci-sha256.txt @@ -0,0 +1 @@ +2f034a1945772983e39dae7143ac2f5ba2c3aa79aa249902fb585960a4582261 decernor_0.1.8_linux_amd64.tar.gz diff --git a/config/release/tagger-identity.txt b/config/release/tagger-identity.txt new file mode 100644 index 0000000..a3a83ac --- /dev/null +++ b/config/release/tagger-identity.txt @@ -0,0 +1 @@ +3 Leaps Infosec Team diff --git a/docs/cicd.md b/docs/cicd.md new file mode 100644 index 0000000..b949a59 --- /dev/null +++ b/docs/cicd.md @@ -0,0 +1,56 @@ +# CI/CD + +## Local gates + +| Target | Runs | +|--------|------| +| `make precommit` | `fmt-check`, `lint` | +| `make check` | format, lint, tests, lean-library check, docs build, `cargo-deny`, `cargo-audit` | +| `make prepush` | `check`, `version-check`, `release-tooling-test` | +| `make release-check` | `version-check`, then packages and verify-builds every publishable crate without publishing | +| `make release-preflight` | pre-tag gate: anchors, clean tree, `prepush`, `release-check`, release notes, `HEAD` == `origin/main` | + +`make release-tooling-test` runs the release guard, signing, asset, crate, +version and release-notes tests. The Decernor test uses `DOCPRIMS_DECERNOR_BIN` +when set, otherwise `decernor` on `PATH`. + +## Workflows + +| Workflow | Trigger | Purpose | +|----------|---------|---------| +| `ci` | push to `main`, PR | quality gates, header drift, MSRV, `publish-check` (package check and release tooling tests with a checksum-pinned Decernor), Go vendored-lib tests | +| `go-bindings` | PR, manual | Go bindings against locally built libraries | +| `typescript-bindings` | push to `main`, PR | TypeScript build and tests on glibc and musl | +| `typescript-napi-prebuilds` | manual | cross-platform `.node` prebuilds and the npm package directory; publishes nothing | +| `go-bindings-prep` | manual | builds the Go prebuilt libraries and opens a PR with them | +| `release` | `v*` tag push, manual | builds and validates the exact release asset set; on a tag push only, verifies the signed tag and opens an unsigned draft release | +| `validate-release` | manual | validates a pushed tag from Go and TypeScript | +| `typescript-npm-publish` | manual | publishes the npm packages with OIDC trusted publishing | + +All workflows default to read-only `contents` permission and pin every action +to a commit SHA. Jobs that write (the release draft, the Go prep PR, npm OIDC) +declare their own permissions. + +## Release workflow + +A manual run of `release` builds every CLI archive, the C header and the SBOM, +then validates the assembled set against `config/release/cli-platforms.txt`. +It creates no release, so the build matrix can be proven before a tag exists. + +On a tag push, `validate` binds the annotated tag to `VERSION` and +`origin/main`, and `verify-signature` checks the tag signature against the +committed public pin and GitHub verification. The draft job runs only after +both pass. + +### Credentials + +The strict release guard fetches the release tag and `origin/main`. The +read-only `validate` job keeps its read-scoped checkout credential for that +guard. The draft job has a write-scoped token, so it provides an authenticated +Git header only while running the guard and removes it before the steps that +create the draft. + +After a tag workflow failure, confirm whether the draft exists and its unsigned +asset inventory is complete before running `make release`. Repair the workflow +and recreate the annotated tag only when the failed workflow did not create a +draft. diff --git a/schemas/release/fingerprint-record.v0.schema.json b/schemas/release/fingerprint-record.v0.schema.json new file mode 100644 index 0000000..e55a2c0 --- /dev/null +++ b/schemas/release/fingerprint-record.v0.schema.json @@ -0,0 +1,246 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.3leaps.dev/decernor/fingerprint-record.v0.schema.json", + "title": "Decernor Fingerprint Record v0", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "class", + "algorithm", + "fingerprint", + "fingerprint_scheme", + "confidence" + ], + "properties": { + "schema_version": { + "const": "v0" + }, + "path": { + "type": "string", + "minLength": 1 + }, + "kind": { + "enum": [ + "gpg", + "ssh", + "minisign" + ] + }, + "class": { + "enum": [ + "public", + "private", + "other" + ] + }, + "algorithm": { + "enum": [ + "openpgp-fingerprint", + "sha256", + "minisign-key-id" + ] + }, + "fingerprint": { + "type": [ + "string", + "null" + ], + "minLength": 1 + }, + "fingerprint_scheme": { + "enum": [ + "openpgp-fingerprint-v1", + "ssh-rfc4253-public-blob-sha256-v1", + "minisign-key-id-v1", + "minisign-public-blob-sha256-v1" + ] + }, + "key_id": { + "type": "string", + "minLength": 1 + }, + "key_role": { + "enum": [ + "primary", + "subkey" + ] + }, + "confidence": { + "enum": [ + "high", + "medium", + "low" + ] + }, + "reason": { + "enum": [ + "encrypted-private-no-public-counterpart", + "helper-unavailable", + "parse-unsupported", + "too-large", + "unreadable", + "unsupported-kind", + "unsupported-version" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "kind": { + "const": "gpg" + }, + "fingerprint": { + "type": "string" + } + }, + "required": [ + "kind", + "fingerprint" + ] + }, + "then": { + "required": [ + "key_role", + "key_id" + ], + "properties": { + "algorithm": { + "const": "openpgp-fingerprint" + }, + "fingerprint_scheme": { + "const": "openpgp-fingerprint-v1" + }, + "key_role": { + "enum": [ + "primary", + "subkey" + ] + }, + "key_id": { + "type": "string", + "pattern": "^[0-9A-F]{16}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[0-9A-F]{40}$" + } + } + }, + "else": { + "not": { + "required": [ + "key_role" + ] + } + } + }, + { + "if": { + "properties": { + "fingerprint_scheme": { + "const": "openpgp-fingerprint-v1" + }, + "fingerprint": { + "type": "string" + } + }, + "required": [ + "fingerprint_scheme", + "fingerprint" + ] + }, + "then": { + "required": [ + "key_role", + "key_id" + ], + "properties": { + "kind": { + "const": "gpg" + }, + "algorithm": { + "const": "openpgp-fingerprint" + }, + "key_role": { + "enum": [ + "primary", + "subkey" + ] + }, + "key_id": { + "type": "string", + "pattern": "^[0-9A-F]{16}$" + }, + "fingerprint": { + "type": "string", + "pattern": "^[0-9A-F]{40}$" + } + } + } + }, + { + "if": { + "properties": { + "fingerprint_scheme": { + "const": "minisign-public-blob-sha256-v1" + }, + "fingerprint": { + "type": "string" + } + }, + "required": [ + "fingerprint_scheme", + "fingerprint" + ] + }, + "then": { + "properties": { + "kind": { + "const": "minisign" + }, + "algorithm": { + "const": "sha256" + }, + "fingerprint": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + }, + { + "if": { + "properties": { + "fingerprint_scheme": { + "const": "ssh-rfc4253-public-blob-sha256-v1" + }, + "fingerprint": { + "type": "string" + } + }, + "required": [ + "fingerprint_scheme", + "fingerprint" + ] + }, + "then": { + "properties": { + "kind": { + "const": "ssh" + }, + "algorithm": { + "const": "sha256" + }, + "fingerprint": { + "type": "string", + "pattern": "^SHA256:[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]$" + } + } + } + } + ] +} diff --git a/scripts/check-draft-release.sh b/scripts/check-draft-release.sh new file mode 100755 index 0000000..43454ee --- /dev/null +++ b/scripts/check-draft-release.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Check whether draft creation is safe, or verify the created exact draft. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" + +mode="${1:?usage: check-draft-release.sh before-or-after}" +tag="$(release_tag)" + +if [[ -n "${GH_REPO+x}" ]]; then + echo "error: GH_REPO must be unset; repository authority is fixed" >&2 + exit 1 +fi + +case "$mode" in +before) + releases="$(mktemp "${TMPDIR:-/tmp}/docprims-release-list.XXXXXX")" + trap 'rm -f "$releases"' EXIT + gh release list --repo "$DOCPRIMS_REPOSITORY" --limit 100 \ + --json tagName >"$releases" + if jq -e --arg tag "$tag" \ + 'any(.[]; .tagName == $tag)' "$releases" >/dev/null; then + assert_github_release_state release_base_assets + else + [[ "$(gh repo view "$DOCPRIMS_REPOSITORY" \ + --json nameWithOwner --jq .nameWithOwner)" == "$DOCPRIMS_REPOSITORY" ]] || { + echo "error: GitHub repository identity mismatch" >&2 + exit 1 + } + fi + ;; +after) + assert_github_release_state release_base_assets + ;; +*) + echo "error: mode must be before or after" >&2 + exit 1 + ;; +esac + +echo "[ok] draft release state is safe for $mode" diff --git a/scripts/download-release-assets.sh b/scripts/download-release-assets.sh index 7a4a5ce..709bf1f 100755 --- a/scripts/download-release-assets.sh +++ b/scripts/download-release-assets.sh @@ -1,25 +1,36 @@ #!/usr/bin/env bash -# Download release assets from GitHub -# Usage: download-release-assets.sh [dest_dir] -# -# Requires: gh CLI authenticated -set -euo pipefail +# Download the exact unsigned CI asset set from the trusted draft release. -TAG=${1:?"usage: download-release-assets.sh [dest_dir]"} -DEST=${2:-dist/release} +set -euo pipefail -echo "Downloading release assets for $TAG to $DEST..." +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" -mkdir -p "$DEST" +destination="${1:-dist/release}" +require_release_guard 1 >/dev/null +tag="$(release_tag)" +assert_github_release_state release_base_assets -# Download all release artifacts except signatures (those are added after signing) -gh release download "$TAG" --dir "$DEST" --clobber \ - --pattern 'docprims-*.tar.gz' \ - --pattern 'docprims-*.zip' \ - --pattern 'docprims-ffi-*.tar.gz' \ - --pattern 'docprims.h' \ - --pattern 'sbom-*.json' \ - --pattern 'LICENSE-*' +root="$(release_repo_root)" +expected="$root/dist/release" +if [[ "$destination" != "$expected" && "$destination" != "dist/release" ]]; then + echo "error: release download destination must be repo-owned dist/release" >&2 + exit 1 +fi +destination="$expected" +mkdir -p "$destination" +if find "$destination" -mindepth 1 -maxdepth 1 -print -quit | grep -q .; then + echo "error: release download requires an empty staging directory" >&2 + exit 1 +fi -echo "Downloaded to $DEST:" -ls -la "$DEST" +args=() +while IFS= read -r asset; do + args+=(--pattern "$asset") +done < <(release_base_assets) +gh release download "$tag" --repo "$DOCPRIMS_REPOSITORY" \ + --dir "$destination" "${args[@]}" +"$SCRIPT_DIR/validate-release-assets.sh" "$destination" base >/dev/null +echo "[ok] exact unsigned draft assets downloaded" diff --git a/scripts/export-release-keys.sh b/scripts/export-release-keys.sh index d10ad80..f3f93ac 100755 --- a/scripts/export-release-keys.sh +++ b/scripts/export-release-keys.sh @@ -1,44 +1,47 @@ #!/usr/bin/env bash -# Export public signing keys to release directory -# Usage: export-release-keys.sh [dir] -# -# Environment variables: -# DOCPRIMS_MINISIGN_PUB - Path to minisign public key (or derives from DOCPRIMS_MINISIGN_KEY) -# DOCPRIMS_MINISIGN_KEY - Path to minisign secret key (used to derive public key location) -# DOCPRIMS_PGP_KEY_ID - PGP key ID for optional export (optional) -# DOCPRIMS_GPG_HOMEDIR - Custom GPG home directory (optional) +# Export explicit public verification material and prove it verifies this cut. + set -euo pipefail -DIR=${1:-dist/release} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" -if [ ! -d "$DIR" ]; then - echo "Error: Directory $DIR does not exist" +directory="${1:-dist/release}" +require_release_guard 1 >/dev/null +release_tag >/dev/null +"$SCRIPT_DIR/validate-release-assets.sh" \ + "$directory" signed-without-keys >/dev/null +: "${DOCPRIMS_MINISIGN_PUB:?load the approved minisign public key}" +[[ -f "$DOCPRIMS_MINISIGN_PUB" && ! -L "$DOCPRIMS_MINISIGN_PUB" ]] || { + echo "error: configured minisign public key is unavailable" >&2 exit 1 -fi - -echo "Exporting public keys to $DIR..." +} +grep -q '^untrusted comment:' "$DOCPRIMS_MINISIGN_PUB" || { + echo "error: minisign public key has an unexpected format" >&2 + exit 1 +} +grep -qi 'secret' "$DOCPRIMS_MINISIGN_PUB" && { + echo "error: refusing public material containing a secret marker" >&2 + exit 1 +} -MINISIGN_PUB="${DOCPRIMS_MINISIGN_PUB:-}" -if [ -z "$MINISIGN_PUB" ] && [ -n "${DOCPRIMS_MINISIGN_KEY:-}" ]; then - MINISIGN_PUB="${DOCPRIMS_MINISIGN_KEY%.key}.pub" -fi +cp "$DOCPRIMS_MINISIGN_PUB" "$directory/docprims-minisign.pub" +chmod 0644 "$directory/docprims-minisign.pub" -if [ -n "$MINISIGN_PUB" ] && [ -f "$MINISIGN_PUB" ]; then - cp "$MINISIGN_PUB" "$DIR/docprims-minisign.pub" - echo "[ok] Exported $DIR/docprims-minisign.pub" -else - echo "[!!] Minisign public key not found" - echo "Set DOCPRIMS_MINISIGN_PUB or ensure .pub file exists alongside .key" +if [[ -n "${DOCPRIMS_PGP_KEY_ID:-}" || -n "${DOCPRIMS_GPG_HOMEDIR:-}" ]]; then + require_complete_pgp_config + gpg --homedir "$DOCPRIMS_GPG_HOMEDIR" --batch --armor \ + --export "$DOCPRIMS_PGP_KEY_ID" \ + >"$directory/docprims-release-signing-key.asc" + [[ -s "$directory/docprims-release-signing-key.asc" ]] || { + echo "error: PGP public-key export is empty" >&2 + exit 1 + } fi -if [ -n "${DOCPRIMS_PGP_KEY_ID:-}" ]; then - GPG_OPTS=() - if [ -n "${DOCPRIMS_GPG_HOMEDIR:-}" ]; then - GPG_OPTS+=("--homedir" "$DOCPRIMS_GPG_HOMEDIR") - fi - - gpg "${GPG_OPTS[@]}" --armor --export "$DOCPRIMS_PGP_KEY_ID" >"$DIR/docprims-release-signing-key.asc" - echo "[ok] Exported $DIR/docprims-release-signing-key.asc" -else - echo "[--] PGP key export skipped (DOCPRIMS_PGP_KEY_ID not set)" -fi +"$SCRIPT_DIR/verify-public-keys.sh" "$directory" >/dev/null +"$SCRIPT_DIR/verify-signatures.sh" "$directory" >/dev/null +"$SCRIPT_DIR/validate-release-assets.sh" "$directory" signed >/dev/null +echo "[ok] public verification keys exported and proven" diff --git a/scripts/generate-checksums.sh b/scripts/generate-checksums.sh index 1c75a30..d343a49 100755 --- a/scripts/generate-checksums.sh +++ b/scripts/generate-checksums.sh @@ -1,58 +1,33 @@ #!/usr/bin/env bash -# Generate SHA256SUMS and SHA512SUMS checksum manifests -# Usage: generate-checksums.sh [dir] -# -# Creates checksums for all release artifacts (excludes signatures and checksum files) -set -euo pipefail - -DIR=${1:-dist/release} - -if [ ! -d "$DIR" ]; then - echo "Error: Directory $DIR does not exist" - exit 1 -fi +# Generate exact SHA256 and SHA512 manifests for one release. -cd "$DIR" - -echo "Generating checksums in $DIR..." +set -euo pipefail -CHECKSUM_PATTERNS=( - '*.tar.gz' - '*.zip' - '*.h' - '*.a' - '*.lib' - '*.so' - '*.dylib' - '*.dll' - '*.json' - 'LICENSE-*' - 'release-notes-*.md' +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" + +directory="${1:-dist/release}" +require_release_guard 1 >/dev/null +release_tag >/dev/null +"$SCRIPT_DIR/validate-release-assets.sh" "$directory" signable >/dev/null + +mapfile_compat() { + local output_name="$1" + shift + local values=() + while IFS= read -r value; do values+=("$value"); done < <("$@") + eval "$output_name=(\"\${values[@]}\")" +} + +assets=() +mapfile_compat assets release_signable_assets +( + cd "$directory" + printf '%s\n' "${assets[@]}" | LC_ALL=C sort | xargs shasum -a 256 >SHA256SUMS + printf '%s\n' "${assets[@]}" | LC_ALL=C sort | xargs shasum -a 512 >SHA512SUMS ) +"$SCRIPT_DIR/validate-release-assets.sh" "$directory" checksummed >/dev/null -FIND_ARGS=() -for pattern in "${CHECKSUM_PATTERNS[@]}"; do - if [ ${#FIND_ARGS[@]} -gt 0 ]; then - FIND_ARGS+=("-o") - fi - FIND_ARGS+=("-name" "$pattern") -done - -find . -maxdepth 1 -type f \( "${FIND_ARGS[@]}" \) \ - ! -name 'SHA*' \ - ! -name '*.minisig' \ - ! -name '*.asc' \ - ! -name '*.pub' \ - -print0 | sort -z | xargs -0 shasum -a 256 >SHA256SUMS - -echo "Generated SHA256SUMS" - -find . -maxdepth 1 -type f \( "${FIND_ARGS[@]}" \) \ - ! -name 'SHA*' \ - ! -name '*.minisig' \ - ! -name '*.asc' \ - ! -name '*.pub' \ - -print0 | sort -z | xargs -0 shasum -a 512 >SHA512SUMS - -echo "Generated SHA512SUMS" -echo "[ok] Checksums generated" +echo "[ok] exact SHA256 and SHA512 manifests generated" diff --git a/scripts/install-release-anchors.sh b/scripts/install-release-anchors.sh new file mode 100755 index 0000000..61c6c42 --- /dev/null +++ b/scripts/install-release-anchors.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Replace a public anchor pair with rollback on error or interruption. +set -euo pipefail +staging="${1:?staging directory required}" +dest="${2:?destination required}" +shift 2 +for name in expected-fingerprints.txt expected-fingerprints.ndjson; do + [[ -s "$staging/$name" && ! -L "$staging/$name" ]] || { + echo 'error: incomplete anchor pair' >&2 + exit 1 + } +done +mkdir -p "$dest" +backup="$(mktemp -d)" +for name in expected-fingerprints.txt expected-fingerprints.ndjson; do + if [[ -f "$dest/$name" ]]; then cp "$dest/$name" "$backup/$name"; fi +done +rollback() { + trap - EXIT INT TERM HUP + for name in expected-fingerprints.txt expected-fingerprints.ndjson; do + rm -f "$dest/$name.new" + if [[ -f "$backup/$name" ]]; then mv -f "$backup/$name" "$dest/$name"; else rm -f "$dest/$name"; fi + done + rm -rf "$backup" +} +trap rollback EXIT +trap 'rollback; exit 130' INT +trap 'rollback; exit 143' TERM +trap 'rollback; exit 129' HUP +for name in expected-fingerprints.txt expected-fingerprints.ndjson; do + cp "$staging/$name" "$dest/$name.new" +done +mv -f "$dest/expected-fingerprints.ndjson.new" "$dest/expected-fingerprints.ndjson" +if [[ "${DOCPRIMS_TEST_FAIL_ANCHOR_INSTALL:-}" == 1 ]]; then + echo 'error: interrupted pair installation' >&2 + exit 1 +fi +mv -f "$dest/expected-fingerprints.txt.new" "$dest/expected-fingerprints.txt" +if [[ "$#" -gt 0 ]]; then + "$@" +fi +trap - EXIT INT TERM HUP +rm -rf "$backup" diff --git a/scripts/release-assets.test.sh b/scripts/release-assets.test.sh new file mode 100755 index 0000000..e5bf64b --- /dev/null +++ b/scripts/release-assets.test.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(git rev-parse --show-toplevel)" +fixture="$(mktemp -d "${TMPDIR:-/tmp}/docprims-release-assets.XXXXXX")" +payload="$(mktemp -d "${TMPDIR:-/tmp}/docprims-release-payload.XXXXXX")" +trap 'rm -rf "$fixture" "$payload"' EXIT + +version="$(cat "$root/VERSION")" +expected_platforms="$(cut -d' ' -f1 "$root/config/release/cli-platforms.txt" | LC_ALL=C sort)" +workflow_platforms="$(awk '$1 == "platform:" { print $2 }' "$root/.github/workflows/release.yml" | LC_ALL=C sort)" +[[ "$expected_platforms" == "$workflow_platforms" ]] || { + echo 'error: CLI release matrix differs from platform inventory' >&2 + exit 1 +} +[[ "$version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || { + echo "error: fixture requires a stable VERSION" >&2 + exit 1 +} +export DOCPRIMS_RELEASE_TAG="v${version}" + +expect_fail() { + if "$@" >/dev/null 2>&1; then + echo "expected failure: $*" >&2 + exit 1 + fi +} + +printf 'license\n' >"$fixture/LICENSE-APACHE" +printf 'license\n' >"$fixture/LICENSE-MIT" +printf '{}\n' >"$fixture/sbom-${version}.cdx.json" +printf 'header\n' >"$fixture/docprims.h" + +# make_archive PLATFORM FORMAT BINARY [MEMBER...]: build an archive from +# regular files named MEMBER (default: the binary and both licenses). +make_archive() { + local platform="$1" format="$2" binary="$3" + shift 3 + local members=("$@") + [[ ${#members[@]} -gt 0 ]] || members=("$binary" LICENSE-APACHE LICENSE-MIT) + local archive="$fixture/docprims-${version}-${platform}.${format}" + rm -rf "$payload" "$archive" + mkdir -p "$payload" + local member + for member in "${members[@]}"; do + mkdir -p "$payload/$(dirname "$member")" + printf 'fixture\n' >"$payload/$member" + done + python3 - "$archive" "$format" "$payload" "${members[@]}" <<'PY' +import sys +import tarfile +import zipfile + +archive, archive_format, payload, *members = sys.argv[1:] +if archive_format == "zip": + with zipfile.ZipFile(archive, "w") as out: + for m in members: + out.write(f"{payload}/{m}", m) +else: + with tarfile.open(archive, "w:gz") as out: + for m in members: + out.add(f"{payload}/{m}", m) +PY +} + +good_archives() { + local platform format binary + while read -r platform format binary; do + make_archive "$platform" "$format" "$binary" + done <"$root/config/release/cli-platforms.txt" +} + +good_archives +"$SCRIPT_DIR/validate-release-assets.sh" "$fixture" base >/dev/null +wrong_tag="v${version%.*}.$((${version##*.} + 1))" +expect_fail env DOCPRIMS_RELEASE_TAG="$wrong_tag" \ + "$SCRIPT_DIR/validate-release-assets.sh" "$fixture" base + +printf 'stale\n' >"$fixture/foreign.txt" +expect_fail "$SCRIPT_DIR/validate-release-assets.sh" "$fixture" base +rm "$fixture/foreign.txt" +rm "$fixture/docprims.h" +expect_fail "$SCRIPT_DIR/validate-release-assets.sh" "$fixture" base +printf 'header\n' >"$fixture/docprims.h" + +# Each malformed archive fails on its own against an otherwise good set. +reject_archive() { + good_archives + "$@" + expect_fail "$SCRIPT_DIR/validate-release-assets.sh" "$fixture" base +} +reject_archive make_archive linux-amd64 tar.gz docprims docprims LICENSE-MIT +reject_archive make_archive linux-amd64 tar.gz docprims docprims LICENSE-APACHE LICENSE-MIT extra +reject_archive make_archive linux-amd64 tar.gz docprims docprims.exe LICENSE-APACHE LICENSE-MIT +reject_archive make_archive linux-amd64 tar.gz docprims bin/docprims LICENSE-APACHE LICENSE-MIT +reject_archive make_archive windows-amd64 zip docprims.exe docprims LICENSE-APACHE LICENSE-MIT +reject_archive make_archive windows-amd64 zip docprims.exe sub/docprims.exe LICENSE-APACHE LICENSE-MIT + +# The layout `tar -C dir .` produces: a ./ directory and ./-prefixed names. +good_archives +rm -rf "$payload" && mkdir -p "$payload" +printf 'fixture\n' | tee "$payload/docprims" "$payload/LICENSE-APACHE" "$payload/LICENSE-MIT" >/dev/null +tar -czf "$fixture/docprims-${version}-darwin-arm64.tar.gz" -C "$payload" . +expect_fail "$SCRIPT_DIR/validate-release-assets.sh" "$fixture" base + +python3 - "$fixture" "$version" <<'PY' +import io +import sys +import tarfile +import zipfile + +fixture, version = sys.argv[1:] +def tar_with(path, special): + with tarfile.open(path, "w:gz") as out: + for name in ("docprims", "LICENSE-APACHE", "LICENSE-MIT"): + info = tarfile.TarInfo(name) + if name == special[0]: + special[1](info) + data = b"fixture\n" if info.isreg() else b"" + info.size = len(data) + out.addfile(info, io.BytesIO(data)) +def symlink(info): + info.type = tarfile.SYMTYPE + info.linkname = "/etc/passwd" +def escape(info): + info.name = "../docprims" +tar_with(f"{fixture}/symlink.tar.gz", ("docprims", symlink)) +tar_with(f"{fixture}/escape.tar.gz", ("docprims", escape)) +with zipfile.ZipFile(f"{fixture}/symlink.zip", "w") as out: + link = zipfile.ZipInfo("docprims.exe") + link.external_attr = (0o120777 << 16) + out.writestr(link, "target") + out.writestr("LICENSE-APACHE", "fixture\n") + out.writestr("LICENSE-MIT", "fixture\n") +with zipfile.ZipFile(f"{fixture}/dir.zip", "w") as out: + out.writestr("docprims.exe", "fixture\n") + out.writestr("LICENSE-APACHE", "fixture\n") + out.writestr("LICENSE-MIT", "fixture\n") + out.writestr(zipfile.ZipInfo("docs/"), "") +PY +for bad in symlink.tar.gz:linux-arm64.tar.gz escape.tar.gz:linux-arm64-musl.tar.gz symlink.zip:windows-amd64.zip dir.zip:windows-amd64.zip; do + good_archives + cp "$fixture/${bad%%:*}" "$fixture/docprims-${version}-${bad#*:}" + mkdir -p "$fixture.hold" && mv "$fixture"/symlink.* "$fixture"/escape.* "$fixture"/dir.* "$fixture.hold/" + expect_fail "$SCRIPT_DIR/validate-release-assets.sh" "$fixture" base + mv "$fixture.hold"/* "$fixture/" && rmdir "$fixture.hold" +done +rm -f "$fixture"/symlink.* "$fixture"/escape.* "$fixture"/dir.* + +good_archives +printf 'notes\n' >"$fixture/release-notes-${DOCPRIMS_RELEASE_TAG}.md" +printf 'gpg fixture\nminisign fixture\n' >"$fixture/expected-fingerprints.txt" +printf '{"fixture":true}\n' >"$fixture/expected-fingerprints.ndjson" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" +signable_assets="$(release_signable_assets)" +( + cd "$fixture" + printf '%s\n' "$signable_assets" | LC_ALL=C sort | xargs shasum -a 256 >SHA256SUMS + printf '%s\n' "$signable_assets" | LC_ALL=C sort | xargs shasum -a 512 >SHA512SUMS +) +"$SCRIPT_DIR/verify-checksums.sh" "$fixture" >/dev/null +cp "$fixture/SHA256SUMS" "$fixture/SHA256SUMS.good" +printf '%s\n' "$(head -n 1 "$fixture/SHA256SUMS")" >>"$fixture/SHA256SUMS" +expect_fail "$SCRIPT_DIR/verify-checksums.sh" "$fixture" +mv "$fixture/SHA256SUMS.good" "$fixture/SHA256SUMS" +sed '1s# # nested/#' "$fixture/SHA256SUMS" >"$fixture/SHA256SUMS.bad" +mv "$fixture/SHA256SUMS.bad" "$fixture/SHA256SUMS" +expect_fail "$SCRIPT_DIR/verify-checksums.sh" "$fixture" + +echo "[ok] exact release asset and archive tests passed" diff --git a/scripts/release-checklist.test.sh b/scripts/release-checklist.test.sh deleted file mode 100755 index df7f7cc..0000000 --- a/scripts/release-checklist.test.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# Every make target the release checklist names must exist in the Makefile. -set -euo pipefail - -root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -missing=0 -while IFS= read -r target; do - grep -Eq "^${target}:" "$root/Makefile" || { - echo "error: RELEASE_CHECKLIST.md names missing target: make $target" >&2 - missing=1 - } -done < <(grep -oE 'make [a-z][a-z0-9-]*' "$root/RELEASE_CHECKLIST.md" | awk '{ print $2 }' | sort -u) -[[ "$missing" -eq 0 ]] -echo '[ok] release checklist names only existing make targets' diff --git a/scripts/release-clean.sh b/scripts/release-clean.sh new file mode 100755 index 0000000..f59b273 --- /dev/null +++ b/scripts/release-clean.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Remove only the repository-owned release staging directory. + +set -euo pipefail + +root="$(git rev-parse --show-toplevel)" +target="${1:-$root/dist/release}" +expected="$root/dist/release" + +if [[ -z "$target" || "$target" == "/" || "$target" != "$expected" ]]; then + echo "error: refusing unsafe release cleanup target" >&2 + exit 1 +fi +if [[ -L "$target" ]]; then + echo "error: refusing symlink release cleanup target" >&2 + exit 1 +fi +if [[ -e "$target" && ! -d "$target" ]]; then + echo "error: release cleanup target is not a directory" >&2 + exit 1 +fi + +if [[ -d "$target" ]]; then + find "$target" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +else + mkdir -p "$target" +fi +echo "[ok] release staging directory is clean" diff --git a/scripts/release-common.sh b/scripts/release-common.sh new file mode 100755 index 0000000..d95613b --- /dev/null +++ b/scripts/release-common.sh @@ -0,0 +1,243 @@ +#!/usr/bin/env bash +# Shared release asset and repository invariants. + +set -euo pipefail + +DOCPRIMS_REPOSITORY="3leaps/docprims" + +release_repo_root() { + git rev-parse --show-toplevel +} + +release_version() { + local root + root="$(release_repo_root)" + local version + version="$(cat "$root/VERSION")" + if [[ ! "$version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "error: VERSION must contain one stable semantic version" >&2 + return 1 + fi + printf '%s\n' "$version" +} + +release_tag() { + local tag="${1:-${DOCPRIMS_RELEASE_TAG:-}}" + if [[ -z "$tag" ]]; then + echo "error: DOCPRIMS_RELEASE_TAG is required" >&2 + return 1 + fi + if [[ ! "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "error: release tag must be canonical vX.Y.Z" >&2 + return 1 + fi + if [[ "$tag" != "v$(release_version)" ]]; then + echo "error: release tag does not match VERSION" >&2 + return 1 + fi + printf '%s\n' "$tag" +} + +require_release_guard() { + local strict="${1:-0}" + local root + root="$(release_repo_root)" + if [[ "$strict" == "1" ]]; then + DOCPRIMS_REQUIRE_TAG=1 "$root/scripts/release-guard-tag-version.sh" + else + "$root/scripts/release-guard-tag-version.sh" + fi +} + +# One line per CLI archive: platform, archive format, binary name. +release_cli_platforms() { + cat "$(release_repo_root)/config/release/cli-platforms.txt" +} + +release_base_assets() { + local version + version="$(release_version)" + printf '%s\n' \ + "LICENSE-APACHE" \ + "LICENSE-MIT" \ + "docprims.h" \ + "sbom-${version}.cdx.json" + local platform format binary + while read -r platform format binary; do + printf 'docprims-%s-%s.%s\n' "$version" "$platform" "$format" + done < <(release_cli_platforms) +} + +release_signable_assets() { + local tag + tag="$(release_tag)" + release_base_assets + printf 'release-notes-%s.md\n' "$tag" + printf '%s\n' expected-fingerprints.txt expected-fingerprints.ndjson +} + +release_checksummed_assets() { + release_signable_assets + printf '%s\n' SHA256SUMS SHA512SUMS +} + +release_signed_without_keys_assets() { + release_checksummed_assets + printf '%s\n' SHA256SUMS.minisig SHA512SUMS.minisig + if [[ -n "${DOCPRIMS_PGP_KEY_ID:-}" || -n "${DOCPRIMS_GPG_HOMEDIR:-}" ]]; then + require_complete_pgp_config + printf '%s\n' SHA256SUMS.asc SHA512SUMS.asc + fi +} + +release_signed_assets() { + release_signed_without_keys_assets + printf '%s\n' docprims-minisign.pub + if [[ -n "${DOCPRIMS_PGP_KEY_ID:-}" || -n "${DOCPRIMS_GPG_HOMEDIR:-}" ]]; then + require_complete_pgp_config + printf '%s\n' docprims-release-signing-key.asc + fi +} + +require_complete_pgp_config() { + if [[ -z "${DOCPRIMS_PGP_KEY_ID:-}" || -z "${DOCPRIMS_GPG_HOMEDIR:-}" ]]; then + echo "error: optional PGP signing requires both PGP variables" >&2 + return 1 + fi +} + +assert_exact_directory_inventory() ( + local directory="$1" + local producer="$2" + if [[ ! -d "$directory" || -L "$directory" ]]; then + echo "error: release directory is absent or unsafe" >&2 + return 1 + fi + + local expected_file actual_file + expected_file="$(mktemp "${TMPDIR:-/tmp}/docprims-expected.XXXXXX")" + actual_file="$(mktemp "${TMPDIR:-/tmp}/docprims-actual.XXXXXX")" + trap 'rm -f "$expected_file" "$actual_file"' EXIT + + "$producer" | LC_ALL=C sort >"$expected_file" + find "$directory" -mindepth 1 -maxdepth 1 -print | + while IFS= read -r entry; do basename "$entry"; done | + LC_ALL=C sort >"$actual_file" + + if ! cmp -s "$expected_file" "$actual_file"; then + echo "error: release directory inventory mismatch" >&2 + diff -u "$expected_file" "$actual_file" >&2 || true + return 1 + fi + while IFS= read -r name; do + if [[ ! -f "$directory/$name" || -L "$directory/$name" ]]; then + echo "error: release asset is not a regular file: $name" >&2 + return 1 + fi + done <"$expected_file" +) + +# A CLI archive holds exactly the binary and both licenses as regular, +# top-level files. +validate_cli_archive() { + local archive="$1" + local filename version platform format binary expected_binary="" + filename="$(basename "$archive")" + version="$(release_version)" + while read -r platform format binary; do + if [[ "$filename" == "docprims-${version}-${platform}.${format}" ]]; then + expected_binary="$binary" + break + fi + done < <(release_cli_platforms) + if [[ -z "$expected_binary" ]]; then + echo "error: unexpected CLI archive name: $filename" >&2 + return 1 + fi + python3 - "$archive" "$format" "$expected_binary" <<'PY' +import stat +import sys +import tarfile +import zipfile + +archive, archive_format, binary = sys.argv[1:] +expected = sorted([binary, "LICENSE-APACHE", "LICENSE-MIT"]) +if archive_format == "tar.gz": + with tarfile.open(archive, "r:gz") as packed: + members = packed.getmembers() + names = [m.name for m in members] + irregular = [m.name for m in members if not m.isreg()] +elif archive_format == "zip": + with zipfile.ZipFile(archive) as packed: + members = packed.infolist() + names = [m.filename for m in members] + irregular = [ + m.filename + for m in members + if m.is_dir() or (m.external_attr >> 16 and not stat.S_ISREG(m.external_attr >> 16)) + ] +else: + sys.exit(f"error: unknown archive format {archive_format}") +if any("/" in n or "\\" in n or n in ("", ".", "..") for n in names): + sys.exit(f"error: unsafe or nested member in {archive}") +if irregular: + sys.exit(f"error: non-regular member in {archive}: {irregular}") +if sorted(names) != expected: + sys.exit(f"error: {archive} members {sorted(names)} != {expected}") +PY +} + +validate_all_cli_archives() { + local directory="$1" + local version platform format binary + version="$(release_version)" + while read -r platform format binary; do + validate_cli_archive "$directory/docprims-${version}-${platform}.${format}" + done < <(release_cli_platforms) +} + +assert_github_release_state() ( + local expected_assets_producer="$1" + local tag root tag_commit + tag="$(release_tag)" + root="$(release_repo_root)" + tag_commit="$(git -C "$root" rev-parse "refs/tags/${tag}^{}")" + + if [[ -n "${GH_REPO+x}" ]]; then + echo "error: GH_REPO must be unset; repository authority is fixed" >&2 + return 1 + fi + for command_name in gh jq; do + command -v "$command_name" >/dev/null 2>&1 || { + echo "error: required release command is unavailable" >&2 + return 1 + } + done + if [[ "$(gh repo view "$DOCPRIMS_REPOSITORY" \ + --json nameWithOwner --jq .nameWithOwner)" != "$DOCPRIMS_REPOSITORY" ]]; then + echo "error: GitHub repository identity mismatch" >&2 + return 1 + fi + + local state expected actual + state="$(mktemp "${TMPDIR:-/tmp}/docprims-release-state.XXXXXX")" + expected="$(mktemp "${TMPDIR:-/tmp}/docprims-remote-expected.XXXXXX")" + actual="$(mktemp "${TMPDIR:-/tmp}/docprims-remote-actual.XXXXXX")" + trap 'rm -f "$state" "$expected" "$actual"' EXIT + gh release view "$tag" --repo "$DOCPRIMS_REPOSITORY" \ + --json tagName,targetCommitish,isDraft,assets >"$state" + + if [[ "$(jq -r .tagName "$state")" != "$tag" || + "$(jq -r .targetCommitish "$state")" != "$tag_commit" || + "$(jq -r .isDraft "$state")" != "true" ]]; then + echo "error: GitHub release tag, target, or draft state mismatch" >&2 + return 1 + fi + "$expected_assets_producer" | LC_ALL=C sort >"$expected" + jq -r '.assets[].name' "$state" | LC_ALL=C sort >"$actual" + if ! cmp -s "$expected" "$actual"; then + echo "error: remote draft asset inventory mismatch" >&2 + diff -u "$expected" "$actual" >&2 || true + return 1 + fi +) diff --git a/scripts/release-decernor.sh b/scripts/release-decernor.sh new file mode 100644 index 0000000..a73d082 --- /dev/null +++ b/scripts/release-decernor.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Source this file, then resolve the Decernor executable once per invocation. +resolve_release_decernor() { + local mode="${1:-general}" candidate short extended version + if [[ "$mode" == ceremony ]]; then + [[ -n "${DOCPRIMS_DECERNOR_BIN:-}" ]] || { + echo 'error: DOCPRIMS_DECERNOR_BIN is required for the release ceremony' >&2 + return 1 + } + candidate="$DOCPRIMS_DECERNOR_BIN" + else + candidate="${DOCPRIMS_DECERNOR_BIN:-${DECERNOR_BIN:-}}" + if [[ -z "$candidate" ]]; then + candidate="$(command -v decernor)" || { + echo 'error: decernor is unavailable' >&2 + return 1 + } + fi + fi + [[ "$candidate" == /* && -f "$candidate" && ! -L "$candidate" && -x "$candidate" ]] || { + echo 'error: Decernor binary must be an absolute executable regular file' >&2 + return 1 + } + short="$("$candidate" version)" || return 1 + extended="$("$candidate" version -e)" || return 1 + [[ "$short" =~ ^decernor\ ([0-9]+\.[0-9]+\.[0-9]+)$ ]] || { + echo 'error: Decernor identity/version mismatch' >&2 + return 1 + } + version="${BASH_REMATCH[1]}" + [[ "$extended" == *$'\n'* && "$extended" == "Version:"* ]] || { + echo 'error: invalid Decernor extended version' >&2 + return 1 + } + python3 - "$version" "$extended" <<'PY' || return 1 +import re +import sys +version, extended = sys.argv[1:] +lines = extended.splitlines() +if len(lines) != 6 or not re.fullmatch(r'Version:\s+' + re.escape(version), lines[0]) or any( + not re.fullmatch(label + r':\s+\S.*', line) + for label, line in zip(('Commit', 'Build Date', 'Go Version', 'Gofulmen', 'Crucible'), lines[1:]) +): + raise SystemExit('error: invalid Decernor extended identity') +if tuple(map(int, version.split('.'))) < (0, 1, 8): + raise SystemExit('error: Decernor >= 0.1.8 required') +PY + # shellcheck disable=SC2034 # Consumed by scripts sourcing this helper. + RELEASE_DECERNOR_BIN="$candidate" +} diff --git a/scripts/release-decernor.test.sh b/scripts/release-decernor.test.sh new file mode 100755 index 0000000..d8222a7 --- /dev/null +++ b/scripts/release-decernor.test.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# Synthetic public exports only; no operator keyring or estate material. +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd -P)" +fixture="$(mktemp -d)" +trap 'rm -rf "$fixture"' EXIT +mkdir -p "$fixture/repo/scripts" "$fixture/repo/docs/security" "$fixture/repo/schemas/release" "$fixture/repo/keys" "$fixture/home" "$fixture/other-home" "$fixture/export" +chmod 700 "$fixture/home" "$fixture/other-home" +cp "$root/scripts/"{release-decernor.sh,release-insert-anchors.sh,install-release-anchors.sh,verify-public-keys.sh} "$fixture/repo/scripts/" +cp "$root/schemas/release/fingerprint-record.v0.schema.json" "$fixture/repo/schemas/release/" +bin="${DOCPRIMS_DECERNOR_BIN:-$(command -v decernor)}" +[[ "$bin" == /* ]] +fail() { + if "$@" >"$fixture/output" 2>&1; then + echo "error: expected rejection: $*" >&2 + exit 1 + fi +} +expect_different() { + local status + if cmp -s "$1" "$2"; then + echo "error: expected distinct synthetic anchor files" >&2 + exit 1 + else + status=$? + [[ "$status" == 1 ]] || exit "$status" + fi +} +gpg --homedir "$fixture/home" --batch --pinentry-mode loopback --passphrase '' \ + --quick-gen-key 'Synthetic release ' ed25519 sign 1d >/dev/null 2>&1 +gpg --homedir "$fixture/home" --batch --armor --export >"$fixture/repo/docs/security/release-signing-keys.asc" +python3 - "$fixture/public.pub" "$fixture/other.pub" <<'PY' +import base64 +import pathlib +import sys +for path, byte in zip(sys.argv[1:], (1, 2)): + blob = b'Ed' + bytes([byte]) * 8 + bytes([byte]) * 32 + pathlib.Path(path).write_text('untrusted comment: synthetic public key\n' + base64.b64encode(blob).decode() + '\n') +PY +export DOCPRIMS_MINISIGN_PUB="$fixture/public.pub" +export DOCPRIMS_DECERNOR_BIN="$bin" +( + # General resolver precedence is distinct from ceremony's explicit binding. + source "$root/scripts/release-decernor.sh" + resolve_release_decernor general + [[ "$RELEASE_DECERNOR_BIN" == "$bin" ]] + unset DOCPRIMS_DECERNOR_BIN + DECERNOR_BIN="$bin" resolve_release_decernor general + [[ "$RELEASE_DECERNOR_BIN" == "$bin" ]] + unset DECERNOR_BIN + mkdir -p "$fixture/path" + cp "$bin" "$fixture/path/decernor" + PATH="$fixture/path:$PATH" + resolve_release_decernor general + [[ "$RELEASE_DECERNOR_BIN" == "$fixture/path/decernor" ]] +) +"$fixture/repo/scripts/release-insert-anchors.sh" >/dev/null +cp "$fixture/repo/keys/expected-fingerprints."{txt,ndjson} "$fixture/export/" +cp "$fixture/repo/docs/security/release-signing-keys.asc" "$fixture/export/docprims-release-signing-key.asc" +cp "$fixture/public.pub" "$fixture/export/docprims-minisign.pub" +"$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" >/dev/null + +fail env -u DOCPRIMS_DECERNOR_BIN DECERNOR_BIN="$bin" "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +fail env -u DOCPRIMS_DECERNOR_BIN DECERNOR_BIN="$bin" "$fixture/repo/scripts/release-insert-anchors.sh" +fail env DOCPRIMS_DECERNOR_BIN=decernor "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +fail env DOCPRIMS_DECERNOR_BIN="$fixture/missing" "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cat >"$fixture/false-version" <<'EOF' +#!/usr/bin/env bash +case "$1 $2" in + 'version ') echo 'decernor 0.1.8' ;; + 'version -e') printf 'Version: 0.1.7\nCommit: fake\nBuild Date: fake\nGo Version: fake\nGofulmen: fake\nCrucible: fake\n' ;; +esac +EOF +chmod +x "$fixture/false-version" +fail env DOCPRIMS_DECERNOR_BIN="$fixture/false-version" "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +sed 's/0\.1\.8/0.1.7/' "$fixture/false-version" >"$fixture/old-version" +chmod +x "$fixture/old-version" +fail env DOCPRIMS_DECERNOR_BIN="$fixture/old-version" "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +sed 's/decernor 0.1.8/other 0.1.8/' "$fixture/false-version" >"$fixture/wrong-identity" +chmod +x "$fixture/wrong-identity" +fail env DOCPRIMS_DECERNOR_BIN="$fixture/wrong-identity" "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cp "$fixture/export/expected-fingerprints.txt" "$fixture/good.txt" +sed 's/^minisign ./minisign Z/' "$fixture/good.txt" >"$fixture/export/expected-fingerprints.txt" +fail "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cp "$fixture/good.txt" "$fixture/export/expected-fingerprints.txt" +printf '\n' >>"$fixture/export/expected-fingerprints.ndjson" +fail "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cp "$fixture/repo/keys/expected-fingerprints.ndjson" "$fixture/export/expected-fingerprints.ndjson" +rm "$fixture/export/docprims-minisign.pub" +fail "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cp "$fixture/public.pub" "$fixture/export/docprims-minisign.pub" +cp "$fixture/other.pub" "$fixture/export/docprims-minisign.pub" +fail "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cp "$fixture/public.pub" "$fixture/export/docprims-minisign.pub" +gpg --homedir "$fixture/other-home" --batch --pinentry-mode loopback --passphrase '' \ + --quick-gen-key 'Other synthetic ' ed25519 sign 1d >/dev/null 2>&1 +gpg --homedir "$fixture/other-home" --batch --armor --export >"$fixture/export/docprims-release-signing-key.asc" +fail "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +gpg --homedir "$fixture/home" --batch --import "$fixture/export/docprims-release-signing-key.asc" >/dev/null 2>&1 +gpg --homedir "$fixture/home" --batch --armor --export >"$fixture/export/docprims-release-signing-key.asc" +fail "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cp "$fixture/repo/docs/security/release-signing-keys.asc" "$fixture/export/docprims-release-signing-key.asc" +truncate -s 0 "$fixture/export/docprims-release-signing-key.asc" +fail "$fixture/repo/scripts/verify-public-keys.sh" "$fixture/export" +cp "$fixture/repo/docs/security/release-signing-keys.asc" "$fixture/export/docprims-release-signing-key.asc" + +# A verifier failure after install must restore both old files, including their bytes. +# First install a different, valid pair so a missing rollback cannot pass by +# leaving the newly generated pair in place. +cp "$fixture/repo/docs/security/release-signing-keys.asc" "$fixture/primary.asc" +gpg --homedir "$fixture/other-home" --batch --armor --export >"$fixture/repo/docs/security/release-signing-keys.asc" +DOCPRIMS_MINISIGN_PUB="$fixture/other.pub" "$fixture/repo/scripts/release-insert-anchors.sh" >/dev/null +cp "$fixture/primary.asc" "$fixture/repo/docs/security/release-signing-keys.asc" +cat >"$fixture/reject-verify" <"$fixture/VERSION" +git init -q -b main "$fixture" +git -C "$fixture" config user.name "release state test" +git -C "$fixture" config user.email "noreply@example.invalid" +git -C "$fixture" add VERSION scripts/release-common.sh +git -C "$fixture" commit -qm "fixture" +git -C "$fixture" tag -a v0.1.0 -m "fixture" +git -C "$fixture" checkout -q --detach v0.1.0 +fixture_commit="$(git -C "$fixture" rev-parse HEAD)" +original_path="$PATH" + +cat >"$fake_bin/gh" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${FAKE_RELEASE_STATE:?}" in + wrong-repo) + if [[ "$1 $2" == "repo view" ]]; then + printf 'other/docprims\n' + exit 0 + fi + ;; +esac +if [[ "$1 $2" == "repo view" ]]; then + printf '3leaps/docprims\n' + exit 0 +fi +if [[ "$1 $2" == "release list" ]]; then + printf '[{"tagName":"v0.1.0"}]\n' + exit 0 +fi +if [[ "$1 $2" != "release view" ]]; then + exit 1 +fi +draft=true +target="$FAKE_COMMIT" +extra="" +case "$FAKE_RELEASE_STATE" in + valid) ;; + published) draft=false ;; + wrong-target) target=0000000000000000000000000000000000000000 ;; + extra-asset) extra=',{"name":"foreign.txt"}' ;; + missing-asset) ;; + *) exit 1 ;; +esac +printf '{"tagName":"v0.1.0","targetCommitish":"%s","isDraft":%s,"assets":[' \ + "$target" "$draft" +printf '%s' \ + '{"name":"LICENSE-APACHE"},' \ + '{"name":"LICENSE-MIT"},' \ + '{"name":"docprims.h"},' \ + '{"name":"sbom-0.1.0.cdx.json"}' +while read -r platform format binary; do + [[ "$FAKE_RELEASE_STATE" == missing-asset && "$platform" == windows-amd64 ]] && continue + printf ',{"name":"docprims-0.1.0-%s.%s"}' "$platform" "$format" +done <"$FAKE_PLATFORM_FILE" +printf '%s]}\n' "$extra" +EOF +chmod +x "$fake_bin/gh" + +expect_fail() { + if "$@" >/dev/null 2>&1; then + echo "expected failure: $*" >&2 + exit 1 + fi +} + +run_check() { + local state="$1" + ( + cd "$fixture" + export PATH="$fake_bin:$original_path" + export DOCPRIMS_RELEASE_TAG=v0.1.0 + export FAKE_COMMIT="$fixture_commit" + export FAKE_RELEASE_STATE="$state" + export FAKE_PLATFORM_FILE="$fixture/config/release/cli-platforms.txt" + unset GH_REPO + # shellcheck source=/dev/null + source scripts/release-common.sh + assert_github_release_state release_base_assets + ) +} + +run_check valid +expect_fail run_check published +expect_fail run_check wrong-target +expect_fail run_check wrong-repo +expect_fail run_check extra-asset +expect_fail run_check missing-asset +expect_fail env GH_REPO=other/repository \ + PATH="$fake_bin:$original_path" \ + DOCPRIMS_RELEASE_TAG=v0.1.0 \ + FAKE_COMMIT="$fixture_commit" \ + FAKE_RELEASE_STATE=valid \ + FAKE_PLATFORM_FILE="$fixture/config/release/cli-platforms.txt" \ + bash -c "cd '$fixture'; source scripts/release-common.sh; assert_github_release_state release_base_assets" + +echo "[ok] GitHub repository, target, draft, and inventory controls passed" diff --git a/scripts/release-guard-tag-ruleset.sh b/scripts/release-guard-tag-ruleset.sh new file mode 100755 index 0000000..a395d5c --- /dev/null +++ b/scripts/release-guard-tag-ruleset.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash + +set -euo pipefail + +readonly expected_repository="3leaps/docprims" +readonly expected_ruleset_name="Tag Publish Protection" + +require_command() { + local name="$1" + if ! command -v "${name}" >/dev/null 2>&1; then + echo "error: ${name} is required on PATH" >&2 + exit 1 + fi +} + +expected_policy_json() { + jq -cnS \ + --arg repository "${expected_repository}" \ + --arg ruleset_name "${expected_ruleset_name}" \ + '{ + repository: $repository, + ruleset_name: $ruleset_name, + source_type: "Repository", + target: "tag", + enforcement: "active", + conditions: {ref_name: {exclude: [], include: ["refs/tags/v*"]}}, + rules: ["creation", "deletion", "non_fast_forward", "update"], + bypass_actors: [{actor_id: null, actor_type: "OrganizationAdmin", bypass_mode: "always"}] + }' +} + +policy_digest() { + if command -v sha256sum >/dev/null 2>&1; then + expected_policy_json | sha256sum | awk '{print $1}' + return + fi + if command -v shasum >/dev/null 2>&1; then + expected_policy_json | shasum -a 256 | awk '{print $1}' + return + fi + echo "error: sha256sum or shasum is required" >&2 + return 1 +} + +policy_attestation() { + printf 'Tag-Publish-Policy-SHA256: %s\n' "$(policy_digest)" +} + +resolve_ruleset() { + local pages ids count id + pages="$(gh api --paginate --slurp "repos/${expected_repository}/rulesets?per_page=100")" + ids="$(printf '%s\n' "${pages}" | jq -r \ + --arg name "${expected_ruleset_name}" \ + 'flatten | map(select(.name == $name)) | .[].id')" + count="$(printf '%s\n' "${ids}" | awk 'NF { count++ } END { print count + 0 }')" + if [ "${count}" -ne 1 ]; then + echo "error: expected exactly one '${expected_ruleset_name}' ruleset; found ${count}" >&2 + return 1 + fi + id="$(printf '%s\n' "${ids}" | awk 'NF { print; exit }')" + gh api "repos/${expected_repository}/rulesets/${id}" +} + +validate_ruleset() { + local ruleset="$1" + local mode="${2:-full}" + if ! printf '%s\n' "${ruleset}" | jq -e \ + --arg name "${expected_ruleset_name}" \ + --arg repository "${expected_repository}" ' + .name == $name and + .source_type == "Repository" and + .source == $repository and + .target == "tag" and + .enforcement == "active" and + .conditions == {"ref_name":{"exclude":[],"include":["refs/tags/v*"]}} and + (.rules | length) == 4 and + ([.rules[].type] | sort) == ["creation","deletion","non_fast_forward","update"] and + all(.rules[]; (keys | sort) == ["type"]) + ' >/dev/null; then + echo "error: live tag ruleset does not match the required publication policy" >&2 + return 1 + fi + local actors='[{"actor_id":null,"actor_type":"OrganizationAdmin","bypass_mode":"always"}]' + case "$mode" in + full) + jq -e --argjson actors "$actors" '.bypass_actors == $actors' >/dev/null <<<"$ruleset" || { + echo 'error: full ruleset must restrict bypass to organization administrators' >&2 + return 1 + } + ;; + read-only) + jq -e --argjson actors "$actors" \ + '(.bypass_actors == null) or (.bypass_actors == []) or (.bypass_actors == $actors)' \ + >/dev/null <<<"$ruleset" || { + echo 'error: unexpected visible bypass actors' >&2 + return 1 + } + ;; + *) + echo 'error: invalid ruleset validation mode' >&2 + return 1 + ;; + esac +} + +main() { + local mode=full + local print_attestation=0 + local expected_attestation=0 + local verify_object="" + while [ "$#" -gt 0 ]; do + case "$1" in + --read-only) mode=read-only ;; + --verify-tag-attestation) + shift + [[ -n "${1:-}" ]] || { + echo 'error: missing tag object' >&2 + exit 1 + } + verify_object="$1" + ;; + --print-attestation) print_attestation=1 ;; + --expected-attestation) expected_attestation=1 ;; + *) + echo "error: unknown argument: $1" >&2 + exit 1 + ;; + esac + shift + done + [[ "$print_attestation" -eq 0 || "$mode" == full ]] || { + echo 'error: full policy required for attestation' >&2 + exit 1 + } + + require_command jq + if [ "${expected_attestation}" -eq 1 ]; then + policy_attestation + exit 0 + fi + require_command gh + + local ruleset + ruleset="$(resolve_ruleset)" + validate_ruleset "${ruleset}" "$mode" + if [ -n "$verify_object" ]; then + [ "$(git cat-file -t "$verify_object" 2>/dev/null)" = tag ] || { + echo 'error: annotated tag required' >&2 + exit 1 + } + local message_file expected_file actual_tail suffix_size + ruleset_tmp="$(mktemp -d)" + trap 'rm -rf "$ruleset_tmp"' EXIT + message_file="$ruleset_tmp/message" + expected_file="$ruleset_tmp/expected" + actual_tail="$ruleset_tmp/tail" + "$(dirname "$0")/release-tag-body.sh" "$verify_object" >"$message_file" + printf '\n\n%s\n' "$(policy_attestation)" >"$expected_file" + suffix_size="$(wc -c <"$expected_file" | tr -d '[:space:]')" + if [[ "$(wc -c <"$message_file" | tr -d '[:space:]')" -lt "$suffix_size" ]]; then + echo 'error: signed policy attestation missing or mismatched' >&2 + exit 1 + fi + tail -c "$suffix_size" "$message_file" >"$actual_tail" + if ! cmp -s "$expected_file" "$actual_tail"; then + echo 'error: signed policy attestation missing or mismatched' >&2 + exit 1 + fi + fi + + if [ "${print_attestation}" -eq 1 ]; then + echo "[ok] tag ruleset matches the full publication policy" >&2 + policy_attestation + else + echo "[ok] tag ruleset matches the $mode publication policy" + fi +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/scripts/release-insert-anchors.sh b/scripts/release-insert-anchors.sh new file mode 100755 index 0000000..7ddc3c7 --- /dev/null +++ b/scripts/release-insert-anchors.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Maintainer-only: derive two public anchors from approved public exports. +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd -P)" +# shellcheck source=scripts/release-decernor.sh +source "$root/scripts/release-decernor.sh" +cd "$root" +resolve_release_decernor ceremony +: "${DOCPRIMS_MINISIGN_PUB:?public minisign export required}" +[[ -s docs/security/release-signing-keys.asc && ! -L docs/security/release-signing-keys.asc && + -s "$DOCPRIMS_MINISIGN_PUB" && ! -L "$DOCPRIMS_MINISIGN_PUB" ]] || { + echo 'error: both public exports required' >&2 + exit 1 +} +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +"$RELEASE_DECERNOR_BIN" fingerprint docs/security/release-signing-keys.asc --class public --kind gpg \ + --format ndjson --path-mode none --gpg-role primary >"$scratch/gpg.ndjson" +"$RELEASE_DECERNOR_BIN" fingerprint "$DOCPRIMS_MINISIGN_PUB" --class public --kind minisign \ + --format ndjson --path-mode none >"$scratch/minisign.ndjson" +python3 - "$scratch" <<'PY' +import json +import pathlib +import sys +base = pathlib.Path(sys.argv[1]) +g = [json.loads(line) for line in (base / 'gpg.ndjson').read_text().splitlines() if line] +m = [json.loads(line) for line in (base / 'minisign.ndjson').read_text().splitlines() if line and json.loads(line).get('fingerprint_scheme') == 'minisign-public-blob-sha256-v1'] +if len(g) != 1 or len(m) != 1 or g[0].get('fingerprint_scheme') != 'openpgp-fingerprint-v1' or g[0].get('key_role') != 'primary': + raise SystemExit('error: expected exactly one primary GPG and one minisign blob fingerprint') +(base / 'expected-fingerprints.ndjson').write_text(''.join(json.dumps(r, separators=(',', ':')) + '\n' for r in (g[0], m[0]))) +(base / 'expected-fingerprints.txt').write_text('gpg ' + g[0]['fingerprint'] + '\nminisign ' + m[0]['fingerprint'] + '\n') +(base / 'gpg.json').write_text(json.dumps(g[0]) + '\n') +(base / 'minisign.json').write_text(json.dumps(m[0]) + '\n') +PY +for kind in gpg minisign; do + "$RELEASE_DECERNOR_BIN" validate --schema "$root/schemas/release/fingerprint-record.v0.schema.json" \ + --data "$scratch/$kind.json" >/dev/null +done +"$RELEASE_DECERNOR_BIN" fingerprint docs/security/release-signing-keys.asc --class public --kind gpg \ + --format ndjson --path-mode none --gpg-role primary >"$scratch/gpg.verify.ndjson" +"$RELEASE_DECERNOR_BIN" fingerprint "$DOCPRIMS_MINISIGN_PUB" --class public --kind minisign \ + --format ndjson --path-mode none >"$scratch/minisign.verify.ndjson" +cmp "$scratch/gpg.ndjson" "$scratch/gpg.verify.ndjson" +cmp "$scratch/minisign.ndjson" "$scratch/minisign.verify.ndjson" +"$root/scripts/install-release-anchors.sh" "$scratch" "$root/keys" \ + "$RELEASE_DECERNOR_BIN" fingerprint verify \ + --anchors "$root/keys/expected-fingerprints.txt" \ + --anchors-ndjson "$root/keys/expected-fingerprints.ndjson" \ + --gpg "$root/docs/security/release-signing-keys.asc" --minisign "$DOCPRIMS_MINISIGN_PUB" >/dev/null +echo '[ok] generated public fingerprint anchors for review' diff --git a/scripts/release-push-tag.sh b/scripts/release-push-tag.sh new file mode 100755 index 0000000..f8b7062 --- /dev/null +++ b/scripts/release-push-tag.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +# shellcheck source=scripts/release-tag-common.sh +source "$(dirname "$0")/release-tag-common.sh" +cd "$tag_root" +"$tag_root/scripts/release-verify-tag.sh" +"$tag_root/scripts/release-guard-tag-ruleset.sh" +[[ -z "$(git ls-remote --tags origin "refs/tags/$DOCPRIMS_RELEASE_TAG")" ]] || tag_die 'remote tag already exists' +git push origin "refs/tags/$DOCPRIMS_RELEASE_TAG" +"$tag_root/scripts/release-verify-remote-tag.sh" diff --git a/scripts/release-safety.test.sh b/scripts/release-safety.test.sh new file mode 100755 index 0000000..4e25631 --- /dev/null +++ b/scripts/release-safety.test.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +fixture="$(mktemp -d "${TMPDIR:-/tmp}/docprims-release-safety.XXXXXX")" +trap 'rm -rf "$fixture"' EXIT + +expect_fail() { + if "$@" >/dev/null 2>&1; then + echo "expected failure: $*" >&2 + exit 1 + fi +} + +root="$fixture/repo" +git init -q -b main "$root" +( + cd "$root" + expect_fail "$SCRIPT_DIR/release-clean.sh" / + mkdir -p dist + ln -s "$fixture/outside" dist/release + expect_fail "$SCRIPT_DIR/release-clean.sh" + [[ -L dist/release ]] +) + +echo "[ok] release cleanup safety tests passed" diff --git a/scripts/release-tag-body.sh b/scripts/release-tag-body.sh new file mode 100755 index 0000000..ec0def3 --- /dev/null +++ b/scripts/release-tag-body.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Emit the annotated tag's message bytes, excluding the armored signature. +set -euo pipefail +object="${1:?tag object required}" +[[ "$(git cat-file -t "$object" 2>/dev/null)" == tag ]] || { + echo 'error: annotated tag required' >&2 + exit 1 +} +git cat-file tag "$object" | python3 -c ' +import sys +raw = sys.stdin.buffer.read() +if b"\n\n" not in raw: + raise SystemExit("error: malformed tag object") +body = raw.split(b"\n\n", 1)[1] +marker = b"-----BEGIN PGP SIGNATURE-----\n" +if body.startswith(marker): + body = b"" +else: + offset = body.find(b"\n" + marker) + if offset >= 0: + body = body[:offset + 1] +sys.stdout.buffer.write(body) +' diff --git a/scripts/release-tag-common.sh b/scripts/release-tag-common.sh new file mode 100755 index 0000000..5603d4e --- /dev/null +++ b/scripts/release-tag-common.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# Signed-tag ceremony invariants. No signing material is stored in this repo. +set -euo pipefail + +tag_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" + +tag_die() { + echo "error: $*" >&2 + return 1 +} + +tag_identity() { + [[ "${DOCPRIMS_TAGGER_NAME:-} <${DOCPRIMS_TAGGER_EMAIL:-}>" == "$(cat "$tag_root/config/release/tagger-identity.txt")" ]] || { + tag_die 'infosec tagger identity required' + return 1 + } + [[ "$DOCPRIMS_TAGGER_NAME" != *$'\n'* && "$DOCPRIMS_TAGGER_NAME" != *$'\r'* && + "$DOCPRIMS_TAGGER_NAME" != *'<'* && "$DOCPRIMS_TAGGER_NAME" != *'>'* ]] || { + tag_die 'invalid tagger name' + return 1 + } + [[ "$DOCPRIMS_TAGGER_EMAIL" != *$'\n'* && "$DOCPRIMS_TAGGER_EMAIL" != *$'\r'* && + "$DOCPRIMS_TAGGER_EMAIL" != *'<'* && "$DOCPRIMS_TAGGER_EMAIL" != *'>'* ]] || { + tag_die 'invalid tagger email' + return 1 + } +} + +tag_version() { + [[ -n "${DOCPRIMS_RELEASE_TAG:-}" ]] || { + tag_die 'DOCPRIMS_RELEASE_TAG required' + return 1 + } + "$tag_root/scripts/release-guard-tag-version.sh" >/dev/null +} + +tag_message_file() { + [[ -n "${DOCPRIMS_TAG_MESSAGE_DIR:-}" && -d "$DOCPRIMS_TAG_MESSAGE_DIR" && + ! -L "$DOCPRIMS_TAG_MESSAGE_DIR" ]] || { + tag_die 'tag message directory required' + return 1 + } + [[ "${DOCPRIMS_TAG_MESSAGE_DIR%/}" == */"$DOCPRIMS_RELEASE_TAG" ]] || { + tag_die 'message directory must end in tag' + return 1 + } + local canonical + canonical="$(cd "$DOCPRIMS_TAG_MESSAGE_DIR" && pwd -P)" + case "$canonical" in "$tag_root" | "$tag_root"/*) + tag_die 'message directory must be outside repository' + return 1 + ;; + esac + local file="${DOCPRIMS_TAG_MESSAGE_DIR%/}/message.txt" + [[ -f "$file" && ! -L "$file" && -s "$file" ]] || { + tag_die 'message.txt required' + return 1 + } + python3 - "$file" <<'PY' || { +import pathlib +import sys +data = pathlib.Path(sys.argv[1]).read_bytes() +try: + data.decode('utf-8') +except UnicodeError: + raise SystemExit(1) +if (b'\x00' in data or b'\r' in data or not data.endswith(b'\n') + or data.endswith(b'\n\n') or any(line.rstrip(b' \t') != line for line in data.splitlines())): + raise SystemExit(1) +PY + tag_die 'message.txt must be UTF-8 with exactly one final newline' + return 1 + } + printf '%s\n' "$file" +} + +tag_origin() { + local url host + url="$(git config --get remote.origin.url)" + case "$url" in + https://github.com/3leaps/docprims | https://github.com/3leaps/docprims.git | \ + git@github.com:3leaps/docprims | git@github.com:3leaps/docprims.git) ;; + git@*:3leaps/docprims | git@*:3leaps/docprims.git) + host="${url#git@}" + host="${host%%:*}" + [[ "$(ssh -G "$host" 2>/dev/null | awk '$1=="hostname" {print $2;exit}')" == github.com ]] || { + tag_die 'origin SSH host must resolve to github.com' + return 1 + } + ;; + *) + tag_die 'origin must be 3leaps/docprims on GitHub' + return 1 + ;; + esac +} + +tag_checkout() { + tag_origin + git fetch --quiet origin '+refs/heads/main:refs/remotes/origin/main' + [[ -z "$(git status --porcelain --untracked-files=normal)" ]] || tag_die 'clean checkout required' + [[ "$(git rev-parse HEAD)" == "$(git rev-parse refs/remotes/origin/main)" ]] || tag_die 'HEAD must match fetched origin/main' +} + +tag_selector_shape() { + [[ "${DOCPRIMS_PGP_KEY_ID:-}" =~ ^[0-9A-F]{40}!$ ]] || { + tag_die 'exact uppercase signing subkey fingerprint with ! required' + return 1 + } + [[ "${DOCPRIMS_GPG_SIGNING_FINGERPRINT:-}" =~ ^[0-9A-F]{40}$ ]] || { + tag_die 'uppercase primary fingerprint required' + return 1 + } +} + +tag_key_selector() { + tag_selector_shape || return 1 + "$tag_root/scripts/validate-release-anchors.sh" >/dev/null + [[ "$(awk '$1=="gpg" {print $2}' "$tag_root/keys/expected-fingerprints.txt")" == "$DOCPRIMS_GPG_SIGNING_FINGERPRINT" ]] || { + tag_die 'operator fingerprint differs from committed anchor' + return 1 + } + [[ -n "${DOCPRIMS_GPG_HOMEDIR:-}" && -d "$DOCPRIMS_GPG_HOMEDIR" ]] || { + tag_die 'external GPG home required' + return 1 + } + local home + home="$(cd "$DOCPRIMS_GPG_HOMEDIR" && pwd -P)" + case "$home" in "$tag_root"/* | "$tag_root") + tag_die 'GPG home must be outside repository' + return 1 + ;; + esac + export GNUPGHOME="$home" + local listing primary subkey + listing="$(gpg --batch --with-colons --fingerprint --with-subkey-fingerprint --list-keys "${DOCPRIMS_PGP_KEY_ID%!}" 2>/dev/null)" || tag_die 'signing subkey unavailable' + primary="$(printf '%s\n' "$listing" | awk -F: '$1=="pub" {p=1;next} p && $1=="fpr" {print $10;exit}')" + subkey="$(printf '%s\n' "$listing" | awk -F: -v f="${DOCPRIMS_PGP_KEY_ID%!}" '$1=="sub" {s=1;cap=$12;next} s && $1=="fpr" {if ($10==f && cap ~ /[sS]/) print $10; s=0}')" + [[ "$primary" == "$DOCPRIMS_GPG_SIGNING_FINGERPRINT" && "$subkey" == "${DOCPRIMS_PGP_KEY_ID%!}" ]] || { + tag_die 'signing subkey not on pinned primary' + return 1 + } +} + +tag_expected_message() { + local file + file="$(tag_message_file)" || return 1 + cat "$file" + printf '\n' + "$tag_root/scripts/release-guard-tag-ruleset.sh" --print-attestation +} + +tag_verify_object() { + local object="$1" expected_file="$2" tagger actual_file + [[ "$(git cat-file -t "$object" 2>/dev/null)" == tag ]] || { + tag_die 'annotated tag object required' + return 1 + } + [[ "$(git cat-file tag "$object" | sed -n 's/^tag //p' | head -1)" == "$DOCPRIMS_RELEASE_TAG" ]] || { + tag_die 'tag name mismatch' + return 1 + } + [[ "$(git rev-parse "${object}^{}")" == "$(git rev-parse HEAD)" ]] || { + tag_die 'tag target mismatch' + return 1 + } + [[ "$(git rev-parse HEAD)" == "$(git rev-parse refs/remotes/origin/main)" ]] || { + tag_die 'tag not at main' + return 1 + } + tagger="$(git cat-file tag "$object" | sed -n 's/^tagger \(.*\) [0-9][0-9]* [+-][0-9][0-9][0-9][0-9]$/\1/p' | head -1)" + [[ "$tagger" == "$DOCPRIMS_TAGGER_NAME <$DOCPRIMS_TAGGER_EMAIL>" ]] || { + tag_die 'tagger identity mismatch' + return 1 + } + actual_file="$(mktemp)" + if ! "$tag_root/scripts/release-tag-body.sh" "$object" >"$actual_file"; then + rm -f "$actual_file" + return 1 + fi + if ! cmp -s "$expected_file" "$actual_file"; then + rm -f "$actual_file" + tag_die 'signed tag message mismatch' + return 1 + fi + rm -f "$actual_file" +} diff --git a/scripts/release-tag-controls.test.sh b/scripts/release-tag-controls.test.sh new file mode 100755 index 0000000..31e2a44 --- /dev/null +++ b/scripts/release-tag-controls.test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2016 +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd -P)" +while IFS= read -r target; do + grep -Eq "^${target}:" "$root/Makefile" || { + echo 'error: release checklist names a missing make target' >&2 + exit 1 + } +done < <(grep -oE 'make [a-z][a-z0-9-]+' "$root/RELEASE_CHECKLIST.md" | awk '{print $2}' | sort -u) +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +mkdir -p "$scratch/anchor-stage" "$scratch/anchor-dest" +for ext in txt ndjson; do + printf 'new\n' >"$scratch/anchor-stage/expected-fingerprints.$ext" + printf 'old\n' >"$scratch/anchor-dest/expected-fingerprints.$ext" +done +if DOCPRIMS_TEST_FAIL_ANCHOR_INSTALL=1 "$root/scripts/install-release-anchors.sh" \ + "$scratch/anchor-stage" "$scratch/anchor-dest" >/dev/null 2>&1; then + echo 'error: expected pair rollback' >&2 + exit 1 +fi +for ext in txt ndjson; do + [[ "$(cat "$scratch/anchor-dest/expected-fingerprints.$ext")" == old ]] +done +"$root/scripts/install-release-anchors.sh" "$scratch/anchor-stage" "$scratch/anchor-dest" +for ext in txt ndjson; do + [[ "$(cat "$scratch/anchor-dest/expected-fingerprints.$ext")" == new ]] +done +mkdir -p "$scratch/v1.2.3" +printf 'Release v1.2.3\n' >"$scratch/v1.2.3/message.txt" +export DOCPRIMS_RELEASE_TAG=v1.2.3 +export DOCPRIMS_TAG_MESSAGE_DIR="$scratch/v1.2.3" +export DOCPRIMS_TAGGER_NAME='3 Leaps Infosec Team' +export DOCPRIMS_TAGGER_EMAIL='infosec@3leaps.net' +source "$root/scripts/release-tag-common.sh" +expect_fail() { if "$@" >/dev/null 2>&1; then + echo "expected rejection: $*" >&2 + exit 1 +fi; } +tag_identity +[[ "$(tag_message_file)" == "$scratch/v1.2.3/message.txt" ]] +printf 'Release v1.2.3' >"$scratch/v1.2.3/message.txt" +expect_fail tag_message_file +printf 'Release v1.2.3 \n' >"$scratch/v1.2.3/message.txt" +expect_fail tag_message_file +printf 'Release v1.2.3\n' >"$scratch/v1.2.3/message.txt" +expect_fail env DOCPRIMS_TAGGER_NAME=$'bad\nname' bash -c 'source "$1"; tag_identity' _ "$root/scripts/release-tag-common.sh" +expect_fail env DOCPRIMS_TAGGER_EMAIL='a' bash -c 'source "$1"; tag_identity' _ "$root/scripts/release-tag-common.sh" +expect_fail env -u DOCPRIMS_TAG_MESSAGE_DIR bash -c 'source "$1"; tag_message_file' _ "$root/scripts/release-tag-common.sh" +expect_fail env DOCPRIMS_TAG_MESSAGE_DIR="$scratch" bash -c 'source "$1"; tag_message_file' _ "$root/scripts/release-tag-common.sh" +export DOCPRIMS_PGP_KEY_ID=0123456789ABCDEF +expect_fail tag_selector_shape +unset DOCPRIMS_PGP_KEY_ID +expect_fail tag_selector_shape +git init -q -b main "$scratch/repo" +git -C "$scratch/repo" config user.name fixture +git -C "$scratch/repo" config user.email fixture@example.invalid +printf 'x\n' >"$scratch/repo/file" +git -C "$scratch/repo" add file +git -C "$scratch/repo" commit -qm fixture +git -C "$scratch/repo" tag v1.2.3 +export DOCPRIMS_PGP_KEY_ID=0123456789ABCDEF0123456789ABCDEF01234567! +( + cd "$scratch/repo" + expect_fail tag_verify_object refs/tags/v1.2.3 'Release v1.2.3' +) +mkdir -p "$scratch/bin" +cat >"$scratch/bin/gh" <<'SH' +#!/usr/bin/env bash +if [[ "$*" == *'rulesets?per_page=100'* ]]; then + printf '[[{"id":7,"name":"Tag Publish Protection"}]]\n' +else + cat "$RULESET_FIXTURE" +fi +SH +chmod +x "$scratch/bin/gh" +export PATH="$scratch/bin:$PATH" RULESET_FIXTURE="$scratch/ruleset.json" +cat >"$RULESET_FIXTURE" <<'JSON' +{"name":"Tag Publish Protection","source_type":"Repository","source":"3leaps/docprims","target":"tag","enforcement":"active","conditions":{"ref_name":{"exclude":[],"include":["refs/tags/v*"]}},"rules":[{"type":"creation"},{"type":"deletion"},{"type":"non_fast_forward"},{"type":"update"}],"bypass_actors":[{"actor_id":null,"actor_type":"OrganizationAdmin","bypass_mode":"always"}]} +JSON +"$root/scripts/release-guard-tag-ruleset.sh" --read-only >/dev/null +for hidden in absent empty; do + if [[ "$hidden" == absent ]]; then + jq 'del(.bypass_actors)' "$scratch/ruleset.json" >"$scratch/hidden.json" + else + jq '.bypass_actors = []' "$scratch/ruleset.json" >"$scratch/hidden.json" + fi + export RULESET_FIXTURE="$scratch/hidden.json" + "$root/scripts/release-guard-tag-ruleset.sh" --read-only >/dev/null + expect_fail "$root/scripts/release-guard-tag-ruleset.sh" +done +export RULESET_FIXTURE="$scratch/ruleset.json" +jq '.bypass_actors = [{"actor_id":2,"actor_type":"Team","bypass_mode":"always"}]' "$RULESET_FIXTURE" >"$scratch/wrong-actor.json" +export RULESET_FIXTURE="$scratch/wrong-actor.json" +expect_fail "$root/scripts/release-guard-tag-ruleset.sh" --read-only +export RULESET_FIXTURE="$scratch/ruleset.json" +attestation="$("$root/scripts/release-guard-tag-ruleset.sh" --print-attestation)" +git -C "$scratch/repo" tag -d v1.2.3 >/dev/null +git -C "$scratch/repo" tag -a v1.2.3 -m 'Release v1.2.3' -m "$attestation" +( + cd "$scratch/repo" + "$root/scripts/release-guard-tag-ruleset.sh" --verify-tag-attestation refs/tags/v1.2.3 >/dev/null +) +git -C "$scratch/repo" update-ref refs/remotes/origin/main "$(git -C "$scratch/repo" rev-parse HEAD)" +git -C "$scratch/repo" tag -d v1.2.3 >/dev/null +printf 'Release v1.2.3\n\n%s\n' "$attestation" >"$scratch/expected-message" +printf 'Changed message\n\n%s\n' "$attestation" >"$scratch/wrong-message" +( + cd "$scratch/repo" + GIT_COMMITTER_NAME="$DOCPRIMS_TAGGER_NAME" GIT_COMMITTER_EMAIL="$DOCPRIMS_TAGGER_EMAIL" \ + git tag -a --cleanup=verbatim v1.2.3 -F "$scratch/expected-message" +) +( + cd "$scratch/repo" + tag_verify_object refs/tags/v1.2.3 "$scratch/expected-message" +) +( + cd "$scratch/repo" + expect_fail tag_verify_object refs/tags/v1.2.3 "$scratch/wrong-message" +) +printf 'Release v1.2.3\n\n%s\n\n\n' "$attestation" >"$scratch/extra-blank-lines" +( + cd "$scratch/repo" + GIT_COMMITTER_NAME="$DOCPRIMS_TAGGER_NAME" GIT_COMMITTER_EMAIL="$DOCPRIMS_TAGGER_EMAIL" \ + git tag -fa --cleanup=verbatim v1.2.3 -F "$scratch/extra-blank-lines" >/dev/null + expect_fail tag_verify_object refs/tags/v1.2.3 "$scratch/expected-message" + expect_fail "$root/scripts/release-guard-tag-ruleset.sh" --verify-tag-attestation refs/tags/v1.2.3 +) +git -C "$scratch/repo" tag -f -a v1.2.3 -m 'Release v1.2.3' -m 'Tag-Publish-Policy-SHA256: stale' >/dev/null +( + cd "$scratch/repo" + expect_fail "$root/scripts/release-guard-tag-ruleset.sh" --verify-tag-attestation refs/tags/v1.2.3 +) +sed 's/"enforcement":"active"/"enforcement":"disabled"/' "$RULESET_FIXTURE" >"$scratch/bad.json" +export RULESET_FIXTURE="$scratch/bad.json" +expect_fail "$root/scripts/release-guard-tag-ruleset.sh" --read-only +echo '[ok] signed-tag negative controls passed' diff --git a/scripts/release-tag.sh b/scripts/release-tag.sh new file mode 100755 index 0000000..7a1c2b4 --- /dev/null +++ b/scripts/release-tag.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Create and verify a signed tag locally; publication is a separate command. +set -euo pipefail +# shellcheck source=scripts/release-tag-common.sh +source "$(dirname "$0")/release-tag-common.sh" +cd "$tag_root" +tag_version +tag_identity +tag_key_selector +tag_checkout +[[ -z "$(git ls-remote --tags origin "refs/tags/$DOCPRIMS_RELEASE_TAG")" ]] || tag_die 'remote tag already exists' +[[ -z "$(git show-ref --tags "refs/tags/$DOCPRIMS_RELEASE_TAG" || true)" ]] || tag_die 'local tag already exists' +message="$(mktemp)" +trap 'rm -f "$message"' EXIT +tag_expected_message >"$message" +export GIT_COMMITTER_NAME="$DOCPRIMS_TAGGER_NAME" GIT_COMMITTER_EMAIL="$DOCPRIMS_TAGGER_EMAIL" +git tag -s -a --cleanup=verbatim -u "$DOCPRIMS_PGP_KEY_ID" -F "$message" "$DOCPRIMS_RELEASE_TAG" HEAD +"$tag_root/scripts/release-verify-tag.sh" +echo '[ok] signed tag is local only' diff --git a/scripts/release-verify-remote-tag.sh b/scripts/release-verify-remote-tag.sh new file mode 100755 index 0000000..77a3d9b --- /dev/null +++ b/scripts/release-verify-remote-tag.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +# shellcheck source=scripts/release-tag-common.sh +source "$(dirname "$0")/release-tag-common.sh" +cd "$tag_root" +"$tag_root/scripts/release-verify-tag.sh" +local_object="$(git rev-parse "refs/tags/$DOCPRIMS_RELEASE_TAG")" +remote_object="$(git ls-remote origin "refs/tags/$DOCPRIMS_RELEASE_TAG" | awk '{print $1}')" +[[ -n "$remote_object" && "$remote_object" == "$local_object" ]] || tag_die 'remote tag object differs from local tag' +tag_ref="$(gh api "repos/3leaps/docprims/git/ref/tags/$DOCPRIMS_RELEASE_TAG")" +[[ "$(jq -r '.object.type' <<<"$tag_ref")" == tag && "$(jq -r '.object.sha' <<<"$tag_ref")" == "$local_object" ]] || tag_die 'GitHub tag ref mismatch' +tag_json="$(gh api "repos/3leaps/docprims/git/tags/$local_object")" +jq -e --arg commit "$(git rev-parse HEAD)" --arg tag "$DOCPRIMS_RELEASE_TAG" \ + '.tag == $tag and .object.type == "commit" and .object.sha == $commit and .verification.verified == true and .verification.reason == "valid"' \ + <<<"$tag_json" >/dev/null || tag_die 'GitHub does not report verified tag at HEAD' +echo '[ok] remote signed tag matches local object and GitHub verification' diff --git a/scripts/release-verify-tag.sh b/scripts/release-verify-tag.sh new file mode 100755 index 0000000..a33b80e --- /dev/null +++ b/scripts/release-verify-tag.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Verify the tag with an isolated keyring imported only from the committed pin. +set -euo pipefail +# shellcheck source=scripts/release-tag-common.sh +source "$(dirname "$0")/release-tag-common.sh" +cd "$tag_root" +tag_version +tag_identity +tag_selector_shape +tag_checkout +object="$(git rev-parse "refs/tags/$DOCPRIMS_RELEASE_TAG")" +expected="$(mktemp)" +trap 'rm -f "$expected"' EXIT +tag_expected_message >"$expected" +tag_verify_object "$object" "$expected" +"$tag_root/scripts/verify-pinned-tag.sh" +[[ "$(awk '$1=="gpg" {print $2}' keys/expected-fingerprints.txt)" == "$DOCPRIMS_GPG_SIGNING_FINGERPRINT" ]] || tag_die 'authorized primary differs from committed anchor' diff --git a/scripts/sign-release-assets.sh b/scripts/sign-release-assets.sh index ab080cd..b9a7540 100755 --- a/scripts/sign-release-assets.sh +++ b/scripts/sign-release-assets.sh @@ -1,70 +1,59 @@ #!/usr/bin/env bash -# Sign release checksum manifests with minisign (and optionally PGP) -# Usage: sign-release-assets.sh [dir] -# -# Environment variables: -# DOCPRIMS_MINISIGN_KEY - Path to minisign secret key (required) -# DOCPRIMS_PGP_KEY_ID - PGP key ID for optional GPG signing (optional) -# DOCPRIMS_GPG_HOMEDIR - Custom GPG home directory (optional) -# -# Requires: minisign, optionally gpg -set -euo pipefail - -TAG=${1:?"usage: sign-release-assets.sh [dir]"} -DIR=${2:-dist/release} +# Sign exact checksum manifests with minisign and optional PGP. -if [ ! -d "$DIR" ]; then - echo "Error: Directory $DIR does not exist" - exit 1 -fi +set -euo pipefail -if [ -z "${DOCPRIMS_MINISIGN_KEY:-}" ]; then - echo "Error: DOCPRIMS_MINISIGN_KEY environment variable not set" - echo "Set to path of your minisign secret key:" - echo " export DOCPRIMS_MINISIGN_KEY=/path/to/docprims.key" - exit 1 -fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" + +directory="${1:-dist/release}" +require_release_guard 1 >/dev/null +tag="$(release_tag)" +"$SCRIPT_DIR/verify-checksums.sh" "$directory" >/dev/null +"$SCRIPT_DIR/validate-release-assets.sh" "$directory" checksummed >/dev/null + +: "${DOCPRIMS_MINISIGN_KEY:?load the approved minisign secret key}" +: "${DOCPRIMS_MINISIGN_PUB:?load the approved minisign public key}" +for key_file in "$DOCPRIMS_MINISIGN_KEY" "$DOCPRIMS_MINISIGN_PUB"; do + [[ -f "$key_file" && ! -L "$key_file" ]] || { + echo "error: configured minisign material is not a regular file" >&2 + exit 1 + } +done -if [ ! -f "$DOCPRIMS_MINISIGN_KEY" ]; then - echo "Error: Minisign key not found: $DOCPRIMS_MINISIGN_KEY" +root="$(release_repo_root)" +secret_parent="$(cd "$(dirname "$DOCPRIMS_MINISIGN_KEY")" && pwd -P)" +secret_real="$secret_parent/$(basename "$DOCPRIMS_MINISIGN_KEY")" +case "$secret_real" in +"$root"/*) + echo "error: minisign secret key must be outside the repository" >&2 exit 1 + ;; +esac + +if [[ -n "${DOCPRIMS_PGP_KEY_ID:-}" || -n "${DOCPRIMS_GPG_HOMEDIR:-}" ]]; then + require_complete_pgp_config + [[ -d "$DOCPRIMS_GPG_HOMEDIR" ]] || { + echo "error: configured GPG home is unavailable" >&2 + exit 1 + } fi -cd "$DIR" - -echo "Signing release $TAG..." - for manifest in SHA256SUMS SHA512SUMS; do - if [ -f "$manifest" ]; then - echo "Signing $manifest with minisign..." - minisign -S -s "$DOCPRIMS_MINISIGN_KEY" \ - -m "$manifest" \ - -t "docprims $TAG - $(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - -x "${manifest}.minisig" - echo "[ok] Created ${manifest}.minisig" + minisign -S -s "$DOCPRIMS_MINISIGN_KEY" \ + -m "$directory/$manifest" \ + -t "docprims $tag" \ + -x "$directory/${manifest}.minisig" + if [[ -n "${DOCPRIMS_PGP_KEY_ID:-}" ]]; then + gpg --homedir "$DOCPRIMS_GPG_HOMEDIR" \ + --batch --armor --detach-sign \ + --local-user "$DOCPRIMS_PGP_KEY_ID" \ + --output "$directory/${manifest}.asc" \ + "$directory/$manifest" fi done - -if [ -n "${DOCPRIMS_PGP_KEY_ID:-}" ]; then - GPG_OPTS=() - if [ -n "${DOCPRIMS_GPG_HOMEDIR:-}" ]; then - GPG_OPTS+=("--homedir" "$DOCPRIMS_GPG_HOMEDIR") - fi - - for manifest in SHA256SUMS SHA512SUMS; do - if [ -f "$manifest" ]; then - echo "Signing $manifest with PGP..." - gpg "${GPG_OPTS[@]}" \ - --armor \ - --detach-sign \ - --local-user "$DOCPRIMS_PGP_KEY_ID" \ - --output "${manifest}.asc" \ - "$manifest" - echo "[ok] Created ${manifest}.asc" - fi - done -else - echo "[--] PGP signing skipped (DOCPRIMS_PGP_KEY_ID not set)" -fi - -echo "[ok] Signing complete" +"$SCRIPT_DIR/validate-release-assets.sh" \ + "$directory" signed-without-keys >/dev/null +echo "[ok] checksum manifests signed" diff --git a/scripts/stage-release-anchors.sh b/scripts/stage-release-anchors.sh new file mode 100755 index 0000000..e4270ec --- /dev/null +++ b/scripts/stage-release-anchors.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd -P)" +dest="${1:-dist/release}" +for ext in txt ndjson; do + source_file="$root/keys/expected-fingerprints.$ext" + [[ -s "$source_file" && ! -L "$source_file" ]] || { + echo "error: missing committed fingerprint pin" >&2 + exit 1 + } + cp "$source_file" "$dest/expected-fingerprints.$ext" +done +echo '[ok] public fingerprint anchors staged' diff --git a/scripts/upload-release-assets.sh b/scripts/upload-release-assets.sh index 1db14af..b67a661 100755 --- a/scripts/upload-release-assets.sh +++ b/scripts/upload-release-assets.sh @@ -1,61 +1,33 @@ #!/usr/bin/env bash -# Upload signed release assets to GitHub -# Usage: upload-release-assets.sh [dir] -# -# Uploads checksum files, signatures, public keys, and release notes. -# Requires: gh CLI authenticated with write permissions -set -euo pipefail - -TAG=${1:?"usage: upload-release-assets.sh [dir]"} -DIR=${2:-dist/release} - -if [ ! -d "$DIR" ]; then - echo "Error: Directory $DIR does not exist" - exit 1 -fi - -cd "$DIR" - -REQUIRED_FILES=( - "SHA256SUMS" - "SHA256SUMS.minisig" - "SHA512SUMS" - "SHA512SUMS.minisig" - "docprims-minisign.pub" -) - -for file in "${REQUIRED_FILES[@]}"; do - if [ ! -f "$file" ]; then - echo "Error: Required file missing: $file" - exit 1 - fi -done +# Verify and upload the exact signed provenance set to the trusted draft. -UPLOAD_FILES=( - "SHA256SUMS" - "SHA256SUMS.minisig" - "SHA512SUMS" - "SHA512SUMS.minisig" - "docprims-minisign.pub" -) +set -euo pipefail -for optional in "SHA256SUMS.asc" "SHA512SUMS.asc" "docprims-release-signing-key.asc"; do - if [ -f "$optional" ]; then - UPLOAD_FILES+=("$optional") +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" + +directory="${1:-dist/release}" +require_release_guard 1 >/dev/null +tag="$(release_tag)" +"$SCRIPT_DIR/validate-release-assets.sh" "$directory" signed >/dev/null +"$SCRIPT_DIR/verify-checksums.sh" "$directory" >/dev/null +"$SCRIPT_DIR/verify-public-keys.sh" "$directory" >/dev/null +"$SCRIPT_DIR/verify-signatures.sh" "$directory" >/dev/null +assert_github_release_state release_base_assets + +# The draft already holds the base assets; upload only what signing added. +upload_files=() +while IFS= read -r asset; do + if ! release_base_assets | grep -Fxq -- "$asset"; then + upload_files+=("$directory/$asset") fi -done - -RELEASE_NOTES="release-notes-${TAG}.md" -if [ -f "$RELEASE_NOTES" ]; then - UPLOAD_FILES+=("$RELEASE_NOTES") -fi - -gh release upload "$TAG" "${UPLOAD_FILES[@]}" --clobber - -if [ -f "$RELEASE_NOTES" ]; then - gh release edit "$TAG" --notes-file "$RELEASE_NOTES" -fi - -gh release edit "$TAG" --draft=false - -echo "[ok] Release $TAG published" +done < <(release_signed_assets) + +gh release upload "$tag" "${upload_files[@]}" \ + --repo "$DOCPRIMS_REPOSITORY" --clobber +gh release edit "$tag" --repo "$DOCPRIMS_REPOSITORY" \ + --notes-file "$directory/release-notes-${tag}.md" +assert_github_release_state release_signed_assets +echo "[ok] signed assets uploaded; GitHub release remains draft" diff --git a/scripts/validate-release-anchors.sh b/scripts/validate-release-anchors.sh new file mode 100755 index 0000000..db9697a --- /dev/null +++ b/scripts/validate-release-anchors.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Cross-check the decernor pair and the pinned public primary. +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd -P)" +cd "$root" +python3 - keys/expected-fingerprints.txt keys/expected-fingerprints.ndjson <<'PY' +import json +import pathlib +import re +import sys +txt, ndjson = (pathlib.Path(path) for path in sys.argv[1:]) +if not txt.is_file() or not ndjson.is_file() or txt.is_symlink() or ndjson.is_symlink(): + raise SystemExit('error: committed fingerprint pair missing') +lines = txt.read_text().splitlines() +if len(lines) != 2: + raise SystemExit('error: expected two fingerprint lines') +records = [json.loads(line) for line in ndjson.read_text().splitlines() if line] +if len(records) != 2: + raise SystemExit('error: expected two fingerprint records') +g, m = records +if g.get('fingerprint_scheme') != 'openpgp-fingerprint-v1' or g.get('key_role') != 'primary': + raise SystemExit('error: expected a primary GPG record') +if m.get('fingerprint_scheme') != 'minisign-public-blob-sha256-v1': + raise SystemExit('error: expected a minisign blob record') +if not re.fullmatch('[0-9A-F]{40}', g.get('fingerprint', '')) or not re.fullmatch('[0-9a-f]{64}', m.get('fingerprint', '')): + raise SystemExit('error: malformed public fingerprint') +if lines != ['gpg ' + g['fingerprint'], 'minisign ' + m['fingerprint']]: + raise SystemExit('error: fingerprint text and records differ') +PY +pin=docs/security/release-signing-keys.asc +[[ -s "$pin" && ! -L "$pin" ]] || { + echo 'error: committed public pin missing' >&2 + exit 1 +} +temp="$(mktemp -d)" +trap 'rm -rf "$temp"' EXIT +chmod 700 "$temp" +GNUPGHOME="$temp" gpg --batch --quiet --import "$pin" +listing="$(GNUPGHOME="$temp" gpg --batch --with-colons --fingerprint --list-keys)" +[[ "$(awk -F: '$1=="pub" {n++} END {print n+0}' <<<"$listing")" == 1 ]] || { + echo 'error: pin must contain exactly one public primary' >&2 + exit 1 +} +[[ "$(awk -F: '$1=="fpr" {print $10;exit}' <<<"$listing")" == "$(awk '$1=="gpg" {print $2}' keys/expected-fingerprints.txt)" ]] || { + echo 'error: primary pin differs from fingerprint pair' >&2 + exit 1 +} +echo '[ok] public pin and decernor anchors agree' diff --git a/scripts/validate-release-assets.sh b/scripts/validate-release-assets.sh new file mode 100755 index 0000000..64236b6 --- /dev/null +++ b/scripts/validate-release-assets.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Validate exact release inventory and CLI archive structure. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" + +directory="${1:-dist/release}" +mode="${2:-base}" + +release_tag >/dev/null + +case "$mode" in +base) + assert_exact_directory_inventory "$directory" release_base_assets + ;; +signable) + assert_exact_directory_inventory "$directory" release_signable_assets + ;; +checksummed) + assert_exact_directory_inventory "$directory" release_checksummed_assets + ;; +signed-without-keys) + assert_exact_directory_inventory "$directory" release_signed_without_keys_assets + ;; +signed) + assert_exact_directory_inventory "$directory" release_signed_assets + ;; +*) + echo "usage: validate-release-assets.sh DIR MODE" >&2 + exit 1 + ;; +esac + +validate_all_cli_archives "$directory" +echo "[ok] exact $mode release asset inventory verified" diff --git a/scripts/verify-checksums.sh b/scripts/verify-checksums.sh new file mode 100755 index 0000000..2042a64 --- /dev/null +++ b/scripts/verify-checksums.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Verify exact, duplicate-free checksum manifests and every signed input. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" + +directory="${1:-dist/release}" +release_tag >/dev/null + +expected="$(mktemp "${TMPDIR:-/tmp}/docprims-checksum-expected.XXXXXX")" +actual="$(mktemp "${TMPDIR:-/tmp}/docprims-checksum-actual.XXXXXX")" +trap 'rm -f "$expected" "$actual"' EXIT +release_signable_assets | LC_ALL=C sort >"$expected" + +for manifest in SHA256SUMS SHA512SUMS; do + [[ -f "$directory/$manifest" && ! -L "$directory/$manifest" ]] || { + echo "error: missing checksum manifest" >&2 + exit 1 + } + awk ' + NF != 2 { exit 1 } + $2 !~ /^\*?[A-Za-z0-9][A-Za-z0-9._-]*$/ { exit 1 } + { + name = $2 + sub(/^\*/, "", name) + print name + } + ' "$directory/$manifest" | LC_ALL=C sort >"$actual" || { + echo "error: malformed or path-bearing checksum entry" >&2 + exit 1 + } + if [[ "$(wc -l <"$actual" | tr -d ' ')" != "$(sort -u "$actual" | wc -l | tr -d ' ')" ]]; then + echo "error: duplicate checksum entry" >&2 + exit 1 + fi + cmp -s "$expected" "$actual" || { + echo "error: checksum manifest inventory mismatch" >&2 + exit 1 + } + algorithm="${manifest#SHA}" + algorithm="${algorithm%SUMS}" + (cd "$directory" && shasum -a "$algorithm" -c "$manifest") +done + +validate_all_cli_archives "$directory" +echo "[ok] exact checksum manifests verified" diff --git a/scripts/verify-pinned-tag.sh b/scripts/verify-pinned-tag.sh new file mode 100755 index 0000000..377dbb5 --- /dev/null +++ b/scripts/verify-pinned-tag.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Verify an annotated tag using only the committed public pin, in an isolated keyring. +set -euo pipefail +die() { + echo "error: $*" >&2 + exit 1 +} +tag="${DOCPRIMS_RELEASE_TAG:-}" +[[ "$tag" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || die 'canonical release tag required' +pin=docs/security/release-signing-keys.asc +anchors=keys/expected-fingerprints.txt +[[ -s "$pin" && ! -L "$pin" && -s "$anchors" && ! -L "$anchors" ]] || die 'committed public pin and anchors required' +"$(dirname "$0")/validate-release-anchors.sh" >/dev/null +object="$(git rev-parse "refs/tags/$tag" 2>/dev/null)" || die 'tag absent' +[[ "$(git cat-file -t "$object")" == tag ]] || die 'annotated tag required' +[[ "$(git rev-parse "${object}^{}")" == "$(git rev-parse HEAD)" ]] || die 'tag target must be HEAD' +[[ "$(git cat-file tag "$object" | sed -n 's/^tag //p' | head -1)" == "$tag" ]] || die 'tag name mismatch' +tagger="$(git cat-file tag "$object" | sed -n 's/^tagger \(.*\) [0-9][0-9]* [+-][0-9][0-9][0-9][0-9]$/\1/p' | head -1)" +[[ "$tagger" == "$(cat config/release/tagger-identity.txt)" ]] || die 'infosec tagger required' +primary="$(awk '$1=="gpg" {print $2}' "$anchors")" +[[ "$primary" =~ ^[0-9A-F]{40}$ ]] || die 'primary anchor malformed' +keyring="$(mktemp -d)" +trap 'rm -rf "$keyring"' EXIT +chmod 700 "$keyring" +GNUPGHOME="$keyring" gpg --batch --quiet --import "$pin" || die 'pin import failed' +[[ "$(GNUPGHOME="$keyring" gpg --batch --with-colons --list-keys | awk -F: '$1=="pub" {count++} END {print count+0}')" == 1 ]] || die 'pin must hold exactly one primary key' +pinned_primary="$(GNUPGHOME="$keyring" gpg --batch --with-colons --fingerprint --list-keys | awk -F: '$1=="fpr" {print $10;exit}')" +[[ "$pinned_primary" == "$primary" ]] || die 'pin does not match primary anchor' +GNUPGHOME="$keyring" git -c gpg.program=gpg verify-tag --raw "$object" >"$keyring/status" 2>&1 || { + awk '$1=="[GNUPG:]" {print "signature status:", $2}' "$keyring/status" >&2 + die 'tag signature invalid under pin' +} +awk -v primary="$primary" -v selector="${DOCPRIMS_PGP_KEY_ID:-}" ' + $1=="[GNUPG:]" && $2=="VALIDSIG" { + valid++; subkey=$3; signer_primary=$NF + } + $1=="[GNUPG:]" && $2=="GOODSIG" {good++} + $1=="[GNUPG:]" && $2 ~ /^(EXPKEYSIG|EXPSIG|REVKEYSIG|KEYREVOKED|BADSIG|ERRSIG)$/ {bad++} + END { + if (bad || good!=1 || valid!=1 || signer_primary!=primary || subkey==primary) exit 1 + if (selector!="" && (selector !~ /^[0-9A-F]{40}!$/ || subkey "!" != selector)) exit 1 + } +' "$keyring/status" || die 'signing subkey, expiry or revocation check failed' +echo '[ok] signed tag verified against committed public pin' diff --git a/scripts/verify-pinned-tag.test.sh b/scripts/verify-pinned-tag.test.sh new file mode 100755 index 0000000..fd21716 --- /dev/null +++ b/scripts/verify-pinned-tag.test.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Synthetic, short-lived signing fixture; all generated material stays outside git. +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd -P)" +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +export GNUPGHOME="$scratch/gpg" +mkdir -m 700 "$GNUPGHOME" +gpg --batch --pinentry-mode loopback --passphrase '' --quick-generate-key \ + '3 Leaps Infosec Team ' ed25519 sign 0 >/dev/null 2>&1 +primary="$(gpg --batch --with-colons --fingerprint --list-keys | awk -F: '$1=="fpr" {print $10;exit}')" +gpg --batch --pinentry-mode loopback --passphrase '' --quick-add-key "$primary" ed25519 sign 1d >/dev/null 2>&1 +subkey="$(gpg --batch --with-colons --with-subkey-fingerprint --list-keys "$primary" | awk -F: '$1=="sub" {s=1;next} s && $1=="fpr" {print $10;exit}')" +fixture="$scratch/repo" +git init -q -b main "$fixture" +mkdir -p "$fixture/docs/security" "$fixture/keys" "$fixture/scripts" "$fixture/config/release" +cp "$root/scripts/verify-pinned-tag.sh" "$root/scripts/validate-release-anchors.sh" "$fixture/scripts/" +cp "$root/config/release/tagger-identity.txt" "$fixture/config/release/" +gpg --batch --armor --export "$primary" >"$fixture/docs/security/release-signing-keys.asc" +printf 'gpg %s\nminisign %064d\n' "$primary" 0 >"$fixture/keys/expected-fingerprints.txt" +python3 - "$primary" "$fixture/keys/expected-fingerprints.ndjson" <<'PY' +import json +import pathlib +import sys +out = pathlib.Path(sys.argv[2]) +out.write_text(json.dumps({'fingerprint_scheme':'openpgp-fingerprint-v1','key_role':'primary','fingerprint':sys.argv[1]})+'\n'+json.dumps({'fingerprint_scheme':'minisign-public-blob-sha256-v1','fingerprint':'0'*64})+'\n') +PY +printf 'fixture\n' >"$fixture/file" +printf 'Fixture release\n' >"$scratch/message.txt" +git -C "$fixture" config user.name '3 Leaps Infosec Team' +git -C "$fixture" config user.email infosec@3leaps.net +git -C "$fixture" add file +git -C "$fixture" commit -qm fixture +export DOCPRIMS_RELEASE_TAG=v1.2.3 +export DOCPRIMS_TAGGER_NAME='3 Leaps Infosec Team' +export DOCPRIMS_TAGGER_EMAIL=infosec@3leaps.net +( + cd "$fixture" + GIT_COMMITTER_NAME='3 Leaps Infosec Team' GIT_COMMITTER_EMAIL=infosec@3leaps.net \ + git tag -s -a --cleanup=verbatim -u "$subkey!" -F "$scratch/message.txt" "$DOCPRIMS_RELEASE_TAG" +) +git -C "$fixture" cat-file tag v1.2.3 | grep -q -- '^-----BEGIN PGP SIGNATURE-----$' || { + echo 'error: fixture tag not signed' >&2 + exit 1 +} +expect_fail() { if "$@" >/dev/null 2>&1; then + echo 'expected signature control failure' >&2 + exit 1 +fi; } +verify() { ( + cd "$fixture" + ./scripts/verify-pinned-tag.sh +); } +verify >/dev/null +DOCPRIMS_PGP_KEY_ID="$subkey!" verify >/dev/null +cp "$fixture/keys/expected-fingerprints.txt" "$scratch/anchors.good" +printf 'gpg %s\nminisign %064d\n' "$primary" 1 >"$fixture/keys/expected-fingerprints.txt" +expect_fail verify +cp "$scratch/anchors.good" "$fixture/keys/expected-fingerprints.txt" +git -C "$fixture" update-ref refs/remotes/origin/main "$(git -C "$fixture" rev-parse HEAD)" +# shellcheck source=scripts/release-tag-common.sh +source "$root/scripts/release-tag-common.sh" +( + cd "$fixture" + tag_verify_object refs/tags/v1.2.3 "$scratch/message.txt" +) +( + cd "$fixture" + printf 'Tampered release\n' >"$scratch/tampered.txt" + expect_fail tag_verify_object refs/tags/v1.2.3 "$scratch/tampered.txt" +) +printf '# Heading\nFixture release\n' >"$scratch/message.txt" +( + cd "$fixture" + GIT_COMMITTER_NAME='3 Leaps Infosec Team' GIT_COMMITTER_EMAIL=infosec@3leaps.net \ + git tag -fs -a --cleanup=verbatim -u "$subkey!" -F "$scratch/message.txt" "$DOCPRIMS_RELEASE_TAG" + tag_verify_object refs/tags/v1.2.3 "$scratch/message.txt" +) +verify >/dev/null +printf 'Fixture release\n' >"$scratch/message.txt" +( + cd "$fixture" + GIT_COMMITTER_NAME='3 Leaps Infosec Team' GIT_COMMITTER_EMAIL=infosec@3leaps.net \ + git tag -fs -a --cleanup=verbatim -u "$subkey!" -F "$scratch/message.txt" "$DOCPRIMS_RELEASE_TAG" +) +mv "$fixture/docs/security/release-signing-keys.asc" "$scratch/approved.asc" +expect_fail verify +cp "$scratch/approved.asc" "$fixture/docs/security/release-signing-keys.asc" +gpg --batch --pinentry-mode loopback --passphrase '' --quick-generate-key \ + 'Fixture Extra ' ed25519 cert 1d >/dev/null 2>&1 +extra="$(gpg --batch --with-colons --fingerprint --list-keys 'Fixture Extra' | awk -F: '$1=="fpr" {print $10;exit}')" +gpg --batch --armor --export "$extra" >>"$fixture/docs/security/release-signing-keys.asc" +expect_fail verify +gpg --batch --armor --export "$extra" >"$fixture/docs/security/release-signing-keys.asc" +expect_fail verify +cp "$scratch/approved.asc" "$fixture/docs/security/release-signing-keys.asc" +( + cd "$fixture" + GIT_COMMITTER_NAME='3 Leaps Infosec Team' GIT_COMMITTER_EMAIL=infosec@3leaps.net \ + git tag -fs -a --cleanup=verbatim -u "$primary!" -F "$scratch/message.txt" "$DOCPRIMS_RELEASE_TAG" +) +expect_fail verify +( + cd "$fixture" + GIT_COMMITTER_NAME='3 Leaps Infosec Team' GIT_COMMITTER_EMAIL=infosec@3leaps.net \ + git tag -fs -a --cleanup=verbatim -u "$subkey!" -F "$scratch/message.txt" "$DOCPRIMS_RELEASE_TAG" +) +verify >/dev/null +real_gpg="$(command -v gpg)" +mkdir -p "$scratch/bin" +cat >"$scratch/bin/gpg" <<'SH' +#!/usr/bin/env bash +if [[ -n "${FIXTURE_FUTURE_TIME:-}" ]]; then + exec "$FIXTURE_REAL_GPG" --faked-system-time "$FIXTURE_FUTURE_TIME" "$@" +fi +exec "$FIXTURE_REAL_GPG" "$@" +SH +chmod +x "$scratch/bin/gpg" +export FIXTURE_REAL_GPG="$real_gpg" PATH="$scratch/bin:$PATH" +export FIXTURE_FUTURE_TIME="$(($(date +%s) + 172800))" +mkdir -m 700 "$scratch/expired-ring" +GNUPGHOME="$scratch/expired-ring" gpg --batch --quiet --import "$fixture/docs/security/release-signing-keys.asc" +( + cd "$fixture" + GNUPGHOME="$scratch/expired-ring" git verify-tag --raw v1.2.3 >"$scratch/expired-status" 2>&1 +) || true +grep -q '^\[GNUPG:\] VALIDSIG ' "$scratch/expired-status" +grep -q '^\[GNUPG:\] EXPKEYSIG ' "$scratch/expired-status" +expect_fail verify +unset FIXTURE_FUTURE_TIME +verify >/dev/null +git -C "$fixture" tag -d "$DOCPRIMS_RELEASE_TAG" >/dev/null +git -C "$fixture" tag "$DOCPRIMS_RELEASE_TAG" +expect_fail verify +echo '[ok] isolated pinned-tag signature controls passed' diff --git a/scripts/verify-public-keys.sh b/scripts/verify-public-keys.sh index 4b50249..312ce67 100755 --- a/scripts/verify-public-keys.sh +++ b/scripts/verify-public-keys.sh @@ -1,49 +1,49 @@ #!/usr/bin/env bash -# Verify that exported keys contain only public material (no secrets) -# Usage: verify-public-keys.sh [dir] +# Reject secret or malformed key material in the release set. + set -euo pipefail -DIR=${1:-dist/release} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/release-decernor.sh +source "$SCRIPT_DIR/release-decernor.sh" +resolve_release_decernor ceremony +directory="${1:-dist/release}" +minisign_public="$directory/docprims-minisign.pub" -if [ ! -d "$DIR" ]; then - echo "Error: Directory $DIR does not exist" +[[ -s "$minisign_public" && ! -L "$minisign_public" ]] || { + echo "error: exported minisign public key is missing or unsafe" >&2 + exit 1 +} +grep -q '^untrusted comment:' "$minisign_public" || { + echo "error: exported minisign public key is malformed" >&2 + exit 1 +} +if grep -qi 'secret' "$minisign_public"; then + echo "error: exported minisign material contains a secret marker" >&2 exit 1 fi -cd "$DIR" - -ERRORS=0 - -if [ -f "docprims-minisign.pub" ]; then - if grep -qi "secret" "docprims-minisign.pub"; then - echo "[!!] DANGER: docprims-minisign.pub may contain secret key material!" - ERRORS=$((ERRORS + 1)) - elif grep -q "^untrusted comment:" "docprims-minisign.pub"; then - echo "[ok] docprims-minisign.pub appears to be a valid public key" - else - echo "[!!] docprims-minisign.pub has unexpected format" - ERRORS=$((ERRORS + 1)) - fi -else - echo "[--] docprims-minisign.pub not found" -fi - -if [ -f "docprims-release-signing-key.asc" ]; then - if grep -q "PRIVATE KEY BLOCK" "docprims-release-signing-key.asc"; then - echo "[!!] DANGER: docprims-release-signing-key.asc contains PRIVATE KEY!" - ERRORS=$((ERRORS + 1)) - elif grep -q "PUBLIC KEY BLOCK" "docprims-release-signing-key.asc"; then - echo "[ok] docprims-release-signing-key.asc is a public key" - else - echo "[!!] docprims-release-signing-key.asc has unexpected format" - ERRORS=$((ERRORS + 1)) - fi -fi - -if [ $ERRORS -eq 0 ]; then - echo "[ok] Public key verification passed" - exit 0 +pgp_public="$directory/docprims-release-signing-key.asc" +[[ -s "$pgp_public" && ! -L "$pgp_public" ]] || { + echo "error: exported PGP key is unsafe" >&2 + exit 1 +} +grep -q 'BEGIN PGP PUBLIC KEY BLOCK' "$pgp_public" || { + echo "error: exported PGP key is malformed" >&2 + exit 1 +} +if grep -q 'PRIVATE KEY BLOCK' "$pgp_public"; then + echo "error: exported PGP material contains a private key" >&2 + exit 1 fi - -echo "[!!] CRITICAL: Found $ERRORS potential secret key exposures!" -exit 1 +for ext in txt ndjson; do + [[ -s "$directory/expected-fingerprints.$ext" && ! -L "$directory/expected-fingerprints.$ext" ]] || { + echo 'error: staged public anchor is missing or unsafe' >&2 + exit 1 + } +done +"$RELEASE_DECERNOR_BIN" fingerprint verify \ + --anchors "$directory/expected-fingerprints.txt" \ + --anchors-ndjson "$directory/expected-fingerprints.ndjson" \ + --gpg "$pgp_public" --minisign "$minisign_public" >/dev/null +echo "[ok] exported public keys contain public material only" diff --git a/scripts/verify-signatures.sh b/scripts/verify-signatures.sh index c33dabe..66a1069 100755 --- a/scripts/verify-signatures.sh +++ b/scripts/verify-signatures.sh @@ -1,60 +1,49 @@ #!/usr/bin/env bash -# Verify signatures on checksum manifests -# Usage: verify-signatures.sh [dir] -set -euo pipefail - -DIR=${1:-dist/release} - -if [ ! -d "$DIR" ]; then - echo "Error: Directory $DIR does not exist" - exit 1 -fi +# Verify minisign and, when configured, PGP signatures for both manifests. -cd "$DIR" - -ERRORS=0 +set -euo pipefail -if [ -f "docprims-minisign.pub" ]; then - for manifest in SHA256SUMS SHA512SUMS; do - if [ -f "$manifest" ] && [ -f "${manifest}.minisig" ]; then - if minisign -Vm "$manifest" -p docprims-minisign.pub; then - echo "[ok] $manifest minisign signature valid" - else - echo "[!!] $manifest minisign signature INVALID" - ERRORS=$((ERRORS + 1)) - fi - elif [ -f "$manifest" ]; then - echo "[!!] Missing signature: ${manifest}.minisig" - ERRORS=$((ERRORS + 1)) - fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release-common.sh +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/release-common.sh" + +directory="${1:-dist/release}" +release_tag >/dev/null +"$SCRIPT_DIR/verify-public-keys.sh" "$directory" >/dev/null + +for manifest in SHA256SUMS SHA512SUMS; do + [[ -f "$directory/${manifest}.minisig" ]] || { + echo "error: required minisign signature is missing" >&2 + exit 1 + } + minisign -Vm "$directory/$manifest" \ + -p "$directory/docprims-minisign.pub" \ + -x "$directory/${manifest}.minisig" >/dev/null +done + +pgp_public="$directory/docprims-release-signing-key.asc" +pgp_present=0 +[[ -e "$pgp_public" ]] && pgp_present=1 +for manifest in SHA256SUMS SHA512SUMS; do + [[ -e "$directory/${manifest}.asc" ]] && pgp_present=1 +done +if [[ "$pgp_present" == "1" ]]; then + for file in "$pgp_public" \ + "$directory/SHA256SUMS.asc" "$directory/SHA512SUMS.asc"; do + [[ -f "$file" ]] || { + echo "error: partial PGP release material is forbidden" >&2 + exit 1 + } done -else - echo "[!!] docprims-minisign.pub not found - cannot verify minisign signatures" - ERRORS=$((ERRORS + 1)) -fi - -if [ -f "docprims-release-signing-key.asc" ]; then - GNUPGHOME=$(mktemp -d) - export GNUPGHOME - trap 'rm -rf "$GNUPGHOME"' EXIT - - gpg --import docprims-release-signing-key.asc 2>/dev/null + temporary_gpg="$(mktemp -d "${TMPDIR:-/tmp}/docprims-gpg.XXXXXX")" + trap 'rm -rf "$temporary_gpg"' EXIT + chmod 0700 "$temporary_gpg" + gpg --homedir "$temporary_gpg" --batch --import "$pgp_public" >/dev/null 2>&1 for manifest in SHA256SUMS SHA512SUMS; do - if [ -f "$manifest" ] && [ -f "${manifest}.asc" ]; then - if gpg --verify "${manifest}.asc" "$manifest" 2>/dev/null; then - echo "[ok] $manifest PGP signature valid" - else - echo "[!!] $manifest PGP signature INVALID" - ERRORS=$((ERRORS + 1)) - fi - fi + gpg --homedir "$temporary_gpg" --batch \ + --verify "$directory/${manifest}.asc" "$directory/$manifest" \ + >/dev/null 2>&1 done fi - -if [ $ERRORS -eq 0 ]; then - echo "[ok] All signatures verified" - exit 0 -fi - -echo "[!!] $ERRORS signature verification errors" -exit 1 +echo "[ok] all configured signatures verified" From 1a7e4864821597d2f8c8f4a0d3ebfddeac05531d Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:05:12 -0400 Subject: [PATCH 26/33] ci: pin actions by commit and default to read-only permissions Every workflow pins actions to a commit SHA, names its Rust toolchain, declares read-only contents permission at the top level and sets a concurrency group. ci gains a publish-check job that packages the crates and runs the release tooling tests with a checksum-pinned Decernor. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .github/workflows/ci.yml | 52 ++++++++++++++-- .github/workflows/go-bindings-prep.yml | 41 +++++++------ .github/workflows/go-bindings.yml | 27 +++++---- .github/workflows/typescript-bindings.yml | 15 +++-- .../workflows/typescript-napi-prebuilds.yml | 59 ++++++++++++------- .github/workflows/typescript-npm-publish.yml | 7 ++- .github/workflows/validate-release.yml | 17 ++++-- 7 files changed, 151 insertions(+), 67 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e694d69..4891968 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,13 +3,19 @@ on: push: branches: ["main"] pull_request: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: 1.94.1 components: rustfmt, clippy - name: Install cargo-deny run: cargo install cargo-deny@0.20.2 --locked @@ -24,11 +30,45 @@ jobs: make cbindgen git diff --exit-code ffi/docprims-ffi/docprims.h diff -u ffi/docprims-ffi/docprims.h bindings/go/docprims/include/docprims.h + publish-check: + name: publish-check + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 + - name: Verify release tooling dependencies + run: | + python3 -c 'import sys; assert sys.version_info >= (3, 11)' + command -v jq + command -v gpg + - name: Install checksum-pinned Decernor for release tests + run: | + tool_dir="$(mktemp -d "$RUNNER_TEMP/decernor.XXXXXX")" + archive="$(awk '{ print $2 }' config/release/decernor-ci-sha256.txt)" + version="${archive#decernor_}" + version="${version%%_*}" + curl --proto '=https' --tlsv1.2 --fail --silent --show-error --location \ + "https://github.com/3leaps/decernor/releases/download/v${version}/${archive}" \ + --output "$tool_dir/$archive" + (cd "$tool_dir" && sha256sum --check "$GITHUB_WORKSPACE/config/release/decernor-ci-sha256.txt") + tar -xzf "$tool_dir/$archive" -C "$tool_dir" decernor + echo "DOCPRIMS_DECERNOR_BIN=$tool_dir/decernor" >> "$GITHUB_ENV" + - name: Package workspace + run: make release-check + - name: Release tooling tests + run: make release-tooling-test msrv: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: toolchain: 1.88.0 - name: Build and test on MSRV @@ -36,8 +76,8 @@ jobs: test-go-vendored: runs-on: macos-14 steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.22.x" - name: Run Go tests (vendored libs) diff --git a/.github/workflows/go-bindings-prep.yml b/.github/workflows/go-bindings-prep.yml index 8560582..048b849 100644 --- a/.github/workflows/go-bindings-prep.yml +++ b/.github/workflows/go-bindings-prep.yml @@ -10,6 +10,11 @@ on: type: string env: CARGO_TERM_COLOR: always +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false jobs: validate: name: Validate Inputs @@ -17,7 +22,7 @@ jobs: outputs: version: ${{ steps.version.outputs.version }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Validate version matches VERSION file id: version run: | @@ -54,7 +59,7 @@ jobs: glibc: "" artifact_suffix: linux-arm64-musl steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Verify cargo-zigbuild is available shell: bash run: | @@ -106,7 +111,7 @@ jobs: cp "target/${TARGET_DIR}/release/libdocprims_ffi.so" "dist/ffi/${{ matrix.artifact_suffix }}/" fi - name: Upload FFI artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ffi-${{ matrix.artifact_suffix }} path: dist/ffi/${{ matrix.artifact_suffix }} @@ -117,9 +122,10 @@ jobs: env: MACOSX_DEPLOYMENT_TARGET: "11.0" steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: 1.94.1 targets: aarch64-apple-darwin - name: Build FFI (arm64) run: | @@ -136,7 +142,7 @@ jobs: # This allows consumers to set rpath or place dylib alongside executable. install_name_tool -id "@rpath/libdocprims_ffi.dylib" dist/ffi/darwin-arm64/libdocprims_ffi.dylib - name: Upload FFI artifact (darwin-arm64) - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ffi-darwin-arm64 path: dist/ffi/darwin-arm64 @@ -145,12 +151,13 @@ jobs: runs-on: windows-latest needs: validate steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: + toolchain: 1.94.1 targets: x86_64-pc-windows-gnu - name: Install MinGW - uses: msys2/setup-msys2@v2 + uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 with: msystem: MINGW64 update: true @@ -170,7 +177,7 @@ jobs: cp target/x86_64-pc-windows-gnu/release/docprims_ffi.dll dist/ffi/windows-amd64/ cp target/x86_64-pc-windows-gnu/release/libdocprims_ffi.dll.a dist/ffi/windows-amd64/ - name: Upload FFI artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ffi-windows-amd64 path: dist/ffi/windows-amd64 @@ -183,14 +190,14 @@ jobs: options: --user root needs: validate steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Generate C header shell: bash run: | set -euo pipefail cbindgen --config cbindgen.toml --crate docprims-ffi --output docprims.h - name: Upload header - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: c-header path: docprims.h @@ -202,16 +209,16 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Download all FFI artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: ffi-* path: ffi-libs - name: Download C header - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: c-header path: ffi-libs @@ -260,9 +267,9 @@ jobs: echo "Go bindings updated for $VERSION" ls -la bindings/go/docprims/lib/*/ || true - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ github.token }} base: main branch: go-bindings/v${{ needs.validate.outputs.version }} title: "feat(bindings): update Go prebuilt libs for v${{ needs.validate.outputs.version }}" diff --git a/.github/workflows/go-bindings.yml b/.github/workflows/go-bindings.yml index 1d4f836..6ca67c1 100644 --- a/.github/workflows/go-bindings.yml +++ b/.github/workflows/go-bindings.yml @@ -2,15 +2,20 @@ name: go-bindings on: pull_request: workflow_dispatch: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: go-test-linux-amd64: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: toolchain: 1.94.1 - - uses: actions/setup-go@v7 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.22.x" - name: Install cbindgen @@ -22,8 +27,8 @@ jobs: go-test-darwin-arm64-vendored: runs-on: macos-14 steps: - - uses: actions/checkout@v7 - - uses: actions/setup-go@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.22.x" - name: Run Go tests (vendored darwin-arm64) @@ -33,11 +38,11 @@ jobs: go-test-darwin-arm64-shared: runs-on: macos-14 steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: toolchain: 1.94.1 - - uses: actions/setup-go@v7 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.22.x" - name: Install cbindgen @@ -48,11 +53,11 @@ jobs: if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 with: toolchain: 1.94.1 - - uses: actions/setup-go@v7 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.22.x" - name: Install toolchain diff --git a/.github/workflows/typescript-bindings.yml b/.github/workflows/typescript-bindings.yml index eb202dd..3e93fbe 100644 --- a/.github/workflows/typescript-bindings.yml +++ b/.github/workflows/typescript-bindings.yml @@ -3,6 +3,11 @@ on: push: branches: ["main"] pull_request: +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: test-typescript: name: Test TypeScript Bindings @@ -12,11 +17,13 @@ jobs: os: [ubuntu-latest, macos-14, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "22" - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 - name: Install deps and run tests run: | cd bindings/typescript/docprims @@ -26,7 +33,7 @@ jobs: name: Test TypeScript Bindings (Alpine/musl) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Run tests in node:22-alpine run: | docker run --rm \ diff --git a/.github/workflows/typescript-napi-prebuilds.yml b/.github/workflows/typescript-napi-prebuilds.yml index 9b23488..8825c50 100644 --- a/.github/workflows/typescript-napi-prebuilds.yml +++ b/.github/workflows/typescript-napi-prebuilds.yml @@ -3,20 +3,27 @@ on: workflow_dispatch: env: CARGO_TERM_COLOR: always +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false jobs: build-linux: name: Build Linux prebuilds runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 - name: Install Zig - uses: goto-bus-stop/setup-zig@v2 + uses: goto-bus-stop/setup-zig@abea47f85e598557f500fa1fd2ab7464fcb39406 # v2.2.1 with: version: 0.12.0 - name: Install Rust targets @@ -55,7 +62,7 @@ jobs: --target aarch64-unknown-linux-musl --cross-compile test -f prebuilds/docprims.linux-arm64-musl.node - name: Upload prebuilds - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ts-prebuilds-linux path: bindings/typescript/docprims/prebuilds/*.node @@ -63,13 +70,15 @@ jobs: name: Build Linux arm64 prebuild (native) runs-on: ubuntu-latest-arm64-s steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims run: npm install @@ -83,7 +92,7 @@ jobs: npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds test -f prebuilds/docprims.linux-arm64-gnu.node - name: Upload prebuild - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ts-prebuilds-linux-arm64-gnu-native path: bindings/typescript/docprims/prebuilds/*.node @@ -91,13 +100,15 @@ jobs: name: Build macOS arm64 prebuild runs-on: macos-14 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims run: npm install @@ -111,7 +122,7 @@ jobs: npx napi build --manifest-path native/Cargo.toml --release --platform --no-js --strip --output-dir prebuilds test -f prebuilds/docprims.darwin-arm64.node - name: Upload prebuild - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ts-prebuilds-darwin-arm64 path: bindings/typescript/docprims/prebuilds/*.node @@ -119,13 +130,15 @@ jobs: name: Build Windows x64 prebuild runs-on: windows-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims run: npm install @@ -140,7 +153,7 @@ jobs: $node = 'prebuilds/docprims.win32-x64-msvc.node' if (-not (Test-Path $node)) { throw "missing $node" } - name: Upload prebuild - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ts-prebuilds-win32-x64-msvc path: bindings/typescript/docprims/prebuilds/*.node @@ -149,13 +162,15 @@ jobs: runs-on: ubuntu-latest needs: [build-linux, build-linux-arm64, build-macos-arm64, build-windows] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims run: npm install @@ -169,7 +184,7 @@ jobs: set -euo pipefail rm -f dist/native/docprims.*.node || true - name: Download prebuilds - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: ts-prebuilds-* path: downloaded-prebuilds @@ -204,7 +219,7 @@ jobs: cp "$f" "$dest" done - name: Upload npm dir - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ts-npm-dir path: bindings/typescript/docprims/npm diff --git a/.github/workflows/typescript-npm-publish.yml b/.github/workflows/typescript-npm-publish.yml index 38c1ec8..080b4df 100644 --- a/.github/workflows/typescript-npm-publish.yml +++ b/.github/workflows/typescript-npm-publish.yml @@ -26,6 +26,9 @@ permissions: id-token: write # Required for OIDC trusted publishing contents: read actions: read # Required to download artifacts from other runs +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false jobs: publish: name: Publish to npm @@ -34,9 +37,9 @@ jobs: steps: # Note: Do NOT use fetch-tags here. When running from a tag ref, # it causes a git fetch conflict. The tag is already checked out. - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Setup Node - uses: actions/setup-node@v7 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '24' registry-url: 'https://registry.npmjs.org' diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 2c44632..c4cb769 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -6,13 +6,18 @@ on: description: "Release tag to validate (e.g. v0.1.0)" required: true type: string +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false jobs: validate-go: name: Validate Go Bindings runs-on: ubuntu-latest steps: - name: Setup Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.22' - name: Test Go module is fetchable @@ -40,13 +45,15 @@ jobs: matrix: os: [ubuntu-latest, macos-14, windows-latest] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.tag }} - - uses: actions/setup-node@v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "22" - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: 1.94.1 - name: Build and test TypeScript bindings run: | cd bindings/typescript/docprims @@ -58,7 +65,7 @@ jobs: container: image: node:22-alpine steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ inputs.tag }} - name: Install build dependencies From 3f6cbc6d980f4463bc4ffe7d7a2573a9d69f199c Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:08:25 -0400 Subject: [PATCH 27/33] build(ts): install with npm ci and add platform packages at publish The committed package.json no longer lists the platform packages as optionalDependencies, so the lock file is complete and every workflow installs with npm ci. The publish workflow writes optionalDependencies for exactly the platform packages it publishes, at the root version, with scripts/set-platform-packages.js. check-version.sh rejects optionalDependencies in the committed manifests. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .github/workflows/typescript-bindings.yml | 4 +- .../workflows/typescript-napi-prebuilds.yml | 10 ++-- .github/workflows/typescript-npm-publish.yml | 10 ++-- .github/workflows/validate-release.yml | 4 +- .../typescript/docprims/package-lock.json | 8 --- bindings/typescript/docprims/package.json | 8 --- .../docprims/scripts/set-platform-packages.js | 54 +++++++++++++++++++ scripts/check-version.sh | 14 +++-- scripts/check-version.test.sh | 3 +- 9 files changed, 78 insertions(+), 37 deletions(-) create mode 100644 bindings/typescript/docprims/scripts/set-platform-packages.js diff --git a/.github/workflows/typescript-bindings.yml b/.github/workflows/typescript-bindings.yml index 3e93fbe..3a6be4e 100644 --- a/.github/workflows/typescript-bindings.yml +++ b/.github/workflows/typescript-bindings.yml @@ -27,7 +27,7 @@ jobs: - name: Install deps and run tests run: | cd bindings/typescript/docprims - npm install + npm ci npm run test:ci test-typescript-alpine: name: Test TypeScript Bindings (Alpine/musl) @@ -45,6 +45,6 @@ jobs: apk add --no-cache bash curl build-base python3 ca-certificates curl -fsSL https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.94.1 . "$HOME/.cargo/env" - npm install + npm ci npm run test:ci ' diff --git a/.github/workflows/typescript-napi-prebuilds.yml b/.github/workflows/typescript-napi-prebuilds.yml index 8825c50..6025dda 100644 --- a/.github/workflows/typescript-napi-prebuilds.yml +++ b/.github/workflows/typescript-napi-prebuilds.yml @@ -38,7 +38,7 @@ jobs: run: cargo install cargo-zigbuild --locked --version 0.23.4 - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm ci - name: Build .node files (zig) working-directory: bindings/typescript/docprims shell: bash @@ -81,7 +81,7 @@ jobs: toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm ci - name: Build .node file working-directory: bindings/typescript/docprims shell: bash @@ -111,7 +111,7 @@ jobs: toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm ci - name: Build .node file working-directory: bindings/typescript/docprims shell: bash @@ -141,7 +141,7 @@ jobs: toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm ci - name: Build .node file working-directory: bindings/typescript/docprims shell: pwsh @@ -173,7 +173,7 @@ jobs: toolchain: 1.94.1 - name: Install Node dependencies working-directory: bindings/typescript/docprims - run: npm install + run: npm ci - name: Build JS dist (root package) working-directory: bindings/typescript/docprims run: npm run clean && npm run build && npm run build:native diff --git a/.github/workflows/typescript-npm-publish.yml b/.github/workflows/typescript-npm-publish.yml index 080b4df..31b3795 100644 --- a/.github/workflows/typescript-npm-publish.yml +++ b/.github/workflows/typescript-npm-publish.yml @@ -237,13 +237,11 @@ jobs: working-directory: bindings/typescript/docprims run: | set -euo pipefail - # Use npm install instead of npm ci: the platform packages listed in - # optionalDependencies are published by this workflow, so they are not - # in the lock file and do not exist yet. npm install skips them. Do not - # pass --omit=optional: the TypeScript compiler binary is an optional - # dependency of typescript. - npm install + npm ci npm run build + - name: Point the root package at the platform packages + working-directory: bindings/typescript/docprims + run: node scripts/set-platform-packages.js ../../../npm-packages - name: Publish (dry run) if: ${{ inputs.dry_run }} working-directory: bindings/typescript/docprims diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index c4cb769..9aec83f 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -57,7 +57,7 @@ jobs: - name: Build and test TypeScript bindings run: | cd bindings/typescript/docprims - npm install + npm ci npm run test:ci validate-typescript-musl: name: Validate TypeScript Bindings (Alpine/musl) @@ -87,5 +87,5 @@ jobs: run: | set -euo pipefail . "$HOME/.cargo/env" - npm install + npm ci npm run test:ci diff --git a/bindings/typescript/docprims/package-lock.json b/bindings/typescript/docprims/package-lock.json index 6b4150b..1f0f46e 100644 --- a/bindings/typescript/docprims/package-lock.json +++ b/bindings/typescript/docprims/package-lock.json @@ -15,14 +15,6 @@ }, "engines": { "node": ">=22" - }, - "optionalDependencies": { - "@3leaps/docprims-darwin-arm64": "0.2.0", - "@3leaps/docprims-linux-arm64-gnu": "0.2.0", - "@3leaps/docprims-linux-arm64-musl": "0.2.0", - "@3leaps/docprims-linux-x64-gnu": "0.2.0", - "@3leaps/docprims-linux-x64-musl": "0.2.0", - "@3leaps/docprims-win32-x64-msvc": "0.2.0" } }, "node_modules/@emnapi/core": { diff --git a/bindings/typescript/docprims/package.json b/bindings/typescript/docprims/package.json index 5f38f7b..ddb90e1 100644 --- a/bindings/typescript/docprims/package.json +++ b/bindings/typescript/docprims/package.json @@ -40,14 +40,6 @@ "typecheck": "tsc -p tsconfig.build.json --noEmit" }, "dependencies": {}, - "optionalDependencies": { - "@3leaps/docprims-linux-x64-gnu": "0.2.0", - "@3leaps/docprims-linux-x64-musl": "0.2.0", - "@3leaps/docprims-linux-arm64-gnu": "0.2.0", - "@3leaps/docprims-linux-arm64-musl": "0.2.0", - "@3leaps/docprims-darwin-arm64": "0.2.0", - "@3leaps/docprims-win32-x64-msvc": "0.2.0" - }, "devDependencies": { "@napi-rs/cli": "^3.10.5", "@types/node": "^22.20.4", diff --git a/bindings/typescript/docprims/scripts/set-platform-packages.js b/bindings/typescript/docprims/scripts/set-platform-packages.js new file mode 100644 index 0000000..82c7eec --- /dev/null +++ b/bindings/typescript/docprims/scripts/set-platform-packages.js @@ -0,0 +1,54 @@ +// Point the root package at the platform packages published with it. +// +// The committed package.json lists no platform packages, so the lock file +// stays installable with `npm ci`. Before publishing, this script writes +// optionalDependencies for exactly the platform packages in the given +// directory (one subdirectory per `napi.targets` entry), at the root version. +// +// Usage: node scripts/set-platform-packages.js + +const fs = require("node:fs"); +const path = require("node:path"); + +function fail(message) { + console.error(`error: ${message}`); + process.exit(1); +} + +function main() { + const dir = process.argv[2]; + if (!dir) fail("usage: set-platform-packages.js "); + + const rootPath = path.resolve(__dirname, "..", "package.json"); + const root = JSON.parse(fs.readFileSync(rootPath, "utf8")); + const targets = root.napi?.targets ?? []; + if (root.optionalDependencies) { + fail("package.json already lists optionalDependencies"); + } + + const names = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (!entry.isDirectory()) fail(`unexpected entry ${entry.name}`); + const manifest = JSON.parse( + fs.readFileSync(path.join(dir, entry.name, "package.json"), "utf8"), + ); + if (manifest.name !== `${root.name}-${entry.name}`) { + fail(`${entry.name}: package name ${manifest.name}`); + } + if (manifest.version !== root.version) { + fail(`${manifest.name}: version ${manifest.version} != ${root.version}`); + } + names.push(manifest.name); + } + if (names.length !== targets.length) { + fail(`${names.length} platform packages for ${targets.length} napi targets`); + } + + root.optionalDependencies = Object.fromEntries( + names.sort().map((name) => [name, root.version]), + ); + fs.writeFileSync(rootPath, `${JSON.stringify(root, null, 2)}\n`); + console.log(`[ok] ${names.length} platform packages at ${root.version}`); +} + +main(); diff --git a/scripts/check-version.sh b/scripts/check-version.sh index f348925..cafc615 100755 --- a/scripts/check-version.sh +++ b/scripts/check-version.sh @@ -78,10 +78,16 @@ fi ok "workspace path-dependency versions match" npm_dir="$PROJECT_ROOT/bindings/typescript/docprims" +# Platform packages are injected at publish time (scripts/set-platform-packages.js); +# committed manifests list none, so the lock file installs with npm ci. +if jq -e -s 'map(.optionalDependencies // .packages[""].optionalDependencies) | any' \ + "$npm_dir/package.json" "$npm_dir/package-lock.json" >/dev/null; then + error "npm manifests must not list optionalDependencies" + failed=1 +fi npm_versions=$( - jq -r '.version, (.optionalDependencies // {} | .[])' "$npm_dir/package.json" - jq -r '.version, .packages[""].version, (.packages[""].optionalDependencies // {} | .[])' \ - "$npm_dir/package-lock.json" + jq -r '.version' "$npm_dir/package.json" + jq -r '.version, .packages[""].version' "$npm_dir/package-lock.json" ) while IFS= read -r npm_version; do if [[ "$npm_version" != "$version" ]]; then @@ -93,5 +99,5 @@ if [[ "$failed" -ne 0 ]]; then error "version consistency check failed" exit 1 fi -ok "npm package and platform package versions match" +ok "npm package versions match" ok "version consistency check passed" diff --git a/scripts/check-version.test.sh b/scripts/check-version.test.sh index 4c55990..770c3da 100755 --- a/scripts/check-version.test.sh +++ b/scripts/check-version.test.sh @@ -25,9 +25,8 @@ for mutation in \ "crates/docprims-ooxml/Cargo.toml|s/^version.workspace = true/version = \"9.9.9\"/" \ "$npm/native/Cargo.toml|s/^version.workspace = true/version = \"9.9.9\"/" \ "$npm/package.json|1,/\"version\": \"[^\"]*\"/s/\"version\": \"[^\"]*\"/\"version\": \"9.9.9\"/" \ - "$npm/package.json|s/\"@3leaps\/docprims-win32-x64-msvc\": \"[^\"]*\"/\"@3leaps\/docprims-win32-x64-msvc\": \"9.9.9\"/" \ "$npm/package-lock.json|1,/\"version\": \"[^\"]*\"/s/\"version\": \"[^\"]*\"/\"version\": \"9.9.9\"/" \ - "$npm/package-lock.json|s/\"@3leaps\/docprims-darwin-arm64\": \"[^\"]*\"/\"@3leaps\/docprims-darwin-arm64\": \"9.9.9\"/"; do + "$npm/package.json|s/\"devDependencies\": {/\"optionalDependencies\": {\"@3leaps\/docprims-linux-x64-gnu\": \"0.0.0\"}, \"devDependencies\": {/"; do file="${mutation%%|*}" expr="${mutation#*|}" fresh_copy From 5bb9743f56c7a82c37569f49a62081a25719383b Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:09:57 -0400 Subject: [PATCH 28/33] docs(decisions): record crates.io publication and public API posture Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- docs/decisions/ADR-0002-crate-structure.md | 6 +- .../PDR-0001-crates-io-publication.md | 110 ++++++++++++++++++ docs/decisions/README.md | 7 ++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 docs/decisions/PDR-0001-crates-io-publication.md diff --git a/docs/decisions/ADR-0002-crate-structure.md b/docs/decisions/ADR-0002-crate-structure.md index bf3418b..9b87066 100644 --- a/docs/decisions/ADR-0002-crate-structure.md +++ b/docs/decisions/ADR-0002-crate-structure.md @@ -1,9 +1,13 @@ # ADR-0002: Crate Structure -> **Status**: Accepted +> **Status**: Accepted; amended by [PDR-0001](PDR-0001-crates-io-publication.md) > **Date**: 2025-01-25 > **Authors**: 3leaps Architecture Council +> **Amendment (v0.2.0):** `docprims-cli` is replaced by the `docprims` crate, the +> library entry point, whose `cli` feature builds the `docprims` binary. See +> PDR-0001. + ## Context docprims needs to support multiple document formats (OOXML, PDF, HTML, Markdown, XML) with: diff --git a/docs/decisions/PDR-0001-crates-io-publication.md b/docs/decisions/PDR-0001-crates-io-publication.md new file mode 100644 index 0000000..68229f7 --- /dev/null +++ b/docs/decisions/PDR-0001-crates-io-publication.md @@ -0,0 +1,110 @@ +--- +id: "PDR-0001" +title: "Publish the docprims crates to crates.io after the release tag" +status: "accepted" +date: "2026-09-25" +deciders: + - "@3leapsdave" +scope: "docprims crate publication, public API and release process" +tags: + - "process" + - "release" + - "crates-io" + - "semver" +relates-to: + - "ADR-0002-crate-structure.md" + - "ADR-0004-stdout-purity.md" + - "RELEASE_CHECKLIST.md" +--- + +# PDR-0001: Publish the docprims crates to crates.io after the release tag + +## Decision + +### One front door, four published crates + +`docprims` is the library's only documented entry point. It re-exports the +public types and dispatches to a parser per format, with each format behind a +feature. The command-line tool is the same crate built with the `cli` feature +(`cargo install docprims --features cli`); the binary name is `docprims`. + +A published crate cannot depend on unpublished path-only crates, so +`docprims-core`, `docprims-text` and `docprims-ooxml` are published too. They +are components of `docprims`: their READMEs and docs point at it, and their +APIs carry no stability promise beyond what `docprims` re-exports. Changing +them to `publish = false` would break `docprims`. + +`docprims-ffi` and `docprims-ts-napi` are not published. The C library and the +Go and npm packages ship through their own channels. + +### Why this shape + +Other 3leaps primitive libraries, such as sysprims, publish only independent +subsystem crates and keep their CLI unpublished. docprims differs on purpose: +its crates are format variants of one capability rather than independent +subsystems, and its CLI is a supported artifact with a versioned JSON output +contract (ADR-0004 and `schemas/v0/extract/`). A single name therefore serves +both `docprims = "0.2"` and `cargo install`. This is not a default for other +repositories. + +Merging all formats into one crate was not chosen: the workspace split keeps +the FFI and Node addon crates small, and a later format can arrive as a +subcrate plus a `docprims` feature without a new top-level name. + +### crates.io after the tag + +Publish manually from a clean, guarded checkout of the pushed release tag, in +the order of `config/release/publishable-crates.txt`: `docprims-core`, +`docprims-text`, `docprims-ooxml`, then `docprims`. Wait for the crates.io +index and API after each upload, including the last. The first registry +version is v0.2.0; earlier tags are not backfilled. + +The maintainer alone holds a short-lived, crate-scoped token outside the +repository and authorizes each irreversible upload. CI has no registry token +and no publish step. `make release-crates-dry-run` exercises each package with +local path patches for earlier unpublished crates; the real upload has no +patches. + +### npm platform packages + +The committed npm manifest lists no platform packages, so its lock file +installs with `npm ci`. The publish workflow adds `optionalDependencies` for +exactly the platform packages it publishes, at the same version. + +## Public API and semver + +- **`serde_json` is a public dependency.** Location hints and metadata expose + `serde_json::Value` and `serde_json::Map`. A `serde_json` major version bump + is a breaking change for docprims. This is deliberate: those fields are open + JSON by design, and `serde_json` has kept one major version. +- **Output structs are not `#[non_exhaustive]`.** The subcrates construct them + across crate boundaries with struct literals. While docprims is 0.x, an + additive schema field is released as a new 0.x minor version. How structs + are constructed is decided before 1.0. +- **Public enums are `#[non_exhaustive]`** (`DocprimsError`, + `DocprimsQualityStatus`, `DocprimsContainerKind`), so matches on them need a + wildcard arm and new variants are not breaking for matching code. `serde` + deserialization still rejects an unknown variant: a consumer that parses + output from a newer docprims with older types gets an error for a new value. + That is acceptable while 0.x minors may break; whether 1.0 adds a catch-all + variant is part of the same pre-1.0 decision. + +## Positioning + +`extract` is a one-shot operation: one document in, one `extract/v0` result +out. docprims does not define a persisted bundle format or a batch, watch or +service mode. A consumer that stores results wraps `extract/v0` in its own +envelope. If docprims later grows a persisted export or a long-running mode, +it adopts the shared contracts for those shapes rather than extending +`extract/v0`. + +## Consequences + +- Package checks on the first registry cut may need a rerun once earlier + crates reach the index. The GitHub draft is not signed until package checks + and the release workflow are green. +- A new library crate needs an entry in the ordered list and an explicit + `publish = true`. The list check fails closed on a mismatch and on a + published FFI or Node addon crate. +- The index and API checks confirm availability, name, version and unyanked + status; they do not prove byte-for-byte identity with a local archive. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index d1056a4..41995b5 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -9,6 +9,7 @@ This directory contains architectural, design, and security decision records for | ADR | Architecture Decision Record | Technical architecture choices | | DDR | Design Decision Record | API design, data structures | | SDR | Security Decision Record | Security-related decisions | +| PDR | Process Decision Record | Release, publication and process | ## Index @@ -28,6 +29,12 @@ This directory contains architectural, design, and security decision records for | -------- | -------------------- | -------- | ---------- | | _None yet_ | | | | +### Process Decision Records (PDR) + +| ID | Title | Status | Date | +| -------- | -------------------- | -------- | ---------- | +| [PDR-0001](PDR-0001-crates-io-publication.md) | Publish the docprims crates to crates.io after the release tag | Accepted | 2026-09-25 | + ### Security Decision Records (SDR) | ID | Title | Status | Date | From 3ff5163d6669727ef45ae91ef0feef8835119e34 Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:10:45 -0400 Subject: [PATCH 29/33] build: add goneat hook and dependency policy configuration The dependency license list mirrors deny.toml. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .goneat/dependencies.yaml | 41 +++++++++++++++++++++++++++++++++++++++ .goneat/hooks.yaml | 24 +++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 .goneat/dependencies.yaml create mode 100644 .goneat/hooks.yaml diff --git a/.goneat/dependencies.yaml b/.goneat/dependencies.yaml new file mode 100644 index 0000000..970fd42 --- /dev/null +++ b/.goneat/dependencies.yaml @@ -0,0 +1,41 @@ +# goneat dependency policy for docprims. +# The license list mirrors deny.toml, which cargo-deny enforces in CI. +version: v1 +licenses: + forbidden: + - GPL-2.0 + - GPL-3.0 + - AGPL-3.0 + - LGPL-2.1 + - LGPL-3.0 + allowed: + - MIT + - Apache-2.0 + - BSD-3-Clause + - ISC + - Zlib + - Unicode-3.0 + - MPL-2.0 + - BSL-1.0 + - Unlicense +# Supply chain cooling: new releases wait before adoption. +cooling: + enabled: true + min_age_days: 7 + min_downloads: 100 + alert_only: false + grace_period_days: 3 + exceptions: + - pattern: "github.com/3leaps/*" + reason: "3leaps packages" + - pattern: "github.com/fulmenhq/*" + reason: "Fulmen ecosystem packages" +policy_engine: + type: embedded +languages: + - language: go + paths: + - bindings/go/docprims/go.mod + - language: rust + paths: + - Cargo.toml diff --git a/.goneat/hooks.yaml b/.goneat/hooks.yaml new file mode 100644 index 0000000..a8f874b --- /dev/null +++ b/.goneat/hooks.yaml @@ -0,0 +1,24 @@ +version: "1.0.0" +hooks: + pre-commit: + - command: "assess" + args: + - "--check" + - "--categories" + - "format,lint,typecheck" + - "--fail-on" + - "critical" + - "--lint-new-from-rev" + - "HEAD~" + priority: 10 + timeout: "5m" + pre-push: + - command: "assess" + args: ["--check", "--categories", "format,lint,typecheck,security,repo-status", "--fail-on", "high"] + priority: 10 + timeout: "5m" +optimization: + cache_results: true + content_source: working + only_changed_files: false + parallel: auto From 06508cb6b96fa622963f587a60f856a34f2ff02f Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:13:57 -0400 Subject: [PATCH 30/33] docs: state extract/v0 stability, output guarantees and sensitivity extract/v0 is classified Evolving. The page states the output guarantees every result meets and that extracted text carries the sensitivity of its input document. The architecture overview carries the same character-set guarantee and now shows parser dispatch by extension or explicit format. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- README.md | 4 ++ docs/architecture/overview.md | 5 +- docs/standards/extract-contract-stability.md | 61 ++++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 docs/standards/extract-contract-stability.md diff --git a/README.md b/README.md index 5d8aa62..d36aafa 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,10 @@ docprims/ └── python/ # Python binding ``` +Every binding returns the same `extract/v0` JSON structure. Its stability +level, output guarantees and data-sensitivity rules are in +[docs/standards/extract-contract-stability.md](docs/standards/extract-contract-stability.md). + ## Development Project documentation conventions (what is canonical vs planning notes): diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index cbc7a0d..a397967 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -30,6 +30,7 @@ docprims is a Rust library (with CLI + FFI bindings) that extracts text from doc |-----------|-----------|-----------| | Defensive parsing | Untrusted input is the norm | ADR-0003 | | Schema-driven contracts | Consumers need stability guarantees | `schemas/v0/extract/` | +| Output guarantees | Extracted text contains only XML 1.0 characters, excluding DEL and C1 controls (U+007F–U+009F); other characters are dropped. Markdown and HTML keep their parsers' U+FFFD replacement for NUL. | [extract/v0 contract](../standards/extract-contract-stability.md) | | Minimal core dependencies | Keep docprims-core embeddable | ADR-0002 | | CLI composability | Stdout purity for pipelines | ADR-0004 | | GPL-free license policy | Safe for commercial embedding | ADR-0001 | @@ -263,7 +264,7 @@ flowchart TD ENC[Encoding validation] end - V --> D[Format Detection] + V --> D[Format Dispatch: extension or explicit format] D --> P{Format Parser} P -->|markdown| TM[docprims-text::markdown] @@ -453,6 +454,8 @@ schemas/ During alpha (`v0.x`), schemas live under `schemas/v0/`. Post-1.0, schemas will be versioned independently. +`extract/v0` is classified **Evolving**; see [extract/v0 contract](../standards/extract-contract-stability.md) for its stability level, output guarantees and data sensitivity. + ## Testing Strategy (Contract First) ### Test Categories diff --git a/docs/standards/extract-contract-stability.md b/docs/standards/extract-contract-stability.md new file mode 100644 index 0000000..ea13b47 --- /dev/null +++ b/docs/standards/extract-contract-stability.md @@ -0,0 +1,61 @@ +# extract/v0 Contract: Stability and Output Guarantees + +This page states what consumers of docprims output can rely on: the stability +level of the `extract/v0` schema, the guarantees every result meets, and how +to treat the sensitivity of extracted text. It applies to the `docprims` crate, +the CLI's JSON output, and the C, Go and TypeScript bindings, which all return +the same structure. + +## Schema stability: Evolving + +`extract/v0` (`schemas/v0/extract/`) is classified **Evolving** under the +3leaps [Schema Stability Classification](https://github.com/3leaps/crucible/blob/main/docs/standards/schema-stability-classification.md) +standard. + +| Aspect | What it means for `extract/v0` | +|--------|--------------------------------| +| Compatibility | Additive changes (new optional fields, new block kinds) keep existing consumers working | +| Breaking changes | Only with notice: a `CHANGELOG.md` entry and a new docprims minor version while docprims is 0.x | +| Versioning | `schema_version` in every result follows the bump rules in `docs/architecture/overview.md` | +| Consumer advice | Safe for development; track the changelog | + +Changes to *what text is extracted* from a given document (for example, which +characters are kept) are contract changes and are announced the same way. + +Stable status (full compatibility within a major version) is planned with +docprims 1.0. The Rust types behind the schema follow +[PDR-0001](../decisions/PDR-0001-crates-io-publication.md). + +## Output guarantees + +Every result meets these properties. Each is covered by tests across all +supported formats. + +- **Character set.** Extracted text contains only XML 1.0 characters, excluding DEL and C1 controls (U+007F–U+009F); other characters are dropped. Markdown and HTML keep their parsers' U+FFFD replacement for NUL. +- **Parser choice.** The parser is chosen from the path or source URI + extension, or from an explicit format. Document content is never inspected + to choose a parser. +- **Byte ranges.** Each block's `doc_text_range` indexes the emitted + `document.text`, and the text is the blocks joined in order by one + separator each. +- **Limits.** `document.text` never exceeds `max_output_bytes`. When a limit + truncates output, `quality.status` is `partial`, and no block is empty. + +The character-set guarantee is not a display-safety claim. Bidirectional and +other format controls (for example U+202E) are legitimate content in +right-to-left text and are kept; presenting untrusted text safely is the +renderer's responsibility. + +## Data sensitivity + +docprims extracts text from documents it does not classify. Under the 3leaps +[Data Sensitivity Classification](https://github.com/3leaps/crucible/blob/main/docs/standards/data-sensitivity-classification.md) +standard: + +- Extracted text and metadata carry the **same sensitivity as the input + document**. Extraction never lowers it. +- Output from an unclassified document is **UNKNOWN** and is handled as such + until the consumer classifies it. + +`extract/v0` has no sensitivity field; consumers that store or forward results +carry the classification alongside them. From a2bcac67ab1e755d6aeed0ca19caabbf72967eeb Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:16:57 -0400 Subject: [PATCH 31/33] build(ts): derive the platform package set from napi.targets set-platform-packages.js maps each napi target to its platform with napi's parseTriple and requires the package directory to match exactly, each package to carry its addon, name and version. --check verifies without writing. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .../docprims/scripts/set-platform-packages.js | 65 ++++++++--- .../docprims/test/platform-packages.test.ts | 107 ++++++++++++++++++ 2 files changed, 155 insertions(+), 17 deletions(-) create mode 100644 bindings/typescript/docprims/test/platform-packages.test.ts diff --git a/bindings/typescript/docprims/scripts/set-platform-packages.js b/bindings/typescript/docprims/scripts/set-platform-packages.js index 82c7eec..5916319 100644 --- a/bindings/typescript/docprims/scripts/set-platform-packages.js +++ b/bindings/typescript/docprims/scripts/set-platform-packages.js @@ -1,51 +1,82 @@ // Point the root package at the platform packages published with it. // // The committed package.json lists no platform packages, so the lock file -// stays installable with `npm ci`. Before publishing, this script writes -// optionalDependencies for exactly the platform packages in the given -// directory (one subdirectory per `napi.targets` entry), at the root version. +// stays installable with `npm ci`. Before publishing, this script checks the +// platform package directory against `napi.targets` and writes +// optionalDependencies for exactly those packages, at the root version. // -// Usage: node scripts/set-platform-packages.js +// Usage: node scripts/set-platform-packages.js [--check] [package.json] +// +// must hold one directory per napi target, named by napi's platform +// suffix (for example linux-x64-gnu), each with a package.json for +// - at the root version and its ..node. +// --check verifies without writing. const fs = require("node:fs"); const path = require("node:path"); +const { parseTriple } = require("@napi-rs/cli"); function fail(message) { console.error(`error: ${message}`); process.exit(1); } +function expectedPlatforms(root) { + const targets = root.napi?.targets ?? []; + if (targets.length === 0) fail("package.json has no napi.targets"); + return targets.map((triple) => parseTriple(triple).platformArchABI).sort(); +} + function main() { - const dir = process.argv[2]; - if (!dir) fail("usage: set-platform-packages.js "); + const args = process.argv.slice(2); + const check = args[0] === "--check"; + if (check) args.shift(); + const [dir, rootArg] = args; + if (!dir || args.length > 2) { + fail("usage: set-platform-packages.js [--check] [package.json]"); + } - const rootPath = path.resolve(__dirname, "..", "package.json"); + const rootPath = rootArg ?? path.resolve(__dirname, "..", "package.json"); const root = JSON.parse(fs.readFileSync(rootPath, "utf8")); - const targets = root.napi?.targets ?? []; if (root.optionalDependencies) { fail("package.json already lists optionalDependencies"); } + const expected = expectedPlatforms(root); + const entries = fs.readdirSync(dir, { withFileTypes: true }); + const unexpected = entries.filter((e) => !e.isDirectory()).map((e) => e.name); + if (unexpected.length > 0) fail(`unexpected entries: ${unexpected.join(", ")}`); + const actual = entries.map((e) => e.name).sort(); + if (actual.join(",") !== expected.join(",")) { + fail(`platform directories [${actual}] != napi.targets [${expected}]`); + } + + const binaryName = root.napi?.binaryName; + if (!binaryName) fail("package.json has no napi.binaryName"); const names = []; - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - if (!entry.isDirectory()) fail(`unexpected entry ${entry.name}`); + for (const platform of actual) { const manifest = JSON.parse( - fs.readFileSync(path.join(dir, entry.name, "package.json"), "utf8"), + fs.readFileSync(path.join(dir, platform, "package.json"), "utf8"), ); - if (manifest.name !== `${root.name}-${entry.name}`) { - fail(`${entry.name}: package name ${manifest.name}`); + if (manifest.name !== `${root.name}-${platform}`) { + fail(`${platform}: package name ${manifest.name}`); } if (manifest.version !== root.version) { fail(`${manifest.name}: version ${manifest.version} != ${root.version}`); } + const addon = path.join(dir, platform, `${binaryName}.${platform}.node`); + if (!fs.existsSync(addon) || fs.statSync(addon).size === 0) { + fail(`${manifest.name}: missing ${path.basename(addon)}`); + } names.push(manifest.name); } - if (names.length !== targets.length) { - fail(`${names.length} platform packages for ${targets.length} napi targets`); - } + if (check) { + console.log(`[ok] ${names.length} platform packages match napi.targets`); + return; + } root.optionalDependencies = Object.fromEntries( - names.sort().map((name) => [name, root.version]), + names.map((name) => [name, root.version]), ); fs.writeFileSync(rootPath, `${JSON.stringify(root, null, 2)}\n`); console.log(`[ok] ${names.length} platform packages at ${root.version}`); diff --git a/bindings/typescript/docprims/test/platform-packages.test.ts b/bindings/typescript/docprims/test/platform-packages.test.ts new file mode 100644 index 0000000..13f2d1b --- /dev/null +++ b/bindings/typescript/docprims/test/platform-packages.test.ts @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +function packageRoot(): string { + let current = __dirname; + for (let i = 0; i < 10; i++) { + if (fs.existsSync(path.join(current, "package.json"))) return current; + current = path.dirname(current); + } + throw new Error("package.json not found"); +} + +const root = packageRoot(); +const script = path.join(root, "scripts", "set-platform-packages.js"); +const manifest = JSON.parse( + fs.readFileSync(path.join(root, "package.json"), "utf8"), +); +const platforms = [ + "darwin-arm64", + "linux-arm64-gnu", + "linux-arm64-musl", + "linux-x64-gnu", + "linux-x64-musl", + "win32-x64-msvc", +]; + +// A platform package directory as the prebuilds workflow stages it. +function stage(dir: string, platform: string, overrides: object = {}): void { + fs.mkdirSync(path.join(dir, platform), { recursive: true }); + fs.writeFileSync( + path.join(dir, platform, "package.json"), + JSON.stringify({ + name: `${manifest.name}-${platform}`, + version: manifest.version, + ...overrides, + }), + ); + fs.writeFileSync(path.join(dir, platform, `docprims.${platform}.node`), "addon"); +} + +function fixture( + edit: (npmDir: string) => void = () => {}, +): { npmDir: string; pkg: string } { + const work = fs.mkdtempSync(path.join(os.tmpdir(), "docprims-platforms-")); + const npmDir = path.join(work, "npm"); + for (const platform of platforms) stage(npmDir, platform); + edit(npmDir); + const pkg = path.join(work, "package.json"); + fs.copyFileSync(path.join(root, "package.json"), pkg); + return { npmDir, pkg }; +} + +function run(...args: string[]) { + return spawnSync(process.execPath, [script, ...args], { encoding: "utf8" }); +} + +test("committed package.json lists no platform packages", () => { + assert.equal(manifest.optionalDependencies, undefined); +}); + +test("writes optionalDependencies for exactly the napi targets", () => { + const { npmDir, pkg } = fixture(); + assert.equal(run("--check", npmDir, pkg).status, 0); + assert.equal(JSON.parse(fs.readFileSync(pkg, "utf8")).optionalDependencies, undefined); + + const result = run(npmDir, pkg); + assert.equal(result.status, 0, result.stderr); + const written = JSON.parse(fs.readFileSync(pkg, "utf8")).optionalDependencies; + assert.deepEqual( + written, + Object.fromEntries( + platforms.map((p) => [`${manifest.name}-${p}`, manifest.version]), + ), + ); + // A second run refuses to overwrite. + assert.notEqual(run(npmDir, pkg).status, 0); +}); + +test("rejects platform directories that differ from napi.targets", () => { + const cases: Array<[string, (npmDir: string) => void]> = [ + ["missing platform", (d) => fs.rmSync(path.join(d, "win32-x64-msvc"), { recursive: true })], + ["extra platform", (d) => stage(d, "darwin-x64")], + ["renamed platform", (d) => fs.renameSync(path.join(d, "linux-x64-gnu"), path.join(d, "linux-x64"))], + ["stray file", (d) => fs.writeFileSync(path.join(d, "README.md"), "")], + ["wrong package name", (d) => stage(d, "darwin-arm64", { name: `${manifest.name}-other` })], + ["wrong version", (d) => stage(d, "darwin-arm64", { version: "0.0.0" })], + ["missing addon", (d) => fs.rmSync(path.join(d, "linux-arm64-musl", "docprims.linux-arm64-musl.node"))], + ["empty addon", (d) => fs.writeFileSync(path.join(d, "linux-arm64-musl", "docprims.linux-arm64-musl.node"), "")], + ]; + for (const [label, edit] of cases) { + const { npmDir, pkg } = fixture(edit); + for (const args of [["--check", npmDir, pkg], [npmDir, pkg]]) { + const result = run(...args); + assert.notEqual(result.status, 0, `${label} (${args[0]}) passed`); + assert.match(result.stderr, /^error: /, label); + } + assert.equal( + JSON.parse(fs.readFileSync(pkg, "utf8")).optionalDependencies, + undefined, + `${label} wrote package.json`, + ); + } +}); From 2a607196265ba7f14579667e5dc9256b2e7159db Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:17:27 -0400 Subject: [PATCH 32/33] test(ts): reject a file in place of a platform package directory Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .../typescript/docprims/test/platform-packages.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bindings/typescript/docprims/test/platform-packages.test.ts b/bindings/typescript/docprims/test/platform-packages.test.ts index 13f2d1b..da6af32 100644 --- a/bindings/typescript/docprims/test/platform-packages.test.ts +++ b/bindings/typescript/docprims/test/platform-packages.test.ts @@ -86,6 +86,13 @@ test("rejects platform directories that differ from napi.targets", () => { ["extra platform", (d) => stage(d, "darwin-x64")], ["renamed platform", (d) => fs.renameSync(path.join(d, "linux-x64-gnu"), path.join(d, "linux-x64"))], ["stray file", (d) => fs.writeFileSync(path.join(d, "README.md"), "")], + [ + "file in place of a platform directory", + (d) => { + fs.rmSync(path.join(d, "win32-x64-msvc"), { recursive: true }); + fs.writeFileSync(path.join(d, "win32-x64-msvc"), ""); + }, + ], ["wrong package name", (d) => stage(d, "darwin-arm64", { name: `${manifest.name}-other` })], ["wrong version", (d) => stage(d, "darwin-arm64", { version: "0.0.0" })], ["missing addon", (d) => fs.rmSync(path.join(d, "linux-arm64-musl", "docprims.linux-arm64-musl.node"))], From 983c6d504ba71fafc5366a34e3929ec29512cbdb Mon Sep 17 00:00:00 2001 From: Dave Thompson Date: Fri, 25 Sep 2026 08:18:12 -0400 Subject: [PATCH 33/33] ci(ts): check platform packages against napi.targets The prebuild and publish workflows verify platform packages with set-platform-packages.js instead of their own platform lists. Generated by Claude Opus 5.5 via Claude Code under supervision of @3leapsdave Co-Authored-By: Claude Opus 5.5 Role: devlead Committer-of-Record: @3leapsdave --- .../workflows/typescript-napi-prebuilds.yml | 12 ++--- .github/workflows/typescript-npm-publish.yml | 44 ++----------------- 2 files changed, 6 insertions(+), 50 deletions(-) diff --git a/.github/workflows/typescript-napi-prebuilds.yml b/.github/workflows/typescript-napi-prebuilds.yml index 6025dda..a40b455 100644 --- a/.github/workflows/typescript-napi-prebuilds.yml +++ b/.github/workflows/typescript-napi-prebuilds.yml @@ -195,15 +195,6 @@ jobs: set -euo pipefail rm -rf npm npx napi create-npm-dirs - - name: Remove unsupported platform packages - working-directory: bindings/typescript/docprims - shell: bash - run: | - set -euo pipefail - # darwin-x64 (Intel Mac) is not supported - # win32-arm64 is not supported (no Go bindings either) - rm -rf npm/darwin-x64 npm/win32-arm64-msvc - echo "Removed unsupported platforms: darwin-x64, win32-arm64-msvc" - name: Stage .node files into npm packages working-directory: bindings/typescript/docprims shell: bash @@ -218,6 +209,9 @@ jobs: mkdir -p "$(dirname "$dest")" cp "$f" "$dest" done + - name: Verify staged platform packages + working-directory: bindings/typescript/docprims + run: node scripts/set-platform-packages.js --check npm - name: Upload npm dir uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/.github/workflows/typescript-npm-publish.yml b/.github/workflows/typescript-npm-publish.yml index 31b3795..97d3724 100644 --- a/.github/workflows/typescript-npm-publish.yml +++ b/.github/workflows/typescript-npm-publish.yml @@ -193,54 +193,16 @@ jobs: echo "Downloaded packages:" ls -la npm-packages/ - - name: Verify packages - run: | - set -euo pipefail - VERSION="${{ env.version }}" - - # Expected platform packages (docprims does not support darwin-x64 or win32-arm64) - PLATFORMS=( - "linux-x64-gnu" - "linux-x64-musl" - "linux-arm64-gnu" - "linux-arm64-musl" - "darwin-arm64" - "win32-x64-msvc" - ) - - echo "Verifying platform packages..." - for plat in "${PLATFORMS[@]}"; do - if [ ! -d "npm-packages/${plat}" ]; then - echo "::error::Missing platform package: ${plat}" - exit 1 - fi - if [ ! -f "npm-packages/${plat}/docprims.${plat}.node" ]; then - echo "::error::Missing .node file in ${plat}" - exit 1 - fi - - PKG_JSON="npm-packages/${plat}/package.json" - if [ ! -f "$PKG_JSON" ]; then - echo "::error::Missing package.json in ${plat}" - exit 1 - fi - PKG_VERSION=$(node -p "require('./' + process.argv[1]).version" "$PKG_JSON") - if [ "$PKG_VERSION" != "$VERSION" ]; then - echo "::error::Version mismatch in ${plat} package.json (${PKG_VERSION} != ${VERSION})" - exit 1 - fi - echo " ${plat}: OK" - done - - echo "All platform packages verified" - name: Build root package working-directory: bindings/typescript/docprims run: | set -euo pipefail npm ci npm run build - - name: Point the root package at the platform packages + - name: Verify platform packages and point the root package at them working-directory: bindings/typescript/docprims + # Fails unless npm-packages holds exactly one package per napi target, + # each with its addon, at the root version. run: node scripts/set-platform-packages.js ../../../npm-packages - name: Publish (dry run) if: ${{ inputs.dry_run }}