diff --git a/.github/scripts/depgraph.js b/.github/scripts/depgraph.js new file mode 100644 index 00000000..16ed7ac5 --- /dev/null +++ b/.github/scripts/depgraph.js @@ -0,0 +1,295 @@ +/*! + * Copyright (c) 2026-present, The Dash Core developers + * SPDX-License-Identifier: MIT + * See the accompanying file LICENSE or https://opensource.org/license/MIT + */ + +// @ts-check + +// Submits `uv.lock` to the dependency graph. GitHub currently natively parses +// `Cargo.lock` but cannot parse `uv.lock`, this script parses it for submission +// to the dependency graph. + +const fs = require("node:fs"); + +// Submission tag, keyed to overwrite autogenerated results from `pyproject.toml`. +const PY_MANIFEST_KEY = "pyproject.toml"; + +// Identification of this script. +const DETECTOR_PROFILE = { + name: "depgraph.js", + version: "1.0.0", + url: "https://github.com/dashpay/base-sdk", +}; + +// Matches `name[extras]==version`, capturing name in 1 and version in 2, ends at whitespace, marker or backslash. +const RE_PIN = /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?==([^\s;\\]+)/; + +// Matches a distribution name, an extras suffix allowed, and nothing else. +const RE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]*\])?$/; + +// Matches an unindented comment. +const RE_HEADER = /^#/; + +// Matches an indented comment. +const RE_OWNED = /^\s+#/; + +// Matches an indented `# via`, capturing what trails it, which may be empty. +const RE_VIA = /^\s+#\s+via\b(.*)$/; + +// Matches an indented comment holding one token, captured. +const RE_VIA_ITEM = /^\s+#\s+(\S.*)$/; + +/** + * PEP 503-style text normalisation. + * + * @param {string} name + * @returns {string} + */ +function normalise(name) { + return name.toLowerCase().replace(/[-_.]+/g, "-"); +} + +/** + * The package URL for a pinned distribution, local version encoded. + * + * @param {string} name normalised name + * @param {string} version + * @returns {string} + */ +function purlFor(name, version) { + return `pkg:pypi/${name}@${version.replace(/\+/g, "%2B")}`; +} + +/** + * Parse a `via` entry. + * + * @param {string} entry + * @param {string} line the line it was read from, named in the error + * @returns {string} normalised name, extras dropped + */ +function viaName(entry, line) { + if (!RE_NAME.test(entry)) { + throw new Error(`unsupported \`via\` entry: ${line.trim()}`); + } + return normalise(entry.replace(/\[.*$/, "")); +} + +/** + * Record *parent* against *pkg*, a marker repeat naming it only once. + * + * @param {{ via: string[] }} pkg + * @param {string} parent + */ +function addVia(pkg, parent) { + if (!pkg.via.includes(parent)) { + pkg.via.push(parent); + } +} + +/** + * Parse `uv export --format requirements-txt --no-hashes` output. + * + * Two shapes are read, a pin and the `# via` beneath it holding a name or list. + * + * A resolution fork states one package once per marker, so an entry is keyed by + * name and version and a repeat merges its `via` into the entry already held. + * + * @param {string} text + * @returns {Map} + */ +function parseExport(text) { + /** @type {Map} */ + const packages = new Map(); + /** @type {{ name: string, version: string, via: string[] } | null} */ + let current = null; + let listing = false; + + for (const raw of text.split("\n")) { + const line = raw.replace(/\r$/, ""); + + if (line.trim() === "" || RE_HEADER.test(line)) { + current = null; + listing = false; + continue; + } + + if (current !== null && RE_OWNED.test(line)) { + const via = RE_VIA.exec(line); + if (via) { + const rest = via[1].trim(); + listing = rest === ""; + if (!listing) { + addVia(current, viaName(rest, line)); + } + continue; + } + + const listed = RE_VIA_ITEM.exec(line); + if (listed && listing) { + addVia(current, viaName(listed[1].trim(), line)); + } + continue; + } + + // Extras are matched so they cannot hide a pin. + const pin = RE_PIN.exec(line); + if (pin === null) { + throw new Error(`unsupported requirement: ${line.trim()}`); + } + + const name = normalise(pin[1]); + const key = `${name}@${pin[2]}`; + let held = packages.get(key); + if (held === undefined) { + held = { name, version: pin[2], via: [] }; + packages.set(key, held); + } + + current = held; + listing = false; + } + + return packages; +} + +/** + * Build the `resolved` map a snapshot carries, keyed and cross-referenced + * by the package URL. + * + * All entries are scoped `development`, since they make up the devshell. + * + * A fork can resolve one name to several versions and `via` names only the + * parent, so an edge is drawn to every version of it rather than guessed at. + * + * @param {Map} packages + * @param {string} project normalised name of the workspace project + * @returns {Record} + */ +function resolveGraph(packages, project) { + /** @type {Map} */ + const byName = new Map(); + for (const pkg of packages.values()) { + const held = byName.get(pkg.name); + if (held === undefined) { + byName.set(pkg.name, [pkg]); + } else { + held.push(pkg); + } + } + + /** @type {Record} */ + const resolved = {}; + + for (const pkg of packages.values()) { + if (pkg.via.length === 0) { + throw new Error(`${pkg.name} has no \`via\`; export --no-emit-project`); + } + for (const parent of pkg.via) { + if (parent !== project && !byName.has(parent)) { + throw new Error( + `${pkg.name} names ${parent}, not a pin nor ${project}`, + ); + } + } + + const purl = purlFor(pkg.name, pkg.version); + resolved[purl] = { + package_url: purl, + relationship: pkg.via.includes(project) ? "direct" : "indirect", + scope: "development", + dependencies: [], + }; + } + + // `via` names parents, a snapshot states children, so invert the edges. + for (const pkg of packages.values()) { + const child = purlFor(pkg.name, pkg.version); + for (const parent of pkg.via) { + for (const owner of byName.get(parent) ?? []) { + const deps = resolved[purlFor(owner.name, owner.version)].dependencies; + if (!deps.includes(child)) { + deps.push(child); + } + } + } + } + + return resolved; +} + +/** + * @param {{ sha: string, ref: string, resolved: Record }} params + * @returns {object} + */ +function buildSnapshot({ sha, ref, resolved }) { + return { + version: 0, + job: { + id: process.env.GITHUB_RUN_ID, + correlator: `${process.env.GITHUB_WORKFLOW}-${process.env.GITHUB_JOB}`, + }, + sha, + ref, + detector: DETECTOR_PROFILE, + scanned: new Date().toISOString(), + manifests: { + [PY_MANIFEST_KEY]: { + name: PY_MANIFEST_KEY, + file: { source_location: PY_MANIFEST_KEY }, + resolved, + }, + }, + }; +} + +/** + * @param {object} params + * @param {ReturnType} params.github + * @param {typeof import("@actions/github").context} params.context + * @param {any} params.core + */ +module.exports = async ({ github, context, core }) => { + const source = process.env.REQUIREMENTS; + if (source === undefined) { + throw new Error("REQUIREMENTS names the export to submit; it is unset"); + } + + const project = process.env.PROJECT; + if (project === undefined) { + throw new Error("PROJECT names the workspace project; it is unset"); + } + + const packages = parseExport(fs.readFileSync(source, "utf8")); + if (packages.size === 0) { + throw new Error(`${source} states no pinned versions`); + } + + const resolved = resolveGraph(packages, normalise(project)); + const snapshot = buildSnapshot({ + sha: context.sha, + ref: context.ref, + resolved, + }); + + const entries = Object.values(resolved); + const direct = entries.filter((e) => e.relationship === "direct").length; + core.info(`submitting ${entries.length} packages, ${direct} direct`); + + const { data } = await github.request( + "POST /repos/{owner}/{repo}/dependency-graph/snapshots", + { + owner: context.repo.owner, + repo: context.repo.repo, + ...snapshot, + }, + ); + if (data.result === "INVALID") { + throw new Error(`snapshot refused: ${data.message}`); + } + core.info(`snapshot ${data.id}: ${data.message}`); +}; + +module.exports.parseExport = parseExport; +module.exports.resolveGraph = resolveGraph; +module.exports.buildSnapshot = buildSnapshot; diff --git a/.github/workflows/build_msrv.yml b/.github/workflows/build_msrv.yml index 009f4aa8..c09c35f5 100644 --- a/.github/workflows/build_msrv.yml +++ b/.github/workflows/build_msrv.yml @@ -45,12 +45,22 @@ jobs: node-version: 24 - name: Set up Python + id: python uses: actions/setup-python@v6 with: python-version-file: pyproject.toml + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: 0.12.9 + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install Python dependencies - run: pip install ".[dev]" + run: | + uv sync --locked --extra dev --python '${{ steps.python.outputs.python-path }}' + echo "${PWD}/.venv/bin" >> "${GITHUB_PATH}" - name: Install CodeQL id: setup-codeql @@ -82,22 +92,24 @@ jobs: uses: actions/cache@v5 with: path: ~/.codeql - key: codeql-packs-${{ hashFiles('contrib/codeql/codeql-pack.lock.yml') }} + key: codeql-packs-${{ hashFiles('maint/codeql/*/codeql-pack.lock.yml') }} - name: Run linters - run: python3 contrib/lint_all.py --exclude lint_codeql + run: | + python3 maint/lint_all.py --exclude lint_codeql + python3 maint/lint/lint_codeql.py check env: RUSTUP_TOOLCHAIN: 1.85.0 - name: Run CodeQL - run: python3 contrib/lint/lint_codeql.py --with-suite=rust-security-and-quality + run: python3 maint/lint/lint_codeql.py run --lang=rust --with-suite=rust-security-and-quality env: RUSTUP_TOOLCHAIN: 1.85.0 - name: Check PR commit messages if: github.event_name == 'pull_request' run: > - python3 contrib/lint/lint_unconv.py + python3 maint/lint/lint_unconv.py -r "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}" build: diff --git a/.github/workflows/build_nightly.yml b/.github/workflows/build_nightly.yml index 0d529d4a..c93e9a93 100644 --- a/.github/workflows/build_nightly.yml +++ b/.github/workflows/build_nightly.yml @@ -88,7 +88,7 @@ jobs: - name: Check formatting if: matrix.config.name == 'full' - run: python contrib/lint/lint_rust.py + run: python maint/lint/lint_rust.py - name: Test package (with coverage) if: matrix.config.name == 'full' diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 7f681835..9be1cde0 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -39,12 +39,22 @@ jobs: run: cargo install wasm-pack@0.15.0 - name: Set up Python + id: python uses: actions/setup-python@v6 with: python-version-file: pyproject.toml + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: 0.12.9 + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install Python dependencies - run: pip install ".[dev]" + run: | + uv sync --locked --extra dev --python '${{ steps.python.outputs.python-path }}' + echo "${PWD}/.venv/bin" >> "${GITHUB_PATH}" - name: Test documentation tooling run: pytest diff --git a/.github/workflows/repo_depgraph.yml b/.github/workflows/repo_depgraph.yml new file mode 100644 index 00000000..ca243a0e --- /dev/null +++ b/.github/workflows/repo_depgraph.yml @@ -0,0 +1,49 @@ +name: Push dependency graph + +on: + push: + branches: [develop] + paths: + - uv.lock + - .github/scripts/depgraph.js + - .github/workflows/repo_depgraph.yml + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + submit: + name: Submit uv lockfile + runs-on: ubuntu-24.04-arm + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: 0.12.9 + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Export resolved dependencies + run: uv export --locked --format requirements-txt --no-emit-project --all-extras --no-hashes -o "${RUNNER_TEMP}/requirements.txt" + + - name: Submit snapshot + uses: actions/github-script@v8 + env: + REQUIREMENTS: ${{ runner.temp }}/requirements.txt + PROJECT: dash-base-sdk + with: + script: | + const script = require("./.github/scripts/depgraph.js"); + await script({ github, context, core }); diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..8795eb28 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,130 @@ +# Development Guide + +## Coding style + +The full guide is at [`docs/dev/guide_rust.md`](./docs/dev/guide_rust.md). Key points: + +- **Formatting**: 2-space indentation, LF line endings, no trailing whitespace, single newline at end of file. Max line + width 120, comment width 80. Enforced by `rustfmt.toml`. +- **Naming**: `UpperCamelCase` for types/traits/enum variants, `snake_case` for functions/variables/modules, + `SCREAMING_SNAKE_CASE` for constants. Acronyms as words (`TxId` not `TXID`). Getters omit `get_` prefix. +- **Type safety**: newtypes over primitives when semantics differ, enums over booleans, make invalid states + unrepresentable. Derive `Clone`, `Debug`, `PartialEq`, `Eq`, `Hash` eagerly. +- **Error handling**: never `.unwrap()` or `.expect()` in library code. Propagate with `?`. Domain error enums implement + `Display`. Lowercase messages without trailing punctuation. Use `#[expect]` over `#[allow]`. +- **Ownership**: prefer borrowing over cloning, accept `&str` over `&String`, `&[T]` over `&Vec`. Let the caller + decide when to clone. +- **Conversions**: `as_` (free, borrow), `to_` (allocates), `into_` (consumes). Implement `From`/`TryFrom`, never `Into` + directly. +- **Comments**: inline comments max 80 chars, 3 lines. Rustdoc summary max 3 lines, don't restate the signature. + Document `# Errors` for `Result`-returning functions. +- **Code segmentation**: organise code through modules (in-file or separate files) and naming prefixes. Never use + decorative separator comments (`// ----`, `// ====`, `// -- Section --`). Latin-1/ISO 8859-1 characters in source + files only; no Unicode dashes, arrows, box drawing, or other decoration in comments or identifiers. +- **Security**: never log secrets, custom `Debug` for sensitive types, constant-time comparison for secrets, zeroize + after use. + +## Crate standards + +### Environment + +- `no_std` + `alloc` is mandatory for all crates. Every crate uses `#![no_std]` with `extern crate alloc`. +- `std` is an optional feature that downstream consumers enable when they need stdlib I/O, `Error` impls, etc. +- No networking code in any crate. The SDK provides encoding, decoding, and data types only. + +### Feature flags + +Every crate must follow this feature layout: + +```toml +[features] +default = [] +std = [...] +full = ["std"] # add "serde" here only if the crate implements Serialize/Deserialize +serde = ["dep:serde"] # only if the crate has serde impls +``` + +No other features should exist unless there is a pressing justification. The `serde` feature is only added to crates +that actually derive or implement `Serialize`/`Deserialize` on public types. + +### Module conventions + +Any shim code that mediates between `alloc` and `std` (e.g. `pub(crate) use alloc::vec::Vec;`) belongs in +`crate::prelude`. + +### File layout + +Every `.rs` file follows this order: + +```rust +// +// Copyright (c) 2026-present, The Dash Core developers +// SPDX-License-Identifier: MIT +// See the accompanying file LICENSE or https://opensource.org/license/MIT +// + +//! One-line module description. + +use crate::some_internal_module; + +use some_external_crate; + +// ... code ... +``` + +1. Copyright header (5-line block, more if additional attribution needed) +2. Blank line +3. Module doc (`//!`): one line, plus an optional 3-line paragraph + for exceptional cases +4. Blank line +5. Internal imports (`use crate::...`, `use super::...`) +6. Blank line +7. External imports (`use some_crate::...`) +8. Blank line +9. Code + +### Test and bench tooling + +- **Tests**: use `rstest` for parametrized and fixture-based tests. +- **Benchmarks**: use `divan` as the benchmark harness. +- **Corpus data**: must be JSON5 (`.json5` files in `corpus/`). JSON5 allows comments for annotating test vectors. + +### Directory layout + +```text +pkgs// + bench/ + corpus/ + src/ + tests/ + Cargo.toml +``` + +`Cargo.toml` must set: + +```toml +[package] +name = "dash-" + +[lints] +workspace = true +``` + +### Internal dependencies + +Crates within this repo must specify `path` and `version`: + +```toml +dash-num = { version = "0.0.0", path = "../num" } +``` + +## Verification + +All changes must pass before merge. Use `full` for the widest coverage. + +```sh +cargo fmt --check +cargo test --features full +cargo bench --features full +cargo clippy --features full --tests +``` diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 8795eb28..00000000 --- a/CLAUDE.md +++ /dev/null @@ -1,130 +0,0 @@ -# Development Guide - -## Coding style - -The full guide is at [`docs/dev/guide_rust.md`](./docs/dev/guide_rust.md). Key points: - -- **Formatting**: 2-space indentation, LF line endings, no trailing whitespace, single newline at end of file. Max line - width 120, comment width 80. Enforced by `rustfmt.toml`. -- **Naming**: `UpperCamelCase` for types/traits/enum variants, `snake_case` for functions/variables/modules, - `SCREAMING_SNAKE_CASE` for constants. Acronyms as words (`TxId` not `TXID`). Getters omit `get_` prefix. -- **Type safety**: newtypes over primitives when semantics differ, enums over booleans, make invalid states - unrepresentable. Derive `Clone`, `Debug`, `PartialEq`, `Eq`, `Hash` eagerly. -- **Error handling**: never `.unwrap()` or `.expect()` in library code. Propagate with `?`. Domain error enums implement - `Display`. Lowercase messages without trailing punctuation. Use `#[expect]` over `#[allow]`. -- **Ownership**: prefer borrowing over cloning, accept `&str` over `&String`, `&[T]` over `&Vec`. Let the caller - decide when to clone. -- **Conversions**: `as_` (free, borrow), `to_` (allocates), `into_` (consumes). Implement `From`/`TryFrom`, never `Into` - directly. -- **Comments**: inline comments max 80 chars, 3 lines. Rustdoc summary max 3 lines, don't restate the signature. - Document `# Errors` for `Result`-returning functions. -- **Code segmentation**: organise code through modules (in-file or separate files) and naming prefixes. Never use - decorative separator comments (`// ----`, `// ====`, `// -- Section --`). Latin-1/ISO 8859-1 characters in source - files only; no Unicode dashes, arrows, box drawing, or other decoration in comments or identifiers. -- **Security**: never log secrets, custom `Debug` for sensitive types, constant-time comparison for secrets, zeroize - after use. - -## Crate standards - -### Environment - -- `no_std` + `alloc` is mandatory for all crates. Every crate uses `#![no_std]` with `extern crate alloc`. -- `std` is an optional feature that downstream consumers enable when they need stdlib I/O, `Error` impls, etc. -- No networking code in any crate. The SDK provides encoding, decoding, and data types only. - -### Feature flags - -Every crate must follow this feature layout: - -```toml -[features] -default = [] -std = [...] -full = ["std"] # add "serde" here only if the crate implements Serialize/Deserialize -serde = ["dep:serde"] # only if the crate has serde impls -``` - -No other features should exist unless there is a pressing justification. The `serde` feature is only added to crates -that actually derive or implement `Serialize`/`Deserialize` on public types. - -### Module conventions - -Any shim code that mediates between `alloc` and `std` (e.g. `pub(crate) use alloc::vec::Vec;`) belongs in -`crate::prelude`. - -### File layout - -Every `.rs` file follows this order: - -```rust -// -// Copyright (c) 2026-present, The Dash Core developers -// SPDX-License-Identifier: MIT -// See the accompanying file LICENSE or https://opensource.org/license/MIT -// - -//! One-line module description. - -use crate::some_internal_module; - -use some_external_crate; - -// ... code ... -``` - -1. Copyright header (5-line block, more if additional attribution needed) -2. Blank line -3. Module doc (`//!`): one line, plus an optional 3-line paragraph - for exceptional cases -4. Blank line -5. Internal imports (`use crate::...`, `use super::...`) -6. Blank line -7. External imports (`use some_crate::...`) -8. Blank line -9. Code - -### Test and bench tooling - -- **Tests**: use `rstest` for parametrized and fixture-based tests. -- **Benchmarks**: use `divan` as the benchmark harness. -- **Corpus data**: must be JSON5 (`.json5` files in `corpus/`). JSON5 allows comments for annotating test vectors. - -### Directory layout - -```text -pkgs// - bench/ - corpus/ - src/ - tests/ - Cargo.toml -``` - -`Cargo.toml` must set: - -```toml -[package] -name = "dash-" - -[lints] -workspace = true -``` - -### Internal dependencies - -Crates within this repo must specify `path` and `version`: - -```toml -dash-num = { version = "0.0.0", path = "../num" } -``` - -## Verification - -All changes must pass before merge. Use `full` for the widest coverage. - -```sh -cargo fmt --check -cargo test --features full -cargo bench --features full -cargo clippy --features full --tests -``` diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index c1fe4675..a748cfa5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ ![GitHub License](https://img.shields.io/github/license/dashpay/base-sdk) ![Minimum Supported Rust Version](https://img.shields.io/badge/v1.85.0-msrv?style=flat&logo=rust&label=MSRV&color=orange) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/dashpay/base-sdk) > [!WARNING] > @@ -33,7 +34,7 @@ > [!NOTE] > Solid lines are build dependencies. Dotted lines are test dependencies. - + ```mermaid %%{init: { "flowchart": { "curve": "basis" } } }%% @@ -64,7 +65,7 @@ graph LR params --> p2p_core ``` - + ## Features diff --git a/contrib/README.md b/contrib/README.md index 8da1b40a..627ac8f0 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -3,7 +3,7 @@ Python 3.x and thus, assume a host capable of running Python. For guidance on installing Python on your host, visit https://www.python.org/downloads/ - + ## Preparing the virtual environment @@ -15,14 +15,11 @@ recommended to create a fresh virtual environment. > your program of choice's documentation if using a different manager. ```bash -# Create a new venv -uv venv .venv +# Create .venv and install the versions uv.lock pins +uv sync --locked --extra dev # Enter venv source .venv/bin/activate - -# Install dependencies -uv pip install -e ".[dev]" ``` > [!WARNING] @@ -90,24 +87,11 @@ sudo apt install git nodejs -y sudo dnf install -y git nodejs24 ``` -## Running linters - -All linters available in [`contrib/lint/`](../contrib/lint) are listed below. The first verb is implied if no verb is -specified at runtime. Verbs may accept arguments of their own, for more information, run an individual lint script with -`--help`. To run all scripts, use [`lint_all.py`](./lint_all.py). + -| Name | Purpose | Verbs | Depends on | -| ---- | ------- | ---------- | ---------- | -| [`lint_cargo.py`](./lint/lint_cargo.py) | Enforce MSRV across Rust build dependency graph, check/format TOML files against [`.taplo.toml`](../.taplo.toml) | `check` , `apply`, `apply-all` | (MSRV enforcement) `cargo` (TOML formatting) `taplo` | -| [`lint_codeql.py`](./lint/lint_codeql.py) | Query Rust sources against [`contrib/codeql/*.ql`](./codeql) | `run` | `codeql`, `rustc` | -| [`lint_javascript.py`](./lint/lint_javascript.py) | Lint Javascript sources against [`eslint.config.mjs`](js/eslint.config.mjs) | *None* | `npx` (part of Node.js), `eslint` (auto-retrieved by script) | -| [`lint_markdown.py`](./lint/lint_markdown.py) | Lint Markdown [documentation](../docs/dev/about_docs.md) | *None* | `pymarkdownlnt` | -| [`lint_python.py`](./lint/lint_python.py) | Lint Python sources against `[tool.ruff]` options in [`pyproject.toml`](../pyproject.toml) | *None* | `ruff` | -| [`lint_rust.py`](./lint/lint_rust.py) | Lint Rust sources against [`rustfmt.toml`](../rustfmt.toml) | *None* | `cargo`, `rustfmt` | -| [`lint_semgrep.py`](./lint/lint_semgrep.py) | Lint Rust sources against [`contrib/semgrep/*.yml`](./semgrep) | *None* | `semgrep` | -| [`lint_unconv.py`](./lint/lint_unconv.py) | Lint commit names in ranges specified against [`unconv.toml`](../unconv.toml) | *None* | `git` | + -### Verifying bisectability +## Verifying bisectability As a general rule of thumb, each commit must individually compile and pass linters. To help with this, we have a helper script, [`git_filter.py`](./git_filter.py) that creates a temporary worktree and executes supplied commands for each @@ -118,7 +102,7 @@ commit in a specified range so the worktree isn't blocked by the validation run. ./contrib/git_filter.py --fast-fail develop branch_name -- bash -c 'cargo clippy --all-targets --no-default-features -- -D warnings && cargo clippy --all-targets --features full -- -D warnings && cargo test --all-targets --features full && -./contrib/lint_all.py' +./maint/lint_all.py' ``` - + diff --git a/contrib/common.py b/contrib/common.py deleted file mode 100644 index 07f9a6ae..00000000 --- a/contrib/common.py +++ /dev/null @@ -1,325 +0,0 @@ -#!/usr/bin/env python3 -# coding: latin-1 - -# -# Copyright (c) 2026-present, The Dash Core developers -# SPDX-License-Identifier: MIT -# See the accompanying file LICENSE or https://opensource.org/license/MIT -# - -"""Shared constants and helpers for lint scripts.""" - -from __future__ import annotations - -import argparse -import os -import re -import shutil -import subprocess -import sys -from functools import cache -from pathlib import Path -from typing import TYPE_CHECKING, NoReturn - -if TYPE_CHECKING: - from collections.abc import Callable, Mapping - from typing import TextIO - -# ANSI escape codes for terminal output. -ANSI_BOLD = "\033[1m" -ANSI_DIM = "\033[2m" -ANSI_GREEN = "\033[32m" -ANSI_RED = "\033[31m" -ANSI_RESET = "\033[0m" - -# Cargo workspace roots, relative to the repository root. -CARGO_WORKSPACES: tuple[str, ...] = (".", "docs/samples") - -# Rust source roots the analysers scan, relative to the repository root. -SOURCE_DIRS: tuple[str, ...] = ("pkgs", "docs/samples") - -# Assumed base branch for codebase. -DEFAULT_BASE = "develop" - -# Return codes. -RETCODE_ERR = 1 -RETCODE_PASS = 0 -RETCODE_SKIP = 77 - -# Matches an address that names something other than a path on disk. -_OFF_DISK_RE = re.compile(r"^(?:[A-Za-z][A-Za-z0-9+.\-]*:|//|#)") - - -class _VerbParser(argparse.ArgumentParser): - """Parser spelling a usage fault in the harness' return codes.""" - - def exit(self, status: int = 0, message: str | None = None) -> NoReturn: - if message: - self._print_message(message, sys.stderr) - sys.exit(RETCODE_ERR if status else RETCODE_PASS) - - -def declare_verbs( - description: str, - verbs: Mapping[str, str], -) -> argparse.ArgumentParser: - """Return a parser taking one of *verbs*. - - *verbs* maps each verb to what it does, and insertion order picks the - default, so the first entry must avoid mutating effects. - """ - if not verbs: - raise ValueError("no verbs declared") - default = next(iter(verbs)) - parser = _VerbParser( - description=description, - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "verb", - choices=tuple(verbs), - default=default, - nargs="?", - help="\n".join( - f"{name}: {what}" + (" (default)" if name == default else "") - for name, what in verbs.items() - ), - ) - return parser - - -def off_disk(target: str) -> bool: - """Whether *target* addresses something other than a file on disk.""" - return _OFF_DISK_RE.match(target) is not None - - -@cache -def _entries(where: Path) -> frozenset[str]: - """Return the names *where* holds, as the filesystem spells them.""" - return frozenset(entry.name for entry in where.iterdir()) - - -def spelt_as_stored(root: Path, target: Path) -> bool: - """Whether *target* is spelt as the filesystem under *root* holds it. - - A case-insensitive filesystem resolves a misspelt path, so a wrong-case - link passes `exists()` on macOS and Windows and then serves a 404 from a - case-sensitive host. Each component is matched against its directory. - """ - # Normalising first drops the `..` a caller may have left in the path, - # which names no directory entry and so would fail the walk outright. - target = target.resolve() - if not target.is_relative_to(root): - return True - probe = root - for part in target.relative_to(root).parts: - if part not in _entries(probe): - return False - probe = probe / part - return True - - -def is_plain_file(root: Path, name: str) -> bool: - """Whether *name* is a regular file inside *root*, reached without links.""" - path = root / name - if not path.is_file(): - return False - try: - relative = path.relative_to(root) - except ValueError: - return False - probe = root - for part in relative.parts: - probe = probe / part - if probe.is_symlink(): - return False - return path.resolve().is_relative_to(root.resolve()) - - -def git_run(cwd: Path | str, *args: str) -> subprocess.CompletedProcess[str]: - """Run a git command in *cwd* and return the result.""" - return subprocess.run( # noqa: S603 - [require_bin("git"), *args], - capture_output=True, - check=False, - cwd=str(cwd), - encoding="utf-8", - errors="replace", - ) - - -def git_out(cwd: Path | str, *args: str) -> str: - """Run a git command in *cwd*, raise on failure, return its output.""" - result = git_run(cwd, *args) - if result.returncode != 0: - fault = result.stderr.strip() or result.stdout.strip() - raise RuntimeError(f"git {args[0]}: {fault or result.returncode}") - return result.stdout.strip() - - -def relay( - text: str, - repo_root: Path, - *, - stream: TextIO | None = None, - drop: Callable[[str], bool] | None = None, -) -> None: - """Print *text* with paths shortened against *repo_root*.""" - prefix = str(repo_root) + "/" - for line in text.splitlines(): - if drop is not None and drop(line): - continue - print(line.replace(prefix, ""), file=stream or sys.stdout) - - -def touched(repo_root: Path, suffixes: tuple[str, ...]) -> list[str]: - """Return the files matching *suffixes* that this branch has changed.""" - base = git_out(repo_root, "merge-base", DEFAULT_BASE, "HEAD") - return [ - name - for name in git_out(repo_root, "diff", "--name-only", base).splitlines() - if name.endswith(suffixes) and is_plain_file(repo_root, name) - ] - - -def format_table( - headers: tuple[str, ...], - rows: list[tuple[str, ...]], - status_colors: dict[str, str] | None = None, -) -> str: - """Render a markdown table with optional color on the last column.""" - colors = status_colors or {} - widths = [ - max(len(h), *(len(r[i]) for r in rows), 0) for i, h in enumerate(headers) - ] - - def fmt(cells: tuple[str, ...], *, color: bool = False) -> str: - parts: list[str] = [] - for i, cell in enumerate(cells): - pre = post = "" - if color and i == len(cells) - 1 and colors: - pre = colors.get(cell, ANSI_DIM) - post = ANSI_RESET - pad = widths[i] - len(cell) - parts.append(f" {pre}{cell}{post}{' ' * pad} ") - return f"|{'|'.join(parts)}|" - - sep = "|" + "|".join("-" * (w + 2) for w in widths) + "|" - return "\n".join( - [ - fmt(headers), - sep, - *(fmt(r, color=True) for r in rows), - ] - ) - - -def find_up( - start: Path, - predicate: Callable[[Path], bool], - label: str = "matching directory", -) -> Path: - """Walk upward from *start*, returning the first matching directory.""" - for directory in (start, *start.parents): - if predicate(directory): - return directory - raise FileNotFoundError(f"{label} not found above {start}") - - -def find_up_file(start: Path, name: str) -> Path | None: - """Walk upward from *start*, returning the first *name* found.""" - for directory in (start, *start.parents): - candidate = directory / name - if candidate.is_file(): - return candidate - return None - - -def is_workspace_root(d: Path) -> bool: - """Return True if *d* looks like a Cargo workspace root.""" - cargo = d / "Cargo.toml" - return ( - cargo.is_file() - and "[workspace]" in cargo.read_text(encoding="utf-8") - and (d / "pkgs").is_dir() - ) - - -def require_bin(name: str, path: str | None = None) -> str: - """Return the path to *name* or raise FileNotFoundError.""" - result = shutil.which(name, path=path) - if result is None and os.name == "nt": - result = shutil.which(f"{name}.exe", path=path) - if result is None: - where = "in expected path" if path else "in PATH" - raise FileNotFoundError(f"error: {name} binary not found {where}") - return result - - -@cache -def root_dir() -> Path: - """Return the workspace root (directory containing Cargo.toml).""" - return find_up( - Path(__file__).resolve().parent, - is_workspace_root, - "workspace Cargo.toml", - ) - - -def usable_threads() -> int: - """Return a conservative thread count (total CPUs minus one).""" - return max(1, (os.cpu_count() or 2) - 1) - - -def usable_mem() -> int: - """Return half the physical RAM in MiB. - - Raises RuntimeError when physical RAM cannot be determined. - """ - total = _physical_ram_bytes() - return total // (2 * 1024 * 1024) - - -def _physical_ram_bytes() -> int: - """Return total physical RAM in bytes.""" - if sys.platform.startswith("linux"): - for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): - if line.startswith("MemTotal:"): - return int(line.split()[1]) * 1024 - raise RuntimeError("MemTotal not found in /proc/meminfo") - if sys.platform == "darwin": - try: - out = subprocess.check_output( - ["sysctl", "-n", "hw.memsize"], # noqa: S607 - ) - return int(out.strip()) - except ( - FileNotFoundError, subprocess.CalledProcessError, ValueError, - ) as exc: - raise RuntimeError( - "could not determine physical RAM on macOS", - ) from exc - if sys.platform == "win32": - try: - out = subprocess.check_output( - ["powershell", "-NoProfile", "-Command", # noqa: S607 - "(Get-CimInstance Win32_ComputerSystem)" - ".TotalPhysicalMemory"], - ).decode() - value = out.strip() - if value.isdigit(): - return int(value) - except (FileNotFoundError, subprocess.CalledProcessError): - pass - try: - out = subprocess.check_output( - ["wmic", "computersystem", "get", # noqa: S607 - "TotalPhysicalMemory", "/value"], - ).decode() - for line in out.splitlines(): - if line.startswith("TotalPhysicalMemory="): - return int(line.split("=", 1)[1].strip()) - except (FileNotFoundError, subprocess.CalledProcessError, ValueError): - pass - raise RuntimeError("could not determine physical RAM on Windows") - raise RuntimeError(f"unsupported platform: {sys.platform}") diff --git a/contrib/common.py b/contrib/common.py new file mode 120000 index 00000000..304e73b7 --- /dev/null +++ b/contrib/common.py @@ -0,0 +1 @@ +../maint/common.py \ No newline at end of file diff --git a/contrib/lint/lint_semgrep.py b/contrib/lint/lint_semgrep.py deleted file mode 100755 index b25b5bc6..00000000 --- a/contrib/lint/lint_semgrep.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# coding: latin-1 - -# -# Copyright (c) 2026-present, The Dash Core developers -# SPDX-License-Identifier: MIT -# See the accompanying file LICENSE or https://opensource.org/license/MIT -# - -"""Runs semgrep rules against the workspace.""" - -from __future__ import annotations - -import subprocess -import sys - -from common import ( - RETCODE_ERR, - RETCODE_PASS, - SOURCE_DIRS, - require_bin, - root_dir, -) - - -def main() -> int: - semgrep_bin = require_bin("semgrep") - - repo_root = root_dir() - config_dir = repo_root / "contrib" / "semgrep" - target_dirs = [repo_root / where for where in SOURCE_DIRS] - - configs: list[str] = [] - for cfg in sorted(config_dir.glob("*.yml")): - configs.extend(["--config", str(cfg)]) - - if not configs: - raise FileNotFoundError( - "no semgrep configs found in contrib/semgrep/", - ) - - result = subprocess.run( # noqa: S603 - [ - semgrep_bin, - "scan", - *configs, - "--error", - *[str(d) for d in target_dirs], - ], - check=False, - ) - return RETCODE_PASS if result.returncode == 0 else RETCODE_ERR - - -if __name__ == "__main__": - try: - sys.exit(main()) - except Exception as exc: # noqa: BLE001 - print(exc, file=sys.stderr) - sys.exit(RETCODE_ERR) diff --git a/docs/README.md b/docs/README.md index a4380544..765886ec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,7 +15,7 @@ utilised by your packages. * Base packages. These packages implement specific algorithms but without chain-distinguishing consensus logic. * Protocol packages. These packages define the Dash protocol as deployed, blocks, transactions, chain parameters. ---8<-- "README.md:crate-graph" + *Note: Solid lines are build dependencies, dotted lines are test dependencies.* diff --git a/docs/common.py b/docs/common.py index 7ddd8eec..304e73b7 120000 --- a/docs/common.py +++ b/docs/common.py @@ -1 +1 @@ -../contrib/common.py \ No newline at end of file +../maint/common.py \ No newline at end of file diff --git a/docs/dev/about_docs.md b/docs/dev/about_docs.md index 28f4e541..d2151c5e 100644 --- a/docs/dev/about_docs.md +++ b/docs/dev/about_docs.md @@ -79,6 +79,37 @@ outside `docs/` resolve to the forge instead. > Zensical treats on-disk `.md` links as documentation and will fail to build if they are located outside `docs/`. > This does not affect non-Markdown files and directories. +### Splicing + +To keep material that would otherwise be duplicated from falling out of sync, a document can be split into reusable +segments that other documents splice in, or to assemble a document outright. This lets the on-disk layout keep +documentation close to the material it describes, while keeping the prose cohesive when it is read as a webpage. + +Splicing happens when the site is built, so a document read on GitHub is the file as stored on-disk, without the +segments spliced in. We use a custom comment syntax to ensure that splice markers are invisible, so they don't interfere +with GitHub and WYSIWYG editors. + +This makes Zensical the primary authority on the _shape_ of a document and it is recommended to [preview](#preview) your +edits and the pages that could be affected by your edits to ensure it remains pleasant to read. + +To splice in a whole document, or a spliced segment of it (like `setup`), the syntax is as below. + +```markdown + + +``` + +To create a spliceable segment, wrap the desired text in `start` and `end` markers carrying its label (like `setup`). + +```markdown + +Carried into the splice. + +``` + +Links in spliced material are resolved against the file that defines them, not the page splicing it in (see +[link processing](#link-processing)). Splices may nest, and a directive inside a code fence is inert. + ## Postprocessing > [!WARNING] diff --git a/docs/dev/getting_started.md b/docs/dev/getting_started.md index 9a9406a2..ca260d6f 100644 --- a/docs/dev/getting_started.md +++ b/docs/dev/getting_started.md @@ -25,4 +25,4 @@ components at the supported versions without further intervention. Should you wa version, please consult the vendor documentation for `RUSTUP_TOOLCHAIN` ([source](https://rust-lang.github.io/rustup/environment-variables.html)). ---8<-- "contrib/README.md:setup" + diff --git a/docs/dev/maintenance.md b/docs/dev/maintenance.md new file mode 100644 index 00000000..7e650eae --- /dev/null +++ b/docs/dev/maintenance.md @@ -0,0 +1,4 @@ +# Maintenance + + + diff --git a/docs/preprocess.py b/docs/preprocess.py index c4cd5c02..0da24d90 100644 --- a/docs/preprocess.py +++ b/docs/preprocess.py @@ -95,7 +95,13 @@ def run(self, lines: list[str]) -> list[str]: _FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$") # Matches a splice from another file, or one named section from it. -_INCLUDE_RE = re.compile(r'^\s*--8<--\s+"([^"]+)"\s*$') +_INCLUDE_RE = re.compile(r"^\s*\s*$") + +# Matches any directive-shaped comment, whatever it names. +_DIRECTIVE_RE = re.compile(r"^\s*\s*$") + +# The directives this module answers for. Anything else is a misspelling. +_DIRECTIVES = frozenset({"include", "start", "end"}) # Matches the target of an inline link, and any title trailing it. _LINK_RE = re.compile( @@ -113,6 +119,13 @@ def run(self, lines: list[str]) -> list[str]: _DEPTH_LIMIT = 8 +def _named_directive(line: str) -> None: + """Raise when *line* holds a directive this module does not answer for.""" + found = _DIRECTIVE_RE.match(line) + if found is not None and found.group(1) not in _DIRECTIVES: + raise ValueError(f"{found.group(1)}: no such directive") + + class _Fences: """Running fence state over a sequence of lines.""" @@ -173,8 +186,11 @@ def _expand( for line in lines: # A directive inside a fence is the syntax being shown, not used. - match = None if fences.covers(line) else _INCLUDE_RE.match(line) + fenced = fences.covers(line) + match = None if fenced else _INCLUDE_RE.match(line) if match is None: + if not fenced: + _named_directive(line) output.append(line) continue if budget <= 0: @@ -328,7 +344,7 @@ def test_include_splices_a_section(self) -> None: with self._scratch( whole="above\n\ninside\n\nbelow\n", ) as home: - out = self._render(f'--8<-- "{home}/whole.md:mid"\n') + out = self._render(f'\n') assert "inside" in out assert "above" not in out assert "below" not in out @@ -338,66 +354,80 @@ def test_include_refuses_an_unknown_section(self) -> None: with self._scratch(whole="nothing marked\n") as home: with pytest.raises(ValueError, match="no such section"): - self._render(f'--8<-- "{home}/whole.md:mid"\n') + self._render(f'\n') def test_include_rejects_an_absolute_path(self) -> None: import pytest with pytest.raises(ValueError, match="outside the repository"): - self._render('--8<-- "/etc/hosts"\n') + self._render('\n') def test_include_rejects_a_traversal(self) -> None: import pytest with pytest.raises(ValueError, match="outside the repository"): - self._render('--8<-- "../../../../etc/hosts"\n') + self._render('\n') def test_include_inside_a_fence_is_left_alone(self) -> None: - out = self._render('```\n--8<-- "unconv.toml"\n```\n') - assert "8<--" in out + out = self._render('```\n\n```\n') + assert "[include:maint/unconv.toml]" in out assert "[global]" not in out + def test_a_misspelt_directive_is_refused(self) -> None: + import pytest + + with pytest.raises(ValueError, match="no such directive"): + self._render("\n") + + def test_a_section_marker_is_a_known_directive(self) -> None: + out = self._render("\nkept\n\n") + assert "kept" in out + + def test_a_misspelt_directive_in_a_fence_is_left_alone(self) -> None: + out = self._render("```\n\n```\n") + assert "[inclde:README.md]" in out + def test_include_nests(self) -> None: with self._scratch( - outer='--8<-- "unconv.toml"\n', + outer='\n', ) as home: - out = self._render(f'--8<-- "{home}/outer.md"\n') - assert "8<--" not in out + out = self._render(f'\n') + assert "[include:" not in out assert "global" in out def test_include_refuses_a_cycle(self) -> None: import pytest with self._scratch(loop="") as home: - spec = f'--8<-- "{home}/loop.md"\n' + spec = f'\n' (root_dir() / home / "loop.md").write_text(spec, encoding="utf-8") with pytest.raises(ValueError, match="nested past the limit"): self._render(spec) def test_titled_link_is_rebased(self) -> None: with self._scratch(page='[a](../README.md "root")\n') as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert f'href="{_REPO}/blob/{_BRANCH}/README.md"' in out assert 'title="root"' in out def test_caged_link_is_rebased(self) -> None: with self._scratch(page="[a](<../README.md>)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert f'href="{_REPO}/blob/{_BRANCH}/README.md"' in out def test_bare_link_is_rebased(self) -> None: - with self._scratch(page="[a](../unconv.toml)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') - assert f'href="{_REPO}/blob/{_BRANCH}/unconv.toml"' in out + with self._scratch(page="[a](../maint/unconv.toml)\n") as home: + out = self._render(f'\n') + assert f'href="{_REPO}/blob/{_BRANCH}/maint/unconv.toml"' in out def test_page_under_docs_is_addressed_from_the_site(self) -> None: with self._scratch(page="[a](../docs/dev/guide_rust.md)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert 'href="/dev/guide_rust/"' in out def test_off_disk_link_is_left_alone(self) -> None: with self._scratch(page="[a](tel:+15551212) [b](irc://x/y)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert 'href="tel:+15551212"' in out assert 'href="irc://x/y"' in out @@ -406,7 +436,7 @@ def test_missing_link_target_is_refused(self) -> None: with self._scratch(page="[a](./nope.md)\n") as home: with pytest.raises(ValueError, match="no such file"): - self._render(f'--8<-- "{home}/page.md"\n') + self._render(f'\n') @staticmethod def _pointer() -> IncludePreprocessor: @@ -437,11 +467,13 @@ def test_wrong_case_link_is_refused(self) -> None: # Refused either as missing or as misspelt, by the host's case rules. with self._scratch(page="[a](../README.MD)\n") as home: with pytest.raises(ValueError, match=r"no such file|not spelt"): - self._render(f'--8<-- "{home}/page.md"\n') + self._render(f'\n') def test_include_survives_a_fenced_info_string(self) -> None: - out = self._render('```\n```text\n--8<-- "unconv.toml"\n```\n') - assert "8<--" in out + out = self._render( + '```\n```text\n\n```\n' + ) + assert "[include:maint/unconv.toml]" in out assert "[global]" not in out def test_alert_survives_a_fenced_info_string(self) -> None: @@ -451,21 +483,21 @@ def test_alert_survives_a_fenced_info_string(self) -> None: def test_site_root_link_is_left_for_postprocessing(self) -> None: with self._scratch(page="[a](/dev/about_docs/)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert 'href="/dev/about_docs/"' in out def test_a_fence_an_include_opens_holds_over_the_parent(self) -> None: with self._scratch(opener="```\n", body="spliced text\n") as home: out = self._pointer().run([ - f'--8<-- "{home}/opener.md"', - f'--8<-- "{home}/body.md"', + f'', + f'', "```", - f'--8<-- "{home}/body.md"', + f'', ]) # Held back while the fence the first splice opened is still open, # then spliced once the parent's own marker closes that fence. assert out[0] == "```" - assert out[1].startswith("--8<--") + assert out[1].startswith("