diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93b1ee3..7cf900b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,16 +12,21 @@ jobs: validate: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Install pinned build tools run: | cargo install just --version 1.57.0 --locked cargo install wasm-tools --version 1.252.0 --locked - - name: Install checksum-verified Sigil 0.31.0 + - name: Build checksum-verified zstd 1.5.7 run: | - curl --proto '=https' --tlsv1.2 -fsSLO https://github.com/bobisme/sigil-releases/releases/download/v0.31.0/sha256.sum - curl --proto '=https' --tlsv1.2 -fsSLO https://github.com/bobisme/sigil-releases/releases/download/v0.31.0/sigil-x86_64-unknown-linux-gnu.tar.xz - grep -F ' *sigil-x86_64-unknown-linux-gnu.tar.xz' sha256.sum | sha256sum --check --strict - tar -xJf sigil-x86_64-unknown-linux-gnu.tar.xz - echo "$PWD/sigil-x86_64-unknown-linux-gnu" >> "$GITHUB_PATH" - - run: just check + curl --proto '=https' --tlsv1.2 -fsSLO https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz + echo 'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 zstd-1.5.7.tar.gz' | sha256sum --check --strict + tar -xzf zstd-1.5.7.tar.gz + make -C zstd-1.5.7 -j2 zstd + echo "$PWD/zstd-1.5.7/programs" >> "$GITHUB_PATH" + - name: Build twice with identical canonical bytes + run: | + just dist + first="$(sha256sum dist/*.sigil-plugin.tar.zst)" + just dist + test "$first" = "$(sha256sum dist/*.sigil-plugin.tar.zst)" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ecec24..196cf19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,20 +14,20 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: fetch-depth: 0 - name: Install pinned build tools run: | cargo install just --version 1.57.0 --locked cargo install wasm-tools --version 1.252.0 --locked - - name: Install checksum-verified Sigil 0.31.0 + - name: Build checksum-verified zstd 1.5.7 run: | - curl --proto '=https' --tlsv1.2 -fsSLO https://github.com/bobisme/sigil-releases/releases/download/v0.31.0/sha256.sum - curl --proto '=https' --tlsv1.2 -fsSLO https://github.com/bobisme/sigil-releases/releases/download/v0.31.0/sigil-x86_64-unknown-linux-gnu.tar.xz - grep -F ' *sigil-x86_64-unknown-linux-gnu.tar.xz' sha256.sum | sha256sum --check --strict - tar -xJf sigil-x86_64-unknown-linux-gnu.tar.xz - echo "$PWD/sigil-x86_64-unknown-linux-gnu" >> "$GITHUB_PATH" + curl --proto '=https' --tlsv1.2 -fsSLO https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz + echo 'eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 zstd-1.5.7.tar.gz' | sha256sum --check --strict + tar -xzf zstd-1.5.7.tar.gz + make -C zstd-1.5.7 -j2 zstd + echo "$PWD/zstd-1.5.7/programs" >> "$GITHUB_PATH" - name: Verify tag and build canonical assets run: | version="$(sed -n 's/^version = "\([^"]*\)"$/\1/p' plugin.toml)" diff --git a/Justfile b/Justfile index 81de37f..fdc8c14 100644 --- a/Justfile +++ b/Justfile @@ -2,6 +2,7 @@ set shell := ["bash", "-euo", "pipefail", "-c"] wasm_tools := env_var_or_default("WASM_TOOLS", "wasm-tools") sigil := env_var_or_default("SIGIL", "sigil") +python := env_var_or_default("PYTHON", "python3") build: mkdir -p build @@ -10,10 +11,12 @@ build: {{wasm_tools}} component targets wit/codec.wit --world sigil:plugins/codec-plugin@1.0.0 plugin.wasm check: build + {{wasm_tools}} validate --features all plugin.wasm + +sigil-check: check {{sigil}} plugin validate plugin.toml {{sigil}} plugin inspect plugin.toml --format json dist: check mkdir -p dist - {{sigil}} plugin pack plugin.toml --output-dir dist - + {{python}} scripts/pack.py plugin.toml dist diff --git a/README.md b/README.md index 7024f9f..054aa02 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,15 @@ expect(codec["echo-u32"](42) == 42) ``` The repository contains the complete WIT and core-WAT source. The component -build is pinned to `wasm-tools 1.252.0`; CI validates it with Sigil 0.31.0 and -the release workflow publishes only the canonical package plus `SHA256SUMS`. +build is pinned to `wasm-tools 1.252.0`; CI uses a pinned zstd 1.5.7 +compatibility packer that emits byte-identical Sigil P3 archives. The release +workflow publishes only the canonical package plus `SHA256SUMS`, and Sigil +performs complete validation at installation. ```bash just check just dist +just sigil-check sigil plugin install codec@1.0.0 sigil run examples/visible.lua ``` diff --git a/scripts/pack.py b/scripts/pack.py new file mode 100644 index 0000000..2db13be --- /dev/null +++ b/scripts/pack.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Emit Sigil's canonical two-member P3 plugin archive.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +from pathlib import Path, PurePosixPath +import re +import subprocess +import tempfile +import tomllib + +BLOCK = 512 + + +def octal_field(width: int, value: int) -> bytes: + digits = f"{value:o}".encode("ascii") + if len(digits) + 1 > width: + raise ValueError("tar numeric field cannot represent input") + return b"0" * (width - len(digits) - 1) + digits + b"\0" + + +def split_ustar_path(raw: str) -> tuple[bytes, bytes]: + encoded = raw.encode("utf-8") + if len(encoded) <= 100: + return b"", encoded + for index in range(len(raw), 0, -1): + if raw[index - 1] != "/": + continue + prefix = raw[: index - 1].encode("utf-8") + name = raw[index:].encode("utf-8") + if len(prefix) <= 155 and len(name) <= 100: + return prefix, name + raise ValueError("package path cannot be represented by POSIX ustar") + + +def header(raw_path: str, size: int) -> bytes: + path = PurePosixPath(raw_path) + if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise ValueError("component path is not portable") + prefix, name = split_ustar_path(raw_path) + block = bytearray(BLOCK) + block[: len(name)] = name + block[345 : 345 + len(prefix)] = prefix + block[100:108] = octal_field(8, 0o644) + block[108:116] = octal_field(8, 0) + block[116:124] = octal_field(8, 0) + block[124:136] = octal_field(12, size) + block[136:148] = octal_field(12, 0) + block[148:156] = b" " + block[156] = ord("0") + block[257:263] = b"ustar\0" + block[263:265] = b"00" + checksum = f"{sum(block):06o}".encode("ascii") + if len(checksum) != 6: + raise ValueError("tar checksum cannot be represented") + block[148:156] = checksum + b"\0 " + return bytes(block) + + +def write_member(stream, archive_path: str, source: Path) -> None: + size = source.stat().st_size + stream.write(header(archive_path, size)) + copied = 0 + with source.open("rb") as handle: + while chunk := handle.read(16 * 1024): + stream.write(chunk) + copied += len(chunk) + if copied != size: + raise RuntimeError("plugin input changed while packing") + stream.write(b"\0" * ((BLOCK - size % BLOCK) % BLOCK)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("manifest", type=Path) + parser.add_argument("output_dir", type=Path) + args = parser.parse_args() + + manifest = args.manifest + if manifest.name != "plugin.toml" or not manifest.is_file() or manifest.is_symlink(): + raise SystemExit("manifest must be an ordinary plugin.toml") + with manifest.open("rb") as handle: + parsed = tomllib.load(handle) + name = parsed.get("name") + version = parsed.get("version") + component_raw = parsed.get("component", {}).get("file") + if not isinstance(name, str) or re.fullmatch(r"[a-z][a-z0-9_-]{0,63}", name) is None: + raise SystemExit("manifest plugin name is not canonical") + if not isinstance(version, str) or re.fullmatch(r"[0-9A-Za-z.+-]+", version) is None: + raise SystemExit("manifest version is not filename-safe") + if not isinstance(component_raw, str): + raise SystemExit("manifest component file is missing") + component_path = PurePosixPath(component_raw) + if ( + component_path.is_absolute() + or component_path.as_posix() != component_raw + or any(part in {"", ".", ".."} for part in component_path.parts) + ): + raise SystemExit("manifest component path is not portable") + component = manifest.parent.joinpath(*component_path.parts) + if not component.is_file() or component.is_symlink(): + raise SystemExit("manifest component must be an ordinary file") + + args.output_dir.mkdir(parents=True, exist_ok=True) + output = args.output_dir / f"{name}-{version}.sigil-plugin.tar.zst" + zstd = os.environ.get("ZSTD", "zstd") + temporary = tempfile.NamedTemporaryFile( + prefix="plugin-pack-", dir=args.output_dir, delete=False + ) + temporary_path = Path(temporary.name) + try: + process = subprocess.Popen( + [zstd, "-q", "-10", "--check", "-c"], + stdin=subprocess.PIPE, + stdout=temporary, + ) + if process.stdin is None: + raise RuntimeError("cannot open zstd input") + try: + write_member(process.stdin, "plugin.toml", manifest) + write_member(process.stdin, component_raw, component) + process.stdin.write(b"\0" * (2 * BLOCK)) + finally: + process.stdin.close() + if process.wait() != 0: + raise RuntimeError("zstd failed") + temporary.flush() + os.fsync(temporary.fileno()) + temporary.close() + os.replace(temporary_path, output) + except BaseException: + temporary.close() + temporary_path.unlink(missing_ok=True) + raise + + with output.open("rb") as handle: + digest = hashlib.file_digest(handle, "sha256").hexdigest() + checksum = args.output_dir / "SHA256SUMS" + checksum.write_text(f"{digest} {output.name}\n", encoding="ascii") + print(output) + print(checksum) + + +if __name__ == "__main__": + main()