diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml new file mode 100644 index 0000000..4db4187 --- /dev/null +++ b/.github/workflows/pr-build.yml @@ -0,0 +1,66 @@ +name: pr-build + +# Build, test, pack, and index every plugin in this repo WITHOUT publishing, so a pull request or a +# branch push gets a downloadable preview of exactly what would ship. Same pipeline as release, driven +# by the one reusable Bespok3d build Action (b3-builder) with publish: 'false': it still builds the bake +# payloads, packs the .b3 set, assembles the sub-list index, and runs each plugin's tests, but cuts no +# GitHub release and registers nothing in the main index. The produced .b3 (plus atoms and the assembled +# index.json) upload as a run artifact, downloadable from the PR's Checks tab and the Actions run page. +# Publishing stays main-only, in release.yml. +on: + pull_request: + push: + branches-ignore: [main] + paths: + - '*/manifest.json' + - '*/files/**' + - '*/doc/**' + - '*/src/**' + - '*/toolchain/**' + - '*/tests/**' + - .github/workflows/pr-build.yml + +# A branch push and the pull_request for the same head would otherwise build twice; collapse them into +# one run per head branch and cancel a superseded build when a newer commit lands. +concurrency: + group: pr-build-${{ github.event.pull_request.head.ref || github.ref }} + cancel-in-progress: true + +jobs: + pr-build: + runs-on: ubuntu-latest + permissions: + contents: read # checkout only, no release / index commit / registration + pull-requests: write # post a sticky comment linking the build artifact on the PR + steps: + - uses: actions/checkout@v4 + + - uses: Bespok3d/b3-builder@main + with: + unit: repo + bake: 'true' + publish: 'false' + atom-repo: ${{ github.repository }} + list-name: 'Reference Python Plugins' + list-publisher: 'PLACEHOLDER' + list-ref-name: 'Reference Python Plugins' + + - uses: actions/upload-artifact@v4 + id: upload + with: + name: b3-packages-${{ github.event.repository.name }} + path: | + dist/*.b3 + dist/*.atom.json + dist/index.json + if-no-files-found: error + + # Surface the build on the PR page, not only the Actions run: a sticky comment linking the + # downloadable artifact. Fires on pull_request events only; a plain branch push has no PR. + - name: link the artifact on the PR + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh pr comment "${{ github.event.pull_request.number }}" --edit-last --create-if-none \ + --body "**b3-packages preview** (build only, not published). Download the packed \`.b3\` set, its atoms, and the assembled index: [b3-packages-${{ github.event.repository.name }}](${{ steps.upload.outputs.artifact-url }}). Rebuilt for commit \`${{ github.sha }}\`." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39d0cd9..604eab8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,9 @@ name: build-and-release -# On a push to main that changes a plugin or the build scripts: pack every plugin's .b3, publish a -# GitHub release per plugin, regenerate this repo's sub-list index.json, and register the sub-list in -# Bespok3d/main-index (a one-time-idempotent lists[] reference) so the official catalog picks it up. -# Each plugin's Python dependencies are baked into its .b3 first (ADR-0036), so the printer never runs -# pip: a requirements.txt becomes files/wheels/ (own-service venv) and a klipper_requirements.txt -# becomes files/site-packages/ (a Klipper/Moonraker extra). These deps are pure Python; a compiled -# dep would build its wheel on an arm64 runner first. +# Build, test, pack, release, index, and register every plugin in this repo through the one reusable +# Bespok3d build Action (b3-builder). This repo no longer carries its own build/pack/atom/list scripts in +# CI: the Action wraps the whole pipeline, so the only thing here is who this publisher is (org identity +# via the Action inputs) and the token to register the sub-list. on: push: branches: [main] @@ -14,7 +11,9 @@ on: - '*/manifest.json' - '*/files/**' - '*/doc/**' - - scripts/** + - '*/src/**' + - '*/toolchain/**' + - '*/tests/**' - .github/workflows/release.yml workflow_dispatch: @@ -25,85 +24,18 @@ jobs: contents: write # create releases + commit index.json on this repo steps: - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 with: - node-version: '20' + fetch-depth: 0 # full history so the index.json push can rebase on a concurrent update - - uses: actions/setup-python@v5 + - uses: Bespok3d/b3-builder@main with: - python-version: '3.11' - - - name: Bake each plugin's Python deps into its files (ADR-0036) - run: sh scripts/bake-deps.sh - - - name: Pack every plugin .b3 - run: sh scripts/pack.sh - - - name: Release each plugin and generate its atom - env: - GH_TOKEN: ${{ github.token }} - run: | - for dir in */; do - [ -f "${dir}manifest.json" ] || continue - id="${dir%/}" - name=$(jq -r '.name' "${dir}manifest.json") - version=$(jq -r '.version' "${dir}manifest.json") - tag="${name}-v${version}" - asset="${name}-${version}.b3" - if ! gh release view "$tag" >/dev/null 2>&1; then - gh release create "$tag" --title "$tag" --notes "Automated release of ${asset}" - fi - gh release upload "$tag" "dist/${asset}" --clobber - # The asset's API URL (.url, not browser_download_url) is what the app fetches with a - # bearer token + Accept: application/octet-stream (works for private repos). - download_url=$(gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${tag}" \ - --jq ".assets[] | select(.name==\"${asset}\") | .url") - node scripts/generate-atom.mjs --plugin "$id" --download-url "$download_url" --repo "$GITHUB_REPOSITORY" - done - - - name: Assemble this repo's sub-list index.json - run: node scripts/assemble-list.mjs - - - name: Commit index.json - run: | - git config user.name "bespok3d-bot" - git config user.email "bot@bespok3d.local" - git add index.json - if git diff --cached --quiet; then - echo "index unchanged; nothing to commit" - else - git commit -m "assemble sub-list index.json" - git push - fi - - - name: Register this sub-list in Bespok3d/main-index (idempotent) - env: - # Fine-grained PAT with contents:write on Bespok3d/main-index (the per-repo GITHUB_TOKEN - # cannot write a sibling repo). Create it in the org and add it as this repo's secret. - MAIN_INDEX_TOKEN: ${{ secrets.MAIN_INDEX_TOKEN }} - run: | - repo_name="${GITHUB_REPOSITORY##*/}" - # github: ref (not a raw.githubusercontent URL): the app fetches sub-lists via the - # authenticated GitHub API, which works for private repos and public alike. Raw URLs - # 404 on private repos (they need an ephemeral per-user token the app cannot store). - list_url="github:${GITHUB_REPOSITORY}/index.json" - git clone "https://x-access-token:${MAIN_INDEX_TOKEN}@github.com/Bespok3d/main-index.git" main-index - mkdir -p main-index/lists - printf '{\n "name": "Reference Python Plugins",\n "url": "%s"\n}\n' "$list_url" \ - > "main-index/lists/${repo_name}.json" - cd main-index - git config user.name "bespok3d-bot" - git config user.email "bot@bespok3d.local" - git add "lists/${repo_name}.json" - if git diff --cached --quiet; then - echo "list reference unchanged; nothing to commit" - else - git commit -m "register ${repo_name} sub-list" - for attempt in 1 2 3 4 5; do - git push && { echo "pushed main-index (attempt ${attempt})"; break; } - [ "$attempt" = 5 ] && { echo "main-index push failed after ${attempt} attempts"; exit 1; } - echo "main-index push rejected; rebasing on remote main (attempt ${attempt})" - git pull --rebase --no-edit origin main - done - fi + unit: repo + bake: 'true' + atom-repo: ${{ github.repository }} + list-name: 'Reference Python Plugins' + list-publisher: 'PLACEHOLDER' + list-ref-name: 'Reference Python Plugins' + main-index-repo: Bespok3d/main-index + # Per-repo Actions secret today (GitHub Free cannot org-secret private repos); becomes ONE org + # secret every repo inherits when the repos go public. See b3-builder README (secrets model). + main-index-token: ${{ secrets.MAIN_INDEX_TOKEN }} diff --git a/README.md b/README.md index 997d82e..a074102 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,31 @@ Both examples use a pure-Python dependency (`humanize`), so any interpreter work is needed. A plugin with a compiled dependency would prebuild that wheel on an arm64 runner first (see `u1-hw-camera`). -## Build +## Build locally -CI (`.github/workflows/release.yml`) runs `scripts/bake-deps.sh` (bakes each plugin's deps by which -requirements file it ships), then `scripts/pack.sh` (one `.b3` per `/`), releases each, -assembles this repo's `index.json` sub-list, and registers it in `Bespok3d/main-index`. The baked -deps are CI artifacts and are gitignored; only the source is committed. +Needs Node.js 20+. Builds run through the shared `Bespok3d/b3-builder` tool: + +```sh +npm install github:Bespok3d/b3-builder +npx b3-builder build --source ./status-feed --atom-repo Bespok3d/reference-python-plugins +# -> dist/status-feed-.b3 + dist/status-feed.atom.json +``` + +Drop `--source` to build every plugin in the repo at once. + +The Action runs with `bake: 'true'`: a plugin that ships a `requirements.txt` or +`klipper_requirements.txt` at its root gets its Python deps downloaded for the printer platform +(aarch64, CPython 3.11) at build time. Pass `--bake` to do the same locally. + +## Releasing + +Bump a plugin's `manifest.json` `version` and push to `main`. CI runs the `Bespok3d/b3-builder` +Action over the whole repo, which packs each `.b3`, cuts a release per plugin, assembles this repo's +`index.json` sub-list as `Reference Python Plugins`, and registers it in `Bespok3d/main-index` +(`lists/.json`). Secret: `MAIN_INDEX_TOKEN` (contents:write on main-index). Signing deferred. > Not yet verified on a physical U1. + ## Maintainership These plugins are published and maintained by the Bespok3d org, and several of them repackage or diff --git a/print-time-human/doc/README.md b/print-time-human/doc/README.md index ca74e0b..c2c63e0 100644 --- a/print-time-human/doc/README.md +++ b/print-time-human/doc/README.md @@ -34,6 +34,8 @@ to Klipper over its API. ## Build ```sh -sh scripts/fetch-deps.sh # pip install --target the deps into files/site-packages -sh scripts/pack.sh # -> dist/print-time-human-.b3 +npm install github:Bespok3d/b3-builder +npx b3-builder build --source ./print-time-human --atom-repo Bespok3d/reference-python-plugins --bake +# --bake installs klipper_requirements.txt into files/site-packages, then packs +# -> dist/print-time-human-.b3 ``` diff --git a/scripts/assemble-list.mjs b/scripts/assemble-list.mjs deleted file mode 100644 index cb82a97..0000000 --- a/scripts/assemble-list.mjs +++ /dev/null @@ -1,72 +0,0 @@ -// Assemble this co-repo's own published sub-list (index.json at the repo root) from the per-plugin -// atoms in dist/. The output is the ADR-0012 federated-index shape, identical to what main-index -// emits, so the app loads it through the same resolver. main-index references this file by URL in -// its lists[], so the official catalog picks these plugins up without copying their atoms. -// Signing (index.json.sig) is deferred. - -import { readdir, readFile, writeFile } from 'node:fs/promises' -import { join, dirname } from 'node:path' -import { fileURLToPath } from 'node:url' - -const LIST_NAME = 'Reference Python Plugins' -const LIST_PUBLISHER = 'PLACEHOLDER' - -function serviceName(provided) { - return typeof provided === 'string' ? provided : provided.service -} - -function requiredServices(atom) { - return (atom.require ?? []).map((requirement) => requirement.service) -} - -function providerByService(atoms) { - const providers = {} - atoms.forEach((atom) => { - ;(atom.provides ?? []).forEach((provided) => { - const service = serviceName(provided) - if (!(service in providers)) providers[service] = atom.name - }) - }) - return providers -} - -function resolveDeps(atom, providers) { - const resolved = [] - requiredServices(atom).forEach((service) => { - const providerId = providers[service] ?? service - if (!resolved.includes(providerId)) resolved.push(providerId) - }) - return resolved -} - -// Drop the internal `require` (only the assembler needs it) and replace it with the resolved `deps`, -// so each published entry matches the catalog entry shape the app expects. This is a leaf list -// (lists: []); main-index is the list-of-lists that references it. -export function assemble(atoms) { - const sorted = [...atoms].sort((earlier, later) => earlier.name.localeCompare(later.name)) - const providers = providerByService(sorted) - const plugins = sorted.map((atom) => { - const { require: _require, ...entry } = atom - return { ...entry, deps: resolveDeps(atom, providers) } - }) - const updated = plugins.reduce((latest, plugin) => (plugin.updated_at > latest ? plugin.updated_at : latest), '') - return { schema_version: 1, name: LIST_NAME, publisher: LIST_PUBLISHER, updated, plugins, lists: [] } -} - -async function main() { - const scriptDir = dirname(fileURLToPath(import.meta.url)) - const repoDir = dirname(scriptDir) - const distDir = join(repoDir, 'dist') - const names = (await readdir(distDir).catch(() => [])).filter((name) => name.endsWith('.atom.json')) - const atoms = await Promise.all(names.map((name) => readFile(join(distDir, name), 'utf8').then(JSON.parse))) - const index = assemble(atoms) - await writeFile(join(repoDir, 'index.json'), `${JSON.stringify(index, null, 2)}\n`) - process.stdout.write(`Wrote index.json (${index.plugins.length} plugins)\n`) -} - -if (process.argv[1] && process.argv[1].endsWith('assemble-list.mjs')) { - main().catch((error) => { - process.stderr.write(`assemble-list failed: ${error.message}\n`) - process.exit(1) - }) -} diff --git a/scripts/bake-deps.sh b/scripts/bake-deps.sh deleted file mode 100755 index 193ff07..0000000 --- a/scripts/bake-deps.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/sh -# Bake each plugin's Python dependencies into its files/ so the printer never runs pip (ADR-0036). -# Two modes, chosen by which requirements file a plugin ships: -# requirements.txt -> wheels for the plugin's OWN service venv. The daemon installs them -# offline (pip install --no-index --find-links files/wheels) into a -# per-plugin venv. We fetch the full closure as wheels into files/wheels. -# klipper_requirements.txt -> unpacked packages for a Klipper/Moonraker EXTRA that must import the -# lib in THAT (system) interpreter. The daemon symlinks each top-level -# package into the system site-packages. We unzip the wheels into -# files/site-packages (a wheel is a zip; unzipping gives a flat -# importable tree). -# -# We fetch wheels for the PRINTER's platform (Snapmaker U1: aarch64, glibc/manylinux2014, CPython -# 3.11), NOT the build runner's, so a compiled dependency is correct on the device even when this -# runs on an x86 CI runner. pip resolves the full closure and picks the aarch64 wheel for compiled -# packages and the universal py3-none-any wheel for pure-Python ones. -# -# --only-binary=:all: makes the bake FAIL LOUDLY if any package in the closure has no compatible -# prebuilt wheel (sdist-only, or no aarch64 build): that is the signal to publish an arm64 wheel for -# that dependency (built once on an arm64 runner) rather than ship a broken .b3 to the printer. -# -# Requires: python3 with pip. -set -e - -# The printer's interpreter. Keep in sync with the U1 runtime (jinni paths.json / the daemon venv). -TARGET_PLATFORM="manylinux2014_aarch64" -TARGET_PYTHON="3.11" -TARGET_IMPL="cp" - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_DIR="$(dirname "$SCRIPT_DIR")" - -# Download the closure of a requirements file as wheels for the printer's platform. The three --abi -# values let one pass collect both compiled (cp311) and universal (none / abi3) wheels. -download_wheels() { # requirements_file dest_dir - python3 -m pip download -r "$1" -d "$2" \ - --only-binary=:all: \ - --platform "$TARGET_PLATFORM" \ - --python-version "$TARGET_PYTHON" \ - --implementation "$TARGET_IMPL" \ - --abi cp311 --abi abi3 --abi none -} - -for dir in "$REPO_DIR"/*/; do - [ -f "${dir}manifest.json" ] || continue - if [ -f "${dir}requirements.txt" ]; then - mkdir -p "${dir}files/wheels" - download_wheels "${dir}requirements.txt" "${dir}files/wheels" - echo "Baked wheels for ${dir}" - fi - if [ -f "${dir}klipper_requirements.txt" ]; then - mkdir -p "${dir}files/site-packages" - wheel_dir="$(mktemp -d)" - download_wheels "${dir}klipper_requirements.txt" "$wheel_dir" - for wheel in "$wheel_dir"/*.whl; do - python3 -m zipfile -e "$wheel" "${dir}files/site-packages" - done - rm -rf "$wheel_dir" - echo "Baked site-packages for ${dir}" - fi -done diff --git a/scripts/generate-atom.mjs b/scripts/generate-atom.mjs deleted file mode 100644 index 4989ee0..0000000 --- a/scripts/generate-atom.mjs +++ /dev/null @@ -1,78 +0,0 @@ -// Emit one plugin's index atom (a single catalog entry) for the federated registry. Same entry -// shape the monorepo's generate-index.mjs produces, plus the raw `require` so a list assembler can -// resolve cross-plugin `deps`. download_url/doc_url are absolute (this is a published list): the -// download_url is the GitHub release asset API URL, injected by CI. -// -// Usage: node scripts/generate-atom.mjs --plugin [--download-url ] [--repo owner/repo] -// Writes dist/.atom.json. Without --download-url it falls back to the local .b3 filename -// (a dry-run inspectable atom, not for publishing). - -import { readFile, writeFile, mkdir } from 'node:fs/promises' -import { join, dirname } from 'node:path' -import { fileURLToPath } from 'node:url' - -function serviceName(provided) { - return typeof provided === 'string' ? provided : provided.service -} - -function copyIfPresent(target, source, keys) { - keys.forEach((key) => { - if (source[key] !== undefined) target[key] = source[key] - }) -} - -export function buildAtom(manifest, downloadUrl, docUrl) { - const entry = { - name: manifest.name, - title: manifest.title, - version: manifest.version, - description: manifest.description, - tagline: manifest.tagline, - category: manifest.category, - channel: manifest.channel, - publisher: manifest.publisher, - printer_specific: manifest.printer_specific ?? false, - published_at: manifest.published_at, - updated_at: manifest.updated_at, - requires: { capabilities: manifest.requires?.capabilities ?? [] }, - provides: (manifest.provides ?? []).map(serviceName), - require: manifest.require ?? [], - conflicts: manifest.conflicts ?? [], - doc_url: docUrl, - download_url: downloadUrl, - } - copyIfPresent(entry, manifest, ['icon', 'min_daemon_version', 'homepage', 'macros', 'config']) - if (manifest.changelog) entry.changelog_url = `${manifest.name}/${manifest.changelog}` - const endpoints = manifest.endpoints ?? [] - if (endpoints.length > 0) entry.endpoints = endpoints - return entry -} - -function arg(flag, fallback) { - const index = process.argv.indexOf(flag) - return index >= 0 && process.argv[index + 1] ? process.argv[index + 1] : fallback -} - -async function main() { - const scriptDir = dirname(fileURLToPath(import.meta.url)) - const repoDir = dirname(scriptDir) - const pluginId = arg('--plugin', undefined) - if (!pluginId) throw new Error('missing required --plugin ') - const manifest = JSON.parse(await readFile(join(repoDir, pluginId, 'manifest.json'), 'utf8')) - const repo = arg('--repo', 'Bespok3d/u1-klipper-config-enhancers') - const downloadUrl = arg('--download-url', `${manifest.name}-${manifest.version}.b3`) - const docUrl = arg('--doc-url', `https://github.com/${repo}/blob/main/${pluginId}/doc/README.md`) - const atom = buildAtom(manifest, downloadUrl, docUrl) - const outDir = join(repoDir, 'dist') - await mkdir(outDir, { recursive: true }) - const outFile = join(outDir, `${manifest.name}.atom.json`) - await writeFile(outFile, `${JSON.stringify(atom, null, 2)}\n`) - process.stdout.write(`Wrote ${outFile} (download_url=${downloadUrl})\n`) -} - -if (process.argv[1] && process.argv[1].endsWith('generate-atom.mjs')) { - main().catch((error) => { - process.stderr.write(`generate-atom failed: ${error.message}\n`) - process.exit(1) - }) -} diff --git a/scripts/pack.sh b/scripts/pack.sh deleted file mode 100755 index d900367..0000000 --- a/scripts/pack.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/sh -# Pack every plugin in this co-repo into a .b3 under dist/. Each plugin lives in its own -# / dir (manifest.json + files/ + optional doc/). A slim port of the monorepo's -# pack-plugins.sh pack step: per-file sha256 + mode go into the manifest files[] array, then -# manifest + files/ + doc/ are zipped. Always-repack (no lockfile/auto-bump); bump a plugin's -# manifest.json version manually to cut a new release. -# -# Requires: zip, jq, and shasum (macOS) or sha256sum (Linux). -set -e - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_DIR="$(dirname "$SCRIPT_DIR")" -DIST_DIR="$REPO_DIR/dist" - -for cmd in zip jq; do - command -v "$cmd" >/dev/null 2>&1 || { echo "ERROR: '$cmd' is required." >&2; exit 1; } -done -command -v shasum >/dev/null 2>&1 || command -v sha256sum >/dev/null 2>&1 \ - || { echo "ERROR: shasum or sha256sum is required." >&2; exit 1; } - -file_sha256() { - if command -v shasum >/dev/null 2>&1; then shasum -a 256 "$1" | awk '{print $1}' - else sha256sum "$1" | awk '{print $1}'; fi -} - -file_mode() { stat -f "%OLp" "$1" 2>/dev/null || stat -c "%a" "$1" 2>/dev/null; } - -# A plugin's Python-dep declaration (ADR-0036) lives at the plugin root, not under files/, but the -# daemon reads it from the unpacked plugin dir to provision the venv / system-site links, so it must -# ship in the .b3 and be listed in the manifest files[] alongside the files/ tree. -dep_declaration_paths() { - plugin_dir="$1" - for req in requirements.txt klipper_requirements.txt; do - [ -f "$plugin_dir/$req" ] && printf '%s\n' "$plugin_dir/$req" - done -} - -# LC_ALL=C forces a byte-order sort so the file list is identical regardless of locale. -build_files_array() { - plugin_dir="$1" - { find "$plugin_dir/files" -type f \ - ! -path '*/__pycache__/*' ! -name '*.pyc' ! -name '.DS_Store' - dep_declaration_paths "$plugin_dir" - } | LC_ALL=C sort | while read -r fpath; do - relpath="${fpath#"$plugin_dir/"}" - sha=$(file_sha256 "$fpath") - mode=$(file_mode "$fpath") - case "$mode" in *7*) mode="755" ;; *) mode="644" ;; esac - printf '{"path":"%s","sha256":"%s","mode":"%s"}\n' "$relpath" "$sha" "$mode" - done -} - -pack_one() { - plugin_dir="$1" - name=$(jq -r '.name' "$plugin_dir/manifest.json") - version=$(jq -r '.version' "$plugin_dir/manifest.json") - output="$DIST_DIR/$name-$version.b3" - tmp_dir=$(mktemp -d) - files_json=$(build_files_array "$plugin_dir" | jq -s '.') - jq --argjson files "$files_json" '.files = $files' "$plugin_dir/manifest.json" > "$tmp_dir/manifest.json" - rm -f "$output" - ( - cd "$plugin_dir" - zip -qr "$output" files/ - if [ -d doc ]; then zip -qr "$output" doc/; fi - for req in requirements.txt klipper_requirements.txt; do - [ -f "$req" ] && zip -q "$output" "$req" - done - cd "$tmp_dir" - zip -q "$output" manifest.json - ) - rm -rf "$tmp_dir" - echo "Packed: $output" - echo " sha256: $(file_sha256 "$output")" -} - -mkdir -p "$DIST_DIR" -packed=0 -for dir in "$REPO_DIR"/*/; do - [ -f "${dir}manifest.json" ] || continue - pack_one "${dir%/}" - packed=$((packed + 1)) -done - -if [ "$packed" -eq 0 ]; then - echo "ERROR: no plugins found (expected /manifest.json dirs)." >&2 - exit 1 -fi -echo "" -echo "Packed $packed plugin(s)."