From e56262d993907ea30c6b3182c64bf03c14d3b1ee Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Mon, 28 Sep 2026 15:26:47 +0100 Subject: [PATCH 01/22] Carry the extension build contract's release checks: per-SDK tags at the built commit, sdkDigest stamp, pinned builder, PR warnings (cherry picked from commit edcfe77ade174832fd50264e6df67bd7c0fc7f95) --- .env.example | 8 +- .github/workflows/builder-image.yml | 14 +++ .github/workflows/extension-ci.yml | 9 ++ .github/workflows/release.yml | 55 +++--------- docker-compose.yml | 12 +-- scripts/_builder.sh | 34 +++++--- scripts/_sdk_digest.sh | 28 ++++++ scripts/build-extension.sh | 20 +++-- scripts/check-extension-pr.sh | 49 +++++++++++ scripts/release-extensions.sh | 77 +++++++++++++++++ tests/test-builder-image.sh | 47 ++++++++++ tests/test-check-extension-pr.sh | 87 +++++++++++++++++++ tests/test-release-extensions.sh | 127 ++++++++++++++++++++++++++++ tests/test-sdk-digest.sh | 58 +++++++++++++ 14 files changed, 558 insertions(+), 67 deletions(-) create mode 100644 scripts/_sdk_digest.sh create mode 100755 scripts/check-extension-pr.sh create mode 100755 scripts/release-extensions.sh create mode 100755 tests/test-builder-image.sh create mode 100755 tests/test-check-extension-pr.sh create mode 100755 tests/test-release-extensions.sh create mode 100755 tests/test-sdk-digest.sh diff --git a/.env.example b/.env.example index 6eb4b14..f7e56c0 100644 --- a/.env.example +++ b/.env.example @@ -46,12 +46,14 @@ QDRANT_API_KEY=devkit-local-qdrant #RUNTIME=docker # Extension build toolchain (.NET SDK 8 + Node 22). Only pulled the first time you build an -# extension — the platform itself doesn't use it. An already-pulled tag is reused without checking the -# registry: BUILDER_PULL=1 forces a re-pull (this tag moves), BUILDER_IMAGE= replaces the image +# extension — the platform itself doesn't use it. The default is pinned by digest in scripts/_builder.sh, +# so every build of a version uses one toolchain. BUILDER_TAG= (e.g. latest) opts out of the pin; an +# already-pulled tag is reused without checking the registry, and BUILDER_PULL=1 forces a re-pull (tags +# move). BUILDER_IMAGE= replaces the image # outright, BUILDER_PLATFORM= forces an architecture. Each build's package caches live in # per-uid named volumes that nothing prunes — ` volume ls | grep duplo_devkit` to find and # ` volume rm` to reclaim them (each holds a full NuGet + npm closure). -BUILDER_TAG=latest +# BUILDER_TAG=latest # Build on THIS machine instead of in the toolchain container. Set to 1 if you already have .NET SDK 8 # and Node 22+ installed and would rather use them — you own keeping them current, and your builds stop diff --git a/.github/workflows/builder-image.yml b/.github/workflows/builder-image.yml index c94faef..7ac672e 100644 --- a/.github/workflows/builder-image.yml +++ b/.github/workflows/builder-image.yml @@ -49,6 +49,7 @@ jobs: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - uses: docker/build-push-action@v7 + id: push with: # `build`, not `.`, to match how scripts/_builder.sh builds this same image locally: the # Dockerfile COPYs nothing from the build context (only --from=node), so sending the whole @@ -61,3 +62,16 @@ jobs: tags: | quay.io/duplocloud/duplo-extension-builder:latest quay.io/duplocloud/duplo-extension-builder:${{ github.sha }} + # scripts/_builder.sh pins the default toolchain by digest, so a publish here changes nothing until that pin + # moves in a reviewed PR. Hand the reviewer the exact value. + - name: Report the digest to pin + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + { + echo "### Builder image published" + echo "To build with it by default, set in scripts/_builder.sh and docker-compose.yml:" + echo '```' + echo "quay.io/duplocloud/duplo-extension-builder@${DIGEST}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/extension-ci.yml b/.github/workflows/extension-ci.yml index b441152..58c764c 100644 --- a/.github/workflows/extension-ci.yml +++ b/.github/workflows/extension-ci.yml @@ -36,6 +36,9 @@ jobs: QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} steps: - uses: actions/checkout@v7 + with: + # The PR's base history, so the release checks below can diff each extension against the merge base. + fetch-depth: 0 - name: Discover extensions id: guard @@ -65,6 +68,12 @@ jobs: if: steps.guard.outputs.build == 'true' run: ./scripts/build-all.sh + # Warnings only: what each changed extension's release will need (version bump, declared skills, Native + # Federation frontend). !cancelled() so a failed build still reports the checks that need no bundle. + - name: Release checks + if: ${{ !cancelled() && github.event_name == 'pull_request' && steps.guard.outputs.build == 'true' }} + run: ./scripts/check-extension-pr.sh "origin/${{ github.base_ref }}" + - name: Upload bundles if: steps.guard.outputs.build == 'true' uses: actions/upload-artifact@v7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 824bfc5..068d562 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,14 @@ # Publishes each extension in this repo as a GitHub Release on push to main. # -# For every extensions// it builds the bundle and, ONLY when manifest.version is new (its -# -v tag doesn't exist yet), cuts a GitHub Release with extension.zip attached. Bumping -# manifest.version is therefore how you publish a new version to the extension library. +# For every extensions// it builds the bundle and publishes one release per build, tagged +# -v-sdk- at the commit that was built. scripts/release-extensions.sh holds the rules: +# a build already released is skipped, and an extension whose content changed at a version that already has a +# release fails the job, so bumping manifest.version is how you publish a new version to the extension library. # # This file ships with the dev-kit. After scripts/init-project.sh makes the repo yours, set these repo secrets # (same as Extension CI): DUPLO_HOST + DUPLO_TOKEN. The build fetches the host SDK feed from DUPLO_HOST, so it -# must be reachable from CI (a public DuploCloud, not localhost). +# must be reachable from CI (a public DuploCloud, not localhost). Turn on immutable releases in the repository's +# settings, too; the publish step warns on every release that lands mutable. # # NOTE: the hot-load/deploy step is intentionally COMMENTED OUT (see below). We publish releases but do not # auto-install onto the portal, so the install-from-registry flow stays testable end to end. Uncomment the @@ -30,6 +32,10 @@ jobs: DUPLO_TOKEN: ${{ secrets.DUPLO_TOKEN }} steps: - uses: actions/checkout@v7 + with: + # Every tag and the history it points at: the publish step compares each released version's commit with + # this one to tell an unchanged extension from one that changed without a version bump. + fetch-depth: 0 - name: Discover extensions id: guard @@ -39,15 +45,7 @@ jobs: if [ ${#dirs[@]} -gt 0 ]; then echo "build=true" >> "$GITHUB_OUTPUT" else echo "build=false" >> "$GITHUB_OUTPUT"; echo "::notice::No extensions/*/manifest.json — nothing to release."; fi - - uses: actions/setup-dotnet@v6 - if: steps.guard.outputs.build == 'true' - with: - dotnet-version: '8.0.x' - - uses: actions/setup-node@v7 - if: steps.guard.outputs.build == 'true' - with: - node-version: '22' - + # Runs in the builder container (scripts/_builder.sh) on the runner's Docker, with the digest-pinned toolchain. - name: Build all extensions if: steps.guard.outputs.build == 'true' run: ./scripts/build-all.sh @@ -58,35 +56,8 @@ jobs: # if: steps.guard.outputs.build == 'true' # run: ./scripts/deploy-all.sh - - name: Publish GitHub releases (only for new manifest.version) + - name: Publish GitHub releases (one per new build) if: steps.guard.outputs.build == 'true' env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - shopt -s nullglob - published=0; skipped=0 - for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do - [ -f "$m" ] || continue - dir="$(dirname "$m")" - zip="$dir/dist/extension.zip" - if [ ! -f "$zip" ]; then - echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping." - continue - fi - name="$(jq -r '.name // (.id)' "$m")" - version="$(jq -r '.version' "$m")" - slug="$(basename "$dir")" - tag="${slug}-v${version}" - if gh release view "$tag" >/dev/null 2>&1; then - echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." - skipped=$((skipped+1)) - continue - fi - echo "==> Publishing release $tag ($name $version)" - gh release create "$tag" "$zip" \ - --title "$name $version" \ - --notes "Automated release of $name v$version from $slug (extension.zip attached)." - published=$((published+1)) - done - echo "==> Released $published, skipped $skipped (already-published versions)." + run: ./scripts/release-extensions.sh diff --git a/docker-compose.yml b/docker-compose.yml index 893fd7f..9de14f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -204,14 +204,14 @@ services: # below). Both routes are live; the compose network is simply the preferred one. builder: profiles: [tools] - # Deliberately ONE level of interpolation, not a nested ${BUILDER_IMAGE:-…:${BUILDER_TAG:-latest}}: + # Deliberately ONE level of interpolation, not a nested ${BUILDER_IMAGE:-…:${BUILDER_TAG:-…}}: # podman-compose 1.5 stops at the first '}' and appends the remainder literally, yielding # "…busybox:1.36}" and an "invalid reference format" error even when BUILDER_IMAGE is set. Nothing - # is lost — BUILDER_TAG is resolved into BUILDER_IMAGE by scripts/_builder.sh:builder_resolve_image, - # which always exports it before compose is invoked, so the default below is only ever reached by a - # hand-run `compose run` with no BUILDER_IMAGE set (where it now means :latest regardless of - # BUILDER_TAG — use the build scripts, or set BUILDER_IMAGE outright, to pin another tag). - image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder:latest} + # is lost — scripts/_builder.sh:builder_resolve_image resolves BUILDER_TAG and exports BUILDER_IMAGE + # before every build, so the default below is only ever reached by a hand-run `compose run` with no + # BUILDER_IMAGE set. It is the digest-pinned toolchain regardless of BUILDER_TAG, and must match + # _BUILDER_PINNED_IMAGE there (use the build scripts, or set BUILDER_IMAGE outright, to pick another). + image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd} # Unset by default, so the build runs on the host's native architecture. Set BUILDER_PLATFORM only # to force one (e.g. linux/amd64 if a publish ever ends up amd64-only) — an unset value is dropped # by compose, whereas copying duplo-ai-studio's linux/amd64 default would silently emulate a .NET diff --git a/scripts/_builder.sh b/scripts/_builder.sh index aae8ea1..2574819 100644 --- a/scripts/_builder.sh +++ b/scripts/_builder.sh @@ -245,6 +245,13 @@ builder_keep_local_copy() { # builder_keep_local_copy echo " $RUNTIME pull said: $(printf '%s' "${1-}" | tail -1)" >&2 } +# The toolchain image a build uses when neither BUILDER_IMAGE nor BUILDER_TAG is set, pinned by digest so two +# builds of one extension version compile with the same toolchain. `:latest` moves on every toolchain publish, +# which would let two builds of one version differ. The digest is the multi-arch index, so it resolves on amd64 +# and arm64 alike. .github/workflows/builder-image.yml prints the new digest after each publish. Moving this pin +# (and docker-compose.yml's matching default) is a reviewed change, since it changes what every extension builds with. +_BUILDER_PINNED_IMAGE="quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd" + builder_resolve_image() { # The raw value is kept for the error message: BUILDER_PULL may have come from .env rather than the # environment, and naming a value the user cannot see in their shell is not a diagnostic. @@ -269,17 +276,22 @@ builder_resolve_image() { return 1 fi - local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")"; tag="${tag:-latest}" - # OCI tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is - # a pull that fails for an unobvious reason and a silent fall-through to building something else. - case "$tag" in - *[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* ) - echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2 - echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2 - echo " Windows line endings." >&2 - return 1 ;; - esac - BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag"; export BUILDER_IMAGE + local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")" + if [ -z "$tag" ]; then + BUILDER_IMAGE="$_BUILDER_PINNED_IMAGE" + else + # OCI tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is + # a pull that fails for an unobvious reason and a silent fall-through to building something else. + case "$tag" in + *[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* ) + echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2 + echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2 + echo " Windows line endings." >&2 + return 1 ;; + esac + BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag" + fi + export BUILDER_IMAGE if [ "$force_pull" != 1 ] && "$RUNTIME" image inspect "$BUILDER_IMAGE" >/dev/null 2>&1; then return 0; fi echo "==> Fetching the build toolchain image ($BUILDER_IMAGE)" >&2 diff --git a/scripts/_sdk_digest.sh b/scripts/_sdk_digest.sh new file mode 100644 index 0000000..86b68ea --- /dev/null +++ b/scripts/_sdk_digest.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash +# Sourced by scripts/build-extension.sh — fingerprints the SDK feed a bundle was compiled against. +# +# sdkVersion alone does not identify an SDK: the host's SDK surface and dependency floors can move under an +# unchanged version, so a bundle that fails at run time could be a broken extension or one built against a drifted +# SDK. build-extension.sh stamps this digest beside sdkVersion so the two can be told apart by recomputing it over +# the host's own sdk-bundle. +# +# Definition (anything recomputing it must match this exactly): for every file in the extracted SDK bundle, the line +# " \n"; the lines sorted bytewise by path; sdkDigest is the lowercase +# sha256 hex of their concatenation. That is `sha256sum` output over the sorted file list, hashed once more. + +# _sdk_sha256 [file] -> lowercase sha256 hex of the file (or stdin). sha256sum on Linux, shasum on macOS. +_sdk_sha256() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@" | cut -d' ' -f1 + else shasum -a 256 "$@" | cut -d' ' -f1; fi +} + +# sdk_digest -> sdkDigest on stdout. Fails on a directory with no files. +sdk_digest() { + local root="${1:?usage: sdk_digest }" files + files="$(cd "$root" && find . -type f | sed 's|^\./||' | LC_ALL=C sort)" + [ -n "$files" ] || { echo "ERROR: no files under $root to digest" >&2; return 1; } + local f + while IFS= read -r f; do + printf '%s %s\n' "$(_sdk_sha256 "$root/$f")" "$f" + done <<<"$files" | _sdk_sha256 +} diff --git a/scripts/build-extension.sh b/scripts/build-extension.sh index e9fe60b..0aed5a5 100755 --- a/scripts/build-extension.sh +++ b/scripts/build-extension.sh @@ -24,6 +24,8 @@ DIR="${DIR%/}" source "$(dirname "$0")/_target.sh" # → BASE_URL (+ TOKEN) from .env / env per DUPLO_TARGET # shellcheck source=scripts/_builder.sh source "$(dirname "$0")/_builder.sh" # → native, or re-launch this build in the builder container +# shellcheck source=scripts/_sdk_digest.sh +source "$(dirname "$0")/_sdk_digest.sh" # → sdk_digest, stamped into the bundle manifest beside sdkVersion BASE_URL="${BASE_URL%/}" # tolerate a trailing slash in DUPLO_HOST (avoids // in URLs) [ -f "$DIR/manifest.json" ] || { echo "No manifest.json in $DIR" >&2; exit 1; } @@ -153,7 +155,13 @@ curl -fsSL -m 120 "${auth[@]}" "$BASE_URL/v1/aiservicedesk/extensions/sdk-bundle || { echo "ERROR: failed to download the SDK bundle from $BASE_URL/v1/aiservicedesk/extensions/sdk-bundle" >&2; exit 1; } unzip -oq "$DIR/dist-sdk.zip" -d "$DIR/backend/sdk-packages" \ || { echo "ERROR: SDK bundle is not a valid zip — got $(file -b "$DIR/dist-sdk.zip" 2>/dev/null). Check DUPLO_HOST." >&2; exit 1; } -rm -f "$DIR/dist-sdk.zip" +# Digest a fresh extraction, not sdk-packages/: that directory accumulates packages from earlier builds, and the +# digest must describe only the SDK this build compiled against. See scripts/_sdk_digest.sh for the definition. +SDK_FEED="$(mktemp -d)" +unzip -oq "$DIR/dist-sdk.zip" -d "$SDK_FEED" +SDK_DIGEST="$(sdk_digest "$SDK_FEED")" || { rm -rf "$SDK_FEED"; echo "ERROR: the SDK bundle from $BASE_URL is empty." >&2; exit 1; } +rm -rf "$SDK_FEED" "$DIR/dist-sdk.zip" +echo " host SDK digest: $SDK_DIGEST" CSPROJ=$(find "$DIR/backend" -maxdepth 1 -name '*.csproj' | head -1) [ -n "$CSPROJ" ] || { echo "No .csproj under $DIR/backend" >&2; exit 1; } @@ -214,9 +222,10 @@ echo " host provides $(wc -l < "$PROVIDED" | tr -d ' ') assemblies (excluded echo "==> Assembling bundle" PKG="$DIR/dist/pkg" rm -rf "$PKG"; mkdir -p "$PKG/backend" "$PKG/fe" "$PKG/skills" -# Bundle manifest with the real host SDK version pinned in, and both // paths derived from -# id+version. The host stages the bundle to /// but reads the entry assembly -# from backend.assemblyDir and each provisioning skill from skills[].folder, so all three have to agree. +# Bundle manifest with the real host SDK version and the digest of the SDK feed it was compiled against pinned +# in, and both // paths derived from id+version. The host stages the bundle to +# /// but reads the entry assembly from backend.assemblyDir and each +# provisioning skill from skills[].folder, so all three have to agree. # Maintained by hand they drift the moment you bump version alone, and what that breaks depends on the target: # # backend.assemblyDir a host that still has the old version on disk keeps running the OLD backend DLL and @@ -230,8 +239,9 @@ rm -rf "$PKG"; mkdir -p "$PKG/backend" "$PKG/fe" "$PKG/skills" # Both are always //…, so compute them here instead of trusting the checked-in values. The skill # tail varies per entry (skills/), so only the first two segments are replaced; a manifest with no # skills array is left alone rather than given an empty one. -jq --arg v "$SDK_VER" ' +jq --arg v "$SDK_VER" --arg d "$SDK_DIGEST" ' .sdkVersion = $v + | .sdkDigest = $d | .backend.assemblyDir = (.id + "/" + .version + "/backend") | .id as $i | .version as $r | if (.skills | type) == "array" then diff --git a/scripts/check-extension-pr.sh b/scripts/check-extension-pr.sh new file mode 100755 index 0000000..1715e0d --- /dev/null +++ b/scripts/check-extension-pr.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Warn on a pull request about what each changed extension's release will need — the PR step of +# .github/workflows/extension-ci.yml. Warnings only, never a failure: authors should learn what shipping needs as +# early as possible without being blocked mid-development. scripts/release-extensions.sh is what refuses a release +# that cannot ship. +# +# For each extension the PR changes (anything under its directory since the merge base): +# - manifest.version is unchanged, so the release job will refuse to publish the new content at the old version. +# - a directory under skills/ has no matching skills[].folder in the manifest, so the host never registers it. +# - the built bundle's frontend is webpack (fe/remoteEntry.js, no fe/remoteEntry.json), whose pages do not open on +# hosts that load Native Federation remotes. Checked only when dist/extension.zip exists, i.e. after a build. +# +# Usage: ./scripts/check-extension-pr.sh # e.g. origin/main; defaults to origin/$GITHUB_BASE_REF +set -euo pipefail +cd "$(dirname "$0")/.." +shopt -s nullglob + +base_ref="${1:-origin/${GITHUB_BASE_REF:?usage: check-extension-pr.sh }}" +base="$(git merge-base "$base_ref" HEAD)" +warned=0 +warn() { echo "::warning file=$1::$2"; warned=$((warned+1)); } + +for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do + [ -f "$m" ] || continue + dir="$(dirname "$m")" + git diff --quiet "$base" HEAD -- "$dir" && continue + + version="$(jq -r '.version' "$m")" + base_version="$(git show "$base:$m" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true)" + if [ -n "$base_version" ] && [ "$base_version" = "$version" ]; then + warn "$m" "$dir changed but manifest.version is still $version. Bump it, or the release job refuses to publish this content." + fi + + for s in "$dir"/skills/*/; do + s="$(basename "$s")" + jq -e --arg s "$s" 'any(.skills[]?.folder // ""; endswith("/skills/" + $s))' "$m" >/dev/null \ + || warn "$m" "$dir ships skills/$s but the manifest declares no skills[].folder for it, so the host never registers it." + done + + zip="$dir/dist/extension.zip" + if [ -f "$zip" ]; then + entries="$(unzip -Z1 "$zip")" + if grep -qx 'fe/remoteEntry.js' <<<"$entries" && ! grep -qx 'fe/remoteEntry.json' <<<"$entries"; then + warn "$m" "$dir builds a webpack frontend (fe/remoteEntry.js). Its pages do not open on hosts that load Native Federation remotes. Migrate with the duplo-extension-ng22-migration skill." + fi + fi +done + +echo "==> $warned release warning(s)." diff --git a/scripts/release-extensions.sh b/scripts/release-extensions.sh new file mode 100755 index 0000000..c55f613 --- /dev/null +++ b/scripts/release-extensions.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Publish every built extension in this repo as a GitHub Release — the publish step of .github/workflows/release.yml, +# kept as a script so tests/test-release-extensions.sh can exercise it. +# +# One release per build, tagged -v-sdk-. is the extension directory's basename, and +# sdkVersion is read from the BUILT bundle's manifest (build-extension.sh stamps it from the host it built against), +# so builds of one version for different SDKs share the version and differ only in the tag. +# +# A version that already has a release, under that tag shape or the legacy -v, is compared with the +# built commit. If the extension is unchanged, an existing build for this SDK is skipped and a missing one is +# published. If it changed, the extension fails rather than skipping green: two different bundles would otherwise +# share one version, and whatever consumes the release could not tell them apart. Legacy tags were cut at main's HEAD +# at publish time rather than at the built commit, so one can report a change that a version bump then clears. +# +# Needs the repo's tags and the history they point at (actions/checkout with fetch-depth: 0), gh authenticated with +# contents: write, jq and unzip. GITHUB_SHA is the built commit; it defaults to HEAD. +# +# Usage: ./scripts/release-extensions.sh +set -euo pipefail +cd "$(dirname "$0")/.." +shopt -s nullglob + +sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" +published=0; skipped=0; failed=0 + +for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do + [ -f "$m" ] || continue + dir="$(dirname "$m")" + zip="$dir/dist/extension.zip" + if [ ! -f "$zip" ]; then + echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping." + continue + fi + name="$(jq -r '.name // .id' "$m")" + version="$(jq -r '.version' "$m")" + slug="$(basename "$dir")" + sdk="$(unzip -p "$zip" manifest.json | jq -r '.sdkVersion // empty')" + if [ -z "$sdk" ]; then + echo "::error::$dir — the bundle's manifest.json carries no sdkVersion, so its release cannot be tagged. Build it with scripts/build-extension.sh." + failed=$((failed+1)); continue + fi + base="${slug}-v${version}" + tag="${base}-sdk-${sdk}" + + changed_since="" + while IFS= read -r t; do + [ -n "$t" ] || continue + git diff --quiet "$t^{commit}" "$sha" -- "$dir" || { changed_since="$t"; break; } + done < <(git tag -l "$base" "$base-sdk-*") + if [ -n "$changed_since" ]; then + echo "::error::$dir — $version is already released as $changed_since and the extension changed since. Bump manifest.version to publish it." + failed=$((failed+1)); continue + fi + + if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." + skipped=$((skipped+1)); continue + fi + + echo "==> Publishing release $tag ($name $version, SDK $sdk)" + if ! gh release create "$tag" "$zip" --target "$sha" \ + --title "$name $version (SDK $sdk)" \ + --notes "Automated release of $name v$version from $slug, built against host SDK $sdk (extension.zip attached)."; then + echo "::error::$dir — gh release create $tag failed." + failed=$((failed+1)); continue + fi + published=$((published+1)) + + # An asset replaced at a mutable tag no longer matches a digest pinned from it, so say so where the owner sees it. + immutable="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.immutable' 2>/dev/null || true)" + if [ "$immutable" != true ]; then + echo "::warning::$tag landed mutable (immutable=${immutable:-unknown}). A repository admin turns on immutable releases under Settings → General → Releases; releases published before that stay mutable." + fi +done + +echo "==> Released $published, skipped $skipped, failed $failed." +[ "$failed" -eq 0 ] diff --git a/tests/test-builder-image.sh b/tests/test-builder-image.sh new file mode 100755 index 0000000..d426ff7 --- /dev/null +++ b/tests/test-builder-image.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Tests for builder_resolve_image in scripts/_builder.sh — which toolchain image a build runs in. +# Uses a stub `docker` on PATH and a throwaway .env; never pulls. Usage: ./tests/test-builder-image.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +mkdir -p "$TMP/bin"; printf '#!/usr/bin/env bash\nexit 0\n' > "$TMP/bin/docker"; chmod +x "$TMP/bin/docker" +export PATH="$TMP/bin:$PATH" + +# Stand-in for _target.sh's .env reader, pointed at a temp file. +ENV="$TMP/.env"; : > "$ENV" +_envv() { grep -E "^$1=" "$ENV" 2>/dev/null | head -1 | cut -d= -f2- || true; } + +. ./scripts/_builder.sh || { echo "scripts/_builder.sh not found"; exit 1; } +# builder_dispatch resolves RUNTIME before it ever calls builder_resolve_image. Calling the resolver directly +# skips that, so point it at the stub here. +RUNTIME=docker + +resolve() { ( unset BUILDER_IMAGE BUILDER_TAG BUILDER_PULL; eval "$1"; builder_resolve_image >/dev/null 2>&1; echo "$BUILDER_IMAGE" ); } + +echo "builder image:" + +t "defaults to the digest-pinned toolchain image" +: > "$ENV"; got="$(resolve :)" +if [[ "$got" =~ ^quay\.io/duplocloud/duplo-extension-builder@sha256:[0-9a-f]{64}$ ]]; then ok; else bad "$got"; fi + +t "docker-compose.yml falls back to the same pinned image" +: > "$ENV"; got="$(resolve :)" +if grep -qF -- "\${BUILDER_IMAGE:-$got}" docker-compose.yml; then ok; else bad "compose does not default to $got"; fi + +t "BUILDER_TAG in .env still selects a tag" +printf 'BUILDER_TAG=latest\n' > "$ENV"; got="$(resolve :)" +if [ "$got" = "quay.io/duplocloud/duplo-extension-builder:latest" ]; then ok; else bad "$got"; fi + +t "BUILDER_IMAGE in the environment still replaces the image outright" +: > "$ENV"; got="$(resolve 'export BUILDER_IMAGE=example.com/mine:1')" +if [ "$got" = "example.com/mine:1" ]; then ok; else bad "$got"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-check-extension-pr.sh b/tests/test-check-extension-pr.sh new file mode 100755 index 0000000..3af736a --- /dev/null +++ b/tests/test-check-extension-pr.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Tests for scripts/check-extension-pr.sh — the PR-time warnings about what a release will need. +# Runs in a throwaway git repo; never touches GitHub. Usage: ./tests/test-check-extension-pr.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +ROOT="$PWD" + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + +# new_repo: extensions/demo at 0.1.0 declaring skill `provision-demo`, committed on main, then a `pr` branch. +new_repo() { + REPO="$TMP/repo.$RANDOM"; D="$REPO/extensions/demo"; mkdir -p "$D/skills/provision-demo" + git -C "$REPO" init -q -b main + git -C "$REPO" config user.email t@example.com; git -C "$REPO" config user.name t + set_manifest 0.1.0 '["duplo.demo/0.1.0/skills/provision-demo"]' + echo skill > "$D/skills/provision-demo/SKILL.md"; echo one > "$D/code.txt" + git -C "$REPO" add -A; git -C "$REPO" commit -qm init + git -C "$REPO" switch -q -c pr +} + +# set_manifest +set_manifest() { + jq -n --arg v "$1" --argjson f "$2" '{id:"duplo.demo",name:"Demo",version:$v,skills:[$f[]|{folder:.,isBuiltIn:true}]}' \ + > "$D/manifest.json" +} + +change() { echo "$1" > "$D/code.txt"; git -C "$REPO" commit -qam "$1"; } + +# bundle : the dist/extension.zip a build leaves, with its frontend entry under fe/. +bundle() { + local p; p="$(mktemp -d)"; mkdir -p "$p/fe"; cp "$D/manifest.json" "$p/"; echo x > "$p/fe/$1" + mkdir -p "$D/dist"; rm -f "$D/dist/extension.zip"; ( cd "$p" && zip -qr "$D/dist/extension.zip" . ); rm -rf "$p" +} + +run() { + mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/check-extension-pr.sh" "$REPO/scripts/" 2>/dev/null + "$REPO/scripts/check-extension-pr.sh" main 2>&1 +} + +echo "check-extension-pr:" + +t "warns when an extension changed with no manifest.version bump" +new_repo; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*extensions/demo.*manifest.version" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about the version when it was bumped" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about an extension the PR does not touch" +new_repo; echo readme > "$REPO/README.md"; git -C "$REPO" add -A; git -C "$REPO" commit -qm readme +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about the version of an extension new in this PR" +new_repo; git -C "$REPO" switch -q main; git -C "$REPO" rm -rq extensions; git -C "$REPO" commit -qm empty +git -C "$REPO" switch -q -c pr2; git -C "$REPO" checkout -q pr -- extensions; git -C "$REPO" commit -qm add +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a shipped skill the manifest does not declare" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]' +mkdir -p "$D/skills/undeclared"; echo s > "$D/skills/undeclared/SKILL.md"; git -C "$REPO" add -A; git -C "$REPO" commit -qm skill +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*skills/undeclared" <<<"$OUT" && ! grep -q "provision-demo" <<<"$OUT" +then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a webpack frontend (fe/remoteEntry.js) in the built bundle" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.js +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*remoteEntry.js" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about a Native Federation frontend (fe/remoteEntry.json)" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.json +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh new file mode 100755 index 0000000..26bbd0d --- /dev/null +++ b/tests/test-release-extensions.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Tests for scripts/release-extensions.sh — the release job's publish step. +# Runs in a throwaway git repo with a stub `gh` on PATH; never touches GitHub. Usage: ./tests/test-release-extensions.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +ROOT="$PWD" + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + +# Stub gh: every call is appended to $GH_LOG. `api` answers the release's immutable flag from $GH_IMMUTABLE. +mkdir -p "$TMP/bin" +cat > "$TMP/bin/gh" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$GH_LOG" +case "$1" in + api) printf '%s\n' "${GH_IMMUTABLE:-true}" ;; + release) [ "$2" = create ] && exit "${GH_CREATE_RC:-0}" ;; +esac +exit 0 +EOF +chmod +x "$TMP/bin/gh" +export PATH="$TMP/bin:$PATH" + +# new_repo: a git repo holding extensions/demo at version 0.1.0, committed. +new_repo() { + REPO="$TMP/repo.$RANDOM"; mkdir -p "$REPO/extensions/demo" + git -C "$REPO" init -q -b main + git -C "$REPO" config user.email t@example.com; git -C "$REPO" config user.name t + printf '{"id":"duplo.demo","name":"Demo","version":"%s"}\n' "${1:-0.1.0}" > "$REPO/extensions/demo/manifest.json" + echo one > "$REPO/extensions/demo/code.txt" + git -C "$REPO" add -A; git -C "$REPO" commit -qm init + export GH_LOG="$REPO.gh.log"; : > "$GH_LOG" +} + +# build_zip [sdkVersion]: the dist/extension.zip build-extension.sh would leave, with sdkVersion stamped. +# An empty argument leaves sdkVersion out of the bundle manifest. +build_zip() { + local d="$REPO/extensions/demo" pkg; pkg="$(mktemp -d)" + if [ -n "${1-}" ]; then jq --arg v "$1" '.sdkVersion = $v' "$d/manifest.json" > "$pkg/manifest.json" + else cp "$d/manifest.json" "$pkg/manifest.json"; fi + mkdir -p "$d/dist"; rm -f "$d/dist/extension.zip" + ( cd "$pkg" && zip -qr "$d/dist/extension.zip" . ); rm -rf "$pkg" +} + +commit_change() { echo "$1" > "$REPO/extensions/demo/code.txt"; git -C "$REPO" commit -qam "$1"; } + +# The script resolves the repo from its own path, as it does when devkit copies it into an extension repo. +run() { + mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/release-extensions.sh" "$REPO/scripts/" 2>/dev/null + GITHUB_SHA="$(git -C "$REPO" rev-parse HEAD)" "$REPO/scripts/release-extensions.sh" 2>&1 +} + +echo "release-extensions:" + +t "publishes a new version under [slug]-v[version]-sdk-[sdkVersion], targeted at the built commit" +new_repo; build_zip 1.0.6; SHA="$(git -C "$REPO" rev-parse HEAD)" +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q -- "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip --target $SHA" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "skips when the -sdk tag exists and the extension is unchanged since it" +new_repo; build_zip 1.0.6; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "already released" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails, naming the version bump, when the extension changed since that version's -sdk release" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; commit_change two; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "manifest.version" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails when the extension changed since that version's legacy (no -sdk) release" +new_repo; git -C "$REPO" tag demo-v0.1.0; commit_change two; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "demo-v0.1.0 " <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "publishes the -sdk build of an unchanged version that has only a legacy release" +new_repo; git -C "$REPO" tag demo-v0.1.0; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "publishes a build for a new SDK of an unchanged version" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.7 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.7" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails when the bundle manifest carries no sdkVersion" +new_repo; build_zip "" +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "sdkVersion" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "warns when the published release landed mutable" +new_repo; build_zip 1.0.6 +OUT="$(GH_IMMUTABLE=false run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning::.*immutable" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "does not warn when the published release is immutable" +new_repo; build_zip 1.0.6 +OUT="$(GH_IMMUTABLE=true run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning::" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "fails when gh release create fails" +new_repo; build_zip 1.0.6 +OUT="$(GH_CREATE_RC=1 run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "still publishes the other extensions when one fails" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; commit_change two; build_zip 1.0.6 +mkdir -p "$REPO/extensions/other/dist" +printf '{"id":"duplo.other","name":"Other","version":"0.2.0"}\n' > "$REPO/extensions/other/manifest.json" +git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm other +( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" + cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "release create other-v0.2.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-sdk-digest.sh b/tests/test-sdk-digest.sh new file mode 100755 index 0000000..87a88b9 --- /dev/null +++ b/tests/test-sdk-digest.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tests for scripts/_sdk_digest.sh — the sdkDigest build-extension.sh stamps into a bundle's manifest. +# Usage: ./tests/test-sdk-digest.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +. ./scripts/_sdk_digest.sh || { echo "scripts/_sdk_digest.sh not found"; exit 1; } + +# The definition, computed independently: sha256 over " \n" per file, sorted by path bytewise. +expected() { + python3 - "$1" <<'PY' +import hashlib, os, sys +root = sys.argv[1]; lines = [] +for d, _, fs in os.walk(root): + for f in fs: + p = os.path.join(d, f); rel = os.path.relpath(p, root) + lines.append((rel.encode(), hashlib.sha256(open(p, "rb").read()).hexdigest())) +lines.sort() +print(hashlib.sha256(b"".join(h.encode() + b" " + r + b"\n" for r, h in lines)).hexdigest()) +PY +} + +mkfeed() { + FEED="$TMP/feed.$RANDOM"; mkdir -p "$FEED/sub" + echo a > "$FEED/Duplocloud.AiHelpdesk.Sdk.1.0.6.nupkg"; echo b > "$FEED/B.nupkg"; echo c > "$FEED/sub/c.nupkg" +} + +echo "sdk digest:" + +t "matches the definition: sha256 of sorted ' ' lines" +mkfeed +if [ "$(sdk_digest "$FEED")" = "$(expected "$FEED")" ]; then ok; else bad "$(sdk_digest "$FEED") != $(expected "$FEED")"; fi + +t "changes when one package's bytes change" +mkfeed; before="$(sdk_digest "$FEED")"; echo changed > "$FEED/B.nupkg" +if [ "$(sdk_digest "$FEED")" != "$before" ]; then ok; else bad "unchanged: $before"; fi + +t "changes when a package is added" +mkfeed; before="$(sdk_digest "$FEED")"; echo d > "$FEED/D.nupkg" +if [ "$(sdk_digest "$FEED")" != "$before" ]; then ok; else bad "unchanged: $before"; fi + +t "does not depend on where the feed sits or the caller's cwd" +mkfeed; a="$(sdk_digest "$FEED")"; cp -R "$FEED" "$TMP/moved"; b="$( cd / && sdk_digest "$TMP/moved" )" +if [ "$a" = "$b" ]; then ok; else bad "$a != $b"; fi + +t "fails on an empty feed rather than stamping the digest of nothing" +mkdir -p "$TMP/empty" +if ! sdk_digest "$TMP/empty" >/dev/null 2>&1; then ok; else bad "returned $(sdk_digest "$TMP/empty")"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] From 8a8c5153008ed9eff2b71a7c8b0544941a9d0fe1 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 13:25:46 +0100 Subject: [PATCH 02/22] Add extension-sign: sign and verify extension.zip.sig against the console's vectors --- .gitignore | 2 + scripts/extension-sign.py | 239 ++++++++++++++++++++++ scripts/extension-sign.requirements.txt | 145 +++++++++++++ tests/extension_sign_vectors.py | 26 +++ tests/test-extension-sign.sh | 67 ++++++ tests/vectors/README.md | 98 +++++++++ tests/vectors/SOURCE | 1 + tests/vectors/alg-hs256.zip.sig | 1 + tests/vectors/alg-none.zip.sig | 1 + tests/vectors/bad-signature.zip.sig | 1 + tests/vectors/cases.json | 200 ++++++++++++++++++ tests/vectors/cert-as-signature.zip.sig | 1 + tests/vectors/extension.zip | Bin 0 -> 167 bytes tests/vectors/extension.zip.sig | 1 + tests/vectors/kid-mismatch.zip.sig | 1 + tests/vectors/missing-cert-header.zip.sig | 1 + tests/vectors/other.pem | 5 + tests/vectors/outside-namespace.zip.sig | 1 + tests/vectors/publisher.pem | 5 + tests/vectors/root.pem | 5 + tests/vectors/root.pub.pem | 4 + tests/vectors/unknown-root.zip.sig | 1 + 22 files changed, 806 insertions(+) create mode 100755 scripts/extension-sign.py create mode 100644 scripts/extension-sign.requirements.txt create mode 100644 tests/extension_sign_vectors.py create mode 100755 tests/test-extension-sign.sh create mode 100644 tests/vectors/README.md create mode 100644 tests/vectors/SOURCE create mode 100644 tests/vectors/alg-hs256.zip.sig create mode 100644 tests/vectors/alg-none.zip.sig create mode 100644 tests/vectors/bad-signature.zip.sig create mode 100644 tests/vectors/cases.json create mode 100644 tests/vectors/cert-as-signature.zip.sig create mode 100644 tests/vectors/extension.zip create mode 100644 tests/vectors/extension.zip.sig create mode 100644 tests/vectors/kid-mismatch.zip.sig create mode 100644 tests/vectors/missing-cert-header.zip.sig create mode 100644 tests/vectors/other.pem create mode 100644 tests/vectors/outside-namespace.zip.sig create mode 100644 tests/vectors/publisher.pem create mode 100644 tests/vectors/root.pem create mode 100644 tests/vectors/root.pub.pem create mode 100644 tests/vectors/unknown-root.zip.sig diff --git a/.gitignore b/.gitignore index 1912f6f..27b10a7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ *.zip !packages/*.tgz !.claude/skills/duplo-extension-dev/templates/helloworld/frontend/vendor/*.tgz +# The console's cross-language signing vectors (tests/extension_sign_vectors.py), not a build artifact. +!tests/vectors/*.zip # The Terraform extension's source, vendored by run.sh (scripts/fetch-terraform-extension.sh). # It is fetched, not authored here, so it does not belong in this repo's history — and without this diff --git a/scripts/extension-sign.py b/scripts/extension-sign.py new file mode 100755 index 0000000..9883d8a --- /dev/null +++ b/scripts/extension-sign.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Sign and verify an extension bundle's signature, extension.zip.sig. + +sign writes one compact ES256 JWS whose claims come from the BUILT zip's manifest.json, since sdkVersion exists only +there, and whose sha256 is over the zip's final bytes. It reproduces the console's reference signer (sign_artifact in +apps/extensions/signing/verify.py). verify ports the console's verify_artifact_signature, check for check and code +for code, so tests/vectors (the console's cross-language vectors) pin both. + +The key and certificate come from EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT, never the command line, so ps on +a shared runner never shows them. EXTENSION_SIGN_NOW overrides the clock for tests. +""" +import argparse, base64, hashlib, json, os, re, sys, time, urllib.request, zipfile + +import jwt +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec + +ALG = "ES256" +SIG_TYP = "duplo-ext-sig+jwt" +CERT_TYP = "duplo-ext-cert+jwt" +CERT_ISSUER = "duplocloud-console" +PLACEHOLDER = "REPLACED_BY_SKILL" +SEMVER = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +WARN_DAYS = 30 +NO_TIME_CHECKS = {"verify_exp": False, "verify_nbf": False, "verify_iat": False, "verify_aud": False, "verify_iss": False} + + +class SignatureError(Exception): + def __init__(self, code, message=""): + super().__init__(message or code) + self.code = code + + +def now_s(): + return int(os.environ.get("EXTENSION_SIGN_NOW") or time.time()) + + +def b64url(data): + return base64.urlsafe_b64encode(data).rstrip(b"=").decode() + + +def coordinate(text): + if not isinstance(text, str): + raise ValueError("JWK coordinate must be a string") + raw = base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)) + if len(raw) != 32 or b64url(raw) != text: + raise ValueError("JWK coordinate must be 32 bytes of canonical base64url") + return raw + + +def pem_from_jwk(jwk): + if not isinstance(jwk, dict) or jwk.get("kty") != "EC" or jwk.get("crv") != "P-256": + raise ValueError("only EC P-256 JWKs are supported") + x = int.from_bytes(coordinate(jwk.get("x")), "big") + y = int.from_bytes(coordinate(jwk.get("y")), "big") + key = ec.EllipticCurvePublicNumbers(x, y, ec.SECP256R1()).public_key() + return key.public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo).decode() + + +def jwk_of(public_key): + n = public_key.public_numbers() + return {"kty": "EC", "crv": "P-256", "x": b64url(n.x.to_bytes(32, "big")), "y": b64url(n.y.to_bytes(32, "big"))} + + +def in_namespaces(manifest_id, prefixes): + return any(manifest_id == p or manifest_id.startswith(p + ".") for p in prefixes) + + +def built_manifest(zip_path): + with zipfile.ZipFile(zip_path) as z: + return json.loads(z.read("manifest.json")) + + +def sha256_of(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def refuse(message): + raise SystemExit(f"::error::{message}") + + +def sign_bytes(zip_path, key_pem, cert, now): + m = built_manifest(zip_path) + manifest_id, version, sdk = m.get("id"), m.get("version"), m.get("sdkVersion") + if not isinstance(sdk, str) or not sdk or sdk.startswith(PLACEHOLDER): + refuse(f"{zip_path}: the built manifest carries no real sdkVersion. Build it with scripts/build-extension.sh") + for name, value in (("version", version), ("sdkVersion", sdk)): + if not isinstance(value, str) or not SEMVER.match(value): + refuse(f"{zip_path}: {name} {value!r} is not strict semver (MAJOR.MINOR.PATCH[-pre], no +build)") + claims = jwt.decode(cert, options={"verify_signature": False}) + if not in_namespaces(manifest_id, claims.get("ns") or []): + refuse(f"manifest id outside this key's namespaces: {manifest_id} is not under {claims.get('ns')}") + key = serialization.load_pem_private_key(key_pem.encode(), password=None) + if jwk_of(key.public_key()) != {k: claims.get("jwk", {}).get(k) for k in ("kty", "crv", "x", "y")}: + refuse("signing key and certificate don't match") + exp = claims.get("exp") + if not isinstance(exp, int) or now >= exp: + refuse("the signing certificate has expired. Reissue it in the console and update CONSOLE_SIGNING_CERT") + if exp - now < WARN_DAYS * 86400: + print(f"::warning::the signing certificate expires in {(exp - now) // 86400} days. Reissue it in the console", + file=sys.stderr) + body = {"manifest_id": manifest_id, "version": version, "sdk_version": sdk, + "sha256": sha256_of(zip_path), "iat": now} + return jwt.encode(body, key, algorithm=ALG, headers={"kid": claims["kid"], "typ": SIG_TYP, "cert": cert}) + + +def header(token, code): + if not isinstance(token, str): + raise SignatureError(code, "token is not a string") + try: + h = jwt.get_unverified_header(token) + except Exception as exc: + raise SignatureError(code, f"unreadable header: {exc}") from exc + if not isinstance(h, dict): + raise SignatureError(code, "header is not an object") + return h + + +def decode(token, public_pem, bad_sig, malformed): + try: + claims = jwt.decode(token, public_pem, algorithms=[ALG], options=NO_TIME_CHECKS) + except jwt.InvalidSignatureError as exc: + raise SignatureError(bad_sig, "signature does not verify") from exc + except Exception as exc: + raise SignatureError(malformed, f"cannot decode token: {exc}") from exc + if not isinstance(claims, dict): + raise SignatureError(malformed, "claims are not an object") + return claims + + +def is_str(v): + return isinstance(v, str) and v != "" + + +def is_int(v): + return isinstance(v, int) and not isinstance(v, bool) + + +def verify_signature(sig, *, root_keys, sha256, manifest_id, version, sdk_version, trusted_publishers, now): + h = header(sig, "malformed") + if h.get("typ") != SIG_TYP: + raise SignatureError("bad_typ", "signature typ") + if h.get("alg") != ALG: + raise SignatureError("malformed", "signature alg") + cert = h.get("cert") + if not isinstance(cert, str) or not cert: + raise SignatureError("malformed", "missing cert header") + ch = header(cert, "bad_cert") + if ch.get("typ") != CERT_TYP: + raise SignatureError("bad_typ", "certificate typ") + if ch.get("alg") != ALG: + raise SignatureError("bad_cert", "certificate alg") + root_kid = ch.get("kid") + if not isinstance(root_kid, str) or root_kid not in root_keys: + raise SignatureError("unknown_root", "certificate is not signed by a known root key") + cc = decode(cert, root_keys[root_kid], "bad_cert", "bad_cert") + ns = cc.get("ns") + if cc.get("iss") != CERT_ISSUER or not (is_str(cc.get("sub")) and is_str(cc.get("kid")) and isinstance(ns, list) + and all(is_str(p) for p in ns) and is_int(cc.get("nbf")) + and is_int(cc.get("exp"))): + raise SignatureError("bad_cert", "certificate claims") + try: + public_pem = pem_from_jwk(cc.get("jwk")) + except Exception as exc: + raise SignatureError("bad_cert", "certificate jwk") from exc + if now < cc["nbf"] or now >= cc["exp"]: + raise SignatureError("cert_expired", "certificate is outside its validity window") + if trusted_publishers is not None and cc["sub"] not in set(trusted_publishers): + raise SignatureError("untrusted_publisher", "publisher is not trusted") + claims = decode(sig, public_pem, "bad_signature", "malformed") + if h.get("kid") != cc["kid"]: + raise SignatureError("kid_mismatch", "signature kid differs from certificate kid") + fields = ("manifest_id", "version", "sdk_version", "sha256") + if not all(is_str(claims.get(f)) for f in fields) or not SHA256.match(claims["sha256"]): + raise SignatureError("malformed", "signature claims") + if not isinstance(sha256, str) or claims["sha256"] != sha256.lower(): + raise SignatureError("hash_mismatch", "sha256 differs") + if (claims["manifest_id"], claims["version"], claims["sdk_version"]) != (manifest_id, version, sdk_version): + raise SignatureError("manifest_mismatch", "manifest_id, version or sdk_version differs") + if not in_namespaces(claims["manifest_id"], ns): + raise SignatureError("outside_namespace", "manifest_id is outside the certificate's namespaces") + return {"publisher": cc["sub"], "kid": cc["kid"], "cert_exp": cc["exp"], "root_kid": root_kid} + + +def console_root_keys(console_url): + with urllib.request.urlopen(f"{console_url.rstrip('/')}/api/extensions/root-keys/", timeout=30) as r: + body = json.load(r) + rows = body.get("results", body) if isinstance(body, dict) else body + return {row["kid"]: row["public_key"] for row in rows if row.get("public_key")} + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = p.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("sign") + s.add_argument("zip") + s.add_argument("--out") + v = sub.add_parser("verify") + v.add_argument("zip") + v.add_argument("sig") + v.add_argument("--root-key", action="append", default=[], help="a local root public key PEM, for tests") + v.add_argument("--console", default="https://console.duplocloud.com") + a = p.parse_args(argv) + if a.cmd == "sign": + key, cert = os.environ.get("EXTENSION_SIGNING_KEY", ""), os.environ.get("EXTENSION_SIGNING_CERT", "") + if not key or not cert: + refuse("EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT must both be set") + token = sign_bytes(a.zip, key, cert, now_s()) + out = a.out or a.zip + ".sig" + with open(out, "w") as f: + f.write(token) + print(f"==> signed {a.zip} -> {out}") + return 0 + sig = open(a.sig).read() + if a.root_key: + # A local root key is addressed by the kid the signature's certificate names, so a test root verifies. + root_kid = jwt.get_unverified_header(jwt.get_unverified_header(sig)["cert"]).get("kid") + roots = {root_kid: open(a.root_key[0]).read()} + else: + roots = console_root_keys(a.console) + m = built_manifest(a.zip) + try: + verify_signature(sig, root_keys=roots, sha256=sha256_of(a.zip), manifest_id=m.get("id"), + version=m.get("version"), sdk_version=m.get("sdkVersion"), trusted_publishers=None, + now=now_s()) + except SignatureError as exc: + print(exc.code) + return 1 + print("ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/extension-sign.requirements.txt b/scripts/extension-sign.requirements.txt new file mode 100644 index 0000000..d9c6de1 --- /dev/null +++ b/scripts/extension-sign.requirements.txt @@ -0,0 +1,145 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes --no-index --output-file=scripts/extension-sign.requirements.txt _tmp_es.in +# +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +cryptography==44.0.0 \ + --hash=sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7 \ + --hash=sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731 \ + --hash=sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b \ + --hash=sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc \ + --hash=sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543 \ + --hash=sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c \ + --hash=sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591 \ + --hash=sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede \ + --hash=sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb \ + --hash=sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f \ + --hash=sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123 \ + --hash=sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c \ + --hash=sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c \ + --hash=sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285 \ + --hash=sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd \ + --hash=sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092 \ + --hash=sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa \ + --hash=sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289 \ + --hash=sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02 \ + --hash=sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64 \ + --hash=sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053 \ + --hash=sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417 \ + --hash=sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e \ + --hash=sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e \ + --hash=sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7 \ + --hash=sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756 \ + --hash=sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4 + # via -r _tmp_es.in +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pyjwt==2.10.1 \ + --hash=sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953 \ + --hash=sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb + # via -r _tmp_es.in diff --git a/tests/extension_sign_vectors.py b/tests/extension_sign_vectors.py new file mode 100644 index 0000000..376dfc1 --- /dev/null +++ b/tests/extension_sign_vectors.py @@ -0,0 +1,26 @@ +"""Runs every case in tests/vectors/cases.json through extension-sign.py's verifier and checks the recorded outcome.""" +import importlib.util, json, pathlib, sys + +HERE = pathlib.Path(__file__).parent +spec = importlib.util.spec_from_file_location("extension_sign", HERE.parent / "scripts" / "extension-sign.py") +es = importlib.util.module_from_spec(spec) +spec.loader.exec_module(es) + +vectors = HERE / "vectors" +data = json.loads((vectors / "cases.json").read_text()) +roots = {kid: (vectors / name).read_text() for kid, name in data["root_keys"].items()} +bad = 0 +for case in data["cases"]: + try: + es.verify_signature((vectors / case["sig"]).read_text(), root_keys=roots, sha256=case["sha256"], + manifest_id=case["manifest_id"], version=case["version"], + sdk_version=case["sdk_version"], trusted_publishers=case["trusted_publishers"], + now=case["now"]) + got = "ok" + except es.SignatureError as exc: + got = exc.code + if got != case["expect"]: + print(f"MISMATCH {case['name']}: expected {case['expect']}, got {got}") + bad += 1 +print(f"{len(data['cases']) - bad} of {len(data['cases'])} vector cases match") +sys.exit(1 if bad else 0) diff --git a/tests/test-extension-sign.sh b/tests/test-extension-sign.sh new file mode 100755 index 0000000..8b77853 --- /dev/null +++ b/tests/test-extension-sign.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Tests for scripts/extension-sign.py against the console's cross-language vectors. Usage: ./tests/test-extension-sign.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +V=tests/vectors; S=scripts/extension-sign.py +# The vectors' certificate rides in each .sig header, so the valid case's header gives it. +CERT=$(python3 -c 'import jwt,sys; print(jwt.get_unverified_header(open(sys.argv[1]).read())["cert"])' $V/extension.zip.sig) +NOW=$(jq -r '.cases[0].now' $V/cases.json) +# mkzip : a zip holding only manifest.json. "-" leaves sdkVersion out. +mkzip() { local d; d=$(mktemp -d); jq -n --arg i "$1" --arg v "$2" --arg s "$3" \ + '{id:$i,version:$v} + (if $s == "-" then {} else {sdkVersion:$s} end)' > "$d/manifest.json" + (cd "$d" && zip -q "$4" manifest.json); rm -rf "$d"; } +sign() { EXTENSION_SIGNING_KEY="$(cat "$1")" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$NOW" python3 $S sign "$2" 2>&1; } + +echo "extension-sign:" + +t "verify reproduces every vector case" +OUT=$(python3 tests/extension_sign_vectors.py 2>&1); [ $? = 0 ] && ok || bad "$OUT" + +t "a fresh signature verifies against the test root" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/a.zip"; OUT=$(sign $V/publisher.pem "$TMP/a.zip") +OUT2=$(EXTENSION_SIGN_NOW="$NOW" python3 $S verify "$TMP/a.zip" "$TMP/a.zip.sig" --root-key $V/root.pub.pem 2>&1) +[ $? = 0 ] && ok || bad "$OUT $OUT2" + +t "the .sig has no trailing newline" +[ "$(tail -c1 "$TMP/a.zip.sig" | od -An -c | tr -d ' ')" != '\n' ] && ok || bad "trailing newline" + +t "refuses a key that does not match the certificate" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/b.zip"; OUT=$(sign $V/other.pem "$TMP/b.zip") +[ $? != 0 ] && grep -q "signing key and certificate don't match" <<<"$OUT" && [ ! -f "$TMP/b.zip.sig" ] && ok || bad "$OUT" + +t "refuses a manifest id outside the certificate's namespaces" +mkzip com.other.thing 1.0.0 1.0.0 "$TMP/c.zip"; OUT=$(sign $V/publisher.pem "$TMP/c.zip") +[ $? != 0 ] && grep -q "manifest id outside this key's namespaces" <<<"$OUT" && ok || bad "$OUT" + +t "refuses a missing sdkVersion" +mkzip io.example.hello 1.0.0 - "$TMP/d.zip"; OUT=$(sign $V/publisher.pem "$TMP/d.zip") +[ $? != 0 ] && grep -q "sdkVersion" <<<"$OUT" && ok || bad "$OUT" + +t "refuses the sdk-version placeholder" +mkzip io.example.hello 1.0.0 REPLACED_BY_SKILL_FROM_sdk-version_ENDPOINT "$TMP/e.zip"; OUT=$(sign $V/publisher.pem "$TMP/e.zip") +[ $? != 0 ] && grep -q "sdkVersion" <<<"$OUT" && ok || bad "$OUT" + +t "refuses a version with build metadata" +mkzip io.example.hello 1.0.0+abc 1.0.0 "$TMP/f.zip"; OUT=$(sign $V/publisher.pem "$TMP/f.zip") +[ $? != 0 ] && grep -q "strict semver" <<<"$OUT" && ok || bad "$OUT" + +t "refuses an expired certificate" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/g.zip" +EXP=$(python3 -c 'import jwt,sys; print(jwt.decode(sys.argv[1], options={"verify_signature": False})["exp"])' "$CERT") +OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$EXP" python3 $S sign "$TMP/g.zip" 2>&1) +[ $? != 0 ] && grep -q "expired" <<<"$OUT" && ok || bad "$OUT" + +t "warns when the certificate expires within 30 days" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/h.zip" +OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$((EXP - 86400))" python3 $S sign "$TMP/h.zip" 2>&1) +[ $? = 0 ] && grep -q "::warning::.*expires" <<<"$OUT" && ok || bad "$OUT" + +t "never prints the key" +grep -q "BEGIN" <<<"$(sign $V/other.pem "$TMP/b.zip")" && bad "key material in output" || ok + +echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] diff --git a/tests/vectors/README.md b/tests/vectors/README.md new file mode 100644 index 0000000..6fe535d --- /dev/null +++ b/tests/vectors/README.md @@ -0,0 +1,98 @@ +# Extension signing test vectors + +**These keys are test-only.** `root.pem`, `publisher.pem` and `other.pem` are throwaway EC P-256 +private keys. They are committed on purpose so that two independent verifiers can be checked +against the same inputs. No console, portal or license trusts them. Never use them for anything +else. + +The .NET helpdesk copies this directory as-is into its own test suite. For every entry in +`cases.json`, its verifier must produce the same outcome as the console's +`verify_artifact_signature` (`apps/extensions/signing/verify.py`). + +## Files + +| File | Contents | +|---|---| +| `root.pem` / `root.pub.pem` | Test root key, private and public. Its kid is the only key of `root_keys` in `cases.json` | +| `publisher.pem` | Test publisher private key. Its root-signed certificate rides in each `.sig` header (`cert`) | +| `other.pem` | An unrelated key. It signs the `bad-signature` and `unknown-root` vectors | +| `extension.zip` | A tiny zip holding `manifest.json`: `{"id":"io.example.hello","version":"1.0.0"}` | +| `*.zip.sig` | One compact JWS per file, with no trailing newline. Each case's `sig` names the file it uses | +| `cases.json` | `root_keys` (a map from root kid to PEM file name in this directory) and `cases` | + +## How to run a case + +Each case is: + +``` +{name, sig, sha256, manifest_id, version, sdk_version, trusted_publishers, now, expect} +``` + +Call the verifier once per case with these inputs: + +- **Signature:** the contents of the file named by `sig`. +- **Root keys:** loaded from `root_keys`. +- **`sha256`, `manifest_id`, `version`, `sdk_version`:** the case's own values. They are the + *observed* values: what the portal would have computed from the zip and read from its + manifest. Feed them **directly** into the claim comparison. Do **not** re-hash + `extension.zip` or re-read its manifest for the case run. Several cases deliberately supply + observed values that differ from the zip. +- **`trusted_publishers`:** the license's `trusted_pub`. `null` means skip the trust check, as + the console does. A list (including an empty list) means the certificate's `sub` must be in + it. +- **`now`:** the current time, as integer epoch seconds. Use it for the certificate's + `nbf`/`exp` check instead of the clock. + +`expect` is `ok` or the error code the verifier must report. + +`extension.zip` is only the source of the valid cases' `sha256`, and a sample for a separate +end-to-end test (hash the zip, read the manifest, then verify `extension.zip.sig`). + +## Check order and codes + +The steps run in this order. The first step that fails decides the code, so a token with several +defects reports the earliest one. + +1. **Signature header** (read unverified, only to route). + - `typ` must be `duplo-ext-sig+jwt`, else `bad_typ`. + - `alg` must be `ES256`, else `malformed`. This covers `none` and `HS256`. + - `cert` must be a non-empty string, else `malformed`. + - A header that can't be read at all is `malformed`. +2. **Certificate.** + - `typ` must be `duplo-ext-cert+jwt`, else `bad_typ`. + - Its header `kid` must be in `root_keys`, else `unknown_root`. + - It must verify as ES256 under that root key, else `bad_cert`. + - `iss` must be `duplocloud-console`. `sub` and `kid` must be strings, `ns` a list of strings + and `nbf`/`exp` integers. `jwk` must be an EC P-256 key with 32-byte canonical base64url + coordinates. Any failure is `bad_cert`. + - Validity has zero leeway: `now < nbf` or `now >= exp` is `cert_expired`. +3. **Trust** (only when `trusted_publishers` is not null): the certificate's `sub` must be in + it, else `untrusted_publisher`. +4. **Signature and key binding.** + - The signature must verify as ES256 under the certificate's `jwk`, else `bad_signature`. + Any other decode failure is `malformed`. + - The signature header `kid` must then equal the certificate's `kid`, else `kid_mismatch`. + - The claims `manifest_id`, `version`, `sdk_version` and `sha256` must be non-empty strings, + with `sha256` as 64 lowercase hex characters, else `malformed`. +5. **Claims.** `sha256` is compared first: a mismatch is `hash_mismatch`, and the observed value + is compared case-insensitively. Then `manifest_id`, `version` and `sdk_version` are compared: + a mismatch is `manifest_mismatch`. +6. **Namespace.** `manifest_id` must equal one of the certificate's `ns` prefixes or start with + `prefix + "."`, else `outside_namespace`. + +All codes: `malformed`, `bad_typ`, `unknown_root`, `bad_cert`, `cert_expired`, +`untrusted_publisher`, `kid_mismatch`, `bad_signature`, `hash_mismatch`, `manifest_mismatch`, +`outside_namespace`. + +## Regenerating + +```bash +make manage ARGS='make_signing_vectors' +``` + +The command reuses the existing private keys, so the kids, the certificate claims and the zip +stay the same. ECDSA signatures are randomised, so the `.sig` files change on every run. Before +writing `cases.json`, the command checks every case against the console verifier. + +The committed files are the source of truth. Only regenerate when the format changes, and copy +the result to the helpdesk again. diff --git a/tests/vectors/SOURCE b/tests/vectors/SOURCE new file mode 100644 index 0000000..2670bfb --- /dev/null +++ b/tests/vectors/SOURCE @@ -0,0 +1 @@ +duplocloud-internal/console apps/extensions/tests/vectors at 412ed03bd41c523e4edcb0be53666841ec9792c6. Test-only keys. diff --git a/tests/vectors/alg-hs256.zip.sig b/tests/vectors/alg-hs256.zip.sig new file mode 100644 index 0000000..79b583a --- /dev/null +++ b/tests/vectors/alg-hs256.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiAiSFMyNTYiLCAia2lkIjogIjEzZTkwN2E0NTMwYjUzNjAiLCAidHlwIjogImR1cGxvLWV4dC1zaWcrand0IiwgImNlcnQiOiAiZXlKaGJHY2lPaUpGVXpJMU5pSXNJbXRwWkNJNkltRTNaRFV3TUdObFlURXlPVGd4TVRFaUxDSjBlWEFpT2lKa2RYQnNieTFsZUhRdFkyVnlkQ3RxZDNRaWZRLmV5SnBjM01pT2lKa2RYQnNiMk5zYjNWa0xXTnZibk52YkdVaUxDSnpkV0lpT2lJd01EQXdNREF3TUMwd01EQXdMVFF3TURBdE9EQXdNQzB3TURBd01EQXdNREF3TURFaUxDSnVZVzFsSWpvaVJYaGhiWEJzWlNCUWRXSnNhWE5vWlhJaUxDSnJhV1FpT2lJeE0yVTVNRGRoTkRVek1HSTFNell3SWl3aWFuZHJJanA3SW10MGVTSTZJa1ZESWl3aVkzSjJJam9pVUMweU5UWWlMQ0o0SWpvaU9WQndjV2xKUVVWTlIwTldSbWw2TVVaUVgzSm9PUzFIVEZaNWVVZG5aWEJJVjB3eVNUTXlSMFpSVVNJc0lua2lPaUoyZW1sU0xYUm1hVmx5VFVkbmFuZGpkVE5xVFRoNGIxTmlZakl0VGxwd2JURktNV2RpVTFObkxXWkJJbjBzSW01eklqcGJJbWx2TG1WNFlXMXdiR1VpWFN3aWFXRjBJam94TnpZM01qSTFOakF3TENKdVltWWlPakUzTmpjeU1qVTJNREFzSW1WNGNDSTZNakE0TWpjMU9EUXdNSDAuX0QtUkl5Rzh5UVBITGpXb2RCMXFxcEhXYzFQempSNkw5QkV5MUpQNk4yR245NWlBTTRkdl85VmVLR2gyS3EtMFhqSXpaUnVuNmd3bGdScllud19RalEifQ.eyJtYW5pZmVzdF9pZCI6ICJpby5leGFtcGxlLmhlbGxvIiwgInZlcnNpb24iOiAiMS4wLjAiLCAic2RrX3ZlcnNpb24iOiAiMS4wLjAiLCAic2hhMjU2IjogIjQ0NTQ3MmZjYmQxMTVlMzNkNjMyNDgzOGQ0MmI5ZjU2N2I1MTY3YzQzNDkyNTBmODVhNjU1NWIyNDI0YTk0MGMiLCAiaWF0IjogMTc2NzIyNTYwMH0.EC8rPsZnPXi1NIrpbWb-cZd6s8JrSMaeR6ZjGH3YymA \ No newline at end of file diff --git a/tests/vectors/alg-none.zip.sig b/tests/vectors/alg-none.zip.sig new file mode 100644 index 0000000..e455638 --- /dev/null +++ b/tests/vectors/alg-none.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiAibm9uZSIsICJraWQiOiAiMTNlOTA3YTQ1MzBiNTM2MCIsICJ0eXAiOiAiZHVwbG8tZXh0LXNpZytqd3QiLCAiY2VydCI6ICJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SW1FM1pEVXdNR05sWVRFeU9UZ3hNVEVpTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5fRC1SSXlHOHlRUEhMaldvZEIxcXFwSFdjMVB6alI2TDlCRXkxSlA2TjJHbjk1aUFNNGR2XzlWZUtHaDJLcS0wWGpJelpSdW42Z3dsZ1JyWW53X1FqUSJ9.eyJtYW5pZmVzdF9pZCI6ICJpby5leGFtcGxlLmhlbGxvIiwgInZlcnNpb24iOiAiMS4wLjAiLCAic2RrX3ZlcnNpb24iOiAiMS4wLjAiLCAic2hhMjU2IjogIjQ0NTQ3MmZjYmQxMTVlMzNkNjMyNDgzOGQ0MmI5ZjU2N2I1MTY3YzQzNDkyNTBmODVhNjU1NWIyNDI0YTk0MGMiLCAiaWF0IjogMTc2NzIyNTYwMH0. \ No newline at end of file diff --git a/tests/vectors/bad-signature.zip.sig b/tests/vectors/bad-signature.zip.sig new file mode 100644 index 0000000..6a33f02 --- /dev/null +++ b/tests/vectors/bad-signature.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImNlcnQiOiJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SW1FM1pEVXdNR05sWVRFeU9UZ3hNVEVpTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5fRC1SSXlHOHlRUEhMaldvZEIxcXFwSFdjMVB6alI2TDlCRXkxSlA2TjJHbjk1aUFNNGR2XzlWZUtHaDJLcS0wWGpJelpSdW42Z3dsZ1JyWW53X1FqUSIsImtpZCI6IjEzZTkwN2E0NTMwYjUzNjAiLCJ0eXAiOiJkdXBsby1leHQtc2lnK2p3dCJ9.eyJtYW5pZmVzdF9pZCI6ImlvLmV4YW1wbGUuaGVsbG8iLCJ2ZXJzaW9uIjoiMS4wLjAiLCJzZGtfdmVyc2lvbiI6IjEuMC4wIiwic2hhMjU2IjoiNDQ1NDcyZmNiZDExNWUzM2Q2MzI0ODM4ZDQyYjlmNTY3YjUxNjdjNDM0OTI1MGY4NWE2NTU1YjI0MjRhOTQwYyIsImlhdCI6MTc2NzIyNTYwMH0.xNhBD0Oco9ghRlHP4QeKHOwdLpT7DEM8K9wckkUufiskjeXZjxxVw7Ne3xcadnTkFL1-TucngEhNmNawI5dYmA \ No newline at end of file diff --git a/tests/vectors/cases.json b/tests/vectors/cases.json new file mode 100644 index 0000000..2bb87e1 --- /dev/null +++ b/tests/vectors/cases.json @@ -0,0 +1,200 @@ +{ + "root_keys": { + "a7d500cea1298111": "root.pub.pem" + }, + "cases": [ + { + "name": "valid", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "ok" + }, + { + "name": "valid-trust-check-skipped", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": null, + "now": 1798761600, + "expect": "ok" + }, + { + "name": "tampered-hash", + "sig": "extension.zip.sig", + "sha256": "8370c756d9c473c4d0c0fc89841fd8bfe37cb50fdc107e09d87583819d987946", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "hash_mismatch" + }, + { + "name": "wrong-version", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.1", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "manifest_mismatch" + }, + { + "name": "wrong-namespace", + "sig": "outside-namespace.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "com.other.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "outside_namespace" + }, + { + "name": "expired", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 2082758400, + "expect": "cert_expired" + }, + { + "name": "not-yet-valid", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1767225599, + "expect": "cert_expired" + }, + { + "name": "untrusted-publisher", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000002" + ], + "now": 1798761600, + "expect": "untrusted_publisher" + }, + { + "name": "bad-signature", + "sig": "bad-signature.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "bad_signature" + }, + { + "name": "kid-mismatch", + "sig": "kid-mismatch.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "kid_mismatch" + }, + { + "name": "unknown-root", + "sig": "unknown-root.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "unknown_root" + }, + { + "name": "cert-as-signature", + "sig": "cert-as-signature.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "bad_typ" + }, + { + "name": "alg-none", + "sig": "alg-none.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "malformed" + }, + { + "name": "alg-hs256", + "sig": "alg-hs256.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "malformed" + }, + { + "name": "missing-cert-header", + "sig": "missing-cert-header.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "malformed" + } + ] +} diff --git a/tests/vectors/cert-as-signature.zip.sig b/tests/vectors/cert-as-signature.zip.sig new file mode 100644 index 0000000..9b4bc27 --- /dev/null +++ b/tests/vectors/cert-as-signature.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImtpZCI6ImE3ZDUwMGNlYTEyOTgxMTEiLCJ0eXAiOiJkdXBsby1leHQtY2VydCtqd3QifQ.eyJpc3MiOiJkdXBsb2Nsb3VkLWNvbnNvbGUiLCJzdWIiOiIwMDAwMDAwMC0wMDAwLTQwMDAtODAwMC0wMDAwMDAwMDAwMDEiLCJuYW1lIjoiRXhhbXBsZSBQdWJsaXNoZXIiLCJraWQiOiIxM2U5MDdhNDUzMGI1MzYwIiwiandrIjp7Imt0eSI6IkVDIiwiY3J2IjoiUC0yNTYiLCJ4IjoiOVBwcWlJQUVNR0NWRml6MUZQX3JoOS1HTFZ5eUdnZXBIV0wySTMyR0ZRUSIsInkiOiJ2emlSLXRmaVlyTUdnandjdTNqTTh4b1NiYjItTlpwbTFKMWdiU1NnLWZBIn0sIm5zIjpbImlvLmV4YW1wbGUiXSwiaWF0IjoxNzY3MjI1NjAwLCJuYmYiOjE3NjcyMjU2MDAsImV4cCI6MjA4Mjc1ODQwMH0._D-RIyG8yQPHLjWodB1qqpHWc1PzjR6L9BEy1JP6N2Gn95iAM4dv_9VeKGh2Kq-0XjIzZRun6gwlgRrYnw_QjQ \ No newline at end of file diff --git a/tests/vectors/extension.zip b/tests/vectors/extension.zip new file mode 100644 index 0000000000000000000000000000000000000000..fd56c9b380119cda54966917d5986bd47b9fac3f GIT binary patch literal 167 zcmWIWW@Zs#fB;2?iT-!ww1FHD<^|&1#JtS3)Z!Aotm6E Date: Tue, 6 Oct 2026 13:34:34 +0100 Subject: [PATCH 03/22] Fix extension-sign verify: parse the console's real root-keys shape and flag local-root verifications --- scripts/extension-sign.py | 13 ++++-- tests/extension_sign_console_shape.py | 59 +++++++++++++++++++++++++++ tests/test-extension-sign.sh | 5 ++- 3 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 tests/extension_sign_console_shape.py diff --git a/scripts/extension-sign.py b/scripts/extension-sign.py index 9883d8a..aa263b7 100755 --- a/scripts/extension-sign.py +++ b/scripts/extension-sign.py @@ -8,6 +8,10 @@ The key and certificate come from EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT, never the command line, so ps on a shared runner never shows them. EXTENSION_SIGN_NOW overrides the clock for tests. + +verify's --root-key substitutes one local root public key for the console's published roots. It exists for tests +only, and its success line says so, so the output is never mistaken for a verification against the console's real +roots. """ import argparse, base64, hashlib, json, os, re, sys, time, urllib.request, zipfile @@ -190,7 +194,8 @@ def verify_signature(sig, *, root_keys, sha256, manifest_id, version, sdk_versio def console_root_keys(console_url): with urllib.request.urlopen(f"{console_url.rstrip('/')}/api/extensions/root-keys/", timeout=30) as r: body = json.load(r) - rows = body.get("results", body) if isinstance(body, dict) else body + rows = body["keys"] if isinstance(body, dict) else body + # A retired root stays here with active=false: certificates it issued are still live until they expire. return {row["kid"]: row["public_key"] for row in rows if row.get("public_key")} @@ -203,7 +208,7 @@ def main(argv=None): v = sub.add_parser("verify") v.add_argument("zip") v.add_argument("sig") - v.add_argument("--root-key", action="append", default=[], help="a local root public key PEM, for tests") + v.add_argument("--root-key", help="a local root public key PEM, for tests only") v.add_argument("--console", default="https://console.duplocloud.com") a = p.parse_args(argv) if a.cmd == "sign": @@ -220,7 +225,7 @@ def main(argv=None): if a.root_key: # A local root key is addressed by the kid the signature's certificate names, so a test root verifies. root_kid = jwt.get_unverified_header(jwt.get_unverified_header(sig)["cert"]).get("kid") - roots = {root_kid: open(a.root_key[0]).read()} + roots = {root_kid: open(a.root_key).read()} else: roots = console_root_keys(a.console) m = built_manifest(a.zip) @@ -231,7 +236,7 @@ def main(argv=None): except SignatureError as exc: print(exc.code) return 1 - print("ok") + print("ok (against the local root key, not the console's roots)" if a.root_key else "ok") return 0 diff --git a/tests/extension_sign_console_shape.py b/tests/extension_sign_console_shape.py new file mode 100644 index 0000000..bec09e5 --- /dev/null +++ b/tests/extension_sign_console_shape.py @@ -0,0 +1,59 @@ +"""Checks console_root_keys against the console's real root-keys response shape, with no network call. + +apps/extensions/api_root_keys.py returns {"keys": [{"kid", "public_key", "active"}]}. This pins that shape (a bare +list is accepted too) and confirms the valid vector's signature verifies against the roots it parses out, including +a retired (active=false) row that must still be kept. +""" +import importlib.util, io, json, pathlib, sys, urllib.request + +HERE = pathlib.Path(__file__).parent +spec = importlib.util.spec_from_file_location("extension_sign", HERE.parent / "scripts" / "extension-sign.py") +es = importlib.util.module_from_spec(spec) +spec.loader.exec_module(es) + +vectors = HERE / "vectors" +data = json.loads((vectors / "cases.json").read_text()) +root_kid = next(iter(data["root_keys"])) +root_pub = (vectors / data["root_keys"][root_kid]).read_text() +body = json.dumps({"keys": [ + {"kid": "retired-root", "public_key": "not a real key", "active": False}, + {"kid": root_kid, "public_key": root_pub, "active": True}, +]}).encode() + + +class FakeResponse: + def __init__(self, payload): + self._buf = io.BytesIO(payload) + + def __enter__(self): + return self._buf + + def __exit__(self, *exc_info): + return False + + +urllib.request.urlopen = lambda url, timeout=30: FakeResponse(body) + +roots = es.console_root_keys("https://console.example.com") +bad = 0 +if roots.get("retired-root") != "not a real key": + print("MISMATCH: a retired (active=false) root key was dropped") + bad += 1 +if roots.get(root_kid) != root_pub: + print("MISMATCH: the active root key was not parsed from the console's {\"keys\": [...]} shape") + bad += 1 + +case = next(c for c in data["cases"] if c["name"] == "valid") +try: + es.verify_signature((vectors / case["sig"]).read_text(), root_keys=roots, sha256=case["sha256"], + manifest_id=case["manifest_id"], version=case["version"], sdk_version=case["sdk_version"], + trusted_publishers=case["trusted_publishers"], now=case["now"]) + got = "ok" +except es.SignatureError as exc: + got = exc.code +if got != "ok": + print(f"MISMATCH: the valid vector did not verify against the parsed console root keys (got {got})") + bad += 1 + +print("console root-keys shape: ok" if not bad else "console root-keys shape: FAIL") +sys.exit(1 if bad else 0) diff --git a/tests/test-extension-sign.sh b/tests/test-extension-sign.sh index 8b77853..154b3b5 100755 --- a/tests/test-extension-sign.sh +++ b/tests/test-extension-sign.sh @@ -22,10 +22,13 @@ echo "extension-sign:" t "verify reproduces every vector case" OUT=$(python3 tests/extension_sign_vectors.py 2>&1); [ $? = 0 ] && ok || bad "$OUT" +t "verify parses the console's real root-keys response shape (no network)" +OUT=$(python3 tests/extension_sign_console_shape.py 2>&1); [ $? = 0 ] && ok || bad "$OUT" + t "a fresh signature verifies against the test root" mkzip io.example.hello 1.0.0 1.0.0 "$TMP/a.zip"; OUT=$(sign $V/publisher.pem "$TMP/a.zip") OUT2=$(EXTENSION_SIGN_NOW="$NOW" python3 $S verify "$TMP/a.zip" "$TMP/a.zip.sig" --root-key $V/root.pub.pem 2>&1) -[ $? = 0 ] && ok || bad "$OUT $OUT2" +[ $? = 0 ] && grep -q "not the console's roots" <<<"$OUT2" && ok || bad "$OUT $OUT2" t "the .sig has no trailing newline" [ "$(tail -c1 "$TMP/a.zip.sig" | od -An -c | tr -d ' ')" != '\n' ] && ok || bad "trailing newline" From 87a597efd161c2420ddf56b1b56adf487fe298ac Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 13:38:54 +0100 Subject: [PATCH 04/22] Add the extension-sign composite action --- .github/actions/extension-sign/action.yml | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/actions/extension-sign/action.yml diff --git a/.github/actions/extension-sign/action.yml b/.github/actions/extension-sign/action.yml new file mode 100644 index 0000000..c8a80ec --- /dev/null +++ b/.github/actions/extension-sign/action.yml @@ -0,0 +1,35 @@ +# Signs one built extension.zip into extension.zip.sig with a publisher key the DuploCloud console certifies. +# For pipelines outside devkit's release job, which calls scripts/extension-sign.py directly. +name: extension-sign +description: Sign an extension bundle for DuploCloud portals that require signed extensions +inputs: + bundle: + description: Path to the built extension.zip + required: true + signing-key: + description: Publisher private key PEM, from a secret + required: true + signing-cert: + description: Publisher certificate (compact JWS) from the console + required: true +outputs: + signature-path: + description: Path of the written .sig + value: ${{ steps.sign.outputs.path }} +runs: + using: composite + steps: + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + - shell: bash + run: pip install --require-hashes -r "$GITHUB_ACTION_PATH/../../../scripts/extension-sign.requirements.txt" + - id: sign + shell: bash + env: + EXTENSION_SIGNING_KEY: ${{ inputs.signing-key }} + EXTENSION_SIGNING_CERT: ${{ inputs.signing-cert }} + BUNDLE: ${{ inputs.bundle }} + run: | + python3 "$GITHUB_ACTION_PATH/../../../scripts/extension-sign.py" sign "$BUNDLE" + echo "path=$BUNDLE.sig" >> "$GITHUB_OUTPUT" From a364f6bb5670b8991725166a3c433e24bd8bf327 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 13:44:48 +0100 Subject: [PATCH 05/22] Add the extension publisher allowlist --- .github/extension-publishers.json | 4 +++ scripts/_publishers.sh | 17 +++++++++++ tests/fixtures/extension-publishers.json | 9 ++++++ tests/test-extension-publishers.sh | 39 ++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 .github/extension-publishers.json create mode 100644 scripts/_publishers.sh create mode 100644 tests/fixtures/extension-publishers.json create mode 100755 tests/test-extension-publishers.sh diff --git a/.github/extension-publishers.json b/.github/extension-publishers.json new file mode 100644 index 0000000..6e961ab --- /dev/null +++ b/.github/extension-publishers.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": 1, + "publishers": [] +} diff --git a/scripts/_publishers.sh b/scripts/_publishers.sh new file mode 100644 index 0000000..3205e51 --- /dev/null +++ b/scripts/_publishers.sh @@ -0,0 +1,17 @@ +# shellcheck shell=bash +# Sourced by scripts/release-extensions.sh and scripts/check-extension-pr.sh. The allowlist names which repository may +# publish each Duplo extension and which console extension record it registers under. ai-release's extension-publisher +# role trusts the same repositories, which is the boundary that matters. This file decides only whether a run tries. +# +# .github/extension-publishers.json ships empty: an entry is added once its extension's console record exists, and +# its repository must already be in ai-release's extension-publisher role trust list. + +publishers_file=".github/extension-publishers.json" + +# publisher_extension_uuid : the console extension UUID when the allowlist maps the id to +# that repository, else nothing and exit 1. +publisher_extension_uuid() { + [ -f "$publishers_file" ] || return 1 + jq -er --arg id "$1" --arg repo "$2" \ + '.publishers[] | select(.manifestId == $id and .repository == $repo) | .consoleExtension' "$publishers_file" +} diff --git a/tests/fixtures/extension-publishers.json b/tests/fixtures/extension-publishers.json new file mode 100644 index 0000000..789d385 --- /dev/null +++ b/tests/fixtures/extension-publishers.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": 1, + "publishers": [ + { "manifestId": "duplo.extensions.alpha", "repository": "duplocloud/alpha", + "consoleExtension": "00000000-0000-4000-8000-000000000001" }, + { "manifestId": "duplo.extensions.beta", "repository": "duplocloud/beta", + "consoleExtension": "00000000-0000-4000-8000-000000000002" } + ] +} diff --git a/tests/test-extension-publishers.sh b/tests/test-extension-publishers.sh new file mode 100755 index 0000000..6b5fb48 --- /dev/null +++ b/tests/test-extension-publishers.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Tests for .github/extension-publishers.json and scripts/_publishers.sh. Usage: ./tests/test-extension-publishers.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } +. ./scripts/_publishers.sh +REAL=.github/extension-publishers.json +FIXTURE=tests/fixtures/extension-publishers.json +echo "extension-publishers:" + +# Structural checks run against both files: the real allowlist can never gain a malformed entry either. +for F in "$REAL" "$FIXTURE"; do + t "$F parses with schemaVersion 1" + [ "$(jq -r .schemaVersion "$F")" = 1 ] && ok || bad "schemaVersion" + t "$F: each manifest id appears once" + [ "$(jq '[.publishers[].manifestId] | length == (unique | length)' "$F")" = true ] && ok || bad "duplicate id" + t "$F: each entry names one repository and a console UUID" + jq -e 'all(.publishers[]; (.repository | test("^[^/]+/[^/]+$")) and (.consoleExtension | test("^[0-9a-f-]{36}$")))' \ + "$F" >/dev/null && ok || bad "shape" +done + +t "the real file is empty or every entry passes the shape check" +jq -e '(.publishers | length == 0) or + all(.publishers[]; (.repository | test("^[^/]+/[^/]+$")) and (.consoleExtension | test("^[0-9a-f-]{36}$")))' \ + "$REAL" >/dev/null && ok || bad "placeholder in the real file" + +publishers_file="$FIXTURE" +t "a mapped id and repository yield the UUID" +id=$(jq -r '.publishers[0].manifestId' "$FIXTURE"); repo=$(jq -r '.publishers[0].repository' "$FIXTURE") +[ "$(publisher_extension_uuid "$id" "$repo")" = "$(jq -r '.publishers[0].consoleExtension' "$FIXTURE")" ] && ok || bad "lookup" +t "an id mapped to another repository is refused" +! publisher_extension_uuid "$id" "someone/else" >/dev/null && ok || bad "accepted another repo" +t "an unknown id is refused" +! publisher_extension_uuid "duplo.nope" "$repo" >/dev/null && ok || bad "accepted unknown id" + +echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] From 2aaa8c91842ac6fc08d5cde9e2d5418e8d4601ba Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 13:53:36 +0100 Subject: [PATCH 06/22] Make publisher_extension_uuid exit exactly 1 on every no-match --- scripts/_publishers.sh | 11 ++++++++--- tests/test-extension-publishers.sh | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/scripts/_publishers.sh b/scripts/_publishers.sh index 3205e51..1381e7d 100644 --- a/scripts/_publishers.sh +++ b/scripts/_publishers.sh @@ -9,9 +9,14 @@ publishers_file=".github/extension-publishers.json" # publisher_extension_uuid : the console extension UUID when the allowlist maps the id to -# that repository, else nothing and exit 1. +# that repository, else nothing and exit 1. The lookup is captured rather than let `jq -e` report its own exit +# status, because that status is not 1 for every miss (no match, an empty list and a malformed file each land on a +# different jq exit code) and callers key off exit 1 specifically to mean "not allowed to publish." publisher_extension_uuid() { [ -f "$publishers_file" ] || return 1 - jq -er --arg id "$1" --arg repo "$2" \ - '.publishers[] | select(.manifestId == $id and .repository == $repo) | .consoleExtension' "$publishers_file" + local uuid + uuid="$(jq -r --arg id "$1" --arg repo "$2" \ + '.publishers[] | select(.manifestId == $id and .repository == $repo) | .consoleExtension' "$publishers_file")" || true + [ -n "$uuid" ] || return 1 + printf '%s\n' "$uuid" } diff --git a/tests/test-extension-publishers.sh b/tests/test-extension-publishers.sh index 6b5fb48..fe71d22 100755 --- a/tests/test-extension-publishers.sh +++ b/tests/test-extension-publishers.sh @@ -6,6 +6,7 @@ PASS=0; FAIL=0 t() { printf ' %s … ' "$1"; } ok() { echo "ok"; PASS=$((PASS+1)); } bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT . ./scripts/_publishers.sh REAL=.github/extension-publishers.json FIXTURE=tests/fixtures/extension-publishers.json @@ -36,4 +37,26 @@ t "an id mapped to another repository is refused" t "an unknown id is refused" ! publisher_extension_uuid "duplo.nope" "$repo" >/dev/null && ok || bad "accepted unknown id" +# Exit-status contract: callers key off exit 1 meaning "not allowed to publish." jq -e itself exits 4 when its +# filter produces no output at all, so the function must capture the lookup rather than pass that code through. +t "an unknown id exits exactly 1" +publisher_extension_uuid "duplo.nope" "$repo" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +t "a wrong repository exits exactly 1" +publisher_extension_uuid "$id" "someone/else" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +publishers_file="$REAL" +t "the empty real file exits exactly 1" +publisher_extension_uuid "duplo.anything" "duplocloud/anything" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +publishers_file="$TMP/missing.json" +t "a missing file exits exactly 1" +publisher_extension_uuid "duplo.anything" "duplocloud/anything" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +printf '{not valid json' > "$TMP/malformed.json" +publishers_file="$TMP/malformed.json" +t "a malformed allowlist file exits exactly 1, not a jq error code" +publisher_extension_uuid "duplo.anything" "duplocloud/anything" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" + echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] From dc841cf5663514c1229a969b596e4219d2f9c98e Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 14:02:09 +0100 Subject: [PATCH 07/22] Sign Duplo releases and attach the signature, with a resume path for an existing release --- scripts/_publish.sh | 10 +++++ scripts/release-extensions.sh | 59 +++++++++++++++++++++++++-- tests/test-release-extensions.sh | 69 ++++++++++++++++++++++++++++++-- 3 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 scripts/_publish.sh diff --git a/scripts/_publish.sh b/scripts/_publish.sh new file mode 100644 index 0000000..d8838db --- /dev/null +++ b/scripts/_publish.sh @@ -0,0 +1,10 @@ +# shellcheck shell=bash +# Sourced by scripts/release-extensions.sh. Signing is mandatory in Duplo's two organizations and skipped anywhere +# else, so a customer copy of the release workflow behaves as before with no Duplo credential. Upload and registration +# run only in Duplo's organizations, for a manifest id the allowlist maps to this repository. + +# duplo_org: exit 0 when this run belongs to one of Duplo's organizations. +duplo_org() { case "${GITHUB_REPOSITORY_OWNER:-}" in duplocloud|duplocloud-internal) return 0 ;; *) return 1 ;; esac; } + +# publish_build : upload and register one released build (Task 8). +publish_build() { echo "==> would publish $1"; } diff --git a/scripts/release-extensions.sh b/scripts/release-extensions.sh index c55f613..0f999d0 100755 --- a/scripts/release-extensions.sh +++ b/scripts/release-extensions.sh @@ -15,11 +15,31 @@ # Needs the repo's tags and the history they point at (actions/checkout with fetch-depth: 0), gh authenticated with # contents: write, jq and unzip. GITHUB_SHA is the built commit; it defaults to HEAD. # +# Signing: a run in one of Duplo's own organizations (duplo_org in scripts/_publish.sh) always signs the built zip +# with EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT before the release is created, and ships extension.zip.sig +# alongside it; a run anywhere else releases the zip alone, as a customer's copy of this workflow carries no Duplo +# signing credential. Within a Duplo organization, the manifest id also has to be in the extension publisher +# allowlist (scripts/_publishers.sh) for the repository, or the release still ships signed but scripts/_publish.sh's +# publish_build is never called for it. +# +# Resuming: a tag that already has a release is never rebuilt, re-signed or re-released. Its own assets, read back +# with gh release view, decide what happens next: a signed release calls publish_build again so a run that stopped +# between releasing and publishing still finishes, and a release that predates signing stays flagged rather than +# retrofitted, since a tag's assets can't gain one after the fact. A new manifest.version is the only way forward. +# # Usage: ./scripts/release-extensions.sh set -euo pipefail cd "$(dirname "$0")/.." shopt -s nullglob +# shellcheck source=scripts/_publishers.sh +. "$(dirname "$0")/_publishers.sh" +# shellcheck source=scripts/_publish.sh +. "$(dirname "$0")/_publish.sh" +publishers_file="${PUBLISHERS_FILE:-$publishers_file}" +sign_cmd="${EXTENSION_SIGN:-python3 $(dirname "$0")/extension-sign.py}" +repo="${GITHUB_REPOSITORY:-}" + sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" published=0; skipped=0; failed=0 @@ -52,15 +72,44 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest failed=$((failed+1)); continue fi + id="$(unzip -p "$zip" manifest.json | jq -r '.id')" + uuid="" + if duplo_org; then + if ! uuid="$(publisher_extension_uuid "$id" "$repo")"; then + uuid="" + echo "::notice::$dir — $id is not in the extension publisher allowlist for $repo, so it is released but not published to the license server." + fi + fi + if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then - echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." + if [ -n "$uuid" ]; then + assets="$(gh release view "$tag" --json assets --jq '[.assets[].name] | join(",")')" + if [[ ",$assets," != *",extension.zip.sig,"* ]]; then + echo "::warning::$tag was released with no extension.zip.sig, and an immutable release cannot gain one. Bump manifest.version to publish this extension." + skipped=$((skipped+1)); continue + fi + publish_build "$tag" "$id" "$version" "$sdk" "$dir" || { failed=$((failed+1)); continue; } + else + echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." + fi skipped=$((skipped+1)); continue fi + assets=("$zip") + if duplo_org; then + if [ -z "${EXTENSION_SIGNING_KEY:-}" ] || [ -z "${EXTENSION_SIGNING_CERT:-}" ]; then + echo "::error::$dir — EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT must be set in $GITHUB_REPOSITORY_OWNER. Duplo releases are always signed." + failed=$((failed+1)); continue + fi + # shellcheck disable=SC2086 # sign_cmd is a command plus its interpreter + if ! $sign_cmd sign "$zip"; then failed=$((failed+1)); continue; fi + assets+=("$zip.sig") + fi + echo "==> Publishing release $tag ($name $version, SDK $sdk)" - if ! gh release create "$tag" "$zip" --target "$sha" \ + if ! gh release create "$tag" "${assets[@]}" --target "$sha" \ --title "$name $version (SDK $sdk)" \ - --notes "Automated release of $name v$version from $slug, built against host SDK $sdk (extension.zip attached)."; then + --notes "Automated release of $name v$version from $slug, built against host SDK $sdk."; then echo "::error::$dir — gh release create $tag failed." failed=$((failed+1)); continue fi @@ -71,6 +120,10 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest if [ "$immutable" != true ]; then echo "::warning::$tag landed mutable (immutable=${immutable:-unknown}). A repository admin turns on immutable releases under Settings → General → Releases; releases published before that stay mutable." fi + + if [ -n "$uuid" ]; then + publish_build "$tag" "$id" "$version" "$sdk" "$dir" || failed=$((failed+1)) + fi done echo "==> Released $published, skipped $skipped, failed $failed." diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh index 26bbd0d..5528975 100755 --- a/tests/test-release-extensions.sh +++ b/tests/test-release-extensions.sh @@ -12,20 +12,40 @@ bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT -# Stub gh: every call is appended to $GH_LOG. `api` answers the release's immutable flag from $GH_IMMUTABLE. +# Stub gh: every call, including every asset name on a `release create`, is appended to $GH_LOG. `api` answers the +# release's immutable flag from $GH_IMMUTABLE, and `release view ... --json assets` answers the asset list a resumed +# publish reads back, from $GH_ASSETS (default both the zip and its signature). mkdir -p "$TMP/bin" cat > "$TMP/bin/gh" <<'EOF' #!/usr/bin/env bash printf '%s\n' "$*" >> "$GH_LOG" case "$1" in api) printf '%s\n' "${GH_IMMUTABLE:-true}" ;; - release) [ "$2" = create ] && exit "${GH_CREATE_RC:-0}" ;; + release) + case "$2" in + create) exit "${GH_CREATE_RC:-0}" ;; + view) printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; + esac + ;; esac exit 0 EOF chmod +x "$TMP/bin/gh" export PATH="$TMP/bin:$PATH" +# Stub signer the release script finds through EXTENSION_SIGN: writes ZIP.sig unless FAKE_SIGN_RC says to refuse. +cat > "$TMP/bin/fake-sign" <<'EOF' +#!/usr/bin/env bash +[ "${FAKE_SIGN_RC:-0}" = 0 ] || { echo "::error::refused"; exit 1; } +printf 'sig-of-%s' "$(basename "$2")" > "$2.sig" +EOF +chmod +x "$TMP/bin/fake-sign" +export EXTENSION_SIGN="$TMP/bin/fake-sign" + +# Test allowlist: duplo.demo may publish from duplocloud/demo. Never the real .github/extension-publishers.json, +# which ships empty on purpose. +jq -n '{schemaVersion:1, publishers:[{manifestId:"duplo.demo", repository:"duplocloud/demo", consoleExtension:"00000000-0000-4000-8000-000000000001"}]}' > "$TMP/pub.json" + # new_repo: a git repo holding extensions/demo at version 0.1.0, committed. new_repo() { REPO="$TMP/repo.$RANDOM"; mkdir -p "$REPO/extensions/demo" @@ -49,9 +69,11 @@ build_zip() { commit_change() { echo "$1" > "$REPO/extensions/demo/code.txt"; git -C "$REPO" commit -qam "$1"; } -# The script resolves the repo from its own path, as it does when devkit copies it into an extension repo. +# The script resolves the repo from its own path, as it does when devkit copies it into an extension repo, so its +# sourced helpers (_publishers.sh, _publish.sh) have to travel with it. run() { mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/release-extensions.sh" "$REPO/scripts/" 2>/dev/null + cp "$ROOT"/scripts/_*.sh "$REPO/scripts/" 2>/dev/null GITHUB_SHA="$(git -C "$REPO" rev-parse HEAD)" "$REPO/scripts/release-extensions.sh" 2>&1 } @@ -122,6 +144,47 @@ git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm oth OUT="$(run)"; RC=$? if [ "$RC" != 0 ] && grep -q "release create other-v0.2.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi +t "outside Duplo's orgs, releases the zip only and never signs" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY_OWNER=acme run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip --target" "$GH_LOG" \ + && ! grep -q 'extension.zip.sig' "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "in a Duplo org, attaches the signature beside the zip" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip extensions/demo/dist/extension.zip.sig --target" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "in a Duplo org with no signing key, fails before releasing" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY_OWNER=duplocloud run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "EXTENSION_SIGNING_KEY" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "a refused signature fails before releasing" +new_repo; build_zip 1.0.6 +OUT="$(FAKE_SIGN_RC=1 GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG")"; fi + +t "a Duplo-org repository outside the allowlist signs and releases, then skips publishing with a notice" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY=duplocloud/not-listed GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::notice::.*allowlist" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" +then ok; else bad "rc=$RC out=$OUT"; fi + +t "an existing signed release resumes publishing without re-signing or re-releasing" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +OUT="$(PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "release create" "$GH_LOG" && [ ! -f "$REPO/extensions/demo/dist/extension.zip.sig" ] \ + && grep -q "would publish demo-v0.1.0-sdk-1.0.6" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "a release cut before signing warns with the bump and does not fail" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +OUT="$(GH_ASSETS=extension.zip PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning::.*no extension.zip.sig.*Bump manifest.version" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" +then ok; else bad "rc=$RC out=$OUT"; fi + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ] From 0f06505f6f323e1b087fa05ad387e67ab1dccc67 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 14:11:19 +0100 Subject: [PATCH 08/22] Fail one extension instead of the whole run when its release-view lookup errors, and assert the unlisted-repo test actually signs --- scripts/release-extensions.sh | 5 ++++- tests/test-release-extensions.sh | 23 +++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scripts/release-extensions.sh b/scripts/release-extensions.sh index 0f999d0..27200d6 100755 --- a/scripts/release-extensions.sh +++ b/scripts/release-extensions.sh @@ -83,7 +83,10 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then if [ -n "$uuid" ]; then - assets="$(gh release view "$tag" --json assets --jq '[.assets[].name] | join(",")')" + if ! assets="$(gh release view "$tag" --json assets --jq '[.assets[].name] | join(",")')"; then + echo "::error::$dir — gh release view $tag failed." + failed=$((failed+1)); continue + fi if [[ ",$assets," != *",extension.zip.sig,"* ]]; then echo "::warning::$tag was released with no extension.zip.sig, and an immutable release cannot gain one. Bump manifest.version to publish this extension." skipped=$((skipped+1)); continue diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh index 5528975..fa9a6d7 100755 --- a/tests/test-release-extensions.sh +++ b/tests/test-release-extensions.sh @@ -14,7 +14,7 @@ TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT # Stub gh: every call, including every asset name on a `release create`, is appended to $GH_LOG. `api` answers the # release's immutable flag from $GH_IMMUTABLE, and `release view ... --json assets` answers the asset list a resumed -# publish reads back, from $GH_ASSETS (default both the zip and its signature). +# publish reads back, from $GH_ASSETS (default both the zip and its signature), or fails outright per $GH_VIEW_RC. mkdir -p "$TMP/bin" cat > "$TMP/bin/gh" <<'EOF' #!/usr/bin/env bash @@ -24,7 +24,8 @@ case "$1" in release) case "$2" in create) exit "${GH_CREATE_RC:-0}" ;; - view) printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; + view) [ "${GH_VIEW_RC:-0}" = 0 ] || exit "${GH_VIEW_RC}" + printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; esac ;; esac @@ -170,8 +171,22 @@ if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG"; then ok; else bad "rc t "a Duplo-org repository outside the allowlist signs and releases, then skips publishing with a notice" new_repo; build_zip 1.0.6 OUT="$(GITHUB_REPOSITORY=duplocloud/not-listed GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? -if [ "$RC" = 0 ] && grep -q "::notice::.*allowlist" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" -then ok; else bad "rc=$RC out=$OUT"; fi +if [ "$RC" = 0 ] && grep -q "::notice::.*allowlist" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" \ + && grep -q "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip extensions/demo/dist/extension.zip.sig --target" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "a failed gh release view fails that extension without stopping the run" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +mkdir -p "$REPO/extensions/other/dist" +printf '{"id":"duplo.other","name":"Other","version":"0.2.0"}\n' > "$REPO/extensions/other/manifest.json" +git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm other +( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" + cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) +OUT="$(GH_VIEW_RC=1 PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "::error::.*gh release view demo-v0.1.0-sdk-1.0.6 failed" <<<"$OUT" \ + && grep -q "release create other-v0.2.0-sdk-1.0.6" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi t "an existing signed release resumes publishing without re-signing or re-releasing" new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 From 75184df46ca6c68fe6beed71e51bdd90880e0c4a Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 14:27:45 +0100 Subject: [PATCH 09/22] Upload and register each signed build from its Release --- scripts/_publish.sh | 64 +++++++++++++++++++++++++++++++- tests/_publish_stubs.sh | 52 ++++++++++++++++++++++++++ tests/test-publish.sh | 59 +++++++++++++++++++++++++++++ tests/test-release-extensions.sh | 41 ++++++++++---------- 4 files changed, 192 insertions(+), 24 deletions(-) create mode 100644 tests/_publish_stubs.sh create mode 100755 tests/test-publish.sh diff --git a/scripts/_publish.sh b/scripts/_publish.sh index d8838db..0627547 100644 --- a/scripts/_publish.sh +++ b/scripts/_publish.sh @@ -6,5 +6,65 @@ # duplo_org: exit 0 when this run belongs to one of Duplo's organizations. duplo_org() { case "${GITHUB_REPOSITORY_OWNER:-}" in duplocloud|duplocloud-internal) return 0 ;; *) return 1 ;; esac; } -# publish_build : upload and register one released build (Task 8). -publish_build() { echo "==> would publish $1"; } +bundle_bucket="${BUNDLE_BUCKET:-duplo-helpdesk-channels}" +console_url="${CONSOLE_URL:-https://console.duplocloud.com}" + +# s3_put_once : create-only upload. A 412 compares bytes, identical counting as already uploaded. +s3_put_once() { + local file=$1 key=$2 out have + if out="$(aws s3api put-object --bucket "$bundle_bucket" --key "$key" --body "$file" --if-none-match '*' 2>&1)"; then + echo "==> uploaded s3://$bundle_bucket/$key"; return 0 + fi + if ! grep -qE 'PreconditionFailed|\(412\)' <<<"$out"; then echo "::error::upload of $key failed: $out"; return 1; fi + have="$(mktemp)" + aws s3api get-object --bucket "$bundle_bucket" --key "$key" "$have" >/dev/null || { rm -f "$have"; return 1; } + if cmp -s "$file" "$have"; then rm -f "$have"; echo "==> $key already uploaded, identical bytes"; return 0; fi + rm -f "$have"; echo "::error::s3://$bundle_bucket/$key already holds different bytes. A published build is never replaced." + return 1 +} + +# console [json]: one console API call. The key reaches curl through a header file, never argv. +console() { + local hdr body + hdr="$(mktemp)"; chmod 600 "$hdr"; printf 'Authorization: Api-Key %s\n' "$CONSOLE_API_KEY" > "$hdr" + if [ -n "${3:-}" ]; then + body="$(curl -sS --fail-with-body -X "$1" -H @"$hdr" -H 'Content-Type: application/json' --data "$3" "$console_url$2")" + else + body="$(curl -sS --fail-with-body -X "$1" -H @"$hdr" "$console_url$2")" + fi + local rc=$?; rm -f "$hdr"; printf '%s' "$body"; return $rc +} + +# publish_build : upload the Release's zip and .sig, then register both. Reads the +# assets from the Release so a re-run reuses the bytes already signed and released, with no rebuild. +publish_build() { + local tag=$1 id=$2 version=$3 sdk=$4 dl sha key vuuid out art + for v in "$id" "$version" "$sdk"; do + [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || { echo "::error::$tag — $v cannot form a bucket key."; return 1; } + done + dl="$(mktemp -d)" + gh release download "$tag" -p extension.zip -p extension.zip.sig -D "$dl" || { echo "::error::$tag — cannot download its assets."; return 1; } + sha="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.assets[] | select(.name == "extension.zip") | .digest // ""' | sed 's/^sha256://')" + [ -n "$sha" ] || sha="$(shasum -a 256 "$dl/extension.zip" | cut -d' ' -f1)" + key="bundles/$id/$version/sdk-$sdk/extension.zip" + s3_put_once "$dl/extension.zip" "$key" && s3_put_once "$dl/extension.zip.sig" "$key.sig" || { rm -rf "$dl"; return 1; } + + vuuid="$(console GET "/api/extensions/$uuid/versions/?version=$version" | jq -r '.[0].uuid // empty')" + if [ -z "$vuuid" ]; then + out="$(console POST "/api/extensions/$uuid/versions/" "$(jq -n --arg v "$version" --arg n "$id $version" '{version: $v, name: $n}')")" \ + || grep -q "already exists" <<<"$out" || { echo "::error::$tag — console refused the version: $out"; rm -rf "$dl"; return 1; } + vuuid="$(console GET "/api/extensions/$uuid/versions/?version=$version" | jq -r '.[0].uuid // empty')" + fi + art="$(jq -n --arg s "$sdk" --arg p "s3://$bundle_bucket/$key" --arg h "$sha" --rawfile g "$dl/extension.zip.sig" \ + '{sdk_version: $s, s3_path: $p, sha256: $h, signature: $g}')" + out="$(console GET "/api/extensions/$uuid/versions/$vuuid/artifacts/?sdk_version=$sdk")" + if [ "$(jq 'length' <<<"$out")" -gt 0 ]; then + if jq -e --argjson a "$art" '.[0] | .s3_path == $a.s3_path and (.sha256 | ascii_downcase) == $a.sha256 and .signature == $a.signature' <<<"$out" >/dev/null; then + echo "==> $tag already registered"; rm -rf "$dl"; return 0 + fi + echo "::error::$tag — the console already holds a build for SDK $sdk that differs from this release."; rm -rf "$dl"; return 1 + fi + out="$(console POST "/api/extensions/$uuid/versions/$vuuid/artifacts/" "$art")" \ + || { echo "::error::$tag — console refused the artifact: $out"; rm -rf "$dl"; return 1; } + echo "==> registered $tag in the license server (unpublished)"; rm -rf "$dl" +} diff --git a/tests/_publish_stubs.sh b/tests/_publish_stubs.sh new file mode 100644 index 0000000..79a10a0 --- /dev/null +++ b/tests/_publish_stubs.sh @@ -0,0 +1,52 @@ +# shellcheck shell=bash +# Fakes for publish_build's dependencies (scripts/_publish.sh), shared by test-publish.sh and the resume tests in +# test-release-extensions.sh. Source this after creating "$TMP/bin" and putting it on PATH. It writes gh, aws and +# curl there and makes them executable. The caller exports ASSETS (a directory holding extension.zip and +# extension.zip.sig), S3 and CONSOLE (scratch directories standing in for the bucket and the console's records), +# and LOG (where every stubbed call is appended). +# +# gh also answers the release-lifecycle calls scripts/release-extensions.sh itself makes (create/view, and the +# immutable-flag api lookup), so one gh stub serves both test files. It tells that lookup apart from publish_build's +# digest lookup, which hits the same `gh api repos/{owner}/{repo}/releases/tags/` shape with a different --jq. + +cat > "$TMP/bin/gh" <<'EOF' +#!/usr/bin/env bash +echo "gh $*" >> "$LOG" +case "$1 $2" in + "release download") while [ $# -gt 0 ]; do [ "$1" = -D ] && cp "$ASSETS"/* "$2"/; shift; done ;; + "release create") exit "${GH_CREATE_RC:-0}" ;; + "release view") [ "${GH_VIEW_RC:-0}" = 0 ] || exit "${GH_VIEW_RC}" + printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; + "api repos/{owner}/{repo}/releases/tags/"*) + if [[ "$*" == *'assets[]'* ]]; then printf '%s\n' "${GH_DIGEST-sha256:$ZSHA}" + else printf '%s\n' "${GH_IMMUTABLE:-true}"; fi ;; +esac +EOF + +cat > "$TMP/bin/aws" <<'EOF' +#!/usr/bin/env bash +echo "aws $*" >> "$LOG" +key=""; body=""; out=""; while [ $# -gt 0 ]; do case "$1" in --key) key=$2 ;; --body) body=$2 ;; esac; out=$1; shift; done +obj="$S3/$(echo "$key" | tr / _)" +if grep -q put-object <<<"$(tail -1 "$LOG")"; then + [ -f "$obj" ] && { echo "An error occurred (PreconditionFailed) when calling the PutObject operation" >&2; exit 254; } + cp "$body" "$obj"; echo '{}' +else cp "$obj" "$out"; fi +EOF + +cat > "$TMP/bin/curl" <<'EOF' +#!/usr/bin/env bash +# Console stub: versions and artifacts live as files under $CONSOLE. Prints body then a final line with the code. +echo "curl $*" | sed -E 's/Api-Key [^ ]+/Api-Key ***/' >> "$LOG" +url=""; data=""; method=GET +while [ $# -gt 0 ]; do case "$1" in -X) method=$2; shift ;; --data|-d) data=$2; method=${method/GET/POST}; shift ;; http*) url=$1 ;; esac; shift; done +case "$method $url" in + "GET "*"/versions/?version="*) cat "$CONSOLE/versions.json" 2>/dev/null || echo '[]' ;; + "POST "*"/versions/") [ -n "${RACE:-}" ] && [ ! -f "$CONSOLE/raced" ] && { touch "$CONSOLE/raced"; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"version":["1.0.0 already exists for this extension."]}'; exit 22; } + echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"uuid":"v-1"}' ;; + "GET "*"/artifacts/?sdk_version="*) cat "$CONSOLE/artifacts.json" 2>/dev/null || echo '[]' ;; + "POST "*"/artifacts/") echo "[$data]" > "$CONSOLE/artifacts.json"; echo "$data" ;; +esac +EOF + +chmod +x "$TMP/bin/gh" "$TMP/bin/aws" "$TMP/bin/curl" diff --git a/tests/test-publish.sh b/tests/test-publish.sh new file mode 100755 index 0000000..98df5d9 --- /dev/null +++ b/tests/test-publish.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Tests for publish_build in scripts/_publish.sh. Usage: ./tests/test-publish.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +mkdir -p "$TMP/bin" "$TMP/s3" "$TMP/assets"; export PATH="$TMP/bin:$PATH" LOG="$TMP/log" +printf 'ZIPBYTES' > "$TMP/assets/extension.zip"; printf 'SIGBYTES' > "$TMP/assets/extension.zip.sig" +ZSHA=$(shasum -a 256 "$TMP/assets/extension.zip" | cut -d' ' -f1) + +# shellcheck source=tests/_publish_stubs.sh +. "$(dirname "$0")/_publish_stubs.sh" +export S3="$TMP/s3" ASSETS="$TMP/assets" ZSHA CONSOLE="$TMP/console" CONSOLE_API_KEY=secret-key uuid=ext-1 +. ./scripts/_publish.sh +fresh() { rm -rf "$S3"/* "$CONSOLE"; mkdir -p "$CONSOLE"; : > "$LOG"; } + +echo "publish:" + +t "uploads both objects create-only and registers version and artifact with the signature" +fresh; OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +if [ $RC = 0 ] && [ "$(grep -c "put-object.*--if-none-match \*" "$LOG")" = 2 ] \ + && jq -e --arg s "$ZSHA" '.[0] | .sha256 == $s and .signature == "SIGBYTES" and (.s3_path | endswith("bundles/duplo.demo/1.0.0/sdk-1.0.6/extension.zip"))' "$CONSOLE/artifacts.json" >/dev/null +then ok; else bad "rc=$RC out=$OUT"; fi + +t "never sets is_published and never logs the key" +grep -q is_published "$LOG" && bad "is_published sent" || { grep -q secret-key "$LOG" "$TMP"/console/* 2>/dev/null && bad "key logged" || ok; } + +t "a re-run of a finished build changes nothing" +: > "$LOG" # Keep S3/CONSOLE state from the prior run. Clear only the log, so this run's own curls are what we check. +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && grep -q "already uploaded" <<<"$OUT" && ! grep -q "POST" <<<"$(grep curl "$LOG")" && ok || bad "rc=$RC out=$OUT" + +t "a re-run after only the zip uploaded uploads the signature" +fresh; cp "$ASSETS/extension.zip" "$S3/bundles_duplo.demo_1.0.0_sdk-1.0.6_extension.zip" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && [ -f "$S3/bundles_duplo.demo_1.0.0_sdk-1.0.6_extension.zip.sig" ] && ok || bad "rc=$RC out=$OUT" + +t "different bytes already at a key fail the job" +fresh; printf 'OTHER' > "$S3/bundles_duplo.demo_1.0.0_sdk-1.0.6_extension.zip" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "different bytes" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" + +t "a version created by a concurrent run is looked up again" +fresh; OUT=$(RACE=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && [ -f "$CONSOLE/artifacts.json" ] && ok || bad "rc=$RC out=$OUT" + +t "an existing artifact with a different sha256 fails the job" +fresh; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '[{"sdk_version":"1.0.6","s3_path":"x","sha256":"0000","signature":"y"}]' > "$CONSOLE/artifacts.json" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "differs" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" + +t "a missing GitHub digest falls back to hashing the downloaded asset" +fresh; OUT=$(GH_DIGEST="" publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && jq -e --arg s "$ZSHA" '.[0].sha256 == $s' "$CONSOLE/artifacts.json" >/dev/null && ok || bad "rc=$RC out=$OUT" + +echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh index fa9a6d7..ef1e749 100755 --- a/tests/test-release-extensions.sh +++ b/tests/test-release-extensions.sh @@ -12,27 +12,16 @@ bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT -# Stub gh: every call, including every asset name on a `release create`, is appended to $GH_LOG. `api` answers the -# release's immutable flag from $GH_IMMUTABLE, and `release view ... --json assets` answers the asset list a resumed -# publish reads back, from $GH_ASSETS (default both the zip and its signature), or fails outright per $GH_VIEW_RC. +# Fakes for gh (release lifecycle, plus the asset download and digest lookup publish_build needs), aws and curl, +# shared with tests/test-publish.sh via tests/_publish_stubs.sh. `api` answers the release's immutable flag from +# $GH_IMMUTABLE, unless the --jq expression is publish_build's digest lookup. `release view ... --json assets` +# answers the asset list a resumed publish reads back, from $GH_ASSETS (default both the zip and its signature), or +# fails outright per $GH_VIEW_RC. Every call is appended to $LOG, aliased below to $GH_LOG so existing assertions +# against $GH_LOG keep working. mkdir -p "$TMP/bin" -cat > "$TMP/bin/gh" <<'EOF' -#!/usr/bin/env bash -printf '%s\n' "$*" >> "$GH_LOG" -case "$1" in - api) printf '%s\n' "${GH_IMMUTABLE:-true}" ;; - release) - case "$2" in - create) exit "${GH_CREATE_RC:-0}" ;; - view) [ "${GH_VIEW_RC:-0}" = 0 ] || exit "${GH_VIEW_RC}" - printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; - esac - ;; -esac -exit 0 -EOF -chmod +x "$TMP/bin/gh" export PATH="$TMP/bin:$PATH" +# shellcheck source=tests/_publish_stubs.sh +. "$ROOT/tests/_publish_stubs.sh" # Stub signer the release script finds through EXTENSION_SIGN: writes ZIP.sig unless FAKE_SIGN_RC says to refuse. cat > "$TMP/bin/fake-sign" <<'EOF' @@ -55,7 +44,7 @@ new_repo() { printf '{"id":"duplo.demo","name":"Demo","version":"%s"}\n' "${1:-0.1.0}" > "$REPO/extensions/demo/manifest.json" echo one > "$REPO/extensions/demo/code.txt" git -C "$REPO" add -A; git -C "$REPO" commit -qm init - export GH_LOG="$REPO.gh.log"; : > "$GH_LOG" + export GH_LOG="$REPO.gh.log" LOG="$REPO.gh.log"; : > "$GH_LOG" } # build_zip [sdkVersion]: the dist/extension.zip build-extension.sh would leave, with sdkVersion stamped. @@ -190,9 +179,17 @@ then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi t "an existing signed release resumes publishing without re-signing or re-releasing" new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 -OUT="$(PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +# The release's own assets (never the local dist/ bundle) are what a resumed publish reads, so plant them separately. +mkdir -p "$TMP/assets.19" "$TMP/s3.19" "$TMP/console.19" +printf 'ZIPBYTES' > "$TMP/assets.19/extension.zip"; printf 'SIGBYTES' > "$TMP/assets.19/extension.zip.sig" +ZSHA19="$(shasum -a 256 "$TMP/assets.19/extension.zip" | cut -d' ' -f1)" +OUT="$(ASSETS="$TMP/assets.19" S3="$TMP/s3.19" CONSOLE="$TMP/console.19" ZSHA="$ZSHA19" CONSOLE_API_KEY=secret-key \ + PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? if [ "$RC" = 0 ] && ! grep -q "release create" "$GH_LOG" && [ ! -f "$REPO/extensions/demo/dist/extension.zip.sig" ] \ - && grep -q "would publish demo-v0.1.0-sdk-1.0.6" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + && grep -q "registered demo-v0.1.0-sdk-1.0.6" <<<"$OUT" \ + && jq -e --arg s "$ZSHA19" '.[0] | .sha256 == $s and .signature == "SIGBYTES"' "$TMP/console.19/artifacts.json" >/dev/null +then ok; else bad "rc=$RC out=$OUT"; fi t "a release cut before signing warns with the bump and does not fail" new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 From c29c84beb2e09cf077c2624f03547afe1f6f7273 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 14:38:31 +0100 Subject: [PATCH 10/22] Verify the release digest, check console lookups, and close publish_build's cleanup and test gaps --- scripts/_publish.sh | 28 ++++++++++++++++++++++------ tests/_publish_stubs.sh | 26 +++++++++++++++++++++----- tests/test-publish.sh | 22 ++++++++++++++++++++-- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/scripts/_publish.sh b/scripts/_publish.sh index 0627547..0b89be5 100644 --- a/scripts/_publish.sh +++ b/scripts/_publish.sh @@ -38,23 +38,39 @@ console() { # publish_build : upload the Release's zip and .sig, then register both. Reads the # assets from the Release so a re-run reuses the bytes already signed and released, with no rebuild. publish_build() { - local tag=$1 id=$2 version=$3 sdk=$4 dl sha key vuuid out art + local tag=$1 id=$2 version=$3 sdk=$4 v dl localsha sha key vuuid out art for v in "$id" "$version" "$sdk"; do [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || { echo "::error::$tag — $v cannot form a bucket key."; return 1; } done dl="$(mktemp -d)" - gh release download "$tag" -p extension.zip -p extension.zip.sig -D "$dl" || { echo "::error::$tag — cannot download its assets."; return 1; } - sha="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.assets[] | select(.name == "extension.zip") | .digest // ""' | sed 's/^sha256://')" - [ -n "$sha" ] || sha="$(shasum -a 256 "$dl/extension.zip" | cut -d' ' -f1)" + gh release download "$tag" -p extension.zip -p extension.zip.sig -D "$dl" \ + || { echo "::error::$tag — cannot download its assets."; rm -rf "$dl"; return 1; } + # The local hash is always computed and is what gets registered. GitHub's own digest, when the release carries + # one, only confirms the download matches what was actually released, before anything is uploaded or registered. + localsha="$(shasum -a 256 "$dl/extension.zip" | cut -d' ' -f1)" + sha="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.assets[] | select(.name == "extension.zip") | .digest // empty')" \ + || { echo "::error::$tag — cannot read its release digest."; rm -rf "$dl"; return 1; } + sha="${sha#sha256:}" + if [ -n "$sha" ]; then + [ "$sha" = "$localsha" ] \ + || { echo "::error::$tag — release digest $sha does not match downloaded extension.zip hash $localsha."; rm -rf "$dl"; return 1; } + else + sha="$localsha" + fi key="bundles/$id/$version/sdk-$sdk/extension.zip" s3_put_once "$dl/extension.zip" "$key" && s3_put_once "$dl/extension.zip.sig" "$key.sig" || { rm -rf "$dl"; return 1; } - vuuid="$(console GET "/api/extensions/$uuid/versions/?version=$version" | jq -r '.[0].uuid // empty')" + out="$(console GET "/api/extensions/$uuid/versions/?version=$version")" \ + || { echo "::error::$tag — console lookup of the version failed: $out"; rm -rf "$dl"; return 1; } + vuuid="$(jq -r '.[0].uuid // empty' <<<"$out")" if [ -z "$vuuid" ]; then out="$(console POST "/api/extensions/$uuid/versions/" "$(jq -n --arg v "$version" --arg n "$id $version" '{version: $v, name: $n}')")" \ || grep -q "already exists" <<<"$out" || { echo "::error::$tag — console refused the version: $out"; rm -rf "$dl"; return 1; } - vuuid="$(console GET "/api/extensions/$uuid/versions/?version=$version" | jq -r '.[0].uuid // empty')" + out="$(console GET "/api/extensions/$uuid/versions/?version=$version")" \ + || { echo "::error::$tag — console lookup of the version failed: $out"; rm -rf "$dl"; return 1; } + vuuid="$(jq -r '.[0].uuid // empty' <<<"$out")" fi + [ -n "$vuuid" ] || { echo "::error::$tag — the console has no version uuid for $version after creating it."; rm -rf "$dl"; return 1; } art="$(jq -n --arg s "$sdk" --arg p "s3://$bundle_bucket/$key" --arg h "$sha" --rawfile g "$dl/extension.zip.sig" \ '{sdk_version: $s, s3_path: $p, sha256: $h, signature: $g}')" out="$(console GET "/api/extensions/$uuid/versions/$vuuid/artifacts/?sdk_version=$sdk")" diff --git a/tests/_publish_stubs.sh b/tests/_publish_stubs.sh index 79a10a0..95c4435 100644 --- a/tests/_publish_stubs.sh +++ b/tests/_publish_stubs.sh @@ -18,6 +18,7 @@ case "$1 $2" in "release view") [ "${GH_VIEW_RC:-0}" = 0 ] || exit "${GH_VIEW_RC}" printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; "api repos/{owner}/{repo}/releases/tags/"*) + [ "${GH_API_RC:-0}" = 0 ] || exit "${GH_API_RC}" if [[ "$*" == *'assets[]'* ]]; then printf '%s\n' "${GH_DIGEST-sha256:$ZSHA}" else printf '%s\n' "${GH_IMMUTABLE:-true}"; fi ;; esac @@ -37,13 +38,28 @@ EOF cat > "$TMP/bin/curl" <<'EOF' #!/usr/bin/env bash # Console stub: versions and artifacts live as files under $CONSOLE. Prints body then a final line with the code. -echo "curl $*" | sed -E 's/Api-Key [^ ]+/Api-Key ***/' >> "$LOG" -url=""; data=""; method=GET -while [ $# -gt 0 ]; do case "$1" in -X) method=$2; shift ;; --data|-d) data=$2; method=${method/GET/POST}; shift ;; http*) url=$1 ;; esac; shift; done +# Logs raw argv, unmasked, so a key leaked onto the command line would show up here the way a real process listing +# or Action log would show it. The key only ever travels in an -H @file argument, so this also records whether this +# call's header file held it (HEADER_FILE_KEY=yes/no), letting a test prove that without the key itself in the log. +echo "curl $*" >> "$LOG" +url=""; data=""; method=GET; hdrfile="" +while [ $# -gt 0 ]; do + case "$1" in + -X) method=$2; shift ;; + --data|-d) data=$2; method=${method/GET/POST}; shift ;; + -H) case "$2" in @*) hdrfile=${2#@} ;; esac; shift ;; + http*) url=$1 ;; + esac + shift +done +if [ -n "$hdrfile" ] && grep -qF -- "$CONSOLE_API_KEY" "$hdrfile" 2>/dev/null +then echo "HEADER_FILE_KEY=yes" >> "$LOG"; else echo "HEADER_FILE_KEY=no" >> "$LOG"; fi case "$method $url" in "GET "*"/versions/?version="*) cat "$CONSOLE/versions.json" 2>/dev/null || echo '[]' ;; - "POST "*"/versions/") [ -n "${RACE:-}" ] && [ ! -f "$CONSOLE/raced" ] && { touch "$CONSOLE/raced"; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"version":["1.0.0 already exists for this extension."]}'; exit 22; } - echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"uuid":"v-1"}' ;; + "POST "*"/versions/") + [ -n "${RACE:-}" ] && [ ! -f "$CONSOLE/raced" ] && { touch "$CONSOLE/raced"; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"version":["1.0.0 already exists for this extension."]}'; exit 22; } + [ -n "${VERSION_VANISHES:-}" ] && { echo '{"uuid":"v-1"}'; exit 0; } + echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"uuid":"v-1"}' ;; "GET "*"/artifacts/?sdk_version="*) cat "$CONSOLE/artifacts.json" 2>/dev/null || echo '[]' ;; "POST "*"/artifacts/") echo "[$data]" > "$CONSOLE/artifacts.json"; echo "$data" ;; esac diff --git a/tests/test-publish.sh b/tests/test-publish.sh index 98df5d9..7dcaec5 100755 --- a/tests/test-publish.sh +++ b/tests/test-publish.sh @@ -26,7 +26,11 @@ if [ $RC = 0 ] && [ "$(grep -c "put-object.*--if-none-match \*" "$LOG")" = 2 ] \ then ok; else bad "rc=$RC out=$OUT"; fi t "never sets is_published and never logs the key" -grep -q is_published "$LOG" && bad "is_published sent" || { grep -q secret-key "$LOG" "$TMP"/console/* 2>/dev/null && bad "key logged" || ok; } +if grep -q is_published "$LOG"; then bad "is_published sent" +elif grep -q secret-key "$LOG" "$TMP"/console/* 2>/dev/null; then bad "key logged" +elif ! grep -q "HEADER_FILE_KEY=yes" "$LOG"; then bad "the header file never carried the key, so this test could not prove a leak" +else ok +fi t "a re-run of a finished build changes nothing" : > "$LOG" # Keep S3/CONSOLE state from the prior run. Clear only the log, so this run's own curls are what we check. @@ -48,7 +52,9 @@ fresh; OUT=$(RACE=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 e [ $RC = 0 ] && [ -f "$CONSOLE/artifacts.json" ] && ok || bad "rc=$RC out=$OUT" t "an existing artifact with a different sha256 fails the job" -fresh; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '[{"sdk_version":"1.0.6","s3_path":"x","sha256":"0000","signature":"y"}]' > "$CONSOLE/artifacts.json" +fresh; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json" +echo '[{"sdk_version":"1.0.6","s3_path":"s3://duplo-helpdesk-channels/bundles/duplo.demo/1.0.0/sdk-1.0.6/extension.zip","sha256":"0000","signature":"SIGBYTES"}]' \ + > "$CONSOLE/artifacts.json" OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? [ $RC != 0 ] && grep -q "differs" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" @@ -56,4 +62,16 @@ t "a missing GitHub digest falls back to hashing the downloaded asset" fresh; OUT=$(GH_DIGEST="" publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? [ $RC = 0 ] && jq -e --arg s "$ZSHA" '.[0].sha256 == $s' "$CONSOLE/artifacts.json" >/dev/null && ok || bad "rc=$RC out=$OUT" +t "a release digest that does not match the downloaded zip fails the job before any upload" +fresh; OUT=$(GH_DIGEST="sha256:deadbeef" publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "deadbeef" <<<"$OUT" && grep -q "$ZSHA" <<<"$OUT" && ! grep -q "put-object" "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "a failed release-digest lookup fails the job before any upload" +fresh; OUT=$(GH_API_RC=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::" <<<"$OUT" && ! grep -q "put-object" "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "a version lookup that still comes back empty after the POST fails the job" +fresh; OUT=$(VERSION_VANISHES=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "no version uuid" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" + echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] From c4e3691a68d7aafb79c756ec4c6d3f8fc894ce99 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 14:46:57 +0100 Subject: [PATCH 11/22] Release workflow: OIDC role, signer install, one run per repository, pinned actions --- .github/workflows/release.yml | 53 +++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 068d562..fbb59c4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,20 @@ # NOTE: the hot-load/deploy step is intentionally COMMENTED OUT (see below). We publish releases but do not # auto-install onto the portal, so the install-from-registry flow stays testable end to end. Uncomment the # "Deploy" step to auto-hot-load on merge. +# +# In Duplo's own organizations (duplocloud, duplocloud-internal) the build is also signed. scripts/extension-sign.py +# writes extension.zip.sig from EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT, mapped below from the org secret +# CONSOLE_SIGNING_KEY and the org variable CONSOLE_SIGNING_CERT, before release-extensions.sh creates the release. +# A missing key fails the job there before anything is written. Outside those organizations the release ships the +# zip alone, since a customer's copy of this workflow holds no Duplo signing credential. +# +# Two guards sit in front of the AWS role. "Check the publisher allowlist" runs only in Duplo's organizations, and +# marks the repository listed only when .github/extension-publishers.json names it as a publisher. The +# configure-aws-credentials step then runs only when that step reports listed=true, since EXTENSION_PUBLISHER_ROLE_ARN +# trusts only those repositories through GitHub OIDC (id-token: write). An unlisted repository's release still ships +# signed, but release-extensions.sh never calls publish_build for it, so the job never reaches S3 or the console and +# never needs AWS credentials. + name: Extension Release on: @@ -21,7 +35,13 @@ on: workflow_dispatch: permissions: - contents: write # required to create GitHub Releases + contents: write # create GitHub Releases + id-token: write # assume the extension-publisher role through GitHub OIDC + +concurrency: + # One release run per repository at a time, never canceled between its writes. + group: extension-release-${{ github.repository }} + cancel-in-progress: false jobs: release: @@ -31,7 +51,7 @@ jobs: DUPLO_HOST: ${{ secrets.DUPLO_HOST }} DUPLO_TOKEN: ${{ secrets.DUPLO_TOKEN }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: # Every tag and the history it points at: the publish step compares each released version's commit with # this one to tell an unchanged extension from one that changed without a version bump. @@ -56,8 +76,37 @@ jobs: # if: steps.guard.outputs.build == 'true' # run: ./scripts/deploy-all.sh + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + if: steps.guard.outputs.build == 'true' + with: + python-version: '3.12' + - name: Install the signer + if: steps.guard.outputs.build == 'true' + run: pip install --require-hashes -r scripts/extension-sign.requirements.txt + + # Upload and registration need the role. Only Duplo's organizations hold it, and only allowlisted repositories + # pass its trust, so the step is skipped everywhere else and release-extensions.sh never reaches S3 there. + - name: Check the publisher allowlist + id: allow + if: >- + steps.guard.outputs.build == 'true' && + (github.repository_owner == 'duplocloud' || github.repository_owner == 'duplocloud-internal') + run: | + jq -e --arg r "$GITHUB_REPOSITORY" 'any(.publishers[]; .repository == $r)' .github/extension-publishers.json >/dev/null \ + && echo "listed=true" >> "$GITHUB_OUTPUT" || echo "listed=false" >> "$GITHUB_OUTPUT" + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + if: steps.allow.outputs.listed == 'true' + with: + role-to-assume: ${{ vars.EXTENSION_PUBLISHER_ROLE_ARN }} + aws-region: us-west-2 + - name: Publish GitHub releases (one per new build) if: steps.guard.outputs.build == 'true' env: GH_TOKEN: ${{ github.token }} + EXTENSION_SIGNING_KEY: ${{ secrets.CONSOLE_SIGNING_KEY }} + EXTENSION_SIGNING_CERT: ${{ vars.CONSOLE_SIGNING_CERT }} + CONSOLE_API_KEY: ${{ secrets.CONSOLE_API_KEY }} + CONSOLE_URL: ${{ vars.CONSOLE_URL }} + BUNDLE_BUCKET: ${{ vars.BUNDLE_BUCKET }} run: ./scripts/release-extensions.sh From 345d088533c9e3d63d42285f1ba8382d640c8903 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 14:59:34 +0100 Subject: [PATCH 12/22] PR checks warn about every refusal the release job would make --- scripts/check-extension-pr.sh | 21 +++++++++++++++++++++ tests/test-check-extension-pr.sh | 29 +++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/check-extension-pr.sh b/scripts/check-extension-pr.sh index 1715e0d..40ffaca 100755 --- a/scripts/check-extension-pr.sh +++ b/scripts/check-extension-pr.sh @@ -9,12 +9,23 @@ # - a directory under skills/ has no matching skills[].folder in the manifest, so the host never registers it. # - the built bundle's frontend is webpack (fe/remoteEntry.js, no fe/remoteEntry.json), whose pages do not open on # hosts that load Native Federation remotes. Checked only when dist/extension.zip exists, i.e. after a build. +# - the manifest id or version cannot form a bucket key, which scripts/_publish.sh's publish_build refuses. +# - the manifest declares no resources, which the host refuses to install. +# - the built bundle is over 268435456 bytes, which the release job refuses to publish. Checked only after a build. +# - in one of Duplo's organizations, the manifest id is not in the extension publisher allowlist +# (scripts/_publishers.sh) for this repository, so the release ships but the license server never gets it. # # Usage: ./scripts/check-extension-pr.sh # e.g. origin/main; defaults to origin/$GITHUB_BASE_REF set -euo pipefail cd "$(dirname "$0")/.." shopt -s nullglob +# shellcheck source=scripts/_publishers.sh +. "$(dirname "$0")/_publishers.sh" +# shellcheck source=scripts/_publish.sh +. "$(dirname "$0")/_publish.sh" +publishers_file="${PUBLISHERS_FILE:-$publishers_file}" + base_ref="${1:-origin/${GITHUB_BASE_REF:?usage: check-extension-pr.sh }}" base="$(git merge-base "$base_ref" HEAD)" warned=0 @@ -31,6 +42,15 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest warn "$m" "$dir changed but manifest.version is still $version. Bump it, or the release job refuses to publish this content." fi + for v in "$(jq -r '.id' "$m")" "$version"; do + [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || warn "$m" "$dir — '$v' cannot form a bucket key, so the release job refuses to publish it." + done + jq -e '(.resources // []) | length > 0' "$m" >/dev/null \ + || warn "$m" "$dir declares no resource, so the host refuses the bundle." + if duplo_org && ! publisher_extension_uuid "$(jq -r '.id' "$m")" "${GITHUB_REPOSITORY:-}" >/dev/null; then + warn "$m" "$dir — $(jq -r '.id' "$m") is not in the extension publisher allowlist for ${GITHUB_REPOSITORY:-this repository}, so a release is not published to the license server." + fi + for s in "$dir"/skills/*/; do s="$(basename "$s")" jq -e --arg s "$s" 'any(.skills[]?.folder // ""; endswith("/skills/" + $s))' "$m" >/dev/null \ @@ -39,6 +59,7 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest zip="$dir/dist/extension.zip" if [ -f "$zip" ]; then + [ "$(wc -c < "$zip")" -le 268435456 ] || warn "$m" "$dir builds a bundle over 268435456 bytes, which the release job refuses." entries="$(unzip -Z1 "$zip")" if grep -qx 'fe/remoteEntry.js' <<<"$entries" && ! grep -qx 'fe/remoteEntry.json' <<<"$entries"; then warn "$m" "$dir builds a webpack frontend (fe/remoteEntry.js). Its pages do not open on hosts that load Native Federation remotes. Migrate with the duplo-extension-ng22-migration skill." diff --git a/tests/test-check-extension-pr.sh b/tests/test-check-extension-pr.sh index 3af736a..847da05 100755 --- a/tests/test-check-extension-pr.sh +++ b/tests/test-check-extension-pr.sh @@ -23,9 +23,11 @@ new_repo() { git -C "$REPO" switch -q -c pr } -# set_manifest +# set_manifest : also declares a resources entry, so only the test that +# removes it sees the no-resource warning. set_manifest() { - jq -n --arg v "$1" --argjson f "$2" '{id:"duplo.demo",name:"Demo",version:$v,skills:[$f[]|{folder:.,isBuiltIn:true}]}' \ + jq -n --arg v "$1" --argjson f "$2" \ + '{id:"duplo.demo",name:"Demo",version:$v,skills:[$f[]|{folder:.,isBuiltIn:true}],resources:[{subType:"demo"}]}' \ > "$D/manifest.json" } @@ -37,8 +39,11 @@ bundle() { mkdir -p "$D/dist"; rm -f "$D/dist/extension.zip"; ( cd "$p" && zip -qr "$D/dist/extension.zip" . ); rm -rf "$p" } +# The script resolves the repo from its own path, as it does when devkit copies it into an extension repo, so its +# sourced helpers (_publishers.sh, _publish.sh) have to travel with it. run() { mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/check-extension-pr.sh" "$REPO/scripts/" 2>/dev/null + cp "$ROOT"/scripts/_*.sh "$REPO/scripts/" 2>/dev/null "$REPO/scripts/check-extension-pr.sh" main 2>&1 } @@ -82,6 +87,26 @@ new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; chang OUT="$(run)"; RC=$? if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi +t "warns about an id outside the publisher allowlist in a Duplo org" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two +jq -n '{schemaVersion:1, publishers:[]}' > "$REPO/pub.json" +OUT="$(PUBLISHERS_FILE="$REPO/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud run)" +grep -q "::warning.*allowlist" <<<"$OUT" && ok || bad "$OUT" + +t "warns about a manifest that declares no resource" +new_repo; jq '.version="0.1.1" | .skills=[{folder:"duplo.demo/0.1.1/skills/provision-demo",isBuiltIn:true}] | del(.resources)' \ + "$D/manifest.json" > "$D/m" && mv "$D/m" "$D/manifest.json"; change two +grep -q "::warning.*resource" <<<"$(run)" && ok || bad "no resource warning" + +t "warns about a bundle over 268435456 bytes" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.json +truncate -s 268435457 "$D/dist/extension.zip" +grep -q "::warning.*268435456" <<<"$(run)" && ok || bad "no size warning" + +t "warns about a version that cannot form a bucket key" +new_repo; set_manifest '0.1.1 beta' '["duplo.demo/0.1.1 beta/skills/provision-demo"]'; change two +grep -q "::warning.*cannot form" <<<"$(run)" && ok || bad "no key warning" + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ] From c5e5f07547acfb802d11d5d0d6fe4374b7b5d913 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 15:15:46 +0100 Subject: [PATCH 13/22] Release job refuses a resourceless or oversized bundle, and PR checks flag non-semver versions and capture their own exit status --- scripts/check-extension-pr.sh | 11 +++++++++-- scripts/release-extensions.sh | 14 ++++++++++++++ tests/test-check-extension-pr.sh | 18 +++++++++++++----- tests/test-release-extensions.sh | 23 +++++++++++++++++++---- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/scripts/check-extension-pr.sh b/scripts/check-extension-pr.sh index 40ffaca..a04a4fb 100755 --- a/scripts/check-extension-pr.sh +++ b/scripts/check-extension-pr.sh @@ -11,9 +11,12 @@ # hosts that load Native Federation remotes. Checked only when dist/extension.zip exists, i.e. after a build. # - the manifest id or version cannot form a bucket key, which scripts/_publish.sh's publish_build refuses. # - the manifest declares no resources, which the host refuses to install. -# - the built bundle is over 268435456 bytes, which the release job refuses to publish. Checked only after a build. +# - the built bundle is over EXTENSION_MAX_BYTES bytes (default 268435456), which the release job refuses to +# publish. Checked only after a build. # - in one of Duplo's organizations, the manifest id is not in the extension publisher allowlist # (scripts/_publishers.sh) for this repository, so the release ships but the license server never gets it. +# - in one of Duplo's organizations, manifest.version is not strict semver (MAJOR.MINOR.PATCH[-pre], no +build), +# the same rule scripts/extension-sign.py enforces before it signs a build. # # Usage: ./scripts/check-extension-pr.sh # e.g. origin/main; defaults to origin/$GITHUB_BASE_REF set -euo pipefail @@ -25,6 +28,7 @@ shopt -s nullglob # shellcheck source=scripts/_publish.sh . "$(dirname "$0")/_publish.sh" publishers_file="${PUBLISHERS_FILE:-$publishers_file}" +max_bytes="${EXTENSION_MAX_BYTES:-268435456}" base_ref="${1:-origin/${GITHUB_BASE_REF:?usage: check-extension-pr.sh }}" base="$(git merge-base "$base_ref" HEAD)" @@ -45,6 +49,9 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest for v in "$(jq -r '.id' "$m")" "$version"; do [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || warn "$m" "$dir — '$v' cannot form a bucket key, so the release job refuses to publish it." done + if duplo_org && ! [[ "$version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?$ ]]; then + warn "$m" "$dir — version '$version' is not strict semver (MAJOR.MINOR.PATCH[-pre], no +build), which scripts/extension-sign.py refuses to sign in Duplo's organizations." + fi jq -e '(.resources // []) | length > 0' "$m" >/dev/null \ || warn "$m" "$dir declares no resource, so the host refuses the bundle." if duplo_org && ! publisher_extension_uuid "$(jq -r '.id' "$m")" "${GITHUB_REPOSITORY:-}" >/dev/null; then @@ -59,7 +66,7 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest zip="$dir/dist/extension.zip" if [ -f "$zip" ]; then - [ "$(wc -c < "$zip")" -le 268435456 ] || warn "$m" "$dir builds a bundle over 268435456 bytes, which the release job refuses." + [ "$(wc -c < "$zip")" -le "$max_bytes" ] || warn "$m" "$dir builds a bundle over $max_bytes bytes, which the release job refuses." entries="$(unzip -Z1 "$zip")" if grep -qx 'fe/remoteEntry.js' <<<"$entries" && ! grep -qx 'fe/remoteEntry.json' <<<"$entries"; then warn "$m" "$dir builds a webpack frontend (fe/remoteEntry.js). Its pages do not open on hosts that load Native Federation remotes. Migrate with the duplo-extension-ng22-migration skill." diff --git a/scripts/release-extensions.sh b/scripts/release-extensions.sh index 27200d6..f39ce83 100755 --- a/scripts/release-extensions.sh +++ b/scripts/release-extensions.sh @@ -27,6 +27,11 @@ # between releasing and publishing still finishes, and a release that predates signing stays flagged rather than # retrofitted, since a tag's assets can't gain one after the fact. A new manifest.version is the only way forward. # +# Two checks run for every build, in every organization, right after confirming the build produced a zip. A missing +# resources entry refuses the build, and so does a bundle over EXTENSION_MAX_BYTES bytes (default 268435456). Either +# failure counts against just that extension and the run moves on, matching scripts/check-extension-pr.sh's PR-time +# warning for the same two rules. +# # Usage: ./scripts/release-extensions.sh set -euo pipefail cd "$(dirname "$0")/.." @@ -39,6 +44,7 @@ shopt -s nullglob publishers_file="${PUBLISHERS_FILE:-$publishers_file}" sign_cmd="${EXTENSION_SIGN:-python3 $(dirname "$0")/extension-sign.py}" repo="${GITHUB_REPOSITORY:-}" +max_bytes="${EXTENSION_MAX_BYTES:-268435456}" sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" published=0; skipped=0; failed=0 @@ -51,6 +57,14 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping." continue fi + if ! jq -e '(.resources // []) | length > 0' "$m" >/dev/null; then + echo "::error::$dir — the manifest declares no resource, so the host refuses the bundle." + failed=$((failed+1)); continue + fi + if [ "$(wc -c < "$zip")" -gt "$max_bytes" ]; then + echo "::error::$dir — the bundle is over $max_bytes bytes, which the release job refuses to publish." + failed=$((failed+1)); continue + fi name="$(jq -r '.name // .id' "$m")" version="$(jq -r '.version' "$m")" slug="$(basename "$dir")" diff --git a/tests/test-check-extension-pr.sh b/tests/test-check-extension-pr.sh index 847da05..fc8d605 100755 --- a/tests/test-check-extension-pr.sh +++ b/tests/test-check-extension-pr.sh @@ -90,22 +90,30 @@ if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC o t "warns about an id outside the publisher allowlist in a Duplo org" new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two jq -n '{schemaVersion:1, publishers:[]}' > "$REPO/pub.json" -OUT="$(PUBLISHERS_FILE="$REPO/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud run)" -grep -q "::warning.*allowlist" <<<"$OUT" && ok || bad "$OUT" +OUT="$(PUBLISHERS_FILE="$REPO/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*allowlist" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi t "warns about a manifest that declares no resource" new_repo; jq '.version="0.1.1" | .skills=[{folder:"duplo.demo/0.1.1/skills/provision-demo",isBuiltIn:true}] | del(.resources)' \ "$D/manifest.json" > "$D/m" && mv "$D/m" "$D/manifest.json"; change two -grep -q "::warning.*resource" <<<"$(run)" && ok || bad "no resource warning" +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*resource" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi t "warns about a bundle over 268435456 bytes" new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.json truncate -s 268435457 "$D/dist/extension.zip" -grep -q "::warning.*268435456" <<<"$(run)" && ok || bad "no size warning" +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*268435456" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi t "warns about a version that cannot form a bucket key" new_repo; set_manifest '0.1.1 beta' '["duplo.demo/0.1.1 beta/skills/provision-demo"]'; change two -grep -q "::warning.*cannot form" <<<"$(run)" && ok || bad "no key warning" +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*cannot form" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a non-semver version in a Duplo org" +new_repo; set_manifest 'v1.0.0' '["duplo.demo/v1.0.0/skills/provision-demo"]'; change two +OUT="$(GITHUB_REPOSITORY_OWNER=duplocloud run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*not strict semver" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi echo echo "passed $PASS, failed $FAIL" diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh index ef1e749..d1982e4 100755 --- a/tests/test-release-extensions.sh +++ b/tests/test-release-extensions.sh @@ -36,12 +36,14 @@ export EXTENSION_SIGN="$TMP/bin/fake-sign" # which ships empty on purpose. jq -n '{schemaVersion:1, publishers:[{manifestId:"duplo.demo", repository:"duplocloud/demo", consoleExtension:"00000000-0000-4000-8000-000000000001"}]}' > "$TMP/pub.json" -# new_repo: a git repo holding extensions/demo at version 0.1.0, committed. +# new_repo: a git repo holding extensions/demo at version 0.1.0, committed. Declares a resources entry, so only the +# tests that remove it see the no-resource refusal. new_repo() { REPO="$TMP/repo.$RANDOM"; mkdir -p "$REPO/extensions/demo" git -C "$REPO" init -q -b main git -C "$REPO" config user.email t@example.com; git -C "$REPO" config user.name t - printf '{"id":"duplo.demo","name":"Demo","version":"%s"}\n' "${1:-0.1.0}" > "$REPO/extensions/demo/manifest.json" + printf '{"id":"duplo.demo","name":"Demo","version":"%s","resources":[{"subType":"demo"}]}\n' "${1:-0.1.0}" \ + > "$REPO/extensions/demo/manifest.json" echo one > "$REPO/extensions/demo/code.txt" git -C "$REPO" add -A; git -C "$REPO" commit -qm init export GH_LOG="$REPO.gh.log" LOG="$REPO.gh.log"; : > "$GH_LOG" @@ -127,7 +129,7 @@ if [ "$RC" != 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6" "$GH_LOG"; t t "still publishes the other extensions when one fails" new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; commit_change two; build_zip 1.0.6 mkdir -p "$REPO/extensions/other/dist" -printf '{"id":"duplo.other","name":"Other","version":"0.2.0"}\n' > "$REPO/extensions/other/manifest.json" +printf '{"id":"duplo.other","name":"Other","version":"0.2.0","resources":[{"subType":"demo"}]}\n' > "$REPO/extensions/other/manifest.json" git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm other ( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) @@ -167,7 +169,7 @@ then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi t "a failed gh release view fails that extension without stopping the run" new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 mkdir -p "$REPO/extensions/other/dist" -printf '{"id":"duplo.other","name":"Other","version":"0.2.0"}\n' > "$REPO/extensions/other/manifest.json" +printf '{"id":"duplo.other","name":"Other","version":"0.2.0","resources":[{"subType":"demo"}]}\n' > "$REPO/extensions/other/manifest.json" git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm other ( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) @@ -197,6 +199,19 @@ OUT="$(GH_ASSETS=extension.zip PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY if [ "$RC" = 0 ] && grep -q "::warning::.*no extension.zip.sig.*Bump manifest.version" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" then ok; else bad "rc=$RC out=$OUT"; fi +t "fails, naming the extension, when the manifest declares no resource" +new_repo; jq 'del(.resources)' "$REPO/extensions/demo/manifest.json" > "$TMP/m.json" && mv "$TMP/m.json" "$REPO/extensions/demo/manifest.json" +build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "::error::.*extensions/demo.*resource" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails, naming the extension, when the bundle is over the size limit" +new_repo; build_zip 1.0.6 +OUT="$(EXTENSION_MAX_BYTES=1 run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "::error::.*extensions/demo.*over 1 bytes" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + echo echo "passed $PASS, failed $FAIL" [ "$FAIL" -eq 0 ] From 80404f30c352c3d62dfc3cb5510437369cbab9d8 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 15:27:57 +0100 Subject: [PATCH 14/22] Document the extension release action: signing, upload and registration --- docs/README.md | 1 + docs/releasing-extensions.md | 152 +++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 docs/releasing-extensions.md diff --git a/docs/README.md b/docs/README.md index 413e47a..286f55d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ reader takes. | [architecture.md](architecture.md) | How the studio, agent, UI, xterm, and Mongo fit together, and how the hot-load loop works | | [configuration.md](configuration.md) | Every `.env` variable — default, effect, and which are managed for you | | [cli-reference.md](cli-reference.md) | `run.sh`, `stop.sh`, `logs.sh`, and every script in `scripts/` — flags and behavior | +| [releasing-extensions.md](releasing-extensions.md) | What `release.yml` publishes, signing, and the extra steps inside DuploCloud's own organizations | | [upgrading.md](upgrading.md) | Taking framework updates, and how image-tag pinning works | | [troubleshooting.md](troubleshooting.md) | Symptoms, causes, fixes | | [faq.md](faq.md) | Licensing, data, platform support, and what happens on upgrade | diff --git a/docs/releasing-extensions.md b/docs/releasing-extensions.md new file mode 100644 index 0000000..7d337d6 --- /dev/null +++ b/docs/releasing-extensions.md @@ -0,0 +1,152 @@ +# Releasing extensions + +`.github/workflows/release.yml` runs on every push to `main` and on a manual dispatch. For each +`extensions/*/manifest.json`, it builds the bundle and hands it to `scripts/release-extensions.sh`, which +publishes one GitHub Release per build. This page covers what that run does, in any organization, and the +extra steps that apply only inside DuploCloud's own organizations, `duplocloud` and `duplocloud-internal`. + +Only one release run executes for a repository at a time. A run that starts while another is still in +flight queues behind it rather than overlapping or being canceled mid-write. + +## What a release produces + +A build is tagged `-v-sdk-`, where `` is the extension directory's name +and `` comes from the built bundle's own `manifest.json` (stamped by `build-extension.sh` from +the host SDK it built against). The release is cut at the built commit with `extension.zip` attached. A +version already released, under this tag shape or the older `-v` one, is skipped if the +extension is unchanged since, and fails the run if it changed, since bumping `manifest.version` is the only +way to tell two different bundles apart. + +Two checks run before any of that, for every build in every organization. A manifest with no `resources` +entry is refused, since the host would refuse the bundle anyway, and so is a zip over 268435456 bytes (256 +MiB). A pull request that touches an extension sees the same checks ahead of time, as a warning rather than +a failure, from `scripts/check-extension-pr.sh`. That script also warns when `manifest.version` has not +moved, when a `skills/` folder has no matching `skills[].folder` entry in the manifest, and when a webpack +frontend bundle ships without the file that Native Federation hosts need. + +In `duplocloud` and `duplocloud-internal`, the release also carries `extension.zip.sig`, and, for an id the +allowlist maps to this repository, the build is uploaded to the channels bucket and registered as a new, +unpublished version with the license server. Nothing in this workflow ever publishes a version. A platform +owner does that separately, once the build has been verified. + +Outside those two organizations, a release ships the zip alone. A customer's copy of this workflow holds no +DuploCloud signing credential, so it signs nothing, uploads nothing, and registers nothing. + +## Signing a build + +Any repository, not only DuploCloud's own, can sign a build with `.github/actions/extension-sign`, a +composite action that wraps `scripts/extension-sign.py sign`. It takes three inputs: + +- `bundle`, the path to the built `extension.zip` +- `signing-key`, the publisher's private key PEM, from a secret +- `signing-cert`, the publisher's certificate, a compact JWS from the console + +It returns `signature-path`, the path of the `.sig` it wrote. The key and certificate reach the script +through the `EXTENSION_SIGNING_KEY` and `EXTENSION_SIGNING_CERT` environment variables, never as a command +argument, so neither shows up in a process listing. + +`sign` writes one compact ES256 JWS next to the zip, built from the zip's own manifest rather than the +source one, since `sdkVersion` only exists in the built copy. Its claims are the manifest's `id`, `version` +and `sdk_version`, the zip's own sha256, and the time it was signed. It refuses, writing nothing, when the +signing key's public half does not match the certificate, when the manifest id falls outside every +namespace the certificate covers, when the certificate has expired, or when `sdkVersion` is missing, still +carries its placeholder value, or either `version` or `sdkVersion` is not strict semver +(`MAJOR.MINOR.PATCH[-pre]`, no build metadata). It warns, but still signs, when the certificate expires +within 30 days. + +`verify [zip] [sig]` checks a signature the way the console does: the certificate's chain to a published +root key, the certificate's own validity window, the signature's claims against the zip and its manifest, +and the namespace. By default it fetches root keys from the console's public `root-keys` endpoint, which +needs no credential. A `--root-key` flag substitutes a local key for testing and says so in its output, so +that run is never mistaken for a check against the console's real roots. + +## In DuploCloud's organizations + +Signing is mandatory in `duplocloud` and `duplocloud-internal`. A repository there sets two secrets and two +organization variables: + +| Name | Kind | Holds | +| --- | --- | --- | +| `CONSOLE_SIGNING_KEY` | Secret, granted per repository | The publisher's private key PEM | +| `CONSOLE_API_KEY` | Secret, granted per repository | The console API key used to register versions and artifacts | +| `CONSOLE_SIGNING_CERT` | Organization variable | The publisher's certificate. The same value for every repository | +| `EXTENSION_PUBLISHER_ROLE_ARN` | Organization variable | The AWS role the workflow assumes, by name, to write to the channels bucket | + +The workflow maps `CONSOLE_SIGNING_KEY` onto `EXTENSION_SIGNING_KEY` and `CONSOLE_SIGNING_CERT` onto +`EXTENSION_SIGNING_CERT` before calling the signer. A repository missing either secret or variable fails +its release job before anything is written, rather than shipping an unsigned build. + +The role is only assumed for a repository the allowlist names, through GitHub's OIDC token +(`permissions: id-token: write`), so a repository that does not publish never needs AWS credentials at all. + +### Joining the allowlist + +`.github/extension-publishers.json` ships empty in devkit. Each entry maps one manifest id to one +repository and one console extension: + +```json +{ "manifestId": "duplo.extensions.your-extension", "repository": "duplocloud/your-repo", "consoleExtension": "[console extension uuid]" } +``` + +The `consoleExtension` uuid comes from the console's extensions page. Without an entry, or with one that +names a different repository, a build still signs and releases, since signing does not depend on the +allowlist, but the release job logs a notice and never uploads or registers it. + +The allowlist alone is not the security boundary. The AWS role's trust policy, held in ai-release's +Terraform (`terraform/modules/release-channels`), names the same repositories, and that trust is what +actually lets a workflow run write to the bucket. A test in ai-release keeps the two lists equal against a +pinned devkit commit, so joining the allowlist takes a PR in both repositories: the entry here, and the +repository added to the role's trust list there. Either alone leaves the id unable to publish, refused by +AWS on one side or skipped by the allowlist check on the other. + +Changes to this file, `release.yml`, `release-extensions.sh` and the signer take two approvals from the +`devkit-maintainers` team, since each one can change what gets signed or where it ends up. + +## The channels bucket and the license server + +A signed build uploads to `s3://duplo-helpdesk-channels/bundles/[id]/[version]/sdk-[sdkVersion]/` in +`us-west-2`, as two objects, `extension.zip` and `extension.zip.sig`. Both the bucket name and the console +URL can be overridden with the `BUNDLE_BUCKET` and `CONSOLE_URL` repository variables. Left unset, they +default to the production bucket and the production console. + +Before uploading, the script downloads both assets from the GitHub Release itself, never rebuilding them, +and recomputes the zip's sha256 locally. When the Release reports its own digest for that asset, the script +compares the two and fails the job on a mismatch, so a corrupted or substituted download is never uploaded. +The locally computed hash, not GitHub's, is what gets registered. + +Every write from here on is create-only. The upload uses `aws s3api put-object --if-none-match '*'`, so it +never overwrites an existing key. When that fails with a 412, the script fetches what is already there and +compares bytes. Identical content counts as already uploaded and the run moves on. Different content fails +the job, since a published build is never replaced. + +Registering the version with the license server follows the same shape. The script looks up the version by +its number before creating it, and a 400 reporting a duplicate just means a concurrent run created it first, +so it looks up again rather than failing. It registers the artifact for this SDK the same way, by looking +it up before creating it. An existing artifact whose path, hash and signature all match counts as already +done. One that differs fails the job. The script never marks a version published. Every version it creates +starts unpublished, and stays that way until a platform owner publishes it. + +## Certificate reissue and key rotation + +Reissuing the certificate alone, which a platform owner does as it nears expiry, changes only the +`CONSOLE_SIGNING_CERT` organization variable, to the new certificate from the console's signing-keys page. +`CONSOLE_SIGNING_KEY` does not change, since the underlying key is the same one the new certificate still +certifies. + +Rotating the key changes both. A new key needs a new certificate that names its public half, so +`CONSOLE_SIGNING_KEY` and `CONSOLE_SIGNING_CERT` have to move together. The signer checks that the key's +public half matches the certificate's before it signs anything, so updating one without the other fails +every release in DuploCloud's organizations until both are in place. + +## Re-running a release + +A tag that already has a release is never rebuilt, re-signed, or recreated. `release-extensions.sh` reads +that release's own assets and resumes from there, so a run that failed partway, during the upload or the +registration, can simply be re-run. Everything it already finished is a create-only no-op, and it picks up +where it stopped. + +A release that was cut before its repository started signing, or outside DuploCloud's organizations, has no +`.sig` asset to resume from. It cannot gain one afterward. Once immutable releases are turned on for a +repository, which this workflow recommends, nothing can be added to a release after it is cut. That release +stays flagged with a warning and is skipped rather than retried. The only way forward is a new +`manifest.version`, which cuts a new tag and a new release that does get signed. From 536e2cda75a5260192215b82ad8d16fbe4c8406f Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 15:37:17 +0100 Subject: [PATCH 15/22] Fix four accuracy issues in the release action doc --- docs/releasing-extensions.md | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/releasing-extensions.md b/docs/releasing-extensions.md index 7d337d6..762349d 100644 --- a/docs/releasing-extensions.md +++ b/docs/releasing-extensions.md @@ -1,7 +1,8 @@ # Releasing extensions `.github/workflows/release.yml` runs on every push to `main` and on a manual dispatch. For each -`extensions/*/manifest.json`, it builds the bundle and hands it to `scripts/release-extensions.sh`, which +`extensions/*/manifest.json`, plus the single-extension layouts `extension/manifest.json` and +`extension/*/manifest.json`, it builds the bundle and hands it to `scripts/release-extensions.sh`, which publishes one GitHub Release per build. This page covers what that run does, in any organization, and the extra steps that apply only inside DuploCloud's own organizations, `duplocloud` and `duplocloud-internal`. @@ -85,7 +86,12 @@ The role is only assumed for a repository the allowlist names, through GitHub's repository and one console extension: ```json -{ "manifestId": "duplo.extensions.your-extension", "repository": "duplocloud/your-repo", "consoleExtension": "[console extension uuid]" } +{ + "schemaVersion": 1, + "publishers": [ + { "manifestId": "duplo.extensions.your-extension", "repository": "duplocloud/your-repo", "consoleExtension": "[console extension uuid]" } + ] +} ``` The `consoleExtension` uuid comes from the console's extensions page. Without an entry, or with one that @@ -99,8 +105,8 @@ pinned devkit commit, so joining the allowlist takes a PR in both repositories: repository added to the role's trust list there. Either alone leaves the id unable to publish, refused by AWS on one side or skipped by the allowlist check on the other. -Changes to this file, `release.yml`, `release-extensions.sh` and the signer take two approvals from the -`devkit-maintainers` team, since each one can change what gets signed or where it ends up. +The design calls for two approvals from the `devkit-maintainers` team on changes to this file, `release.yml`, +`release-extensions.sh` and the signer, though the ruleset enforcing that is not configured yet. ## The channels bucket and the license server @@ -120,11 +126,11 @@ compares bytes. Identical content counts as already uploaded and the run moves o the job, since a published build is never replaced. Registering the version with the license server follows the same shape. The script looks up the version by -its number before creating it, and a 400 reporting a duplicate just means a concurrent run created it first, -so it looks up again rather than failing. It registers the artifact for this SDK the same way, by looking -it up before creating it. An existing artifact whose path, hash and signature all match counts as already -done. One that differs fails the job. The script never marks a version published. Every version it creates -starts unpublished, and stays that way until a platform owner publishes it. +its number before creating it, and an error saying the record already exists just means a concurrent run +created it first, so it looks up again rather than failing. It registers the artifact for this SDK the same +way, by looking it up before creating it. An existing artifact whose path, hash and signature all match +counts as already done. One that differs fails the job. The script never marks a version published. Every +version it creates starts unpublished, and stays that way until a platform owner publishes it. ## Certificate reissue and key rotation From ef841f9107785425a495005df2a1cc8125db4041 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 16:03:34 +0100 Subject: [PATCH 16/22] Short-circuit a finished build at the console lookup, guard the artifact lookup, and fail an allowlisted extension with no CONSOLE_API_KEY before any write --- docs/releasing-extensions.md | 22 ++++++++++---- scripts/_publish.sh | 50 ++++++++++++++++++++++++++------ scripts/release-extensions.sh | 8 ++++- tests/_publish_stubs.sh | 8 ++++- tests/test-publish.sh | 26 +++++++++++++++-- tests/test-release-extensions.sh | 20 +++++++++++-- 6 files changed, 112 insertions(+), 22 deletions(-) diff --git a/docs/releasing-extensions.md b/docs/releasing-extensions.md index 762349d..33aef55 100644 --- a/docs/releasing-extensions.md +++ b/docs/releasing-extensions.md @@ -74,8 +74,12 @@ organization variables: | `EXTENSION_PUBLISHER_ROLE_ARN` | Organization variable | The AWS role the workflow assumes, by name, to write to the channels bucket | The workflow maps `CONSOLE_SIGNING_KEY` onto `EXTENSION_SIGNING_KEY` and `CONSOLE_SIGNING_CERT` onto -`EXTENSION_SIGNING_CERT` before calling the signer. A repository missing either secret or variable fails -its release job before anything is written, rather than shipping an unsigned build. +`EXTENSION_SIGNING_CERT` before calling the signer. Each of the four fails the job before anything is +written. A missing signing key or certificate fails each extension before it is signed or released, rather +than shipping an unsigned build. An allowlisted extension with no `CONSOLE_API_KEY` fails the same way, +before it is signed or released, since that build could never be registered. A missing +`EXTENSION_PUBLISHER_ROLE_ARN` fails the credentials step, which runs before the publish step. Only +`CONSOLE_SIGNING_KEY` and `CONSOLE_SIGNING_CERT` matter to a repository the allowlist does not name. The role is only assumed for a repository the allowlist names, through GitHub's OIDC token (`permissions: id-token: write`), so a repository that does not publish never needs AWS credentials at all. @@ -115,10 +119,16 @@ A signed build uploads to `s3://duplo-helpdesk-channels/bundles/[id]/[version]/s URL can be overridden with the `BUNDLE_BUCKET` and `CONSOLE_URL` repository variables. Left unset, they default to the production bucket and the production console. -Before uploading, the script downloads both assets from the GitHub Release itself, never rebuilding them, -and recomputes the zip's sha256 locally. When the Release reports its own digest for that asset, the script -compares the two and fails the job on a mismatch, so a corrupted or substituted download is never uploaded. -The locally computed hash, not GitHub's, is what gets registered. +A build that is already finished stops before any download of the zip or any S3 call. The script reads the +Release's digest for `extension.zip`, downloads only `extension.zip.sig`, and looks up the version and its +artifact for this SDK in the license server. An artifact whose path, hash and signature match is reported +as already published. Registration only ever follows both uploads, so a matching artifact means both objects +are already in the bucket. A failed lookup fails the job. Anything else takes the full path below. + +Otherwise, before uploading, the script downloads both assets from the GitHub Release itself, never +rebuilding them, and recomputes the zip's sha256 locally. When the Release reports its own digest for that +asset, the script compares the two and fails the job on a mismatch, so a corrupted or substituted download +is never uploaded. The locally computed hash, not GitHub's, is what gets registered. Every write from here on is create-only. The upload uses `aws s3api put-object --if-none-match '*'`, so it never overwrites an existing key. When that fails with a 412, the script fetches what is already there and diff --git a/scripts/_publish.sh b/scripts/_publish.sh index 0b89be5..e2eae12 100644 --- a/scripts/_publish.sh +++ b/scripts/_publish.sh @@ -35,29 +35,61 @@ console() { local rc=$?; rm -f "$hdr"; printf '%s' "$body"; return $rc } +# artifact_body : the artifact record publish_build registers, and compares an existing one to. +artifact_body() { + jq -n --arg p "s3://$bundle_bucket/$1" --arg s "$2" --arg h "$3" --rawfile g "$4" \ + '{sdk_version: $s, s3_path: $p, sha256: $h, signature: $g}' +} + +# artifact_matches : exit 0 when the lookup's first artifact has the same path, hash and signature. +artifact_matches() { + jq -e --argjson a "$2" '.[0] | .s3_path == $a.s3_path and (.sha256 | ascii_downcase) == $a.sha256 and .signature == $a.signature' \ + <<<"$1" >/dev/null +} + # publish_build : upload the Release's zip and .sig, then register both. Reads the # assets from the Release so a re-run reuses the bytes already signed and released, with no rebuild. +# +# A finished build stops early. Registration runs only after both uploads, so a registered artifact matching the +# Release's digest and .sig means both objects are already in the bucket, and the run needs neither the zip nor S3. publish_build() { local tag=$1 id=$2 version=$3 sdk=$4 v dl localsha sha key vuuid out art for v in "$id" "$version" "$sdk"; do [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || { echo "::error::$tag — $v cannot form a bucket key."; return 1; } done + key="bundles/$id/$version/sdk-$sdk/extension.zip" + sha="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.assets[] | select(.name == "extension.zip") | .digest // empty')" \ + || { echo "::error::$tag — cannot read its release digest."; return 1; } + sha="${sha#sha256:}" dl="$(mktemp -d)" - gh release download "$tag" -p extension.zip -p extension.zip.sig -D "$dl" \ + gh release download "$tag" -p extension.zip.sig -D "$dl" \ + || { echo "::error::$tag — cannot download its signature."; rm -rf "$dl"; return 1; } + + # Without a digest there is no hash to compare before the zip is downloaded, so the full path below runs. + if [ -n "$sha" ]; then + out="$(console GET "/api/extensions/$uuid/versions/?version=$version")" \ + || { echo "::error::$tag — console lookup of the version failed: $out"; rm -rf "$dl"; return 1; } + vuuid="$(jq -r '.[0].uuid // empty' <<<"$out")" + if [ -n "$vuuid" ]; then + out="$(console GET "/api/extensions/$uuid/versions/$vuuid/artifacts/?sdk_version=$sdk")" \ + || { echo "::error::$tag — console lookup of the artifact failed: $out"; rm -rf "$dl"; return 1; } + if artifact_matches "$out" "$(artifact_body "$key" "$sdk" "$sha" "$dl/extension.zip.sig")"; then + echo "==> $tag already published"; rm -rf "$dl"; return 0 + fi + fi + fi + + gh release download "$tag" -p extension.zip -D "$dl" \ || { echo "::error::$tag — cannot download its assets."; rm -rf "$dl"; return 1; } # The local hash is always computed and is what gets registered. GitHub's own digest, when the release carries # one, only confirms the download matches what was actually released, before anything is uploaded or registered. localsha="$(shasum -a 256 "$dl/extension.zip" | cut -d' ' -f1)" - sha="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.assets[] | select(.name == "extension.zip") | .digest // empty')" \ - || { echo "::error::$tag — cannot read its release digest."; rm -rf "$dl"; return 1; } - sha="${sha#sha256:}" if [ -n "$sha" ]; then [ "$sha" = "$localsha" ] \ || { echo "::error::$tag — release digest $sha does not match downloaded extension.zip hash $localsha."; rm -rf "$dl"; return 1; } else sha="$localsha" fi - key="bundles/$id/$version/sdk-$sdk/extension.zip" s3_put_once "$dl/extension.zip" "$key" && s3_put_once "$dl/extension.zip.sig" "$key.sig" || { rm -rf "$dl"; return 1; } out="$(console GET "/api/extensions/$uuid/versions/?version=$version")" \ @@ -71,11 +103,11 @@ publish_build() { vuuid="$(jq -r '.[0].uuid // empty' <<<"$out")" fi [ -n "$vuuid" ] || { echo "::error::$tag — the console has no version uuid for $version after creating it."; rm -rf "$dl"; return 1; } - art="$(jq -n --arg s "$sdk" --arg p "s3://$bundle_bucket/$key" --arg h "$sha" --rawfile g "$dl/extension.zip.sig" \ - '{sdk_version: $s, s3_path: $p, sha256: $h, signature: $g}')" - out="$(console GET "/api/extensions/$uuid/versions/$vuuid/artifacts/?sdk_version=$sdk")" + art="$(artifact_body "$key" "$sdk" "$sha" "$dl/extension.zip.sig")" + out="$(console GET "/api/extensions/$uuid/versions/$vuuid/artifacts/?sdk_version=$sdk")" \ + || { echo "::error::$tag — console lookup of the artifact failed: $out"; rm -rf "$dl"; return 1; } if [ "$(jq 'length' <<<"$out")" -gt 0 ]; then - if jq -e --argjson a "$art" '.[0] | .s3_path == $a.s3_path and (.sha256 | ascii_downcase) == $a.sha256 and .signature == $a.signature' <<<"$out" >/dev/null; then + if artifact_matches "$out" "$art"; then echo "==> $tag already registered"; rm -rf "$dl"; return 0 fi echo "::error::$tag — the console already holds a build for SDK $sdk that differs from this release."; rm -rf "$dl"; return 1 diff --git a/scripts/release-extensions.sh b/scripts/release-extensions.sh index f39ce83..3692328 100755 --- a/scripts/release-extensions.sh +++ b/scripts/release-extensions.sh @@ -20,7 +20,8 @@ # alongside it; a run anywhere else releases the zip alone, as a customer's copy of this workflow carries no Duplo # signing credential. Within a Duplo organization, the manifest id also has to be in the extension publisher # allowlist (scripts/_publishers.sh) for the repository, or the release still ships signed but scripts/_publish.sh's -# publish_build is never called for it. +# publish_build is never called for it. An allowlisted id with no CONSOLE_API_KEY fails before it is signed or +# released, since that build could be released but never registered. # # Resuming: a tag that already has a release is never rebuilt, re-signed or re-released. Its own assets, read back # with gh release view, decide what happens next: a signed release calls publish_build again so a run that stopped @@ -94,6 +95,11 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest echo "::notice::$dir — $id is not in the extension publisher allowlist for $repo, so it is released but not published to the license server." fi fi + # Registration needs the console key, so a missing one fails here, before a release that could never be registered. + if [ -n "$uuid" ] && [ -z "${CONSOLE_API_KEY:-}" ]; then + echo "::error::$dir — $id is allowlisted for $repo, but CONSOLE_API_KEY is not set, so it cannot be registered." + failed=$((failed+1)); continue + fi if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then if [ -n "$uuid" ]; then diff --git a/tests/_publish_stubs.sh b/tests/_publish_stubs.sh index 95c4435..e721da6 100644 --- a/tests/_publish_stubs.sh +++ b/tests/_publish_stubs.sh @@ -13,7 +13,11 @@ cat > "$TMP/bin/gh" <<'EOF' #!/usr/bin/env bash echo "gh $*" >> "$LOG" case "$1 $2" in - "release download") while [ $# -gt 0 ]; do [ "$1" = -D ] && cp "$ASSETS"/* "$2"/; shift; done ;; + "release download") + # Copies only the assets named by -p, so a test can tell which ones a run downloaded. + pats=(); dest="" + while [ $# -gt 0 ]; do case "$1" in -p) pats+=("$2"); shift ;; -D) dest=$2; shift ;; esac; shift; done + for p in "${pats[@]}"; do cp "$ASSETS/$p" "$dest"/; done ;; "release create") exit "${GH_CREATE_RC:-0}" ;; "release view") [ "${GH_VIEW_RC:-0}" = 0 ] || exit "${GH_VIEW_RC}" printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; @@ -54,6 +58,8 @@ while [ $# -gt 0 ]; do done if [ -n "$hdrfile" ] && grep -qF -- "$CONSOLE_API_KEY" "$hdrfile" 2>/dev/null then echo "HEADER_FILE_KEY=yes" >> "$LOG"; else echo "HEADER_FILE_KEY=no" >> "$LOG"; fi +# CONSOLE_FAIL, a regex matched against the URL, makes that call fail the way curl --fail-with-body does. +if [ -n "${CONSOLE_FAIL:-}" ] && [[ "$url" =~ $CONSOLE_FAIL ]]; then echo '{"detail":"server error"}'; exit 22; fi case "$method $url" in "GET "*"/versions/?version="*) cat "$CONSOLE/versions.json" 2>/dev/null || echo '[]' ;; "POST "*"/versions/") diff --git a/tests/test-publish.sh b/tests/test-publish.sh index 7dcaec5..7bf4902 100755 --- a/tests/test-publish.sh +++ b/tests/test-publish.sh @@ -32,10 +32,26 @@ elif ! grep -q "HEADER_FILE_KEY=yes" "$LOG"; then bad "the header file never car else ok fi -t "a re-run of a finished build changes nothing" -: > "$LOG" # Keep S3/CONSOLE state from the prior run. Clear only the log, so this run's own curls are what we check. +t "a re-run of a finished build stops at the console lookup, with no zip download and no aws call" +: > "$LOG" # Keep S3/CONSOLE state from the prior run. Clear only the log, so this run's own calls are what we check. OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? -[ $RC = 0 ] && grep -q "already uploaded" <<<"$OUT" && ! grep -q "POST" <<<"$(grep curl "$LOG")" && ok || bad "rc=$RC out=$OUT" +[ $RC = 0 ] && grep -q "already published" <<<"$OUT" && ! grep -q "^aws " "$LOG" && ! grep -q "POST" <<<"$(grep curl "$LOG")" \ + && grep -q -- "-p extension.zip.sig" "$LOG" && ! grep -qE -- "-p extension\.zip( |$)" "$LOG" && ok || bad "rc=$RC out=$OUT log=$(cat "$LOG")" + +t "a console lookup failure on a finished build fails the job, with no aws call" +: > "$LOG" +OUT=$(CONSOLE_FAIL='/versions/\?version=' publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::.*lookup of the version failed" <<<"$OUT" && ! grep -q "^aws " "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "an artifact lookup failure on a finished build fails the job, with no aws call" +: > "$LOG" +OUT=$(CONSOLE_FAIL='/artifacts/\?sdk_version=' publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::.*lookup of the artifact failed" <<<"$OUT" && ! grep -q "^aws " "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "a registered artifact that differs from the release takes the full path and fails there" +: > "$LOG"; jq '.[0].signature = "OTHERSIG"' "$CONSOLE/artifacts.json" > "$TMP/a.json" && mv "$TMP/a.json" "$CONSOLE/artifacts.json" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "differs" <<<"$OUT" && grep -qE -- "-p extension\.zip( |$)" "$LOG" && ok || bad "rc=$RC out=$OUT" t "a re-run after only the zip uploaded uploads the signature" fresh; cp "$ASSETS/extension.zip" "$S3/bundles_duplo.demo_1.0.0_sdk-1.0.6_extension.zip" @@ -70,6 +86,10 @@ t "a failed release-digest lookup fails the job before any upload" fresh; OUT=$(GH_API_RC=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? [ $RC != 0 ] && grep -q "::error::" <<<"$OUT" && ! grep -q "put-object" "$LOG" && ok || bad "rc=$RC out=$OUT" +t "an artifact lookup failure on a new build fails the job" +fresh; OUT=$(CONSOLE_FAIL='/artifacts/\?sdk_version=' publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::.*lookup of the artifact failed" <<<"$OUT" && [ ! -f "$CONSOLE/artifacts.json" ] && ok || bad "rc=$RC out=$OUT" + t "a version lookup that still comes back empty after the POST fails the job" fresh; OUT=$(VERSION_VANISHES=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? [ $RC != 0 ] && grep -q "no version uuid" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh index d1982e4..46d9126 100755 --- a/tests/test-release-extensions.sh +++ b/tests/test-release-extensions.sh @@ -174,11 +174,26 @@ git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm oth ( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) OUT="$(GH_VIEW_RC=1 PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ - EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? + CONSOLE_API_KEY=secret-key EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? if [ "$RC" != 0 ] && grep -q "::error::.*gh release view demo-v0.1.0-sdk-1.0.6 failed" <<<"$OUT" \ && grep -q "release create other-v0.2.0-sdk-1.0.6" "$GH_LOG" then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi +t "an allowlisted extension with no CONSOLE_API_KEY fails before signing or releasing" +new_repo; build_zip 1.0.6 +OUT="$(PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + CONSOLE_API_KEY= EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "::error::.*extensions/demo.*CONSOLE_API_KEY" <<<"$OUT" && ! grep -q "release create" "$GH_LOG" \ + && [ ! -f "$REPO/extensions/demo/dist/extension.zip.sig" ] +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "an allowlisted, already released extension with no CONSOLE_API_KEY fails before reading its release" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +OUT="$(PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + CONSOLE_API_KEY= EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "::error::.*CONSOLE_API_KEY" <<<"$OUT" && ! grep -q "release view" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + t "an existing signed release resumes publishing without re-signing or re-releasing" new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 # The release's own assets (never the local dist/ bundle) are what a resumed publish reads, so plant them separately. @@ -195,7 +210,8 @@ then ok; else bad "rc=$RC out=$OUT"; fi t "a release cut before signing warns with the bump and does not fail" new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 -OUT="$(GH_ASSETS=extension.zip PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +OUT="$(GH_ASSETS=extension.zip PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + CONSOLE_API_KEY=secret-key EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? if [ "$RC" = 0 ] && grep -q "::warning::.*no extension.zip.sig.*Bump manifest.version" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" then ok; else bad "rc=$RC out=$OUT"; fi From 53e86363a7beabeb186d1da5cf4a67adad221e0f Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Tue, 6 Oct 2026 16:03:34 +0100 Subject: [PATCH 17/22] Run the release job only on main and keep the signer tests from writing bytecode --- .github/workflows/release.yml | 2 ++ tests/test-extension-sign.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fbb59c4..2c132cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,6 +45,8 @@ concurrency: jobs: release: + # A dispatch from any other branch skips the job, so only main's code ever releases, signs or publishes. + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest env: DUPLO_TARGET: remote diff --git a/tests/test-extension-sign.sh b/tests/test-extension-sign.sh index 154b3b5..ef509e9 100755 --- a/tests/test-extension-sign.sh +++ b/tests/test-extension-sign.sh @@ -7,6 +7,8 @@ t() { printf ' %s … ' "$1"; } ok() { echo "ok"; PASS=$((PASS+1)); } bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +# The vector tests import the signer as a module. Without this, Python leaves scripts/__pycache__/ in the checkout. +export PYTHONDONTWRITEBYTECODE=1 V=tests/vectors; S=scripts/extension-sign.py # The vectors' certificate rides in each .sig header, so the valid case's header gives it. CERT=$(python3 -c 'import jwt,sys; print(jwt.get_unverified_header(open(sys.argv[1]).read())["cert"])' $V/extension.zip.sig) From 20fb4ae3b5d29071f780b1b73dadf7d1e1e60c7e Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Wed, 7 Oct 2026 13:35:30 +0100 Subject: [PATCH 18/22] Read BUILDER_TAG=latest as unset so older .env files keep the pinned toolchain --- .env.example | 11 ++++++----- scripts/_builder.sh | 4 +++- tests/test-builder-image.sh | 7 ++++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index f7e56c0..add02c1 100644 --- a/.env.example +++ b/.env.example @@ -47,13 +47,14 @@ QDRANT_API_KEY=devkit-local-qdrant # Extension build toolchain (.NET SDK 8 + Node 22). Only pulled the first time you build an # extension — the platform itself doesn't use it. The default is pinned by digest in scripts/_builder.sh, -# so every build of a version uses one toolchain. BUILDER_TAG= (e.g. latest) opts out of the pin; an -# already-pulled tag is reused without checking the registry, and BUILDER_PULL=1 forces a re-pull (tags -# move). BUILDER_IMAGE= replaces the image -# outright, BUILDER_PLATFORM= forces an architecture. Each build's package caches live in +# so every build of a version uses one toolchain. BUILDER_TAG= opts out of the pin for any tag but +# latest, which reads as unset; an already-pulled tag is reused without checking the registry, and +# BUILDER_PULL=1 forces a re-pull (tags move). BUILDER_IMAGE= replaces the image outright, so +# BUILDER_IMAGE=quay.io/duplocloud/duplo-extension-builder:latest builds on the moving tag. +# BUILDER_PLATFORM= forces an architecture. Each build's package caches live in # per-uid named volumes that nothing prunes — ` volume ls | grep duplo_devkit` to find and # ` volume rm` to reclaim them (each holds a full NuGet + npm closure). -# BUILDER_TAG=latest +# BUILDER_TAG=1.2.3 # Build on THIS machine instead of in the toolchain container. Set to 1 if you already have .NET SDK 8 # and Node 22+ installed and would rather use them — you own keeping them current, and your builds stop diff --git a/scripts/_builder.sh b/scripts/_builder.sh index 2574819..7939089 100644 --- a/scripts/_builder.sh +++ b/scripts/_builder.sh @@ -277,7 +277,9 @@ builder_resolve_image() { fi local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")" - if [ -z "$tag" ]; then + # `latest` was the default before the pin, so older .env files still carry it. Reading it as unset keeps those + # repos on the pinned toolchain. BUILDER_IMAGE=...:latest still selects the moving tag on purpose. + if [ -z "$tag" ] || [ "$tag" = latest ]; then BUILDER_IMAGE="$_BUILDER_PINNED_IMAGE" else # OCI tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is diff --git a/tests/test-builder-image.sh b/tests/test-builder-image.sh index d426ff7..f171378 100755 --- a/tests/test-builder-image.sh +++ b/tests/test-builder-image.sh @@ -35,8 +35,13 @@ t "docker-compose.yml falls back to the same pinned image" if grep -qF -- "\${BUILDER_IMAGE:-$got}" docker-compose.yml; then ok; else bad "compose does not default to $got"; fi t "BUILDER_TAG in .env still selects a tag" +printf 'BUILDER_TAG=1.2.3\n' > "$ENV"; got="$(resolve :)" +if [ "$got" = "quay.io/duplocloud/duplo-extension-builder:1.2.3" ]; then ok; else bad "$got"; fi + +t "BUILDER_TAG=latest, left over in an older .env, still takes the pinned image" +: > "$ENV"; pinned="$(resolve :)" printf 'BUILDER_TAG=latest\n' > "$ENV"; got="$(resolve :)" -if [ "$got" = "quay.io/duplocloud/duplo-extension-builder:latest" ]; then ok; else bad "$got"; fi +if [ "$got" = "$pinned" ]; then ok; else bad "$got"; fi t "BUILDER_IMAGE in the environment still replaces the image outright" : > "$ENV"; got="$(resolve 'export BUILDER_IMAGE=example.com/mine:1')" From 01d09597fdfad8963d3828434cc409ede478aeab Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Wed, 7 Oct 2026 13:35:30 +0100 Subject: [PATCH 19/22] Warn about an unbumped version only when it already has a release tag --- scripts/check-extension-pr.sh | 6 ++++-- tests/test-check-extension-pr.sh | 9 +++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/check-extension-pr.sh b/scripts/check-extension-pr.sh index a04a4fb..6c8ce0c 100755 --- a/scripts/check-extension-pr.sh +++ b/scripts/check-extension-pr.sh @@ -42,8 +42,10 @@ for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest version="$(jq -r '.version' "$m")" base_version="$(git show "$base:$m" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true)" - if [ -n "$base_version" ] && [ "$base_version" = "$version" ]; then - warn "$m" "$dir changed but manifest.version is still $version. Bump it, or the release job refuses to publish this content." + # The release job refuses only a version that already has a release tag, under either tag shape, so warn only then. + slug="$(basename "$dir")" + if [ -n "$base_version" ] && [ "$base_version" = "$version" ] && [ -n "$(git tag -l "$slug-v$version" "$slug-v$version-sdk-*")" ]; then + warn "$m" "$dir changed but manifest.version is still $version, which is already released. Bump it, or the release job refuses to publish this content." fi for v in "$(jq -r '.id' "$m")" "$version"; do diff --git a/tests/test-check-extension-pr.sh b/tests/test-check-extension-pr.sh index fc8d605..60176d6 100755 --- a/tests/test-check-extension-pr.sh +++ b/tests/test-check-extension-pr.sh @@ -49,11 +49,16 @@ run() { echo "check-extension-pr:" -t "warns when an extension changed with no manifest.version bump" -new_repo; change two +t "warns when a released extension changed with no manifest.version bump" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6 main; change two OUT="$(run)"; RC=$? if [ "$RC" = 0 ] && grep -q "::warning.*extensions/demo.*manifest.version" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi +t "is quiet about an unbumped version that has no release yet" +new_repo; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "manifest.version" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + t "is quiet about the version when it was bumped" new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two OUT="$(run)"; RC=$? From 7eaa7ce02e08d34f0b47a7d861ae94ce275b9a83 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Wed, 7 Oct 2026 13:35:30 +0100 Subject: [PATCH 20/22] Refuse to sign with a certificate that is not valid yet --- scripts/extension-sign.py | 3 +++ tests/test-extension-sign.sh | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/scripts/extension-sign.py b/scripts/extension-sign.py index aa263b7..029bb2c 100755 --- a/scripts/extension-sign.py +++ b/scripts/extension-sign.py @@ -102,6 +102,9 @@ def sign_bytes(zip_path, key_pem, cert, now): key = serialization.load_pem_private_key(key_pem.encode(), password=None) if jwk_of(key.public_key()) != {k: claims.get("jwk", {}).get(k) for k in ("kty", "crv", "x", "y")}: refuse("signing key and certificate don't match") + nbf = claims.get("nbf") + if not isinstance(nbf, int) or now < nbf: + refuse("the signing certificate is not valid yet. Check the runner clock, or wait for its start time") exp = claims.get("exp") if not isinstance(exp, int) or now >= exp: refuse("the signing certificate has expired. Reissue it in the console and update CONSOLE_SIGNING_CERT") diff --git a/tests/test-extension-sign.sh b/tests/test-extension-sign.sh index ef509e9..e7b5a93 100755 --- a/tests/test-extension-sign.sh +++ b/tests/test-extension-sign.sh @@ -61,6 +61,12 @@ EXP=$(python3 -c 'import jwt,sys; print(jwt.decode(sys.argv[1], options={"verify OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$EXP" python3 $S sign "$TMP/g.zip" 2>&1) [ $? != 0 ] && grep -q "expired" <<<"$OUT" && ok || bad "$OUT" +t "refuses a certificate that is not valid yet" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/n.zip" +NBF=$(python3 -c 'import jwt,sys; print(jwt.decode(sys.argv[1], options={"verify_signature": False})["nbf"])' "$CERT") +OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$((NBF - 1))" python3 $S sign "$TMP/n.zip" 2>&1) +[ $? != 0 ] && grep -q "not valid yet" <<<"$OUT" && [ ! -f "$TMP/n.zip.sig" ] && ok || bad "$OUT" + t "warns when the certificate expires within 30 days" mkzip io.example.hello 1.0.0 1.0.0 "$TMP/h.zip" OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$((EXP - 86400))" python3 $S sign "$TMP/h.zip" 2>&1) From f0de420038c0d4e531e53f531df97329ae8aec5a Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Wed, 7 Oct 2026 16:21:41 +0100 Subject: [PATCH 21/22] Document the one-time console setup a release depends on --- docs/releasing-extensions.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/releasing-extensions.md b/docs/releasing-extensions.md index 33aef55..4de056e 100644 --- a/docs/releasing-extensions.md +++ b/docs/releasing-extensions.md @@ -84,6 +84,25 @@ before it is signed or released, since that build could never be registered. A m The role is only assumed for a repository the allowlist names, through GitHub's OIDC token (`permissions: id-token: write`), so a repository that does not publish never needs AWS credentials at all. +### One-time console setup + +A platform owner sets up each of these once in the console. Each one then serves every release. + +1. **The API key behind `CONSOLE_API_KEY`.** In the platform-owner team, create a custom role holding + **View extension versions** and **Manage extension versions**, create a service account with that role, + and store its API key as the organization secret. The release job reads and creates only versions and + artifacts, and the allowlist supplies each extension's uuid, so the key needs no other capability. The + console's own guide covers the service-account workflow in `docs/platform_owner/extension-versions.md`. +2. **The signing key behind `CONSOLE_SIGNING_KEY` and `CONSOLE_SIGNING_CERT`.** The publishing team needs a + namespace covering its manifest ids, and a signing key created under that team. The console returns the + key's private PEM once, at creation, and that goes into the secret. The key's certificate goes into the + organization variable. +3. **One extension record per extension.** Create it under the same publishing team, with a manifest id + that equals the `id` in the extension's `manifest.json`. The console accepts a signed artifact only when + the signature's manifest id matches the record and the signing key belongs to the record's team. The + console's API only reads extension records, so this step happens in the console, and the record's uuid is + what the allowlist entry names. + ### Joining the allowlist `.github/extension-publishers.json` ships empty in devkit. Each entry maps one manifest id to one From 8d43bb9803d5a1146a435e2347703ea3761c6449 Mon Sep 17 00:00:00 2001 From: amaechiabuah Date: Wed, 7 Oct 2026 16:26:57 +0100 Subject: [PATCH 22/22] Pin every workflow action by commit SHA --- .github/workflows/builder-image.yml | 10 +++++----- .github/workflows/extension-ci.yml | 6 +++--- .github/workflows/image-arch-check.yml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/builder-image.yml b/.github/workflows/builder-image.yml index 7ac672e..cfbbdb3 100644 --- a/.github/workflows/builder-image.yml +++ b/.github/workflows/builder-image.yml @@ -36,19 +36,19 @@ jobs: # re-dispatch the workflow. if: github.repository == 'duplocloud/devkit' steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 # QEMU emulates arm64 for the cross-build; buildx provides the multi-platform driver. - - uses: docker/setup-qemu-action@v4 - - uses: docker/setup-buildx-action@v4 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 # Keep this step and its `with:` block adjacent. A dependabot merge once reordered these two `uses:` # lines and left the credentials attached to setup-buildx-action, which silently ignores them — # login-action then ran with no inputs and every publish died on "Username and password required". - - uses: docker/login-action@v4 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - - uses: docker/build-push-action@v7 + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 id: push with: # `build`, not `.`, to match how scripts/_builder.sh builds this same image locally: the diff --git a/.github/workflows/extension-ci.yml b/.github/workflows/extension-ci.yml index 58c764c..83f933b 100644 --- a/.github/workflows/extension-ci.yml +++ b/.github/workflows/extension-ci.yml @@ -35,7 +35,7 @@ jobs: # in an `if:` expression, but `env` is. A fork PR gets no org secrets, so this is empty there. QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: # The PR's base history, so the release checks below can diff each extension against the merge base. fetch-depth: 0 @@ -58,7 +58,7 @@ jobs: # image build. See the env: block above for why this reads env rather than secrets. - name: Log in to quay.io (for the builder image) if: steps.guard.outputs.build == 'true' && env.QUAY_USERNAME != '' - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} @@ -76,7 +76,7 @@ jobs: - name: Upload bundles if: steps.guard.outputs.build == 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: extension-bundles path: | diff --git a/.github/workflows/image-arch-check.yml b/.github/workflows/image-arch-check.yml index aec8060..c0a9d1d 100644 --- a/.github/workflows/image-arch-check.yml +++ b/.github/workflows/image-arch-check.yml @@ -30,6 +30,6 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Check duplocloud images publish amd64 and arm64 run: ./scripts/check-image-arches.sh