diff --git a/.env.example b/.env.example index c8de660..0c415ae 100644 --- a/.env.example +++ b/.env.example @@ -23,12 +23,14 @@ QDRANT_PORT=6333 QDRANT_API_KEY=devkit-local-qdrant # Extension build toolchain (.NET SDK 8 + Node 22). Only pulled the first time you build an -# extension — the platform itself doesn't use it. An already-pulled tag is reused without checking the -# registry: BUILDER_PULL=1 forces a re-pull (this tag moves), BUILDER_IMAGE= replaces the image +# extension — the platform itself doesn't use it. The default is pinned by digest in scripts/_builder.sh, +# so every build of a version uses one toolchain. BUILDER_TAG= (e.g. latest) opts out of the pin; an +# already-pulled tag is reused without checking the registry, and BUILDER_PULL=1 forces a re-pull (tags +# move). BUILDER_IMAGE= replaces the image # outright, BUILDER_PLATFORM= forces an architecture. Each build's package caches live in # per-uid named volumes that nothing prunes — `docker volume ls | grep duplo_devkit` to find and # `docker volume rm` to reclaim them (each holds a full NuGet + npm closure). -BUILDER_TAG=latest +# BUILDER_TAG=latest # Build on THIS machine instead of in the toolchain container. Set to 1 if you already have .NET SDK 8 # and Node 22+ installed and would rather use them — you own keeping them current, and your builds stop diff --git a/.github/workflows/builder-image.yml b/.github/workflows/builder-image.yml index c94faef..7ac672e 100644 --- a/.github/workflows/builder-image.yml +++ b/.github/workflows/builder-image.yml @@ -49,6 +49,7 @@ jobs: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - uses: docker/build-push-action@v7 + id: push with: # `build`, not `.`, to match how scripts/_builder.sh builds this same image locally: the # Dockerfile COPYs nothing from the build context (only --from=node), so sending the whole @@ -61,3 +62,16 @@ jobs: tags: | quay.io/duplocloud/duplo-extension-builder:latest quay.io/duplocloud/duplo-extension-builder:${{ github.sha }} + # scripts/_builder.sh pins the default toolchain by digest, so a publish here changes nothing until that pin + # moves in a reviewed PR. Hand the reviewer the exact value. + - name: Report the digest to pin + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + { + echo "### Builder image published" + echo "To build with it by default, set in scripts/_builder.sh and docker-compose.yml:" + echo '```' + echo "quay.io/duplocloud/duplo-extension-builder@${DIGEST}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/extension-ci.yml b/.github/workflows/extension-ci.yml index b441152..58c764c 100644 --- a/.github/workflows/extension-ci.yml +++ b/.github/workflows/extension-ci.yml @@ -36,6 +36,9 @@ jobs: QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} steps: - uses: actions/checkout@v7 + with: + # The PR's base history, so the release checks below can diff each extension against the merge base. + fetch-depth: 0 - name: Discover extensions id: guard @@ -65,6 +68,12 @@ jobs: if: steps.guard.outputs.build == 'true' run: ./scripts/build-all.sh + # Warnings only: what each changed extension's release will need (version bump, declared skills, Native + # Federation frontend). !cancelled() so a failed build still reports the checks that need no bundle. + - name: Release checks + if: ${{ !cancelled() && github.event_name == 'pull_request' && steps.guard.outputs.build == 'true' }} + run: ./scripts/check-extension-pr.sh "origin/${{ github.base_ref }}" + - name: Upload bundles if: steps.guard.outputs.build == 'true' uses: actions/upload-artifact@v7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 824bfc5..068d562 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,14 @@ # Publishes each extension in this repo as a GitHub Release on push to main. # -# For every extensions// it builds the bundle and, ONLY when manifest.version is new (its -# -v tag doesn't exist yet), cuts a GitHub Release with extension.zip attached. Bumping -# manifest.version is therefore how you publish a new version to the extension library. +# For every extensions// it builds the bundle and publishes one release per build, tagged +# -v-sdk- at the commit that was built. scripts/release-extensions.sh holds the rules: +# a build already released is skipped, and an extension whose content changed at a version that already has a +# release fails the job, so bumping manifest.version is how you publish a new version to the extension library. # # This file ships with the dev-kit. After scripts/init-project.sh makes the repo yours, set these repo secrets # (same as Extension CI): DUPLO_HOST + DUPLO_TOKEN. The build fetches the host SDK feed from DUPLO_HOST, so it -# must be reachable from CI (a public DuploCloud, not localhost). +# must be reachable from CI (a public DuploCloud, not localhost). Turn on immutable releases in the repository's +# settings, too; the publish step warns on every release that lands mutable. # # NOTE: the hot-load/deploy step is intentionally COMMENTED OUT (see below). We publish releases but do not # auto-install onto the portal, so the install-from-registry flow stays testable end to end. Uncomment the @@ -30,6 +32,10 @@ jobs: DUPLO_TOKEN: ${{ secrets.DUPLO_TOKEN }} steps: - uses: actions/checkout@v7 + with: + # Every tag and the history it points at: the publish step compares each released version's commit with + # this one to tell an unchanged extension from one that changed without a version bump. + fetch-depth: 0 - name: Discover extensions id: guard @@ -39,15 +45,7 @@ jobs: if [ ${#dirs[@]} -gt 0 ]; then echo "build=true" >> "$GITHUB_OUTPUT" else echo "build=false" >> "$GITHUB_OUTPUT"; echo "::notice::No extensions/*/manifest.json — nothing to release."; fi - - uses: actions/setup-dotnet@v6 - if: steps.guard.outputs.build == 'true' - with: - dotnet-version: '8.0.x' - - uses: actions/setup-node@v7 - if: steps.guard.outputs.build == 'true' - with: - node-version: '22' - + # Runs in the builder container (scripts/_builder.sh) on the runner's Docker, with the digest-pinned toolchain. - name: Build all extensions if: steps.guard.outputs.build == 'true' run: ./scripts/build-all.sh @@ -58,35 +56,8 @@ jobs: # if: steps.guard.outputs.build == 'true' # run: ./scripts/deploy-all.sh - - name: Publish GitHub releases (only for new manifest.version) + - name: Publish GitHub releases (one per new build) if: steps.guard.outputs.build == 'true' env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - shopt -s nullglob - published=0; skipped=0 - for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do - [ -f "$m" ] || continue - dir="$(dirname "$m")" - zip="$dir/dist/extension.zip" - if [ ! -f "$zip" ]; then - echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping." - continue - fi - name="$(jq -r '.name // (.id)' "$m")" - version="$(jq -r '.version' "$m")" - slug="$(basename "$dir")" - tag="${slug}-v${version}" - if gh release view "$tag" >/dev/null 2>&1; then - echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." - skipped=$((skipped+1)) - continue - fi - echo "==> Publishing release $tag ($name $version)" - gh release create "$tag" "$zip" \ - --title "$name $version" \ - --notes "Automated release of $name v$version from $slug (extension.zip attached)." - published=$((published+1)) - done - echo "==> Released $published, skipped $skipped (already-published versions)." + run: ./scripts/release-extensions.sh diff --git a/docker-compose.yml b/docker-compose.yml index b3e32fd..9b2574c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -170,7 +170,9 @@ services: # below). Both routes are live; the compose network is simply the preferred one. builder: profiles: [tools] - image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder:${BUILDER_TAG:-latest}} + # scripts/_builder.sh resolves BUILDER_TAG and exports BUILDER_IMAGE before every build. This default is only + # for a hand-run `docker compose run builder`, and must match _BUILDER_PINNED_IMAGE there. + image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd} # Unset by default, so the build runs on the host's native architecture. Set BUILDER_PLATFORM only # to force one (e.g. linux/amd64 if a publish ever ends up amd64-only) — an unset value is dropped # by compose, whereas copying duplo-ai-studio's linux/amd64 default would silently emulate a .NET diff --git a/scripts/_builder.sh b/scripts/_builder.sh index 06efb4d..7665ef2 100644 --- a/scripts/_builder.sh +++ b/scripts/_builder.sh @@ -160,6 +160,13 @@ builder_keep_local_copy() { # builder_keep_local_copy echo " docker pull said: $(printf '%s' "${1-}" | tail -1)" >&2 } +# The toolchain image a build uses when neither BUILDER_IMAGE nor BUILDER_TAG is set, pinned by digest so two +# builds of one extension version compile with the same toolchain. `:latest` moves on every toolchain publish, +# which would let two builds of one version differ. The digest is the multi-arch index, so it resolves on amd64 +# and arm64 alike. .github/workflows/builder-image.yml prints the new digest after each publish. Moving this pin +# (and docker-compose.yml's matching default) is a reviewed change, since it changes what every extension builds with. +_BUILDER_PINNED_IMAGE="quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd" + builder_resolve_image() { # The raw value is kept for the error message: BUILDER_PULL may have come from .env rather than the # environment, and naming a value the user cannot see in their shell is not a diagnostic. @@ -184,17 +191,22 @@ builder_resolve_image() { return 1 fi - local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")"; tag="${tag:-latest}" - # Docker's own tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is - # a pull that fails for an unobvious reason and a silent fall-through to building something else. - case "$tag" in - *[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* ) - echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2 - echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2 - echo " Windows line endings." >&2 - return 1 ;; - esac - BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag"; export BUILDER_IMAGE + local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")" + if [ -z "$tag" ]; then + BUILDER_IMAGE="$_BUILDER_PINNED_IMAGE" + else + # Docker's own tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is + # a pull that fails for an unobvious reason and a silent fall-through to building something else. + case "$tag" in + *[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* ) + echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2 + echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2 + echo " Windows line endings." >&2 + return 1 ;; + esac + BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag" + fi + export BUILDER_IMAGE if [ "$force_pull" != 1 ] && docker image inspect "$BUILDER_IMAGE" >/dev/null 2>&1; then return 0; fi echo "==> Fetching the build toolchain image ($BUILDER_IMAGE)" >&2 diff --git a/scripts/_sdk_digest.sh b/scripts/_sdk_digest.sh new file mode 100644 index 0000000..86b68ea --- /dev/null +++ b/scripts/_sdk_digest.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash +# Sourced by scripts/build-extension.sh — fingerprints the SDK feed a bundle was compiled against. +# +# sdkVersion alone does not identify an SDK: the host's SDK surface and dependency floors can move under an +# unchanged version, so a bundle that fails at run time could be a broken extension or one built against a drifted +# SDK. build-extension.sh stamps this digest beside sdkVersion so the two can be told apart by recomputing it over +# the host's own sdk-bundle. +# +# Definition (anything recomputing it must match this exactly): for every file in the extracted SDK bundle, the line +# " \n"; the lines sorted bytewise by path; sdkDigest is the lowercase +# sha256 hex of their concatenation. That is `sha256sum` output over the sorted file list, hashed once more. + +# _sdk_sha256 [file] -> lowercase sha256 hex of the file (or stdin). sha256sum on Linux, shasum on macOS. +_sdk_sha256() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@" | cut -d' ' -f1 + else shasum -a 256 "$@" | cut -d' ' -f1; fi +} + +# sdk_digest -> sdkDigest on stdout. Fails on a directory with no files. +sdk_digest() { + local root="${1:?usage: sdk_digest }" files + files="$(cd "$root" && find . -type f | sed 's|^\./||' | LC_ALL=C sort)" + [ -n "$files" ] || { echo "ERROR: no files under $root to digest" >&2; return 1; } + local f + while IFS= read -r f; do + printf '%s %s\n' "$(_sdk_sha256 "$root/$f")" "$f" + done <<<"$files" | _sdk_sha256 +} diff --git a/scripts/build-extension.sh b/scripts/build-extension.sh index da36b75..fc9c435 100755 --- a/scripts/build-extension.sh +++ b/scripts/build-extension.sh @@ -24,6 +24,8 @@ DIR="${DIR%/}" source "$(dirname "$0")/_target.sh" # → BASE_URL (+ TOKEN) from .env / env per DUPLO_TARGET # shellcheck source=scripts/_builder.sh source "$(dirname "$0")/_builder.sh" # → native, or re-launch this build in the builder container +# shellcheck source=scripts/_sdk_digest.sh +source "$(dirname "$0")/_sdk_digest.sh" # → sdk_digest, stamped into the bundle manifest beside sdkVersion BASE_URL="${BASE_URL%/}" # tolerate a trailing slash in DUPLO_HOST (avoids // in URLs) [ -f "$DIR/manifest.json" ] || { echo "No manifest.json in $DIR" >&2; exit 1; } @@ -153,7 +155,13 @@ curl -fsSL -m 120 "${auth[@]}" "$BASE_URL/v1/aiservicedesk/extensions/sdk-bundle || { echo "ERROR: failed to download the SDK bundle from $BASE_URL/v1/aiservicedesk/extensions/sdk-bundle" >&2; exit 1; } unzip -oq "$DIR/dist-sdk.zip" -d "$DIR/backend/sdk-packages" \ || { echo "ERROR: SDK bundle is not a valid zip — got $(file -b "$DIR/dist-sdk.zip" 2>/dev/null). Check DUPLO_HOST." >&2; exit 1; } -rm -f "$DIR/dist-sdk.zip" +# Digest a fresh extraction, not sdk-packages/: that directory accumulates packages from earlier builds, and the +# digest must describe only the SDK this build compiled against. See scripts/_sdk_digest.sh for the definition. +SDK_FEED="$(mktemp -d)" +unzip -oq "$DIR/dist-sdk.zip" -d "$SDK_FEED" +SDK_DIGEST="$(sdk_digest "$SDK_FEED")" || { rm -rf "$SDK_FEED"; echo "ERROR: the SDK bundle from $BASE_URL is empty." >&2; exit 1; } +rm -rf "$SDK_FEED" "$DIR/dist-sdk.zip" +echo " host SDK digest: $SDK_DIGEST" CSPROJ=$(find "$DIR/backend" -maxdepth 1 -name '*.csproj' | head -1) [ -n "$CSPROJ" ] || { echo "No .csproj under $DIR/backend" >&2; exit 1; } @@ -214,8 +222,8 @@ echo " host provides $(wc -l < "$PROVIDED" | tr -d ' ') assemblies (excluded echo "==> Assembling bundle" PKG="$DIR/dist/pkg" rm -rf "$PKG"; mkdir -p "$PKG/backend" "$PKG/fe" "$PKG/skills" -# Bundle manifest with the real host SDK version pinned in. -jq --arg v "$SDK_VER" '.sdkVersion = $v' "$DIR/manifest.json" > "$PKG/manifest.json" +# Bundle manifest with the real host SDK version pinned in, and the digest of the SDK feed it was compiled against. +jq --arg v "$SDK_VER" --arg d "$SDK_DIGEST" '.sdkVersion = $v | .sdkDigest = $d' "$DIR/manifest.json" > "$PKG/manifest.json" # Ship only backend assemblies the host does NOT already provide, plus each kept DLL's sidecars. Native # runtimes/ are host-provided too (Mongo/AWS natives load via the host's Default-ALC copies), so skip them. kept=0; dropped=0 diff --git a/scripts/check-extension-pr.sh b/scripts/check-extension-pr.sh new file mode 100755 index 0000000..1715e0d --- /dev/null +++ b/scripts/check-extension-pr.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Warn on a pull request about what each changed extension's release will need — the PR step of +# .github/workflows/extension-ci.yml. Warnings only, never a failure: authors should learn what shipping needs as +# early as possible without being blocked mid-development. scripts/release-extensions.sh is what refuses a release +# that cannot ship. +# +# For each extension the PR changes (anything under its directory since the merge base): +# - manifest.version is unchanged, so the release job will refuse to publish the new content at the old version. +# - a directory under skills/ has no matching skills[].folder in the manifest, so the host never registers it. +# - the built bundle's frontend is webpack (fe/remoteEntry.js, no fe/remoteEntry.json), whose pages do not open on +# hosts that load Native Federation remotes. Checked only when dist/extension.zip exists, i.e. after a build. +# +# Usage: ./scripts/check-extension-pr.sh # e.g. origin/main; defaults to origin/$GITHUB_BASE_REF +set -euo pipefail +cd "$(dirname "$0")/.." +shopt -s nullglob + +base_ref="${1:-origin/${GITHUB_BASE_REF:?usage: check-extension-pr.sh }}" +base="$(git merge-base "$base_ref" HEAD)" +warned=0 +warn() { echo "::warning file=$1::$2"; warned=$((warned+1)); } + +for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do + [ -f "$m" ] || continue + dir="$(dirname "$m")" + git diff --quiet "$base" HEAD -- "$dir" && continue + + version="$(jq -r '.version' "$m")" + base_version="$(git show "$base:$m" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true)" + if [ -n "$base_version" ] && [ "$base_version" = "$version" ]; then + warn "$m" "$dir changed but manifest.version is still $version. Bump it, or the release job refuses to publish this content." + fi + + for s in "$dir"/skills/*/; do + s="$(basename "$s")" + jq -e --arg s "$s" 'any(.skills[]?.folder // ""; endswith("/skills/" + $s))' "$m" >/dev/null \ + || warn "$m" "$dir ships skills/$s but the manifest declares no skills[].folder for it, so the host never registers it." + done + + zip="$dir/dist/extension.zip" + if [ -f "$zip" ]; then + entries="$(unzip -Z1 "$zip")" + if grep -qx 'fe/remoteEntry.js' <<<"$entries" && ! grep -qx 'fe/remoteEntry.json' <<<"$entries"; then + warn "$m" "$dir builds a webpack frontend (fe/remoteEntry.js). Its pages do not open on hosts that load Native Federation remotes. Migrate with the duplo-extension-ng22-migration skill." + fi + fi +done + +echo "==> $warned release warning(s)." diff --git a/scripts/release-extensions.sh b/scripts/release-extensions.sh new file mode 100755 index 0000000..c55f613 --- /dev/null +++ b/scripts/release-extensions.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Publish every built extension in this repo as a GitHub Release — the publish step of .github/workflows/release.yml, +# kept as a script so tests/test-release-extensions.sh can exercise it. +# +# One release per build, tagged -v-sdk-. is the extension directory's basename, and +# sdkVersion is read from the BUILT bundle's manifest (build-extension.sh stamps it from the host it built against), +# so builds of one version for different SDKs share the version and differ only in the tag. +# +# A version that already has a release, under that tag shape or the legacy -v, is compared with the +# built commit. If the extension is unchanged, an existing build for this SDK is skipped and a missing one is +# published. If it changed, the extension fails rather than skipping green: two different bundles would otherwise +# share one version, and whatever consumes the release could not tell them apart. Legacy tags were cut at main's HEAD +# at publish time rather than at the built commit, so one can report a change that a version bump then clears. +# +# Needs the repo's tags and the history they point at (actions/checkout with fetch-depth: 0), gh authenticated with +# contents: write, jq and unzip. GITHUB_SHA is the built commit; it defaults to HEAD. +# +# Usage: ./scripts/release-extensions.sh +set -euo pipefail +cd "$(dirname "$0")/.." +shopt -s nullglob + +sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" +published=0; skipped=0; failed=0 + +for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do + [ -f "$m" ] || continue + dir="$(dirname "$m")" + zip="$dir/dist/extension.zip" + if [ ! -f "$zip" ]; then + echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping." + continue + fi + name="$(jq -r '.name // .id' "$m")" + version="$(jq -r '.version' "$m")" + slug="$(basename "$dir")" + sdk="$(unzip -p "$zip" manifest.json | jq -r '.sdkVersion // empty')" + if [ -z "$sdk" ]; then + echo "::error::$dir — the bundle's manifest.json carries no sdkVersion, so its release cannot be tagged. Build it with scripts/build-extension.sh." + failed=$((failed+1)); continue + fi + base="${slug}-v${version}" + tag="${base}-sdk-${sdk}" + + changed_since="" + while IFS= read -r t; do + [ -n "$t" ] || continue + git diff --quiet "$t^{commit}" "$sha" -- "$dir" || { changed_since="$t"; break; } + done < <(git tag -l "$base" "$base-sdk-*") + if [ -n "$changed_since" ]; then + echo "::error::$dir — $version is already released as $changed_since and the extension changed since. Bump manifest.version to publish it." + failed=$((failed+1)); continue + fi + + if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." + skipped=$((skipped+1)); continue + fi + + echo "==> Publishing release $tag ($name $version, SDK $sdk)" + if ! gh release create "$tag" "$zip" --target "$sha" \ + --title "$name $version (SDK $sdk)" \ + --notes "Automated release of $name v$version from $slug, built against host SDK $sdk (extension.zip attached)."; then + echo "::error::$dir — gh release create $tag failed." + failed=$((failed+1)); continue + fi + published=$((published+1)) + + # An asset replaced at a mutable tag no longer matches a digest pinned from it, so say so where the owner sees it. + immutable="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.immutable' 2>/dev/null || true)" + if [ "$immutable" != true ]; then + echo "::warning::$tag landed mutable (immutable=${immutable:-unknown}). A repository admin turns on immutable releases under Settings → General → Releases; releases published before that stay mutable." + fi +done + +echo "==> Released $published, skipped $skipped, failed $failed." +[ "$failed" -eq 0 ] diff --git a/tests/test-builder-image.sh b/tests/test-builder-image.sh new file mode 100755 index 0000000..49ae4a4 --- /dev/null +++ b/tests/test-builder-image.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Tests for builder_resolve_image in scripts/_builder.sh — which toolchain image a build runs in. +# Uses a stub `docker` on PATH and a throwaway .env; never pulls. Usage: ./tests/test-builder-image.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +mkdir -p "$TMP/bin"; printf '#!/usr/bin/env bash\nexit 0\n' > "$TMP/bin/docker"; chmod +x "$TMP/bin/docker" +export PATH="$TMP/bin:$PATH" + +# Stand-in for _target.sh's .env reader, pointed at a temp file. +ENV="$TMP/.env"; : > "$ENV" +_envv() { grep -E "^$1=" "$ENV" 2>/dev/null | head -1 | cut -d= -f2- || true; } + +. ./scripts/_builder.sh || { echo "scripts/_builder.sh not found"; exit 1; } + +resolve() { ( unset BUILDER_IMAGE BUILDER_TAG BUILDER_PULL; eval "$1"; builder_resolve_image >/dev/null 2>&1; echo "$BUILDER_IMAGE" ); } + +echo "builder image:" + +t "defaults to the digest-pinned toolchain image" +: > "$ENV"; got="$(resolve :)" +if [[ "$got" =~ ^quay\.io/duplocloud/duplo-extension-builder@sha256:[0-9a-f]{64}$ ]]; then ok; else bad "$got"; fi + +t "docker-compose.yml falls back to the same pinned image" +: > "$ENV"; got="$(resolve :)" +if grep -qF -- "\${BUILDER_IMAGE:-$got}" docker-compose.yml; then ok; else bad "compose does not default to $got"; fi + +t "BUILDER_TAG in .env still selects a tag" +printf 'BUILDER_TAG=latest\n' > "$ENV"; got="$(resolve :)" +if [ "$got" = "quay.io/duplocloud/duplo-extension-builder:latest" ]; then ok; else bad "$got"; fi + +t "BUILDER_IMAGE in the environment still replaces the image outright" +: > "$ENV"; got="$(resolve 'export BUILDER_IMAGE=example.com/mine:1')" +if [ "$got" = "example.com/mine:1" ]; then ok; else bad "$got"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-check-extension-pr.sh b/tests/test-check-extension-pr.sh new file mode 100755 index 0000000..3af736a --- /dev/null +++ b/tests/test-check-extension-pr.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Tests for scripts/check-extension-pr.sh — the PR-time warnings about what a release will need. +# Runs in a throwaway git repo; never touches GitHub. Usage: ./tests/test-check-extension-pr.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +ROOT="$PWD" + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + +# new_repo: extensions/demo at 0.1.0 declaring skill `provision-demo`, committed on main, then a `pr` branch. +new_repo() { + REPO="$TMP/repo.$RANDOM"; D="$REPO/extensions/demo"; mkdir -p "$D/skills/provision-demo" + git -C "$REPO" init -q -b main + git -C "$REPO" config user.email t@example.com; git -C "$REPO" config user.name t + set_manifest 0.1.0 '["duplo.demo/0.1.0/skills/provision-demo"]' + echo skill > "$D/skills/provision-demo/SKILL.md"; echo one > "$D/code.txt" + git -C "$REPO" add -A; git -C "$REPO" commit -qm init + git -C "$REPO" switch -q -c pr +} + +# set_manifest +set_manifest() { + jq -n --arg v "$1" --argjson f "$2" '{id:"duplo.demo",name:"Demo",version:$v,skills:[$f[]|{folder:.,isBuiltIn:true}]}' \ + > "$D/manifest.json" +} + +change() { echo "$1" > "$D/code.txt"; git -C "$REPO" commit -qam "$1"; } + +# bundle : the dist/extension.zip a build leaves, with its frontend entry under fe/. +bundle() { + local p; p="$(mktemp -d)"; mkdir -p "$p/fe"; cp "$D/manifest.json" "$p/"; echo x > "$p/fe/$1" + mkdir -p "$D/dist"; rm -f "$D/dist/extension.zip"; ( cd "$p" && zip -qr "$D/dist/extension.zip" . ); rm -rf "$p" +} + +run() { + mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/check-extension-pr.sh" "$REPO/scripts/" 2>/dev/null + "$REPO/scripts/check-extension-pr.sh" main 2>&1 +} + +echo "check-extension-pr:" + +t "warns when an extension changed with no manifest.version bump" +new_repo; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*extensions/demo.*manifest.version" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about the version when it was bumped" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about an extension the PR does not touch" +new_repo; echo readme > "$REPO/README.md"; git -C "$REPO" add -A; git -C "$REPO" commit -qm readme +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about the version of an extension new in this PR" +new_repo; git -C "$REPO" switch -q main; git -C "$REPO" rm -rq extensions; git -C "$REPO" commit -qm empty +git -C "$REPO" switch -q -c pr2; git -C "$REPO" checkout -q pr -- extensions; git -C "$REPO" commit -qm add +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a shipped skill the manifest does not declare" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]' +mkdir -p "$D/skills/undeclared"; echo s > "$D/skills/undeclared/SKILL.md"; git -C "$REPO" add -A; git -C "$REPO" commit -qm skill +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*skills/undeclared" <<<"$OUT" && ! grep -q "provision-demo" <<<"$OUT" +then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a webpack frontend (fe/remoteEntry.js) in the built bundle" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.js +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*remoteEntry.js" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about a Native Federation frontend (fe/remoteEntry.json)" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.json +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh new file mode 100755 index 0000000..26bbd0d --- /dev/null +++ b/tests/test-release-extensions.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Tests for scripts/release-extensions.sh — the release job's publish step. +# Runs in a throwaway git repo with a stub `gh` on PATH; never touches GitHub. Usage: ./tests/test-release-extensions.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +ROOT="$PWD" + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + +# Stub gh: every call is appended to $GH_LOG. `api` answers the release's immutable flag from $GH_IMMUTABLE. +mkdir -p "$TMP/bin" +cat > "$TMP/bin/gh" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$GH_LOG" +case "$1" in + api) printf '%s\n' "${GH_IMMUTABLE:-true}" ;; + release) [ "$2" = create ] && exit "${GH_CREATE_RC:-0}" ;; +esac +exit 0 +EOF +chmod +x "$TMP/bin/gh" +export PATH="$TMP/bin:$PATH" + +# new_repo: a git repo holding extensions/demo at version 0.1.0, committed. +new_repo() { + REPO="$TMP/repo.$RANDOM"; mkdir -p "$REPO/extensions/demo" + git -C "$REPO" init -q -b main + git -C "$REPO" config user.email t@example.com; git -C "$REPO" config user.name t + printf '{"id":"duplo.demo","name":"Demo","version":"%s"}\n' "${1:-0.1.0}" > "$REPO/extensions/demo/manifest.json" + echo one > "$REPO/extensions/demo/code.txt" + git -C "$REPO" add -A; git -C "$REPO" commit -qm init + export GH_LOG="$REPO.gh.log"; : > "$GH_LOG" +} + +# build_zip [sdkVersion]: the dist/extension.zip build-extension.sh would leave, with sdkVersion stamped. +# An empty argument leaves sdkVersion out of the bundle manifest. +build_zip() { + local d="$REPO/extensions/demo" pkg; pkg="$(mktemp -d)" + if [ -n "${1-}" ]; then jq --arg v "$1" '.sdkVersion = $v' "$d/manifest.json" > "$pkg/manifest.json" + else cp "$d/manifest.json" "$pkg/manifest.json"; fi + mkdir -p "$d/dist"; rm -f "$d/dist/extension.zip" + ( cd "$pkg" && zip -qr "$d/dist/extension.zip" . ); rm -rf "$pkg" +} + +commit_change() { echo "$1" > "$REPO/extensions/demo/code.txt"; git -C "$REPO" commit -qam "$1"; } + +# The script resolves the repo from its own path, as it does when devkit copies it into an extension repo. +run() { + mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/release-extensions.sh" "$REPO/scripts/" 2>/dev/null + GITHUB_SHA="$(git -C "$REPO" rev-parse HEAD)" "$REPO/scripts/release-extensions.sh" 2>&1 +} + +echo "release-extensions:" + +t "publishes a new version under [slug]-v[version]-sdk-[sdkVersion], targeted at the built commit" +new_repo; build_zip 1.0.6; SHA="$(git -C "$REPO" rev-parse HEAD)" +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q -- "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip --target $SHA" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "skips when the -sdk tag exists and the extension is unchanged since it" +new_repo; build_zip 1.0.6; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "already released" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails, naming the version bump, when the extension changed since that version's -sdk release" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; commit_change two; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "manifest.version" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails when the extension changed since that version's legacy (no -sdk) release" +new_repo; git -C "$REPO" tag demo-v0.1.0; commit_change two; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "demo-v0.1.0 " <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "publishes the -sdk build of an unchanged version that has only a legacy release" +new_repo; git -C "$REPO" tag demo-v0.1.0; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "publishes a build for a new SDK of an unchanged version" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.7 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.7" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails when the bundle manifest carries no sdkVersion" +new_repo; build_zip "" +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "sdkVersion" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "warns when the published release landed mutable" +new_repo; build_zip 1.0.6 +OUT="$(GH_IMMUTABLE=false run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning::.*immutable" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "does not warn when the published release is immutable" +new_repo; build_zip 1.0.6 +OUT="$(GH_IMMUTABLE=true run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning::" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "fails when gh release create fails" +new_repo; build_zip 1.0.6 +OUT="$(GH_CREATE_RC=1 run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "still publishes the other extensions when one fails" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; commit_change two; build_zip 1.0.6 +mkdir -p "$REPO/extensions/other/dist" +printf '{"id":"duplo.other","name":"Other","version":"0.2.0"}\n' > "$REPO/extensions/other/manifest.json" +git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm other +( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" + cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "release create other-v0.2.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-sdk-digest.sh b/tests/test-sdk-digest.sh new file mode 100755 index 0000000..87a88b9 --- /dev/null +++ b/tests/test-sdk-digest.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tests for scripts/_sdk_digest.sh — the sdkDigest build-extension.sh stamps into a bundle's manifest. +# Usage: ./tests/test-sdk-digest.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +. ./scripts/_sdk_digest.sh || { echo "scripts/_sdk_digest.sh not found"; exit 1; } + +# The definition, computed independently: sha256 over " \n" per file, sorted by path bytewise. +expected() { + python3 - "$1" <<'PY' +import hashlib, os, sys +root = sys.argv[1]; lines = [] +for d, _, fs in os.walk(root): + for f in fs: + p = os.path.join(d, f); rel = os.path.relpath(p, root) + lines.append((rel.encode(), hashlib.sha256(open(p, "rb").read()).hexdigest())) +lines.sort() +print(hashlib.sha256(b"".join(h.encode() + b" " + r + b"\n" for r, h in lines)).hexdigest()) +PY +} + +mkfeed() { + FEED="$TMP/feed.$RANDOM"; mkdir -p "$FEED/sub" + echo a > "$FEED/Duplocloud.AiHelpdesk.Sdk.1.0.6.nupkg"; echo b > "$FEED/B.nupkg"; echo c > "$FEED/sub/c.nupkg" +} + +echo "sdk digest:" + +t "matches the definition: sha256 of sorted ' ' lines" +mkfeed +if [ "$(sdk_digest "$FEED")" = "$(expected "$FEED")" ]; then ok; else bad "$(sdk_digest "$FEED") != $(expected "$FEED")"; fi + +t "changes when one package's bytes change" +mkfeed; before="$(sdk_digest "$FEED")"; echo changed > "$FEED/B.nupkg" +if [ "$(sdk_digest "$FEED")" != "$before" ]; then ok; else bad "unchanged: $before"; fi + +t "changes when a package is added" +mkfeed; before="$(sdk_digest "$FEED")"; echo d > "$FEED/D.nupkg" +if [ "$(sdk_digest "$FEED")" != "$before" ]; then ok; else bad "unchanged: $before"; fi + +t "does not depend on where the feed sits or the caller's cwd" +mkfeed; a="$(sdk_digest "$FEED")"; cp -R "$FEED" "$TMP/moved"; b="$( cd / && sdk_digest "$TMP/moved" )" +if [ "$a" = "$b" ]; then ok; else bad "$a != $b"; fi + +t "fails on an empty feed rather than stamping the digest of nothing" +mkdir -p "$TMP/empty" +if ! sdk_digest "$TMP/empty" >/dev/null 2>&1; then ok; else bad "returned $(sdk_digest "$TMP/empty")"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ]