Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@ QDRANT_PORT=6333
QDRANT_API_KEY=devkit-local-qdrant

# Extension build toolchain (.NET SDK 8 + Node 22). Only pulled the first time you build an
# extension — the platform itself doesn't use it. An already-pulled tag is reused without checking the
# registry: BUILDER_PULL=1 forces a re-pull (this tag moves), BUILDER_IMAGE=<ref> replaces the image
# extension — the platform itself doesn't use it. The default is pinned by digest in scripts/_builder.sh,
# so every build of a version uses one toolchain. BUILDER_TAG=<tag> (e.g. latest) opts out of the pin; an
# already-pulled tag is reused without checking the registry, and BUILDER_PULL=1 forces a re-pull (tags
# move). BUILDER_IMAGE=<ref> replaces the image
# outright, BUILDER_PLATFORM=<os/arch> forces an architecture. Each build's package caches live in
# per-uid named volumes that nothing prunes — `docker volume ls | grep duplo_devkit` to find and
# `docker volume rm` to reclaim them (each holds a full NuGet + npm closure).
BUILDER_TAG=latest
# BUILDER_TAG=latest

# Build on THIS machine instead of in the toolchain container. Set to 1 if you already have .NET SDK 8
# and Node 22+ installed and would rather use them — you own keeping them current, and your builds stop
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/builder-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
username: ${{ secrets.QUAY_USERNAME }}
password: ${{ secrets.QUAY_PASSWORD }}
- uses: docker/build-push-action@v7
id: push
with:
# `build`, not `.`, to match how scripts/_builder.sh builds this same image locally: the
# Dockerfile COPYs nothing from the build context (only --from=node), so sending the whole
Expand All @@ -61,3 +62,16 @@ jobs:
tags: |
quay.io/duplocloud/duplo-extension-builder:latest
quay.io/duplocloud/duplo-extension-builder:${{ github.sha }}
# scripts/_builder.sh pins the default toolchain by digest, so a publish here changes nothing until that pin
# moves in a reviewed PR. Hand the reviewer the exact value.
- name: Report the digest to pin
env:
DIGEST: ${{ steps.push.outputs.digest }}
run: |
{
echo "### Builder image published"
echo "To build with it by default, set in scripts/_builder.sh and docker-compose.yml:"
echo '```'
echo "quay.io/duplocloud/duplo-extension-builder@${DIGEST}"
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
9 changes: 9 additions & 0 deletions .github/workflows/extension-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ jobs:
QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
steps:
- uses: actions/checkout@v7
with:
# The PR's base history, so the release checks below can diff each extension against the merge base.
fetch-depth: 0

- name: Discover extensions
id: guard
Expand Down Expand Up @@ -65,6 +68,12 @@ jobs:
if: steps.guard.outputs.build == 'true'
run: ./scripts/build-all.sh

# Warnings only: what each changed extension's release will need (version bump, declared skills, Native
# Federation frontend). !cancelled() so a failed build still reports the checks that need no bundle.
- name: Release checks
if: ${{ !cancelled() && github.event_name == 'pull_request' && steps.guard.outputs.build == 'true' }}
run: ./scripts/check-extension-pr.sh "origin/${{ github.base_ref }}"

- name: Upload bundles
if: steps.guard.outputs.build == 'true'
uses: actions/upload-artifact@v7
Expand Down
55 changes: 13 additions & 42 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
# Publishes each extension in this repo as a GitHub Release on push to main.
#
# For every extensions/<name>/ it builds the bundle and, ONLY when manifest.version is new (its
# <name>-v<version> tag doesn't exist yet), cuts a GitHub Release with extension.zip attached. Bumping
# manifest.version is therefore how you publish a new version to the extension library.
# For every extensions/<name>/ it builds the bundle and publishes one release per build, tagged
# <name>-v<version>-sdk-<sdkVersion> at the commit that was built. scripts/release-extensions.sh holds the rules:
# a build already released is skipped, and an extension whose content changed at a version that already has a
# release fails the job, so bumping manifest.version is how you publish a new version to the extension library.
#
# This file ships with the dev-kit. After scripts/init-project.sh makes the repo yours, set these repo secrets
# (same as Extension CI): DUPLO_HOST + DUPLO_TOKEN. The build fetches the host SDK feed from DUPLO_HOST, so it
# must be reachable from CI (a public DuploCloud, not localhost).
# must be reachable from CI (a public DuploCloud, not localhost). Turn on immutable releases in the repository's
# settings, too; the publish step warns on every release that lands mutable.
#
# NOTE: the hot-load/deploy step is intentionally COMMENTED OUT (see below). We publish releases but do not
# auto-install onto the portal, so the install-from-registry flow stays testable end to end. Uncomment the
Expand All @@ -30,6 +32,10 @@ jobs:
DUPLO_TOKEN: ${{ secrets.DUPLO_TOKEN }}
steps:
- uses: actions/checkout@v7
with:
# Every tag and the history it points at: the publish step compares each released version's commit with
# this one to tell an unchanged extension from one that changed without a version bump.
fetch-depth: 0

- name: Discover extensions
id: guard
Expand All @@ -39,15 +45,7 @@ jobs:
if [ ${#dirs[@]} -gt 0 ]; then echo "build=true" >> "$GITHUB_OUTPUT"
else echo "build=false" >> "$GITHUB_OUTPUT"; echo "::notice::No extensions/*/manifest.json — nothing to release."; fi

- uses: actions/setup-dotnet@v6
if: steps.guard.outputs.build == 'true'
with:
dotnet-version: '8.0.x'
- uses: actions/setup-node@v7
if: steps.guard.outputs.build == 'true'
with:
node-version: '22'

# Runs in the builder container (scripts/_builder.sh) on the runner's Docker, with the digest-pinned toolchain.
- name: Build all extensions
if: steps.guard.outputs.build == 'true'
run: ./scripts/build-all.sh
Expand All @@ -58,35 +56,8 @@ jobs:
# if: steps.guard.outputs.build == 'true'
# run: ./scripts/deploy-all.sh

- name: Publish GitHub releases (only for new manifest.version)
- name: Publish GitHub releases (one per new build)
if: steps.guard.outputs.build == 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
shopt -s nullglob
published=0; skipped=0
for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do
[ -f "$m" ] || continue
dir="$(dirname "$m")"
zip="$dir/dist/extension.zip"
if [ ! -f "$zip" ]; then
echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping."
continue
fi
name="$(jq -r '.name // (.id)' "$m")"
version="$(jq -r '.version' "$m")"
slug="$(basename "$dir")"
tag="${slug}-v${version}"
if gh release view "$tag" >/dev/null 2>&1; then
echo "==> $tag already released — skipping (bump manifest.version to publish a new one)."
skipped=$((skipped+1))
continue
fi
echo "==> Publishing release $tag ($name $version)"
gh release create "$tag" "$zip" \
--title "$name $version" \
--notes "Automated release of $name v$version from $slug (extension.zip attached)."
published=$((published+1))
done
echo "==> Released $published, skipped $skipped (already-published versions)."
run: ./scripts/release-extensions.sh
4 changes: 3 additions & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,9 @@ services:
# below). Both routes are live; the compose network is simply the preferred one.
builder:
profiles: [tools]
image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder:${BUILDER_TAG:-latest}}
# scripts/_builder.sh resolves BUILDER_TAG and exports BUILDER_IMAGE before every build. This default is only
# for a hand-run `docker compose run builder`, and must match _BUILDER_PINNED_IMAGE there.
image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd}
# Unset by default, so the build runs on the host's native architecture. Set BUILDER_PLATFORM only
# to force one (e.g. linux/amd64 if a publish ever ends up amd64-only) — an unset value is dropped
# by compose, whereas copying duplo-ai-studio's linux/amd64 default would silently emulate a .NET
Expand Down
34 changes: 23 additions & 11 deletions scripts/_builder.sh
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,13 @@ builder_keep_local_copy() { # builder_keep_local_copy <pull-error-text>
echo " docker pull said: $(printf '%s' "${1-}" | tail -1)" >&2
}

# The toolchain image a build uses when neither BUILDER_IMAGE nor BUILDER_TAG is set, pinned by digest so two
# builds of one extension version compile with the same toolchain. `:latest` moves on every toolchain publish,
# which would let two builds of one version differ. The digest is the multi-arch index, so it resolves on amd64
# and arm64 alike. .github/workflows/builder-image.yml prints the new digest after each publish. Moving this pin
# (and docker-compose.yml's matching default) is a reviewed change, since it changes what every extension builds with.
_BUILDER_PINNED_IMAGE="quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd"

builder_resolve_image() {
# The raw value is kept for the error message: BUILDER_PULL may have come from .env rather than the
# environment, and naming a value the user cannot see in their shell is not a diagnostic.
Expand All @@ -184,17 +191,22 @@ builder_resolve_image() {
return 1
fi

local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")"; tag="${tag:-latest}"
# Docker's own tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is
# a pull that fails for an unobvious reason and a silent fall-through to building something else.
case "$tag" in
*[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* )
echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2
echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2
echo " Windows line endings." >&2
return 1 ;;
esac
BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag"; export BUILDER_IMAGE
local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")"
if [ -z "$tag" ]; then
BUILDER_IMAGE="$_BUILDER_PINNED_IMAGE"
else
# Docker's own tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is
# a pull that fails for an unobvious reason and a silent fall-through to building something else.
case "$tag" in
*[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* )
echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2
echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2
echo " Windows line endings." >&2
return 1 ;;
esac
BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag"
fi
export BUILDER_IMAGE
if [ "$force_pull" != 1 ] && docker image inspect "$BUILDER_IMAGE" >/dev/null 2>&1; then return 0; fi

echo "==> Fetching the build toolchain image ($BUILDER_IMAGE)" >&2
Expand Down
28 changes: 28 additions & 0 deletions scripts/_sdk_digest.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# shellcheck shell=bash
# Sourced by scripts/build-extension.sh — fingerprints the SDK feed a bundle was compiled against.
#
# sdkVersion alone does not identify an SDK: the host's SDK surface and dependency floors can move under an
# unchanged version, so a bundle that fails at run time could be a broken extension or one built against a drifted
# SDK. build-extension.sh stamps this digest beside sdkVersion so the two can be told apart by recomputing it over
# the host's own sdk-bundle.
#
# Definition (anything recomputing it must match this exactly): for every file in the extracted SDK bundle, the line
# "<sha256 hex> <path relative to the bundle root>\n"; the lines sorted bytewise by path; sdkDigest is the lowercase
# sha256 hex of their concatenation. That is `sha256sum` output over the sorted file list, hashed once more.

# _sdk_sha256 [file] -> lowercase sha256 hex of the file (or stdin). sha256sum on Linux, shasum on macOS.
_sdk_sha256() {
if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@" | cut -d' ' -f1
else shasum -a 256 "$@" | cut -d' ' -f1; fi
}

# sdk_digest <extracted-sdk-bundle-dir> -> sdkDigest on stdout. Fails on a directory with no files.
sdk_digest() {
local root="${1:?usage: sdk_digest <dir>}" files
files="$(cd "$root" && find . -type f | sed 's|^\./||' | LC_ALL=C sort)"
[ -n "$files" ] || { echo "ERROR: no files under $root to digest" >&2; return 1; }
local f
while IFS= read -r f; do
printf '%s %s\n' "$(_sdk_sha256 "$root/$f")" "$f"
done <<<"$files" | _sdk_sha256
}
14 changes: 11 additions & 3 deletions scripts/build-extension.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ DIR="${DIR%/}"
source "$(dirname "$0")/_target.sh" # → BASE_URL (+ TOKEN) from .env / env per DUPLO_TARGET
# shellcheck source=scripts/_builder.sh
source "$(dirname "$0")/_builder.sh" # → native, or re-launch this build in the builder container
# shellcheck source=scripts/_sdk_digest.sh
source "$(dirname "$0")/_sdk_digest.sh" # → sdk_digest, stamped into the bundle manifest beside sdkVersion
BASE_URL="${BASE_URL%/}" # tolerate a trailing slash in DUPLO_HOST (avoids // in URLs)
[ -f "$DIR/manifest.json" ] || { echo "No manifest.json in $DIR" >&2; exit 1; }

Expand Down Expand Up @@ -153,7 +155,13 @@ curl -fsSL -m 120 "${auth[@]}" "$BASE_URL/v1/aiservicedesk/extensions/sdk-bundle
|| { echo "ERROR: failed to download the SDK bundle from $BASE_URL/v1/aiservicedesk/extensions/sdk-bundle" >&2; exit 1; }
unzip -oq "$DIR/dist-sdk.zip" -d "$DIR/backend/sdk-packages" \
|| { echo "ERROR: SDK bundle is not a valid zip — got $(file -b "$DIR/dist-sdk.zip" 2>/dev/null). Check DUPLO_HOST." >&2; exit 1; }
rm -f "$DIR/dist-sdk.zip"
# Digest a fresh extraction, not sdk-packages/: that directory accumulates packages from earlier builds, and the
# digest must describe only the SDK this build compiled against. See scripts/_sdk_digest.sh for the definition.
SDK_FEED="$(mktemp -d)"
unzip -oq "$DIR/dist-sdk.zip" -d "$SDK_FEED"
SDK_DIGEST="$(sdk_digest "$SDK_FEED")" || { rm -rf "$SDK_FEED"; echo "ERROR: the SDK bundle from $BASE_URL is empty." >&2; exit 1; }
rm -rf "$SDK_FEED" "$DIR/dist-sdk.zip"
echo " host SDK digest: $SDK_DIGEST"

CSPROJ=$(find "$DIR/backend" -maxdepth 1 -name '*.csproj' | head -1)
[ -n "$CSPROJ" ] || { echo "No .csproj under $DIR/backend" >&2; exit 1; }
Expand Down Expand Up @@ -214,8 +222,8 @@ echo " host provides $(wc -l < "$PROVIDED" | tr -d ' ') assemblies (excluded
echo "==> Assembling bundle"
PKG="$DIR/dist/pkg"
rm -rf "$PKG"; mkdir -p "$PKG/backend" "$PKG/fe" "$PKG/skills"
# Bundle manifest with the real host SDK version pinned in.
jq --arg v "$SDK_VER" '.sdkVersion = $v' "$DIR/manifest.json" > "$PKG/manifest.json"
# Bundle manifest with the real host SDK version pinned in, and the digest of the SDK feed it was compiled against.
jq --arg v "$SDK_VER" --arg d "$SDK_DIGEST" '.sdkVersion = $v | .sdkDigest = $d' "$DIR/manifest.json" > "$PKG/manifest.json"
# Ship only backend assemblies the host does NOT already provide, plus each kept DLL's sidecars. Native
# runtimes/ are host-provided too (Mongo/AWS natives load via the host's Default-ALC copies), so skip them.
kept=0; dropped=0
Expand Down
49 changes: 49 additions & 0 deletions scripts/check-extension-pr.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Warn on a pull request about what each changed extension's release will need — the PR step of
# .github/workflows/extension-ci.yml. Warnings only, never a failure: authors should learn what shipping needs as
# early as possible without being blocked mid-development. scripts/release-extensions.sh is what refuses a release
# that cannot ship.
#
# For each extension the PR changes (anything under its directory since the merge base):
# - manifest.version is unchanged, so the release job will refuse to publish the new content at the old version.
# - a directory under skills/ has no matching skills[].folder in the manifest, so the host never registers it.
# - the built bundle's frontend is webpack (fe/remoteEntry.js, no fe/remoteEntry.json), whose pages do not open on
# hosts that load Native Federation remotes. Checked only when dist/extension.zip exists, i.e. after a build.
#
# Usage: ./scripts/check-extension-pr.sh <base-ref> # e.g. origin/main; defaults to origin/$GITHUB_BASE_REF
set -euo pipefail
cd "$(dirname "$0")/.."
shopt -s nullglob

base_ref="${1:-origin/${GITHUB_BASE_REF:?usage: check-extension-pr.sh <base-ref>}}"
base="$(git merge-base "$base_ref" HEAD)"
warned=0
warn() { echo "::warning file=$1::$2"; warned=$((warned+1)); }

for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do
[ -f "$m" ] || continue
dir="$(dirname "$m")"
git diff --quiet "$base" HEAD -- "$dir" && continue

version="$(jq -r '.version' "$m")"
base_version="$(git show "$base:$m" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true)"
if [ -n "$base_version" ] && [ "$base_version" = "$version" ]; then
warn "$m" "$dir changed but manifest.version is still $version. Bump it, or the release job refuses to publish this content."
fi

for s in "$dir"/skills/*/; do
s="$(basename "$s")"
jq -e --arg s "$s" 'any(.skills[]?.folder // ""; endswith("/skills/" + $s))' "$m" >/dev/null \
|| warn "$m" "$dir ships skills/$s but the manifest declares no skills[].folder for it, so the host never registers it."
done

zip="$dir/dist/extension.zip"
if [ -f "$zip" ]; then
entries="$(unzip -Z1 "$zip")"
if grep -qx 'fe/remoteEntry.js' <<<"$entries" && ! grep -qx 'fe/remoteEntry.json' <<<"$entries"; then
warn "$m" "$dir builds a webpack frontend (fe/remoteEntry.js). Its pages do not open on hosts that load Native Federation remotes. Migrate with the duplo-extension-ng22-migration skill."
fi
fi
done

echo "==> $warned release warning(s)."
Loading
Loading