diff --git a/.env.example b/.env.example index 6eb4b14..add02c1 100644 --- a/.env.example +++ b/.env.example @@ -46,12 +46,15 @@ QDRANT_API_KEY=devkit-local-qdrant #RUNTIME=docker # Extension build toolchain (.NET SDK 8 + Node 22). Only pulled the first time you build an -# extension — the platform itself doesn't use it. An already-pulled tag is reused without checking the -# registry: BUILDER_PULL=1 forces a re-pull (this tag moves), BUILDER_IMAGE= replaces the image -# outright, BUILDER_PLATFORM= forces an architecture. Each build's package caches live in +# extension — the platform itself doesn't use it. The default is pinned by digest in scripts/_builder.sh, +# so every build of a version uses one toolchain. BUILDER_TAG= opts out of the pin for any tag but +# latest, which reads as unset; an already-pulled tag is reused without checking the registry, and +# BUILDER_PULL=1 forces a re-pull (tags move). BUILDER_IMAGE= replaces the image outright, so +# BUILDER_IMAGE=quay.io/duplocloud/duplo-extension-builder:latest builds on the moving tag. +# BUILDER_PLATFORM= forces an architecture. Each build's package caches live in # per-uid named volumes that nothing prunes — ` volume ls | grep duplo_devkit` to find and # ` volume rm` to reclaim them (each holds a full NuGet + npm closure). -BUILDER_TAG=latest +# BUILDER_TAG=1.2.3 # Build on THIS machine instead of in the toolchain container. Set to 1 if you already have .NET SDK 8 # and Node 22+ installed and would rather use them — you own keeping them current, and your builds stop diff --git a/.github/actions/extension-sign/action.yml b/.github/actions/extension-sign/action.yml new file mode 100644 index 0000000..c8a80ec --- /dev/null +++ b/.github/actions/extension-sign/action.yml @@ -0,0 +1,35 @@ +# Signs one built extension.zip into extension.zip.sig with a publisher key the DuploCloud console certifies. +# For pipelines outside devkit's release job, which calls scripts/extension-sign.py directly. +name: extension-sign +description: Sign an extension bundle for DuploCloud portals that require signed extensions +inputs: + bundle: + description: Path to the built extension.zip + required: true + signing-key: + description: Publisher private key PEM, from a secret + required: true + signing-cert: + description: Publisher certificate (compact JWS) from the console + required: true +outputs: + signature-path: + description: Path of the written .sig + value: ${{ steps.sign.outputs.path }} +runs: + using: composite + steps: + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + - shell: bash + run: pip install --require-hashes -r "$GITHUB_ACTION_PATH/../../../scripts/extension-sign.requirements.txt" + - id: sign + shell: bash + env: + EXTENSION_SIGNING_KEY: ${{ inputs.signing-key }} + EXTENSION_SIGNING_CERT: ${{ inputs.signing-cert }} + BUNDLE: ${{ inputs.bundle }} + run: | + python3 "$GITHUB_ACTION_PATH/../../../scripts/extension-sign.py" sign "$BUNDLE" + echo "path=$BUNDLE.sig" >> "$GITHUB_OUTPUT" diff --git a/.github/extension-publishers.json b/.github/extension-publishers.json new file mode 100644 index 0000000..6e961ab --- /dev/null +++ b/.github/extension-publishers.json @@ -0,0 +1,4 @@ +{ + "schemaVersion": 1, + "publishers": [] +} diff --git a/.github/workflows/builder-image.yml b/.github/workflows/builder-image.yml index c94faef..cfbbdb3 100644 --- a/.github/workflows/builder-image.yml +++ b/.github/workflows/builder-image.yml @@ -36,19 +36,20 @@ jobs: # re-dispatch the workflow. if: github.repository == 'duplocloud/devkit' steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 # QEMU emulates arm64 for the cross-build; buildx provides the multi-platform driver. - - uses: docker/setup-qemu-action@v4 - - uses: docker/setup-buildx-action@v4 + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4 # Keep this step and its `with:` block adjacent. A dependabot merge once reordered these two `uses:` # lines and left the credentials attached to setup-buildx-action, which silently ignores them — # login-action then ran with no inputs and every publish died on "Username and password required". - - uses: docker/login-action@v4 + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - - uses: docker/build-push-action@v7 + - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7 + id: push with: # `build`, not `.`, to match how scripts/_builder.sh builds this same image locally: the # Dockerfile COPYs nothing from the build context (only --from=node), so sending the whole @@ -61,3 +62,16 @@ jobs: tags: | quay.io/duplocloud/duplo-extension-builder:latest quay.io/duplocloud/duplo-extension-builder:${{ github.sha }} + # scripts/_builder.sh pins the default toolchain by digest, so a publish here changes nothing until that pin + # moves in a reviewed PR. Hand the reviewer the exact value. + - name: Report the digest to pin + env: + DIGEST: ${{ steps.push.outputs.digest }} + run: | + { + echo "### Builder image published" + echo "To build with it by default, set in scripts/_builder.sh and docker-compose.yml:" + echo '```' + echo "quay.io/duplocloud/duplo-extension-builder@${DIGEST}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/extension-ci.yml b/.github/workflows/extension-ci.yml index b441152..83f933b 100644 --- a/.github/workflows/extension-ci.yml +++ b/.github/workflows/extension-ci.yml @@ -35,7 +35,10 @@ jobs: # in an `if:` expression, but `env` is. A fork PR gets no org secrets, so this is empty there. QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # The PR's base history, so the release checks below can diff each extension against the merge base. + fetch-depth: 0 - name: Discover extensions id: guard @@ -55,7 +58,7 @@ jobs: # image build. See the env: block above for why this reads env rather than secrets. - name: Log in to quay.io (for the builder image) if: steps.guard.outputs.build == 'true' && env.QUAY_USERNAME != '' - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 with: registry: quay.io username: ${{ secrets.QUAY_USERNAME }} @@ -65,9 +68,15 @@ jobs: if: steps.guard.outputs.build == 'true' run: ./scripts/build-all.sh + # Warnings only: what each changed extension's release will need (version bump, declared skills, Native + # Federation frontend). !cancelled() so a failed build still reports the checks that need no bundle. + - name: Release checks + if: ${{ !cancelled() && github.event_name == 'pull_request' && steps.guard.outputs.build == 'true' }} + run: ./scripts/check-extension-pr.sh "origin/${{ github.base_ref }}" + - name: Upload bundles if: steps.guard.outputs.build == 'true' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: extension-bundles path: | diff --git a/.github/workflows/image-arch-check.yml b/.github/workflows/image-arch-check.yml index aec8060..c0a9d1d 100644 --- a/.github/workflows/image-arch-check.yml +++ b/.github/workflows/image-arch-check.yml @@ -30,6 +30,6 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Check duplocloud images publish amd64 and arm64 run: ./scripts/check-image-arches.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 824bfc5..2c132cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,16 +1,32 @@ # Publishes each extension in this repo as a GitHub Release on push to main. # -# For every extensions// it builds the bundle and, ONLY when manifest.version is new (its -# -v tag doesn't exist yet), cuts a GitHub Release with extension.zip attached. Bumping -# manifest.version is therefore how you publish a new version to the extension library. +# For every extensions// it builds the bundle and publishes one release per build, tagged +# -v-sdk- at the commit that was built. scripts/release-extensions.sh holds the rules: +# a build already released is skipped, and an extension whose content changed at a version that already has a +# release fails the job, so bumping manifest.version is how you publish a new version to the extension library. # # This file ships with the dev-kit. After scripts/init-project.sh makes the repo yours, set these repo secrets # (same as Extension CI): DUPLO_HOST + DUPLO_TOKEN. The build fetches the host SDK feed from DUPLO_HOST, so it -# must be reachable from CI (a public DuploCloud, not localhost). +# must be reachable from CI (a public DuploCloud, not localhost). Turn on immutable releases in the repository's +# settings, too; the publish step warns on every release that lands mutable. # # NOTE: the hot-load/deploy step is intentionally COMMENTED OUT (see below). We publish releases but do not # auto-install onto the portal, so the install-from-registry flow stays testable end to end. Uncomment the # "Deploy" step to auto-hot-load on merge. +# +# In Duplo's own organizations (duplocloud, duplocloud-internal) the build is also signed. scripts/extension-sign.py +# writes extension.zip.sig from EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT, mapped below from the org secret +# CONSOLE_SIGNING_KEY and the org variable CONSOLE_SIGNING_CERT, before release-extensions.sh creates the release. +# A missing key fails the job there before anything is written. Outside those organizations the release ships the +# zip alone, since a customer's copy of this workflow holds no Duplo signing credential. +# +# Two guards sit in front of the AWS role. "Check the publisher allowlist" runs only in Duplo's organizations, and +# marks the repository listed only when .github/extension-publishers.json names it as a publisher. The +# configure-aws-credentials step then runs only when that step reports listed=true, since EXTENSION_PUBLISHER_ROLE_ARN +# trusts only those repositories through GitHub OIDC (id-token: write). An unlisted repository's release still ships +# signed, but release-extensions.sh never calls publish_build for it, so the job never reaches S3 or the console and +# never needs AWS credentials. + name: Extension Release on: @@ -19,17 +35,29 @@ on: workflow_dispatch: permissions: - contents: write # required to create GitHub Releases + contents: write # create GitHub Releases + id-token: write # assume the extension-publisher role through GitHub OIDC + +concurrency: + # One release run per repository at a time, never canceled between its writes. + group: extension-release-${{ github.repository }} + cancel-in-progress: false jobs: release: + # A dispatch from any other branch skips the job, so only main's code ever releases, signs or publishes. + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest env: DUPLO_TARGET: remote DUPLO_HOST: ${{ secrets.DUPLO_HOST }} DUPLO_TOKEN: ${{ secrets.DUPLO_TOKEN }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + # Every tag and the history it points at: the publish step compares each released version's commit with + # this one to tell an unchanged extension from one that changed without a version bump. + fetch-depth: 0 - name: Discover extensions id: guard @@ -39,15 +67,7 @@ jobs: if [ ${#dirs[@]} -gt 0 ]; then echo "build=true" >> "$GITHUB_OUTPUT" else echo "build=false" >> "$GITHUB_OUTPUT"; echo "::notice::No extensions/*/manifest.json — nothing to release."; fi - - uses: actions/setup-dotnet@v6 - if: steps.guard.outputs.build == 'true' - with: - dotnet-version: '8.0.x' - - uses: actions/setup-node@v7 - if: steps.guard.outputs.build == 'true' - with: - node-version: '22' - + # Runs in the builder container (scripts/_builder.sh) on the runner's Docker, with the digest-pinned toolchain. - name: Build all extensions if: steps.guard.outputs.build == 'true' run: ./scripts/build-all.sh @@ -58,35 +78,37 @@ jobs: # if: steps.guard.outputs.build == 'true' # run: ./scripts/deploy-all.sh - - name: Publish GitHub releases (only for new manifest.version) + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + if: steps.guard.outputs.build == 'true' + with: + python-version: '3.12' + - name: Install the signer + if: steps.guard.outputs.build == 'true' + run: pip install --require-hashes -r scripts/extension-sign.requirements.txt + + # Upload and registration need the role. Only Duplo's organizations hold it, and only allowlisted repositories + # pass its trust, so the step is skipped everywhere else and release-extensions.sh never reaches S3 there. + - name: Check the publisher allowlist + id: allow + if: >- + steps.guard.outputs.build == 'true' && + (github.repository_owner == 'duplocloud' || github.repository_owner == 'duplocloud-internal') + run: | + jq -e --arg r "$GITHUB_REPOSITORY" 'any(.publishers[]; .repository == $r)' .github/extension-publishers.json >/dev/null \ + && echo "listed=true" >> "$GITHUB_OUTPUT" || echo "listed=false" >> "$GITHUB_OUTPUT" + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4 + if: steps.allow.outputs.listed == 'true' + with: + role-to-assume: ${{ vars.EXTENSION_PUBLISHER_ROLE_ARN }} + aws-region: us-west-2 + + - name: Publish GitHub releases (one per new build) if: steps.guard.outputs.build == 'true' env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - shopt -s nullglob - published=0; skipped=0 - for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do - [ -f "$m" ] || continue - dir="$(dirname "$m")" - zip="$dir/dist/extension.zip" - if [ ! -f "$zip" ]; then - echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping." - continue - fi - name="$(jq -r '.name // (.id)' "$m")" - version="$(jq -r '.version' "$m")" - slug="$(basename "$dir")" - tag="${slug}-v${version}" - if gh release view "$tag" >/dev/null 2>&1; then - echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." - skipped=$((skipped+1)) - continue - fi - echo "==> Publishing release $tag ($name $version)" - gh release create "$tag" "$zip" \ - --title "$name $version" \ - --notes "Automated release of $name v$version from $slug (extension.zip attached)." - published=$((published+1)) - done - echo "==> Released $published, skipped $skipped (already-published versions)." + EXTENSION_SIGNING_KEY: ${{ secrets.CONSOLE_SIGNING_KEY }} + EXTENSION_SIGNING_CERT: ${{ vars.CONSOLE_SIGNING_CERT }} + CONSOLE_API_KEY: ${{ secrets.CONSOLE_API_KEY }} + CONSOLE_URL: ${{ vars.CONSOLE_URL }} + BUNDLE_BUCKET: ${{ vars.BUNDLE_BUCKET }} + run: ./scripts/release-extensions.sh diff --git a/.gitignore b/.gitignore index 1912f6f..27b10a7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ *.zip !packages/*.tgz !.claude/skills/duplo-extension-dev/templates/helloworld/frontend/vendor/*.tgz +# The console's cross-language signing vectors (tests/extension_sign_vectors.py), not a build artifact. +!tests/vectors/*.zip # The Terraform extension's source, vendored by run.sh (scripts/fetch-terraform-extension.sh). # It is fetched, not authored here, so it does not belong in this repo's history — and without this diff --git a/docker-compose.yml b/docker-compose.yml index 893fd7f..9de14f2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -204,14 +204,14 @@ services: # below). Both routes are live; the compose network is simply the preferred one. builder: profiles: [tools] - # Deliberately ONE level of interpolation, not a nested ${BUILDER_IMAGE:-…:${BUILDER_TAG:-latest}}: + # Deliberately ONE level of interpolation, not a nested ${BUILDER_IMAGE:-…:${BUILDER_TAG:-…}}: # podman-compose 1.5 stops at the first '}' and appends the remainder literally, yielding # "…busybox:1.36}" and an "invalid reference format" error even when BUILDER_IMAGE is set. Nothing - # is lost — BUILDER_TAG is resolved into BUILDER_IMAGE by scripts/_builder.sh:builder_resolve_image, - # which always exports it before compose is invoked, so the default below is only ever reached by a - # hand-run `compose run` with no BUILDER_IMAGE set (where it now means :latest regardless of - # BUILDER_TAG — use the build scripts, or set BUILDER_IMAGE outright, to pin another tag). - image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder:latest} + # is lost — scripts/_builder.sh:builder_resolve_image resolves BUILDER_TAG and exports BUILDER_IMAGE + # before every build, so the default below is only ever reached by a hand-run `compose run` with no + # BUILDER_IMAGE set. It is the digest-pinned toolchain regardless of BUILDER_TAG, and must match + # _BUILDER_PINNED_IMAGE there (use the build scripts, or set BUILDER_IMAGE outright, to pick another). + image: ${BUILDER_IMAGE:-quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd} # Unset by default, so the build runs on the host's native architecture. Set BUILDER_PLATFORM only # to force one (e.g. linux/amd64 if a publish ever ends up amd64-only) — an unset value is dropped # by compose, whereas copying duplo-ai-studio's linux/amd64 default would silently emulate a .NET diff --git a/docs/README.md b/docs/README.md index 413e47a..286f55d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ reader takes. | [architecture.md](architecture.md) | How the studio, agent, UI, xterm, and Mongo fit together, and how the hot-load loop works | | [configuration.md](configuration.md) | Every `.env` variable — default, effect, and which are managed for you | | [cli-reference.md](cli-reference.md) | `run.sh`, `stop.sh`, `logs.sh`, and every script in `scripts/` — flags and behavior | +| [releasing-extensions.md](releasing-extensions.md) | What `release.yml` publishes, signing, and the extra steps inside DuploCloud's own organizations | | [upgrading.md](upgrading.md) | Taking framework updates, and how image-tag pinning works | | [troubleshooting.md](troubleshooting.md) | Symptoms, causes, fixes | | [faq.md](faq.md) | Licensing, data, platform support, and what happens on upgrade | diff --git a/docs/releasing-extensions.md b/docs/releasing-extensions.md new file mode 100644 index 0000000..4de056e --- /dev/null +++ b/docs/releasing-extensions.md @@ -0,0 +1,187 @@ +# Releasing extensions + +`.github/workflows/release.yml` runs on every push to `main` and on a manual dispatch. For each +`extensions/*/manifest.json`, plus the single-extension layouts `extension/manifest.json` and +`extension/*/manifest.json`, it builds the bundle and hands it to `scripts/release-extensions.sh`, which +publishes one GitHub Release per build. This page covers what that run does, in any organization, and the +extra steps that apply only inside DuploCloud's own organizations, `duplocloud` and `duplocloud-internal`. + +Only one release run executes for a repository at a time. A run that starts while another is still in +flight queues behind it rather than overlapping or being canceled mid-write. + +## What a release produces + +A build is tagged `-v-sdk-`, where `` is the extension directory's name +and `` comes from the built bundle's own `manifest.json` (stamped by `build-extension.sh` from +the host SDK it built against). The release is cut at the built commit with `extension.zip` attached. A +version already released, under this tag shape or the older `-v` one, is skipped if the +extension is unchanged since, and fails the run if it changed, since bumping `manifest.version` is the only +way to tell two different bundles apart. + +Two checks run before any of that, for every build in every organization. A manifest with no `resources` +entry is refused, since the host would refuse the bundle anyway, and so is a zip over 268435456 bytes (256 +MiB). A pull request that touches an extension sees the same checks ahead of time, as a warning rather than +a failure, from `scripts/check-extension-pr.sh`. That script also warns when `manifest.version` has not +moved, when a `skills/` folder has no matching `skills[].folder` entry in the manifest, and when a webpack +frontend bundle ships without the file that Native Federation hosts need. + +In `duplocloud` and `duplocloud-internal`, the release also carries `extension.zip.sig`, and, for an id the +allowlist maps to this repository, the build is uploaded to the channels bucket and registered as a new, +unpublished version with the license server. Nothing in this workflow ever publishes a version. A platform +owner does that separately, once the build has been verified. + +Outside those two organizations, a release ships the zip alone. A customer's copy of this workflow holds no +DuploCloud signing credential, so it signs nothing, uploads nothing, and registers nothing. + +## Signing a build + +Any repository, not only DuploCloud's own, can sign a build with `.github/actions/extension-sign`, a +composite action that wraps `scripts/extension-sign.py sign`. It takes three inputs: + +- `bundle`, the path to the built `extension.zip` +- `signing-key`, the publisher's private key PEM, from a secret +- `signing-cert`, the publisher's certificate, a compact JWS from the console + +It returns `signature-path`, the path of the `.sig` it wrote. The key and certificate reach the script +through the `EXTENSION_SIGNING_KEY` and `EXTENSION_SIGNING_CERT` environment variables, never as a command +argument, so neither shows up in a process listing. + +`sign` writes one compact ES256 JWS next to the zip, built from the zip's own manifest rather than the +source one, since `sdkVersion` only exists in the built copy. Its claims are the manifest's `id`, `version` +and `sdk_version`, the zip's own sha256, and the time it was signed. It refuses, writing nothing, when the +signing key's public half does not match the certificate, when the manifest id falls outside every +namespace the certificate covers, when the certificate has expired, or when `sdkVersion` is missing, still +carries its placeholder value, or either `version` or `sdkVersion` is not strict semver +(`MAJOR.MINOR.PATCH[-pre]`, no build metadata). It warns, but still signs, when the certificate expires +within 30 days. + +`verify [zip] [sig]` checks a signature the way the console does: the certificate's chain to a published +root key, the certificate's own validity window, the signature's claims against the zip and its manifest, +and the namespace. By default it fetches root keys from the console's public `root-keys` endpoint, which +needs no credential. A `--root-key` flag substitutes a local key for testing and says so in its output, so +that run is never mistaken for a check against the console's real roots. + +## In DuploCloud's organizations + +Signing is mandatory in `duplocloud` and `duplocloud-internal`. A repository there sets two secrets and two +organization variables: + +| Name | Kind | Holds | +| --- | --- | --- | +| `CONSOLE_SIGNING_KEY` | Secret, granted per repository | The publisher's private key PEM | +| `CONSOLE_API_KEY` | Secret, granted per repository | The console API key used to register versions and artifacts | +| `CONSOLE_SIGNING_CERT` | Organization variable | The publisher's certificate. The same value for every repository | +| `EXTENSION_PUBLISHER_ROLE_ARN` | Organization variable | The AWS role the workflow assumes, by name, to write to the channels bucket | + +The workflow maps `CONSOLE_SIGNING_KEY` onto `EXTENSION_SIGNING_KEY` and `CONSOLE_SIGNING_CERT` onto +`EXTENSION_SIGNING_CERT` before calling the signer. Each of the four fails the job before anything is +written. A missing signing key or certificate fails each extension before it is signed or released, rather +than shipping an unsigned build. An allowlisted extension with no `CONSOLE_API_KEY` fails the same way, +before it is signed or released, since that build could never be registered. A missing +`EXTENSION_PUBLISHER_ROLE_ARN` fails the credentials step, which runs before the publish step. Only +`CONSOLE_SIGNING_KEY` and `CONSOLE_SIGNING_CERT` matter to a repository the allowlist does not name. + +The role is only assumed for a repository the allowlist names, through GitHub's OIDC token +(`permissions: id-token: write`), so a repository that does not publish never needs AWS credentials at all. + +### One-time console setup + +A platform owner sets up each of these once in the console. Each one then serves every release. + +1. **The API key behind `CONSOLE_API_KEY`.** In the platform-owner team, create a custom role holding + **View extension versions** and **Manage extension versions**, create a service account with that role, + and store its API key as the organization secret. The release job reads and creates only versions and + artifacts, and the allowlist supplies each extension's uuid, so the key needs no other capability. The + console's own guide covers the service-account workflow in `docs/platform_owner/extension-versions.md`. +2. **The signing key behind `CONSOLE_SIGNING_KEY` and `CONSOLE_SIGNING_CERT`.** The publishing team needs a + namespace covering its manifest ids, and a signing key created under that team. The console returns the + key's private PEM once, at creation, and that goes into the secret. The key's certificate goes into the + organization variable. +3. **One extension record per extension.** Create it under the same publishing team, with a manifest id + that equals the `id` in the extension's `manifest.json`. The console accepts a signed artifact only when + the signature's manifest id matches the record and the signing key belongs to the record's team. The + console's API only reads extension records, so this step happens in the console, and the record's uuid is + what the allowlist entry names. + +### Joining the allowlist + +`.github/extension-publishers.json` ships empty in devkit. Each entry maps one manifest id to one +repository and one console extension: + +```json +{ + "schemaVersion": 1, + "publishers": [ + { "manifestId": "duplo.extensions.your-extension", "repository": "duplocloud/your-repo", "consoleExtension": "[console extension uuid]" } + ] +} +``` + +The `consoleExtension` uuid comes from the console's extensions page. Without an entry, or with one that +names a different repository, a build still signs and releases, since signing does not depend on the +allowlist, but the release job logs a notice and never uploads or registers it. + +The allowlist alone is not the security boundary. The AWS role's trust policy, held in ai-release's +Terraform (`terraform/modules/release-channels`), names the same repositories, and that trust is what +actually lets a workflow run write to the bucket. A test in ai-release keeps the two lists equal against a +pinned devkit commit, so joining the allowlist takes a PR in both repositories: the entry here, and the +repository added to the role's trust list there. Either alone leaves the id unable to publish, refused by +AWS on one side or skipped by the allowlist check on the other. + +The design calls for two approvals from the `devkit-maintainers` team on changes to this file, `release.yml`, +`release-extensions.sh` and the signer, though the ruleset enforcing that is not configured yet. + +## The channels bucket and the license server + +A signed build uploads to `s3://duplo-helpdesk-channels/bundles/[id]/[version]/sdk-[sdkVersion]/` in +`us-west-2`, as two objects, `extension.zip` and `extension.zip.sig`. Both the bucket name and the console +URL can be overridden with the `BUNDLE_BUCKET` and `CONSOLE_URL` repository variables. Left unset, they +default to the production bucket and the production console. + +A build that is already finished stops before any download of the zip or any S3 call. The script reads the +Release's digest for `extension.zip`, downloads only `extension.zip.sig`, and looks up the version and its +artifact for this SDK in the license server. An artifact whose path, hash and signature match is reported +as already published. Registration only ever follows both uploads, so a matching artifact means both objects +are already in the bucket. A failed lookup fails the job. Anything else takes the full path below. + +Otherwise, before uploading, the script downloads both assets from the GitHub Release itself, never +rebuilding them, and recomputes the zip's sha256 locally. When the Release reports its own digest for that +asset, the script compares the two and fails the job on a mismatch, so a corrupted or substituted download +is never uploaded. The locally computed hash, not GitHub's, is what gets registered. + +Every write from here on is create-only. The upload uses `aws s3api put-object --if-none-match '*'`, so it +never overwrites an existing key. When that fails with a 412, the script fetches what is already there and +compares bytes. Identical content counts as already uploaded and the run moves on. Different content fails +the job, since a published build is never replaced. + +Registering the version with the license server follows the same shape. The script looks up the version by +its number before creating it, and an error saying the record already exists just means a concurrent run +created it first, so it looks up again rather than failing. It registers the artifact for this SDK the same +way, by looking it up before creating it. An existing artifact whose path, hash and signature all match +counts as already done. One that differs fails the job. The script never marks a version published. Every +version it creates starts unpublished, and stays that way until a platform owner publishes it. + +## Certificate reissue and key rotation + +Reissuing the certificate alone, which a platform owner does as it nears expiry, changes only the +`CONSOLE_SIGNING_CERT` organization variable, to the new certificate from the console's signing-keys page. +`CONSOLE_SIGNING_KEY` does not change, since the underlying key is the same one the new certificate still +certifies. + +Rotating the key changes both. A new key needs a new certificate that names its public half, so +`CONSOLE_SIGNING_KEY` and `CONSOLE_SIGNING_CERT` have to move together. The signer checks that the key's +public half matches the certificate's before it signs anything, so updating one without the other fails +every release in DuploCloud's organizations until both are in place. + +## Re-running a release + +A tag that already has a release is never rebuilt, re-signed, or recreated. `release-extensions.sh` reads +that release's own assets and resumes from there, so a run that failed partway, during the upload or the +registration, can simply be re-run. Everything it already finished is a create-only no-op, and it picks up +where it stopped. + +A release that was cut before its repository started signing, or outside DuploCloud's organizations, has no +`.sig` asset to resume from. It cannot gain one afterward. Once immutable releases are turned on for a +repository, which this workflow recommends, nothing can be added to a release after it is cut. That release +stays flagged with a warning and is skipped rather than retried. The only way forward is a new +`manifest.version`, which cuts a new tag and a new release that does get signed. diff --git a/scripts/_builder.sh b/scripts/_builder.sh index aae8ea1..7939089 100644 --- a/scripts/_builder.sh +++ b/scripts/_builder.sh @@ -245,6 +245,13 @@ builder_keep_local_copy() { # builder_keep_local_copy echo " $RUNTIME pull said: $(printf '%s' "${1-}" | tail -1)" >&2 } +# The toolchain image a build uses when neither BUILDER_IMAGE nor BUILDER_TAG is set, pinned by digest so two +# builds of one extension version compile with the same toolchain. `:latest` moves on every toolchain publish, +# which would let two builds of one version differ. The digest is the multi-arch index, so it resolves on amd64 +# and arm64 alike. .github/workflows/builder-image.yml prints the new digest after each publish. Moving this pin +# (and docker-compose.yml's matching default) is a reviewed change, since it changes what every extension builds with. +_BUILDER_PINNED_IMAGE="quay.io/duplocloud/duplo-extension-builder@sha256:8185e6a74379ae99e145f55608b3bc17e3037e95e64a612a226fe0eb15c0dbcd" + builder_resolve_image() { # The raw value is kept for the error message: BUILDER_PULL may have come from .env rather than the # environment, and naming a value the user cannot see in their shell is not a diagnostic. @@ -269,17 +276,24 @@ builder_resolve_image() { return 1 fi - local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")"; tag="${tag:-latest}" - # OCI tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is - # a pull that fails for an unobvious reason and a silent fall-through to building something else. - case "$tag" in - *[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* ) - echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2 - echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2 - echo " Windows line endings." >&2 - return 1 ;; - esac - BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag"; export BUILDER_IMAGE + local tag; tag="$(builder_clean_value "$(_envv BUILDER_TAG)")" + # `latest` was the default before the pin, so older .env files still carry it. Reading it as unset keeps those + # repos on the pinned toolchain. BUILDER_IMAGE=...:latest still selects the moving tag on purpose. + if [ -z "$tag" ] || [ "$tag" = latest ]; then + BUILDER_IMAGE="$_BUILDER_PINNED_IMAGE" + else + # OCI tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127}. Checked here because the alternative is + # a pull that fails for an unobvious reason and a silent fall-through to building something else. + case "$tag" in + *[!A-Za-z0-9._-]* | [!A-Za-z0-9_]* ) + echo "ERROR: BUILDER_TAG is not a valid image tag: $(printf '%q' "$tag")" >&2 + echo " Tags are [A-Za-z0-9_][A-Za-z0-9._-]* — check .env for quotes, stray whitespace or" >&2 + echo " Windows line endings." >&2 + return 1 ;; + esac + BUILDER_IMAGE="quay.io/duplocloud/duplo-extension-builder:$tag" + fi + export BUILDER_IMAGE if [ "$force_pull" != 1 ] && "$RUNTIME" image inspect "$BUILDER_IMAGE" >/dev/null 2>&1; then return 0; fi echo "==> Fetching the build toolchain image ($BUILDER_IMAGE)" >&2 diff --git a/scripts/_publish.sh b/scripts/_publish.sh new file mode 100644 index 0000000..e2eae12 --- /dev/null +++ b/scripts/_publish.sh @@ -0,0 +1,118 @@ +# shellcheck shell=bash +# Sourced by scripts/release-extensions.sh. Signing is mandatory in Duplo's two organizations and skipped anywhere +# else, so a customer copy of the release workflow behaves as before with no Duplo credential. Upload and registration +# run only in Duplo's organizations, for a manifest id the allowlist maps to this repository. + +# duplo_org: exit 0 when this run belongs to one of Duplo's organizations. +duplo_org() { case "${GITHUB_REPOSITORY_OWNER:-}" in duplocloud|duplocloud-internal) return 0 ;; *) return 1 ;; esac; } + +bundle_bucket="${BUNDLE_BUCKET:-duplo-helpdesk-channels}" +console_url="${CONSOLE_URL:-https://console.duplocloud.com}" + +# s3_put_once : create-only upload. A 412 compares bytes, identical counting as already uploaded. +s3_put_once() { + local file=$1 key=$2 out have + if out="$(aws s3api put-object --bucket "$bundle_bucket" --key "$key" --body "$file" --if-none-match '*' 2>&1)"; then + echo "==> uploaded s3://$bundle_bucket/$key"; return 0 + fi + if ! grep -qE 'PreconditionFailed|\(412\)' <<<"$out"; then echo "::error::upload of $key failed: $out"; return 1; fi + have="$(mktemp)" + aws s3api get-object --bucket "$bundle_bucket" --key "$key" "$have" >/dev/null || { rm -f "$have"; return 1; } + if cmp -s "$file" "$have"; then rm -f "$have"; echo "==> $key already uploaded, identical bytes"; return 0; fi + rm -f "$have"; echo "::error::s3://$bundle_bucket/$key already holds different bytes. A published build is never replaced." + return 1 +} + +# console [json]: one console API call. The key reaches curl through a header file, never argv. +console() { + local hdr body + hdr="$(mktemp)"; chmod 600 "$hdr"; printf 'Authorization: Api-Key %s\n' "$CONSOLE_API_KEY" > "$hdr" + if [ -n "${3:-}" ]; then + body="$(curl -sS --fail-with-body -X "$1" -H @"$hdr" -H 'Content-Type: application/json' --data "$3" "$console_url$2")" + else + body="$(curl -sS --fail-with-body -X "$1" -H @"$hdr" "$console_url$2")" + fi + local rc=$?; rm -f "$hdr"; printf '%s' "$body"; return $rc +} + +# artifact_body : the artifact record publish_build registers, and compares an existing one to. +artifact_body() { + jq -n --arg p "s3://$bundle_bucket/$1" --arg s "$2" --arg h "$3" --rawfile g "$4" \ + '{sdk_version: $s, s3_path: $p, sha256: $h, signature: $g}' +} + +# artifact_matches : exit 0 when the lookup's first artifact has the same path, hash and signature. +artifact_matches() { + jq -e --argjson a "$2" '.[0] | .s3_path == $a.s3_path and (.sha256 | ascii_downcase) == $a.sha256 and .signature == $a.signature' \ + <<<"$1" >/dev/null +} + +# publish_build : upload the Release's zip and .sig, then register both. Reads the +# assets from the Release so a re-run reuses the bytes already signed and released, with no rebuild. +# +# A finished build stops early. Registration runs only after both uploads, so a registered artifact matching the +# Release's digest and .sig means both objects are already in the bucket, and the run needs neither the zip nor S3. +publish_build() { + local tag=$1 id=$2 version=$3 sdk=$4 v dl localsha sha key vuuid out art + for v in "$id" "$version" "$sdk"; do + [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || { echo "::error::$tag — $v cannot form a bucket key."; return 1; } + done + key="bundles/$id/$version/sdk-$sdk/extension.zip" + sha="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.assets[] | select(.name == "extension.zip") | .digest // empty')" \ + || { echo "::error::$tag — cannot read its release digest."; return 1; } + sha="${sha#sha256:}" + dl="$(mktemp -d)" + gh release download "$tag" -p extension.zip.sig -D "$dl" \ + || { echo "::error::$tag — cannot download its signature."; rm -rf "$dl"; return 1; } + + # Without a digest there is no hash to compare before the zip is downloaded, so the full path below runs. + if [ -n "$sha" ]; then + out="$(console GET "/api/extensions/$uuid/versions/?version=$version")" \ + || { echo "::error::$tag — console lookup of the version failed: $out"; rm -rf "$dl"; return 1; } + vuuid="$(jq -r '.[0].uuid // empty' <<<"$out")" + if [ -n "$vuuid" ]; then + out="$(console GET "/api/extensions/$uuid/versions/$vuuid/artifacts/?sdk_version=$sdk")" \ + || { echo "::error::$tag — console lookup of the artifact failed: $out"; rm -rf "$dl"; return 1; } + if artifact_matches "$out" "$(artifact_body "$key" "$sdk" "$sha" "$dl/extension.zip.sig")"; then + echo "==> $tag already published"; rm -rf "$dl"; return 0 + fi + fi + fi + + gh release download "$tag" -p extension.zip -D "$dl" \ + || { echo "::error::$tag — cannot download its assets."; rm -rf "$dl"; return 1; } + # The local hash is always computed and is what gets registered. GitHub's own digest, when the release carries + # one, only confirms the download matches what was actually released, before anything is uploaded or registered. + localsha="$(shasum -a 256 "$dl/extension.zip" | cut -d' ' -f1)" + if [ -n "$sha" ]; then + [ "$sha" = "$localsha" ] \ + || { echo "::error::$tag — release digest $sha does not match downloaded extension.zip hash $localsha."; rm -rf "$dl"; return 1; } + else + sha="$localsha" + fi + s3_put_once "$dl/extension.zip" "$key" && s3_put_once "$dl/extension.zip.sig" "$key.sig" || { rm -rf "$dl"; return 1; } + + out="$(console GET "/api/extensions/$uuid/versions/?version=$version")" \ + || { echo "::error::$tag — console lookup of the version failed: $out"; rm -rf "$dl"; return 1; } + vuuid="$(jq -r '.[0].uuid // empty' <<<"$out")" + if [ -z "$vuuid" ]; then + out="$(console POST "/api/extensions/$uuid/versions/" "$(jq -n --arg v "$version" --arg n "$id $version" '{version: $v, name: $n}')")" \ + || grep -q "already exists" <<<"$out" || { echo "::error::$tag — console refused the version: $out"; rm -rf "$dl"; return 1; } + out="$(console GET "/api/extensions/$uuid/versions/?version=$version")" \ + || { echo "::error::$tag — console lookup of the version failed: $out"; rm -rf "$dl"; return 1; } + vuuid="$(jq -r '.[0].uuid // empty' <<<"$out")" + fi + [ -n "$vuuid" ] || { echo "::error::$tag — the console has no version uuid for $version after creating it."; rm -rf "$dl"; return 1; } + art="$(artifact_body "$key" "$sdk" "$sha" "$dl/extension.zip.sig")" + out="$(console GET "/api/extensions/$uuid/versions/$vuuid/artifacts/?sdk_version=$sdk")" \ + || { echo "::error::$tag — console lookup of the artifact failed: $out"; rm -rf "$dl"; return 1; } + if [ "$(jq 'length' <<<"$out")" -gt 0 ]; then + if artifact_matches "$out" "$art"; then + echo "==> $tag already registered"; rm -rf "$dl"; return 0 + fi + echo "::error::$tag — the console already holds a build for SDK $sdk that differs from this release."; rm -rf "$dl"; return 1 + fi + out="$(console POST "/api/extensions/$uuid/versions/$vuuid/artifacts/" "$art")" \ + || { echo "::error::$tag — console refused the artifact: $out"; rm -rf "$dl"; return 1; } + echo "==> registered $tag in the license server (unpublished)"; rm -rf "$dl" +} diff --git a/scripts/_publishers.sh b/scripts/_publishers.sh new file mode 100644 index 0000000..1381e7d --- /dev/null +++ b/scripts/_publishers.sh @@ -0,0 +1,22 @@ +# shellcheck shell=bash +# Sourced by scripts/release-extensions.sh and scripts/check-extension-pr.sh. The allowlist names which repository may +# publish each Duplo extension and which console extension record it registers under. ai-release's extension-publisher +# role trusts the same repositories, which is the boundary that matters. This file decides only whether a run tries. +# +# .github/extension-publishers.json ships empty: an entry is added once its extension's console record exists, and +# its repository must already be in ai-release's extension-publisher role trust list. + +publishers_file=".github/extension-publishers.json" + +# publisher_extension_uuid : the console extension UUID when the allowlist maps the id to +# that repository, else nothing and exit 1. The lookup is captured rather than let `jq -e` report its own exit +# status, because that status is not 1 for every miss (no match, an empty list and a malformed file each land on a +# different jq exit code) and callers key off exit 1 specifically to mean "not allowed to publish." +publisher_extension_uuid() { + [ -f "$publishers_file" ] || return 1 + local uuid + uuid="$(jq -r --arg id "$1" --arg repo "$2" \ + '.publishers[] | select(.manifestId == $id and .repository == $repo) | .consoleExtension' "$publishers_file")" || true + [ -n "$uuid" ] || return 1 + printf '%s\n' "$uuid" +} diff --git a/scripts/_sdk_digest.sh b/scripts/_sdk_digest.sh new file mode 100644 index 0000000..86b68ea --- /dev/null +++ b/scripts/_sdk_digest.sh @@ -0,0 +1,28 @@ +# shellcheck shell=bash +# Sourced by scripts/build-extension.sh — fingerprints the SDK feed a bundle was compiled against. +# +# sdkVersion alone does not identify an SDK: the host's SDK surface and dependency floors can move under an +# unchanged version, so a bundle that fails at run time could be a broken extension or one built against a drifted +# SDK. build-extension.sh stamps this digest beside sdkVersion so the two can be told apart by recomputing it over +# the host's own sdk-bundle. +# +# Definition (anything recomputing it must match this exactly): for every file in the extracted SDK bundle, the line +# " \n"; the lines sorted bytewise by path; sdkDigest is the lowercase +# sha256 hex of their concatenation. That is `sha256sum` output over the sorted file list, hashed once more. + +# _sdk_sha256 [file] -> lowercase sha256 hex of the file (or stdin). sha256sum on Linux, shasum on macOS. +_sdk_sha256() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$@" | cut -d' ' -f1 + else shasum -a 256 "$@" | cut -d' ' -f1; fi +} + +# sdk_digest -> sdkDigest on stdout. Fails on a directory with no files. +sdk_digest() { + local root="${1:?usage: sdk_digest }" files + files="$(cd "$root" && find . -type f | sed 's|^\./||' | LC_ALL=C sort)" + [ -n "$files" ] || { echo "ERROR: no files under $root to digest" >&2; return 1; } + local f + while IFS= read -r f; do + printf '%s %s\n' "$(_sdk_sha256 "$root/$f")" "$f" + done <<<"$files" | _sdk_sha256 +} diff --git a/scripts/build-extension.sh b/scripts/build-extension.sh index e9fe60b..0aed5a5 100755 --- a/scripts/build-extension.sh +++ b/scripts/build-extension.sh @@ -24,6 +24,8 @@ DIR="${DIR%/}" source "$(dirname "$0")/_target.sh" # → BASE_URL (+ TOKEN) from .env / env per DUPLO_TARGET # shellcheck source=scripts/_builder.sh source "$(dirname "$0")/_builder.sh" # → native, or re-launch this build in the builder container +# shellcheck source=scripts/_sdk_digest.sh +source "$(dirname "$0")/_sdk_digest.sh" # → sdk_digest, stamped into the bundle manifest beside sdkVersion BASE_URL="${BASE_URL%/}" # tolerate a trailing slash in DUPLO_HOST (avoids // in URLs) [ -f "$DIR/manifest.json" ] || { echo "No manifest.json in $DIR" >&2; exit 1; } @@ -153,7 +155,13 @@ curl -fsSL -m 120 "${auth[@]}" "$BASE_URL/v1/aiservicedesk/extensions/sdk-bundle || { echo "ERROR: failed to download the SDK bundle from $BASE_URL/v1/aiservicedesk/extensions/sdk-bundle" >&2; exit 1; } unzip -oq "$DIR/dist-sdk.zip" -d "$DIR/backend/sdk-packages" \ || { echo "ERROR: SDK bundle is not a valid zip — got $(file -b "$DIR/dist-sdk.zip" 2>/dev/null). Check DUPLO_HOST." >&2; exit 1; } -rm -f "$DIR/dist-sdk.zip" +# Digest a fresh extraction, not sdk-packages/: that directory accumulates packages from earlier builds, and the +# digest must describe only the SDK this build compiled against. See scripts/_sdk_digest.sh for the definition. +SDK_FEED="$(mktemp -d)" +unzip -oq "$DIR/dist-sdk.zip" -d "$SDK_FEED" +SDK_DIGEST="$(sdk_digest "$SDK_FEED")" || { rm -rf "$SDK_FEED"; echo "ERROR: the SDK bundle from $BASE_URL is empty." >&2; exit 1; } +rm -rf "$SDK_FEED" "$DIR/dist-sdk.zip" +echo " host SDK digest: $SDK_DIGEST" CSPROJ=$(find "$DIR/backend" -maxdepth 1 -name '*.csproj' | head -1) [ -n "$CSPROJ" ] || { echo "No .csproj under $DIR/backend" >&2; exit 1; } @@ -214,9 +222,10 @@ echo " host provides $(wc -l < "$PROVIDED" | tr -d ' ') assemblies (excluded echo "==> Assembling bundle" PKG="$DIR/dist/pkg" rm -rf "$PKG"; mkdir -p "$PKG/backend" "$PKG/fe" "$PKG/skills" -# Bundle manifest with the real host SDK version pinned in, and both // paths derived from -# id+version. The host stages the bundle to /// but reads the entry assembly -# from backend.assemblyDir and each provisioning skill from skills[].folder, so all three have to agree. +# Bundle manifest with the real host SDK version and the digest of the SDK feed it was compiled against pinned +# in, and both // paths derived from id+version. The host stages the bundle to +# /// but reads the entry assembly from backend.assemblyDir and each +# provisioning skill from skills[].folder, so all three have to agree. # Maintained by hand they drift the moment you bump version alone, and what that breaks depends on the target: # # backend.assemblyDir a host that still has the old version on disk keeps running the OLD backend DLL and @@ -230,8 +239,9 @@ rm -rf "$PKG"; mkdir -p "$PKG/backend" "$PKG/fe" "$PKG/skills" # Both are always //…, so compute them here instead of trusting the checked-in values. The skill # tail varies per entry (skills/), so only the first two segments are replaced; a manifest with no # skills array is left alone rather than given an empty one. -jq --arg v "$SDK_VER" ' +jq --arg v "$SDK_VER" --arg d "$SDK_DIGEST" ' .sdkVersion = $v + | .sdkDigest = $d | .backend.assemblyDir = (.id + "/" + .version + "/backend") | .id as $i | .version as $r | if (.skills | type) == "array" then diff --git a/scripts/check-extension-pr.sh b/scripts/check-extension-pr.sh new file mode 100755 index 0000000..6c8ce0c --- /dev/null +++ b/scripts/check-extension-pr.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Warn on a pull request about what each changed extension's release will need — the PR step of +# .github/workflows/extension-ci.yml. Warnings only, never a failure: authors should learn what shipping needs as +# early as possible without being blocked mid-development. scripts/release-extensions.sh is what refuses a release +# that cannot ship. +# +# For each extension the PR changes (anything under its directory since the merge base): +# - manifest.version is unchanged, so the release job will refuse to publish the new content at the old version. +# - a directory under skills/ has no matching skills[].folder in the manifest, so the host never registers it. +# - the built bundle's frontend is webpack (fe/remoteEntry.js, no fe/remoteEntry.json), whose pages do not open on +# hosts that load Native Federation remotes. Checked only when dist/extension.zip exists, i.e. after a build. +# - the manifest id or version cannot form a bucket key, which scripts/_publish.sh's publish_build refuses. +# - the manifest declares no resources, which the host refuses to install. +# - the built bundle is over EXTENSION_MAX_BYTES bytes (default 268435456), which the release job refuses to +# publish. Checked only after a build. +# - in one of Duplo's organizations, the manifest id is not in the extension publisher allowlist +# (scripts/_publishers.sh) for this repository, so the release ships but the license server never gets it. +# - in one of Duplo's organizations, manifest.version is not strict semver (MAJOR.MINOR.PATCH[-pre], no +build), +# the same rule scripts/extension-sign.py enforces before it signs a build. +# +# Usage: ./scripts/check-extension-pr.sh # e.g. origin/main; defaults to origin/$GITHUB_BASE_REF +set -euo pipefail +cd "$(dirname "$0")/.." +shopt -s nullglob + +# shellcheck source=scripts/_publishers.sh +. "$(dirname "$0")/_publishers.sh" +# shellcheck source=scripts/_publish.sh +. "$(dirname "$0")/_publish.sh" +publishers_file="${PUBLISHERS_FILE:-$publishers_file}" +max_bytes="${EXTENSION_MAX_BYTES:-268435456}" + +base_ref="${1:-origin/${GITHUB_BASE_REF:?usage: check-extension-pr.sh }}" +base="$(git merge-base "$base_ref" HEAD)" +warned=0 +warn() { echo "::warning file=$1::$2"; warned=$((warned+1)); } + +for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do + [ -f "$m" ] || continue + dir="$(dirname "$m")" + git diff --quiet "$base" HEAD -- "$dir" && continue + + version="$(jq -r '.version' "$m")" + base_version="$(git show "$base:$m" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true)" + # The release job refuses only a version that already has a release tag, under either tag shape, so warn only then. + slug="$(basename "$dir")" + if [ -n "$base_version" ] && [ "$base_version" = "$version" ] && [ -n "$(git tag -l "$slug-v$version" "$slug-v$version-sdk-*")" ]; then + warn "$m" "$dir changed but manifest.version is still $version, which is already released. Bump it, or the release job refuses to publish this content." + fi + + for v in "$(jq -r '.id' "$m")" "$version"; do + [[ "$v" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || warn "$m" "$dir — '$v' cannot form a bucket key, so the release job refuses to publish it." + done + if duplo_org && ! [[ "$version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?$ ]]; then + warn "$m" "$dir — version '$version' is not strict semver (MAJOR.MINOR.PATCH[-pre], no +build), which scripts/extension-sign.py refuses to sign in Duplo's organizations." + fi + jq -e '(.resources // []) | length > 0' "$m" >/dev/null \ + || warn "$m" "$dir declares no resource, so the host refuses the bundle." + if duplo_org && ! publisher_extension_uuid "$(jq -r '.id' "$m")" "${GITHUB_REPOSITORY:-}" >/dev/null; then + warn "$m" "$dir — $(jq -r '.id' "$m") is not in the extension publisher allowlist for ${GITHUB_REPOSITORY:-this repository}, so a release is not published to the license server." + fi + + for s in "$dir"/skills/*/; do + s="$(basename "$s")" + jq -e --arg s "$s" 'any(.skills[]?.folder // ""; endswith("/skills/" + $s))' "$m" >/dev/null \ + || warn "$m" "$dir ships skills/$s but the manifest declares no skills[].folder for it, so the host never registers it." + done + + zip="$dir/dist/extension.zip" + if [ -f "$zip" ]; then + [ "$(wc -c < "$zip")" -le "$max_bytes" ] || warn "$m" "$dir builds a bundle over $max_bytes bytes, which the release job refuses." + entries="$(unzip -Z1 "$zip")" + if grep -qx 'fe/remoteEntry.js' <<<"$entries" && ! grep -qx 'fe/remoteEntry.json' <<<"$entries"; then + warn "$m" "$dir builds a webpack frontend (fe/remoteEntry.js). Its pages do not open on hosts that load Native Federation remotes. Migrate with the duplo-extension-ng22-migration skill." + fi + fi +done + +echo "==> $warned release warning(s)." diff --git a/scripts/extension-sign.py b/scripts/extension-sign.py new file mode 100755 index 0000000..029bb2c --- /dev/null +++ b/scripts/extension-sign.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Sign and verify an extension bundle's signature, extension.zip.sig. + +sign writes one compact ES256 JWS whose claims come from the BUILT zip's manifest.json, since sdkVersion exists only +there, and whose sha256 is over the zip's final bytes. It reproduces the console's reference signer (sign_artifact in +apps/extensions/signing/verify.py). verify ports the console's verify_artifact_signature, check for check and code +for code, so tests/vectors (the console's cross-language vectors) pin both. + +The key and certificate come from EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT, never the command line, so ps on +a shared runner never shows them. EXTENSION_SIGN_NOW overrides the clock for tests. + +verify's --root-key substitutes one local root public key for the console's published roots. It exists for tests +only, and its success line says so, so the output is never mistaken for a verification against the console's real +roots. +""" +import argparse, base64, hashlib, json, os, re, sys, time, urllib.request, zipfile + +import jwt +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec + +ALG = "ES256" +SIG_TYP = "duplo-ext-sig+jwt" +CERT_TYP = "duplo-ext-cert+jwt" +CERT_ISSUER = "duplocloud-console" +PLACEHOLDER = "REPLACED_BY_SKILL" +SEMVER = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +WARN_DAYS = 30 +NO_TIME_CHECKS = {"verify_exp": False, "verify_nbf": False, "verify_iat": False, "verify_aud": False, "verify_iss": False} + + +class SignatureError(Exception): + def __init__(self, code, message=""): + super().__init__(message or code) + self.code = code + + +def now_s(): + return int(os.environ.get("EXTENSION_SIGN_NOW") or time.time()) + + +def b64url(data): + return base64.urlsafe_b64encode(data).rstrip(b"=").decode() + + +def coordinate(text): + if not isinstance(text, str): + raise ValueError("JWK coordinate must be a string") + raw = base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)) + if len(raw) != 32 or b64url(raw) != text: + raise ValueError("JWK coordinate must be 32 bytes of canonical base64url") + return raw + + +def pem_from_jwk(jwk): + if not isinstance(jwk, dict) or jwk.get("kty") != "EC" or jwk.get("crv") != "P-256": + raise ValueError("only EC P-256 JWKs are supported") + x = int.from_bytes(coordinate(jwk.get("x")), "big") + y = int.from_bytes(coordinate(jwk.get("y")), "big") + key = ec.EllipticCurvePublicNumbers(x, y, ec.SECP256R1()).public_key() + return key.public_bytes(serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo).decode() + + +def jwk_of(public_key): + n = public_key.public_numbers() + return {"kty": "EC", "crv": "P-256", "x": b64url(n.x.to_bytes(32, "big")), "y": b64url(n.y.to_bytes(32, "big"))} + + +def in_namespaces(manifest_id, prefixes): + return any(manifest_id == p or manifest_id.startswith(p + ".") for p in prefixes) + + +def built_manifest(zip_path): + with zipfile.ZipFile(zip_path) as z: + return json.loads(z.read("manifest.json")) + + +def sha256_of(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def refuse(message): + raise SystemExit(f"::error::{message}") + + +def sign_bytes(zip_path, key_pem, cert, now): + m = built_manifest(zip_path) + manifest_id, version, sdk = m.get("id"), m.get("version"), m.get("sdkVersion") + if not isinstance(sdk, str) or not sdk or sdk.startswith(PLACEHOLDER): + refuse(f"{zip_path}: the built manifest carries no real sdkVersion. Build it with scripts/build-extension.sh") + for name, value in (("version", version), ("sdkVersion", sdk)): + if not isinstance(value, str) or not SEMVER.match(value): + refuse(f"{zip_path}: {name} {value!r} is not strict semver (MAJOR.MINOR.PATCH[-pre], no +build)") + claims = jwt.decode(cert, options={"verify_signature": False}) + if not in_namespaces(manifest_id, claims.get("ns") or []): + refuse(f"manifest id outside this key's namespaces: {manifest_id} is not under {claims.get('ns')}") + key = serialization.load_pem_private_key(key_pem.encode(), password=None) + if jwk_of(key.public_key()) != {k: claims.get("jwk", {}).get(k) for k in ("kty", "crv", "x", "y")}: + refuse("signing key and certificate don't match") + nbf = claims.get("nbf") + if not isinstance(nbf, int) or now < nbf: + refuse("the signing certificate is not valid yet. Check the runner clock, or wait for its start time") + exp = claims.get("exp") + if not isinstance(exp, int) or now >= exp: + refuse("the signing certificate has expired. Reissue it in the console and update CONSOLE_SIGNING_CERT") + if exp - now < WARN_DAYS * 86400: + print(f"::warning::the signing certificate expires in {(exp - now) // 86400} days. Reissue it in the console", + file=sys.stderr) + body = {"manifest_id": manifest_id, "version": version, "sdk_version": sdk, + "sha256": sha256_of(zip_path), "iat": now} + return jwt.encode(body, key, algorithm=ALG, headers={"kid": claims["kid"], "typ": SIG_TYP, "cert": cert}) + + +def header(token, code): + if not isinstance(token, str): + raise SignatureError(code, "token is not a string") + try: + h = jwt.get_unverified_header(token) + except Exception as exc: + raise SignatureError(code, f"unreadable header: {exc}") from exc + if not isinstance(h, dict): + raise SignatureError(code, "header is not an object") + return h + + +def decode(token, public_pem, bad_sig, malformed): + try: + claims = jwt.decode(token, public_pem, algorithms=[ALG], options=NO_TIME_CHECKS) + except jwt.InvalidSignatureError as exc: + raise SignatureError(bad_sig, "signature does not verify") from exc + except Exception as exc: + raise SignatureError(malformed, f"cannot decode token: {exc}") from exc + if not isinstance(claims, dict): + raise SignatureError(malformed, "claims are not an object") + return claims + + +def is_str(v): + return isinstance(v, str) and v != "" + + +def is_int(v): + return isinstance(v, int) and not isinstance(v, bool) + + +def verify_signature(sig, *, root_keys, sha256, manifest_id, version, sdk_version, trusted_publishers, now): + h = header(sig, "malformed") + if h.get("typ") != SIG_TYP: + raise SignatureError("bad_typ", "signature typ") + if h.get("alg") != ALG: + raise SignatureError("malformed", "signature alg") + cert = h.get("cert") + if not isinstance(cert, str) or not cert: + raise SignatureError("malformed", "missing cert header") + ch = header(cert, "bad_cert") + if ch.get("typ") != CERT_TYP: + raise SignatureError("bad_typ", "certificate typ") + if ch.get("alg") != ALG: + raise SignatureError("bad_cert", "certificate alg") + root_kid = ch.get("kid") + if not isinstance(root_kid, str) or root_kid not in root_keys: + raise SignatureError("unknown_root", "certificate is not signed by a known root key") + cc = decode(cert, root_keys[root_kid], "bad_cert", "bad_cert") + ns = cc.get("ns") + if cc.get("iss") != CERT_ISSUER or not (is_str(cc.get("sub")) and is_str(cc.get("kid")) and isinstance(ns, list) + and all(is_str(p) for p in ns) and is_int(cc.get("nbf")) + and is_int(cc.get("exp"))): + raise SignatureError("bad_cert", "certificate claims") + try: + public_pem = pem_from_jwk(cc.get("jwk")) + except Exception as exc: + raise SignatureError("bad_cert", "certificate jwk") from exc + if now < cc["nbf"] or now >= cc["exp"]: + raise SignatureError("cert_expired", "certificate is outside its validity window") + if trusted_publishers is not None and cc["sub"] not in set(trusted_publishers): + raise SignatureError("untrusted_publisher", "publisher is not trusted") + claims = decode(sig, public_pem, "bad_signature", "malformed") + if h.get("kid") != cc["kid"]: + raise SignatureError("kid_mismatch", "signature kid differs from certificate kid") + fields = ("manifest_id", "version", "sdk_version", "sha256") + if not all(is_str(claims.get(f)) for f in fields) or not SHA256.match(claims["sha256"]): + raise SignatureError("malformed", "signature claims") + if not isinstance(sha256, str) or claims["sha256"] != sha256.lower(): + raise SignatureError("hash_mismatch", "sha256 differs") + if (claims["manifest_id"], claims["version"], claims["sdk_version"]) != (manifest_id, version, sdk_version): + raise SignatureError("manifest_mismatch", "manifest_id, version or sdk_version differs") + if not in_namespaces(claims["manifest_id"], ns): + raise SignatureError("outside_namespace", "manifest_id is outside the certificate's namespaces") + return {"publisher": cc["sub"], "kid": cc["kid"], "cert_exp": cc["exp"], "root_kid": root_kid} + + +def console_root_keys(console_url): + with urllib.request.urlopen(f"{console_url.rstrip('/')}/api/extensions/root-keys/", timeout=30) as r: + body = json.load(r) + rows = body["keys"] if isinstance(body, dict) else body + # A retired root stays here with active=false: certificates it issued are still live until they expire. + return {row["kid"]: row["public_key"] for row in rows if row.get("public_key")} + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = p.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("sign") + s.add_argument("zip") + s.add_argument("--out") + v = sub.add_parser("verify") + v.add_argument("zip") + v.add_argument("sig") + v.add_argument("--root-key", help="a local root public key PEM, for tests only") + v.add_argument("--console", default="https://console.duplocloud.com") + a = p.parse_args(argv) + if a.cmd == "sign": + key, cert = os.environ.get("EXTENSION_SIGNING_KEY", ""), os.environ.get("EXTENSION_SIGNING_CERT", "") + if not key or not cert: + refuse("EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT must both be set") + token = sign_bytes(a.zip, key, cert, now_s()) + out = a.out or a.zip + ".sig" + with open(out, "w") as f: + f.write(token) + print(f"==> signed {a.zip} -> {out}") + return 0 + sig = open(a.sig).read() + if a.root_key: + # A local root key is addressed by the kid the signature's certificate names, so a test root verifies. + root_kid = jwt.get_unverified_header(jwt.get_unverified_header(sig)["cert"]).get("kid") + roots = {root_kid: open(a.root_key).read()} + else: + roots = console_root_keys(a.console) + m = built_manifest(a.zip) + try: + verify_signature(sig, root_keys=roots, sha256=sha256_of(a.zip), manifest_id=m.get("id"), + version=m.get("version"), sdk_version=m.get("sdkVersion"), trusted_publishers=None, + now=now_s()) + except SignatureError as exc: + print(exc.code) + return 1 + print("ok (against the local root key, not the console's roots)" if a.root_key else "ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/extension-sign.requirements.txt b/scripts/extension-sign.requirements.txt new file mode 100644 index 0000000..d9c6de1 --- /dev/null +++ b/scripts/extension-sign.requirements.txt @@ -0,0 +1,145 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --generate-hashes --no-index --output-file=scripts/extension-sign.requirements.txt _tmp_es.in +# +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via cryptography +cryptography==44.0.0 \ + --hash=sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7 \ + --hash=sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731 \ + --hash=sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b \ + --hash=sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc \ + --hash=sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543 \ + --hash=sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c \ + --hash=sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591 \ + --hash=sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede \ + --hash=sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb \ + --hash=sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f \ + --hash=sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123 \ + --hash=sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c \ + --hash=sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c \ + --hash=sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285 \ + --hash=sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd \ + --hash=sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092 \ + --hash=sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa \ + --hash=sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289 \ + --hash=sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02 \ + --hash=sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64 \ + --hash=sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053 \ + --hash=sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417 \ + --hash=sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e \ + --hash=sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e \ + --hash=sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7 \ + --hash=sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756 \ + --hash=sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4 + # via -r _tmp_es.in +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pyjwt==2.10.1 \ + --hash=sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953 \ + --hash=sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb + # via -r _tmp_es.in diff --git a/scripts/release-extensions.sh b/scripts/release-extensions.sh new file mode 100755 index 0000000..3692328 --- /dev/null +++ b/scripts/release-extensions.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Publish every built extension in this repo as a GitHub Release — the publish step of .github/workflows/release.yml, +# kept as a script so tests/test-release-extensions.sh can exercise it. +# +# One release per build, tagged -v-sdk-. is the extension directory's basename, and +# sdkVersion is read from the BUILT bundle's manifest (build-extension.sh stamps it from the host it built against), +# so builds of one version for different SDKs share the version and differ only in the tag. +# +# A version that already has a release, under that tag shape or the legacy -v, is compared with the +# built commit. If the extension is unchanged, an existing build for this SDK is skipped and a missing one is +# published. If it changed, the extension fails rather than skipping green: two different bundles would otherwise +# share one version, and whatever consumes the release could not tell them apart. Legacy tags were cut at main's HEAD +# at publish time rather than at the built commit, so one can report a change that a version bump then clears. +# +# Needs the repo's tags and the history they point at (actions/checkout with fetch-depth: 0), gh authenticated with +# contents: write, jq and unzip. GITHUB_SHA is the built commit; it defaults to HEAD. +# +# Signing: a run in one of Duplo's own organizations (duplo_org in scripts/_publish.sh) always signs the built zip +# with EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT before the release is created, and ships extension.zip.sig +# alongside it; a run anywhere else releases the zip alone, as a customer's copy of this workflow carries no Duplo +# signing credential. Within a Duplo organization, the manifest id also has to be in the extension publisher +# allowlist (scripts/_publishers.sh) for the repository, or the release still ships signed but scripts/_publish.sh's +# publish_build is never called for it. An allowlisted id with no CONSOLE_API_KEY fails before it is signed or +# released, since that build could be released but never registered. +# +# Resuming: a tag that already has a release is never rebuilt, re-signed or re-released. Its own assets, read back +# with gh release view, decide what happens next: a signed release calls publish_build again so a run that stopped +# between releasing and publishing still finishes, and a release that predates signing stays flagged rather than +# retrofitted, since a tag's assets can't gain one after the fact. A new manifest.version is the only way forward. +# +# Two checks run for every build, in every organization, right after confirming the build produced a zip. A missing +# resources entry refuses the build, and so does a bundle over EXTENSION_MAX_BYTES bytes (default 268435456). Either +# failure counts against just that extension and the run moves on, matching scripts/check-extension-pr.sh's PR-time +# warning for the same two rules. +# +# Usage: ./scripts/release-extensions.sh +set -euo pipefail +cd "$(dirname "$0")/.." +shopt -s nullglob + +# shellcheck source=scripts/_publishers.sh +. "$(dirname "$0")/_publishers.sh" +# shellcheck source=scripts/_publish.sh +. "$(dirname "$0")/_publish.sh" +publishers_file="${PUBLISHERS_FILE:-$publishers_file}" +sign_cmd="${EXTENSION_SIGN:-python3 $(dirname "$0")/extension-sign.py}" +repo="${GITHUB_REPOSITORY:-}" +max_bytes="${EXTENSION_MAX_BYTES:-268435456}" + +sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" +published=0; skipped=0; failed=0 + +for m in extensions/*/manifest.json extension/*/manifest.json extension/manifest.json; do + [ -f "$m" ] || continue + dir="$(dirname "$m")" + zip="$dir/dist/extension.zip" + if [ ! -f "$zip" ]; then + echo "::warning::$dir — no dist/extension.zip (build produced nothing); skipping." + continue + fi + if ! jq -e '(.resources // []) | length > 0' "$m" >/dev/null; then + echo "::error::$dir — the manifest declares no resource, so the host refuses the bundle." + failed=$((failed+1)); continue + fi + if [ "$(wc -c < "$zip")" -gt "$max_bytes" ]; then + echo "::error::$dir — the bundle is over $max_bytes bytes, which the release job refuses to publish." + failed=$((failed+1)); continue + fi + name="$(jq -r '.name // .id' "$m")" + version="$(jq -r '.version' "$m")" + slug="$(basename "$dir")" + sdk="$(unzip -p "$zip" manifest.json | jq -r '.sdkVersion // empty')" + if [ -z "$sdk" ]; then + echo "::error::$dir — the bundle's manifest.json carries no sdkVersion, so its release cannot be tagged. Build it with scripts/build-extension.sh." + failed=$((failed+1)); continue + fi + base="${slug}-v${version}" + tag="${base}-sdk-${sdk}" + + changed_since="" + while IFS= read -r t; do + [ -n "$t" ] || continue + git diff --quiet "$t^{commit}" "$sha" -- "$dir" || { changed_since="$t"; break; } + done < <(git tag -l "$base" "$base-sdk-*") + if [ -n "$changed_since" ]; then + echo "::error::$dir — $version is already released as $changed_since and the extension changed since. Bump manifest.version to publish it." + failed=$((failed+1)); continue + fi + + id="$(unzip -p "$zip" manifest.json | jq -r '.id')" + uuid="" + if duplo_org; then + if ! uuid="$(publisher_extension_uuid "$id" "$repo")"; then + uuid="" + echo "::notice::$dir — $id is not in the extension publisher allowlist for $repo, so it is released but not published to the license server." + fi + fi + # Registration needs the console key, so a missing one fails here, before a release that could never be registered. + if [ -n "$uuid" ] && [ -z "${CONSOLE_API_KEY:-}" ]; then + echo "::error::$dir — $id is allowlisted for $repo, but CONSOLE_API_KEY is not set, so it cannot be registered." + failed=$((failed+1)); continue + fi + + if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + if [ -n "$uuid" ]; then + if ! assets="$(gh release view "$tag" --json assets --jq '[.assets[].name] | join(",")')"; then + echo "::error::$dir — gh release view $tag failed." + failed=$((failed+1)); continue + fi + if [[ ",$assets," != *",extension.zip.sig,"* ]]; then + echo "::warning::$tag was released with no extension.zip.sig, and an immutable release cannot gain one. Bump manifest.version to publish this extension." + skipped=$((skipped+1)); continue + fi + publish_build "$tag" "$id" "$version" "$sdk" "$dir" || { failed=$((failed+1)); continue; } + else + echo "==> $tag already released — skipping (bump manifest.version to publish a new one)." + fi + skipped=$((skipped+1)); continue + fi + + assets=("$zip") + if duplo_org; then + if [ -z "${EXTENSION_SIGNING_KEY:-}" ] || [ -z "${EXTENSION_SIGNING_CERT:-}" ]; then + echo "::error::$dir — EXTENSION_SIGNING_KEY and EXTENSION_SIGNING_CERT must be set in $GITHUB_REPOSITORY_OWNER. Duplo releases are always signed." + failed=$((failed+1)); continue + fi + # shellcheck disable=SC2086 # sign_cmd is a command plus its interpreter + if ! $sign_cmd sign "$zip"; then failed=$((failed+1)); continue; fi + assets+=("$zip.sig") + fi + + echo "==> Publishing release $tag ($name $version, SDK $sdk)" + if ! gh release create "$tag" "${assets[@]}" --target "$sha" \ + --title "$name $version (SDK $sdk)" \ + --notes "Automated release of $name v$version from $slug, built against host SDK $sdk."; then + echo "::error::$dir — gh release create $tag failed." + failed=$((failed+1)); continue + fi + published=$((published+1)) + + # An asset replaced at a mutable tag no longer matches a digest pinned from it, so say so where the owner sees it. + immutable="$(gh api "repos/{owner}/{repo}/releases/tags/$tag" --jq '.immutable' 2>/dev/null || true)" + if [ "$immutable" != true ]; then + echo "::warning::$tag landed mutable (immutable=${immutable:-unknown}). A repository admin turns on immutable releases under Settings → General → Releases; releases published before that stay mutable." + fi + + if [ -n "$uuid" ]; then + publish_build "$tag" "$id" "$version" "$sdk" "$dir" || failed=$((failed+1)) + fi +done + +echo "==> Released $published, skipped $skipped, failed $failed." +[ "$failed" -eq 0 ] diff --git a/tests/_publish_stubs.sh b/tests/_publish_stubs.sh new file mode 100644 index 0000000..e721da6 --- /dev/null +++ b/tests/_publish_stubs.sh @@ -0,0 +1,74 @@ +# shellcheck shell=bash +# Fakes for publish_build's dependencies (scripts/_publish.sh), shared by test-publish.sh and the resume tests in +# test-release-extensions.sh. Source this after creating "$TMP/bin" and putting it on PATH. It writes gh, aws and +# curl there and makes them executable. The caller exports ASSETS (a directory holding extension.zip and +# extension.zip.sig), S3 and CONSOLE (scratch directories standing in for the bucket and the console's records), +# and LOG (where every stubbed call is appended). +# +# gh also answers the release-lifecycle calls scripts/release-extensions.sh itself makes (create/view, and the +# immutable-flag api lookup), so one gh stub serves both test files. It tells that lookup apart from publish_build's +# digest lookup, which hits the same `gh api repos/{owner}/{repo}/releases/tags/` shape with a different --jq. + +cat > "$TMP/bin/gh" <<'EOF' +#!/usr/bin/env bash +echo "gh $*" >> "$LOG" +case "$1 $2" in + "release download") + # Copies only the assets named by -p, so a test can tell which ones a run downloaded. + pats=(); dest="" + while [ $# -gt 0 ]; do case "$1" in -p) pats+=("$2"); shift ;; -D) dest=$2; shift ;; esac; shift; done + for p in "${pats[@]}"; do cp "$ASSETS/$p" "$dest"/; done ;; + "release create") exit "${GH_CREATE_RC:-0}" ;; + "release view") [ "${GH_VIEW_RC:-0}" = 0 ] || exit "${GH_VIEW_RC}" + printf '%s\n' "${GH_ASSETS:-extension.zip,extension.zip.sig}" ;; + "api repos/{owner}/{repo}/releases/tags/"*) + [ "${GH_API_RC:-0}" = 0 ] || exit "${GH_API_RC}" + if [[ "$*" == *'assets[]'* ]]; then printf '%s\n' "${GH_DIGEST-sha256:$ZSHA}" + else printf '%s\n' "${GH_IMMUTABLE:-true}"; fi ;; +esac +EOF + +cat > "$TMP/bin/aws" <<'EOF' +#!/usr/bin/env bash +echo "aws $*" >> "$LOG" +key=""; body=""; out=""; while [ $# -gt 0 ]; do case "$1" in --key) key=$2 ;; --body) body=$2 ;; esac; out=$1; shift; done +obj="$S3/$(echo "$key" | tr / _)" +if grep -q put-object <<<"$(tail -1 "$LOG")"; then + [ -f "$obj" ] && { echo "An error occurred (PreconditionFailed) when calling the PutObject operation" >&2; exit 254; } + cp "$body" "$obj"; echo '{}' +else cp "$obj" "$out"; fi +EOF + +cat > "$TMP/bin/curl" <<'EOF' +#!/usr/bin/env bash +# Console stub: versions and artifacts live as files under $CONSOLE. Prints body then a final line with the code. +# Logs raw argv, unmasked, so a key leaked onto the command line would show up here the way a real process listing +# or Action log would show it. The key only ever travels in an -H @file argument, so this also records whether this +# call's header file held it (HEADER_FILE_KEY=yes/no), letting a test prove that without the key itself in the log. +echo "curl $*" >> "$LOG" +url=""; data=""; method=GET; hdrfile="" +while [ $# -gt 0 ]; do + case "$1" in + -X) method=$2; shift ;; + --data|-d) data=$2; method=${method/GET/POST}; shift ;; + -H) case "$2" in @*) hdrfile=${2#@} ;; esac; shift ;; + http*) url=$1 ;; + esac + shift +done +if [ -n "$hdrfile" ] && grep -qF -- "$CONSOLE_API_KEY" "$hdrfile" 2>/dev/null +then echo "HEADER_FILE_KEY=yes" >> "$LOG"; else echo "HEADER_FILE_KEY=no" >> "$LOG"; fi +# CONSOLE_FAIL, a regex matched against the URL, makes that call fail the way curl --fail-with-body does. +if [ -n "${CONSOLE_FAIL:-}" ] && [[ "$url" =~ $CONSOLE_FAIL ]]; then echo '{"detail":"server error"}'; exit 22; fi +case "$method $url" in + "GET "*"/versions/?version="*) cat "$CONSOLE/versions.json" 2>/dev/null || echo '[]' ;; + "POST "*"/versions/") + [ -n "${RACE:-}" ] && [ ! -f "$CONSOLE/raced" ] && { touch "$CONSOLE/raced"; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"version":["1.0.0 already exists for this extension."]}'; exit 22; } + [ -n "${VERSION_VANISHES:-}" ] && { echo '{"uuid":"v-1"}'; exit 0; } + echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json"; echo '{"uuid":"v-1"}' ;; + "GET "*"/artifacts/?sdk_version="*) cat "$CONSOLE/artifacts.json" 2>/dev/null || echo '[]' ;; + "POST "*"/artifacts/") echo "[$data]" > "$CONSOLE/artifacts.json"; echo "$data" ;; +esac +EOF + +chmod +x "$TMP/bin/gh" "$TMP/bin/aws" "$TMP/bin/curl" diff --git a/tests/extension_sign_console_shape.py b/tests/extension_sign_console_shape.py new file mode 100644 index 0000000..bec09e5 --- /dev/null +++ b/tests/extension_sign_console_shape.py @@ -0,0 +1,59 @@ +"""Checks console_root_keys against the console's real root-keys response shape, with no network call. + +apps/extensions/api_root_keys.py returns {"keys": [{"kid", "public_key", "active"}]}. This pins that shape (a bare +list is accepted too) and confirms the valid vector's signature verifies against the roots it parses out, including +a retired (active=false) row that must still be kept. +""" +import importlib.util, io, json, pathlib, sys, urllib.request + +HERE = pathlib.Path(__file__).parent +spec = importlib.util.spec_from_file_location("extension_sign", HERE.parent / "scripts" / "extension-sign.py") +es = importlib.util.module_from_spec(spec) +spec.loader.exec_module(es) + +vectors = HERE / "vectors" +data = json.loads((vectors / "cases.json").read_text()) +root_kid = next(iter(data["root_keys"])) +root_pub = (vectors / data["root_keys"][root_kid]).read_text() +body = json.dumps({"keys": [ + {"kid": "retired-root", "public_key": "not a real key", "active": False}, + {"kid": root_kid, "public_key": root_pub, "active": True}, +]}).encode() + + +class FakeResponse: + def __init__(self, payload): + self._buf = io.BytesIO(payload) + + def __enter__(self): + return self._buf + + def __exit__(self, *exc_info): + return False + + +urllib.request.urlopen = lambda url, timeout=30: FakeResponse(body) + +roots = es.console_root_keys("https://console.example.com") +bad = 0 +if roots.get("retired-root") != "not a real key": + print("MISMATCH: a retired (active=false) root key was dropped") + bad += 1 +if roots.get(root_kid) != root_pub: + print("MISMATCH: the active root key was not parsed from the console's {\"keys\": [...]} shape") + bad += 1 + +case = next(c for c in data["cases"] if c["name"] == "valid") +try: + es.verify_signature((vectors / case["sig"]).read_text(), root_keys=roots, sha256=case["sha256"], + manifest_id=case["manifest_id"], version=case["version"], sdk_version=case["sdk_version"], + trusted_publishers=case["trusted_publishers"], now=case["now"]) + got = "ok" +except es.SignatureError as exc: + got = exc.code +if got != "ok": + print(f"MISMATCH: the valid vector did not verify against the parsed console root keys (got {got})") + bad += 1 + +print("console root-keys shape: ok" if not bad else "console root-keys shape: FAIL") +sys.exit(1 if bad else 0) diff --git a/tests/extension_sign_vectors.py b/tests/extension_sign_vectors.py new file mode 100644 index 0000000..376dfc1 --- /dev/null +++ b/tests/extension_sign_vectors.py @@ -0,0 +1,26 @@ +"""Runs every case in tests/vectors/cases.json through extension-sign.py's verifier and checks the recorded outcome.""" +import importlib.util, json, pathlib, sys + +HERE = pathlib.Path(__file__).parent +spec = importlib.util.spec_from_file_location("extension_sign", HERE.parent / "scripts" / "extension-sign.py") +es = importlib.util.module_from_spec(spec) +spec.loader.exec_module(es) + +vectors = HERE / "vectors" +data = json.loads((vectors / "cases.json").read_text()) +roots = {kid: (vectors / name).read_text() for kid, name in data["root_keys"].items()} +bad = 0 +for case in data["cases"]: + try: + es.verify_signature((vectors / case["sig"]).read_text(), root_keys=roots, sha256=case["sha256"], + manifest_id=case["manifest_id"], version=case["version"], + sdk_version=case["sdk_version"], trusted_publishers=case["trusted_publishers"], + now=case["now"]) + got = "ok" + except es.SignatureError as exc: + got = exc.code + if got != case["expect"]: + print(f"MISMATCH {case['name']}: expected {case['expect']}, got {got}") + bad += 1 +print(f"{len(data['cases']) - bad} of {len(data['cases'])} vector cases match") +sys.exit(1 if bad else 0) diff --git a/tests/fixtures/extension-publishers.json b/tests/fixtures/extension-publishers.json new file mode 100644 index 0000000..789d385 --- /dev/null +++ b/tests/fixtures/extension-publishers.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": 1, + "publishers": [ + { "manifestId": "duplo.extensions.alpha", "repository": "duplocloud/alpha", + "consoleExtension": "00000000-0000-4000-8000-000000000001" }, + { "manifestId": "duplo.extensions.beta", "repository": "duplocloud/beta", + "consoleExtension": "00000000-0000-4000-8000-000000000002" } + ] +} diff --git a/tests/test-builder-image.sh b/tests/test-builder-image.sh new file mode 100755 index 0000000..f171378 --- /dev/null +++ b/tests/test-builder-image.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Tests for builder_resolve_image in scripts/_builder.sh — which toolchain image a build runs in. +# Uses a stub `docker` on PATH and a throwaway .env; never pulls. Usage: ./tests/test-builder-image.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +mkdir -p "$TMP/bin"; printf '#!/usr/bin/env bash\nexit 0\n' > "$TMP/bin/docker"; chmod +x "$TMP/bin/docker" +export PATH="$TMP/bin:$PATH" + +# Stand-in for _target.sh's .env reader, pointed at a temp file. +ENV="$TMP/.env"; : > "$ENV" +_envv() { grep -E "^$1=" "$ENV" 2>/dev/null | head -1 | cut -d= -f2- || true; } + +. ./scripts/_builder.sh || { echo "scripts/_builder.sh not found"; exit 1; } +# builder_dispatch resolves RUNTIME before it ever calls builder_resolve_image. Calling the resolver directly +# skips that, so point it at the stub here. +RUNTIME=docker + +resolve() { ( unset BUILDER_IMAGE BUILDER_TAG BUILDER_PULL; eval "$1"; builder_resolve_image >/dev/null 2>&1; echo "$BUILDER_IMAGE" ); } + +echo "builder image:" + +t "defaults to the digest-pinned toolchain image" +: > "$ENV"; got="$(resolve :)" +if [[ "$got" =~ ^quay\.io/duplocloud/duplo-extension-builder@sha256:[0-9a-f]{64}$ ]]; then ok; else bad "$got"; fi + +t "docker-compose.yml falls back to the same pinned image" +: > "$ENV"; got="$(resolve :)" +if grep -qF -- "\${BUILDER_IMAGE:-$got}" docker-compose.yml; then ok; else bad "compose does not default to $got"; fi + +t "BUILDER_TAG in .env still selects a tag" +printf 'BUILDER_TAG=1.2.3\n' > "$ENV"; got="$(resolve :)" +if [ "$got" = "quay.io/duplocloud/duplo-extension-builder:1.2.3" ]; then ok; else bad "$got"; fi + +t "BUILDER_TAG=latest, left over in an older .env, still takes the pinned image" +: > "$ENV"; pinned="$(resolve :)" +printf 'BUILDER_TAG=latest\n' > "$ENV"; got="$(resolve :)" +if [ "$got" = "$pinned" ]; then ok; else bad "$got"; fi + +t "BUILDER_IMAGE in the environment still replaces the image outright" +: > "$ENV"; got="$(resolve 'export BUILDER_IMAGE=example.com/mine:1')" +if [ "$got" = "example.com/mine:1" ]; then ok; else bad "$got"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-check-extension-pr.sh b/tests/test-check-extension-pr.sh new file mode 100755 index 0000000..60176d6 --- /dev/null +++ b/tests/test-check-extension-pr.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Tests for scripts/check-extension-pr.sh — the PR-time warnings about what a release will need. +# Runs in a throwaway git repo; never touches GitHub. Usage: ./tests/test-check-extension-pr.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +ROOT="$PWD" + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + +# new_repo: extensions/demo at 0.1.0 declaring skill `provision-demo`, committed on main, then a `pr` branch. +new_repo() { + REPO="$TMP/repo.$RANDOM"; D="$REPO/extensions/demo"; mkdir -p "$D/skills/provision-demo" + git -C "$REPO" init -q -b main + git -C "$REPO" config user.email t@example.com; git -C "$REPO" config user.name t + set_manifest 0.1.0 '["duplo.demo/0.1.0/skills/provision-demo"]' + echo skill > "$D/skills/provision-demo/SKILL.md"; echo one > "$D/code.txt" + git -C "$REPO" add -A; git -C "$REPO" commit -qm init + git -C "$REPO" switch -q -c pr +} + +# set_manifest : also declares a resources entry, so only the test that +# removes it sees the no-resource warning. +set_manifest() { + jq -n --arg v "$1" --argjson f "$2" \ + '{id:"duplo.demo",name:"Demo",version:$v,skills:[$f[]|{folder:.,isBuiltIn:true}],resources:[{subType:"demo"}]}' \ + > "$D/manifest.json" +} + +change() { echo "$1" > "$D/code.txt"; git -C "$REPO" commit -qam "$1"; } + +# bundle : the dist/extension.zip a build leaves, with its frontend entry under fe/. +bundle() { + local p; p="$(mktemp -d)"; mkdir -p "$p/fe"; cp "$D/manifest.json" "$p/"; echo x > "$p/fe/$1" + mkdir -p "$D/dist"; rm -f "$D/dist/extension.zip"; ( cd "$p" && zip -qr "$D/dist/extension.zip" . ); rm -rf "$p" +} + +# The script resolves the repo from its own path, as it does when devkit copies it into an extension repo, so its +# sourced helpers (_publishers.sh, _publish.sh) have to travel with it. +run() { + mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/check-extension-pr.sh" "$REPO/scripts/" 2>/dev/null + cp "$ROOT"/scripts/_*.sh "$REPO/scripts/" 2>/dev/null + "$REPO/scripts/check-extension-pr.sh" main 2>&1 +} + +echo "check-extension-pr:" + +t "warns when a released extension changed with no manifest.version bump" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6 main; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*extensions/demo.*manifest.version" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about an unbumped version that has no release yet" +new_repo; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "manifest.version" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about the version when it was bumped" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about an extension the PR does not touch" +new_repo; echo readme > "$REPO/README.md"; git -C "$REPO" add -A; git -C "$REPO" commit -qm readme +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about the version of an extension new in this PR" +new_repo; git -C "$REPO" switch -q main; git -C "$REPO" rm -rq extensions; git -C "$REPO" commit -qm empty +git -C "$REPO" switch -q -c pr2; git -C "$REPO" checkout -q pr -- extensions; git -C "$REPO" commit -qm add +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a shipped skill the manifest does not declare" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]' +mkdir -p "$D/skills/undeclared"; echo s > "$D/skills/undeclared/SKILL.md"; git -C "$REPO" add -A; git -C "$REPO" commit -qm skill +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*skills/undeclared" <<<"$OUT" && ! grep -q "provision-demo" <<<"$OUT" +then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a webpack frontend (fe/remoteEntry.js) in the built bundle" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.js +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*remoteEntry.js" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "is quiet about a Native Federation frontend (fe/remoteEntry.json)" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.json +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about an id outside the publisher allowlist in a Duplo org" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two +jq -n '{schemaVersion:1, publishers:[]}' > "$REPO/pub.json" +OUT="$(PUBLISHERS_FILE="$REPO/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*allowlist" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a manifest that declares no resource" +new_repo; jq '.version="0.1.1" | .skills=[{folder:"duplo.demo/0.1.1/skills/provision-demo",isBuiltIn:true}] | del(.resources)' \ + "$D/manifest.json" > "$D/m" && mv "$D/m" "$D/manifest.json"; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*resource" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a bundle over 268435456 bytes" +new_repo; set_manifest 0.1.1 '["duplo.demo/0.1.1/skills/provision-demo"]'; change two; bundle remoteEntry.json +truncate -s 268435457 "$D/dist/extension.zip" +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*268435456" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a version that cannot form a bucket key" +new_repo; set_manifest '0.1.1 beta' '["duplo.demo/0.1.1 beta/skills/provision-demo"]'; change two +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*cannot form" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "warns about a non-semver version in a Duplo org" +new_repo; set_manifest 'v1.0.0' '["duplo.demo/v1.0.0/skills/provision-demo"]'; change two +OUT="$(GITHUB_REPOSITORY_OWNER=duplocloud run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning.*not strict semver" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-extension-publishers.sh b/tests/test-extension-publishers.sh new file mode 100755 index 0000000..fe71d22 --- /dev/null +++ b/tests/test-extension-publishers.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Tests for .github/extension-publishers.json and scripts/_publishers.sh. Usage: ./tests/test-extension-publishers.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +. ./scripts/_publishers.sh +REAL=.github/extension-publishers.json +FIXTURE=tests/fixtures/extension-publishers.json +echo "extension-publishers:" + +# Structural checks run against both files: the real allowlist can never gain a malformed entry either. +for F in "$REAL" "$FIXTURE"; do + t "$F parses with schemaVersion 1" + [ "$(jq -r .schemaVersion "$F")" = 1 ] && ok || bad "schemaVersion" + t "$F: each manifest id appears once" + [ "$(jq '[.publishers[].manifestId] | length == (unique | length)' "$F")" = true ] && ok || bad "duplicate id" + t "$F: each entry names one repository and a console UUID" + jq -e 'all(.publishers[]; (.repository | test("^[^/]+/[^/]+$")) and (.consoleExtension | test("^[0-9a-f-]{36}$")))' \ + "$F" >/dev/null && ok || bad "shape" +done + +t "the real file is empty or every entry passes the shape check" +jq -e '(.publishers | length == 0) or + all(.publishers[]; (.repository | test("^[^/]+/[^/]+$")) and (.consoleExtension | test("^[0-9a-f-]{36}$")))' \ + "$REAL" >/dev/null && ok || bad "placeholder in the real file" + +publishers_file="$FIXTURE" +t "a mapped id and repository yield the UUID" +id=$(jq -r '.publishers[0].manifestId' "$FIXTURE"); repo=$(jq -r '.publishers[0].repository' "$FIXTURE") +[ "$(publisher_extension_uuid "$id" "$repo")" = "$(jq -r '.publishers[0].consoleExtension' "$FIXTURE")" ] && ok || bad "lookup" +t "an id mapped to another repository is refused" +! publisher_extension_uuid "$id" "someone/else" >/dev/null && ok || bad "accepted another repo" +t "an unknown id is refused" +! publisher_extension_uuid "duplo.nope" "$repo" >/dev/null && ok || bad "accepted unknown id" + +# Exit-status contract: callers key off exit 1 meaning "not allowed to publish." jq -e itself exits 4 when its +# filter produces no output at all, so the function must capture the lookup rather than pass that code through. +t "an unknown id exits exactly 1" +publisher_extension_uuid "duplo.nope" "$repo" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +t "a wrong repository exits exactly 1" +publisher_extension_uuid "$id" "someone/else" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +publishers_file="$REAL" +t "the empty real file exits exactly 1" +publisher_extension_uuid "duplo.anything" "duplocloud/anything" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +publishers_file="$TMP/missing.json" +t "a missing file exits exactly 1" +publisher_extension_uuid "duplo.anything" "duplocloud/anything" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" +printf '{not valid json' > "$TMP/malformed.json" +publishers_file="$TMP/malformed.json" +t "a malformed allowlist file exits exactly 1, not a jq error code" +publisher_extension_uuid "duplo.anything" "duplocloud/anything" >/dev/null 2>&1; rc=$? +[ "$rc" -eq 1 ] && ok || bad "exit $rc" + +echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] diff --git a/tests/test-extension-sign.sh b/tests/test-extension-sign.sh new file mode 100755 index 0000000..e7b5a93 --- /dev/null +++ b/tests/test-extension-sign.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Tests for scripts/extension-sign.py against the console's cross-language vectors. Usage: ./tests/test-extension-sign.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +# The vector tests import the signer as a module. Without this, Python leaves scripts/__pycache__/ in the checkout. +export PYTHONDONTWRITEBYTECODE=1 +V=tests/vectors; S=scripts/extension-sign.py +# The vectors' certificate rides in each .sig header, so the valid case's header gives it. +CERT=$(python3 -c 'import jwt,sys; print(jwt.get_unverified_header(open(sys.argv[1]).read())["cert"])' $V/extension.zip.sig) +NOW=$(jq -r '.cases[0].now' $V/cases.json) +# mkzip : a zip holding only manifest.json. "-" leaves sdkVersion out. +mkzip() { local d; d=$(mktemp -d); jq -n --arg i "$1" --arg v "$2" --arg s "$3" \ + '{id:$i,version:$v} + (if $s == "-" then {} else {sdkVersion:$s} end)' > "$d/manifest.json" + (cd "$d" && zip -q "$4" manifest.json); rm -rf "$d"; } +sign() { EXTENSION_SIGNING_KEY="$(cat "$1")" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$NOW" python3 $S sign "$2" 2>&1; } + +echo "extension-sign:" + +t "verify reproduces every vector case" +OUT=$(python3 tests/extension_sign_vectors.py 2>&1); [ $? = 0 ] && ok || bad "$OUT" + +t "verify parses the console's real root-keys response shape (no network)" +OUT=$(python3 tests/extension_sign_console_shape.py 2>&1); [ $? = 0 ] && ok || bad "$OUT" + +t "a fresh signature verifies against the test root" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/a.zip"; OUT=$(sign $V/publisher.pem "$TMP/a.zip") +OUT2=$(EXTENSION_SIGN_NOW="$NOW" python3 $S verify "$TMP/a.zip" "$TMP/a.zip.sig" --root-key $V/root.pub.pem 2>&1) +[ $? = 0 ] && grep -q "not the console's roots" <<<"$OUT2" && ok || bad "$OUT $OUT2" + +t "the .sig has no trailing newline" +[ "$(tail -c1 "$TMP/a.zip.sig" | od -An -c | tr -d ' ')" != '\n' ] && ok || bad "trailing newline" + +t "refuses a key that does not match the certificate" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/b.zip"; OUT=$(sign $V/other.pem "$TMP/b.zip") +[ $? != 0 ] && grep -q "signing key and certificate don't match" <<<"$OUT" && [ ! -f "$TMP/b.zip.sig" ] && ok || bad "$OUT" + +t "refuses a manifest id outside the certificate's namespaces" +mkzip com.other.thing 1.0.0 1.0.0 "$TMP/c.zip"; OUT=$(sign $V/publisher.pem "$TMP/c.zip") +[ $? != 0 ] && grep -q "manifest id outside this key's namespaces" <<<"$OUT" && ok || bad "$OUT" + +t "refuses a missing sdkVersion" +mkzip io.example.hello 1.0.0 - "$TMP/d.zip"; OUT=$(sign $V/publisher.pem "$TMP/d.zip") +[ $? != 0 ] && grep -q "sdkVersion" <<<"$OUT" && ok || bad "$OUT" + +t "refuses the sdk-version placeholder" +mkzip io.example.hello 1.0.0 REPLACED_BY_SKILL_FROM_sdk-version_ENDPOINT "$TMP/e.zip"; OUT=$(sign $V/publisher.pem "$TMP/e.zip") +[ $? != 0 ] && grep -q "sdkVersion" <<<"$OUT" && ok || bad "$OUT" + +t "refuses a version with build metadata" +mkzip io.example.hello 1.0.0+abc 1.0.0 "$TMP/f.zip"; OUT=$(sign $V/publisher.pem "$TMP/f.zip") +[ $? != 0 ] && grep -q "strict semver" <<<"$OUT" && ok || bad "$OUT" + +t "refuses an expired certificate" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/g.zip" +EXP=$(python3 -c 'import jwt,sys; print(jwt.decode(sys.argv[1], options={"verify_signature": False})["exp"])' "$CERT") +OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$EXP" python3 $S sign "$TMP/g.zip" 2>&1) +[ $? != 0 ] && grep -q "expired" <<<"$OUT" && ok || bad "$OUT" + +t "refuses a certificate that is not valid yet" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/n.zip" +NBF=$(python3 -c 'import jwt,sys; print(jwt.decode(sys.argv[1], options={"verify_signature": False})["nbf"])' "$CERT") +OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$((NBF - 1))" python3 $S sign "$TMP/n.zip" 2>&1) +[ $? != 0 ] && grep -q "not valid yet" <<<"$OUT" && [ ! -f "$TMP/n.zip.sig" ] && ok || bad "$OUT" + +t "warns when the certificate expires within 30 days" +mkzip io.example.hello 1.0.0 1.0.0 "$TMP/h.zip" +OUT=$(EXTENSION_SIGNING_KEY="$(cat $V/publisher.pem)" EXTENSION_SIGNING_CERT="$CERT" EXTENSION_SIGN_NOW="$((EXP - 86400))" python3 $S sign "$TMP/h.zip" 2>&1) +[ $? = 0 ] && grep -q "::warning::.*expires" <<<"$OUT" && ok || bad "$OUT" + +t "never prints the key" +grep -q "BEGIN" <<<"$(sign $V/other.pem "$TMP/b.zip")" && bad "key material in output" || ok + +echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] diff --git a/tests/test-publish.sh b/tests/test-publish.sh new file mode 100755 index 0000000..7bf4902 --- /dev/null +++ b/tests/test-publish.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# Tests for publish_build in scripts/_publish.sh. Usage: ./tests/test-publish.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +mkdir -p "$TMP/bin" "$TMP/s3" "$TMP/assets"; export PATH="$TMP/bin:$PATH" LOG="$TMP/log" +printf 'ZIPBYTES' > "$TMP/assets/extension.zip"; printf 'SIGBYTES' > "$TMP/assets/extension.zip.sig" +ZSHA=$(shasum -a 256 "$TMP/assets/extension.zip" | cut -d' ' -f1) + +# shellcheck source=tests/_publish_stubs.sh +. "$(dirname "$0")/_publish_stubs.sh" +export S3="$TMP/s3" ASSETS="$TMP/assets" ZSHA CONSOLE="$TMP/console" CONSOLE_API_KEY=secret-key uuid=ext-1 +. ./scripts/_publish.sh +fresh() { rm -rf "$S3"/* "$CONSOLE"; mkdir -p "$CONSOLE"; : > "$LOG"; } + +echo "publish:" + +t "uploads both objects create-only and registers version and artifact with the signature" +fresh; OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +if [ $RC = 0 ] && [ "$(grep -c "put-object.*--if-none-match \*" "$LOG")" = 2 ] \ + && jq -e --arg s "$ZSHA" '.[0] | .sha256 == $s and .signature == "SIGBYTES" and (.s3_path | endswith("bundles/duplo.demo/1.0.0/sdk-1.0.6/extension.zip"))' "$CONSOLE/artifacts.json" >/dev/null +then ok; else bad "rc=$RC out=$OUT"; fi + +t "never sets is_published and never logs the key" +if grep -q is_published "$LOG"; then bad "is_published sent" +elif grep -q secret-key "$LOG" "$TMP"/console/* 2>/dev/null; then bad "key logged" +elif ! grep -q "HEADER_FILE_KEY=yes" "$LOG"; then bad "the header file never carried the key, so this test could not prove a leak" +else ok +fi + +t "a re-run of a finished build stops at the console lookup, with no zip download and no aws call" +: > "$LOG" # Keep S3/CONSOLE state from the prior run. Clear only the log, so this run's own calls are what we check. +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && grep -q "already published" <<<"$OUT" && ! grep -q "^aws " "$LOG" && ! grep -q "POST" <<<"$(grep curl "$LOG")" \ + && grep -q -- "-p extension.zip.sig" "$LOG" && ! grep -qE -- "-p extension\.zip( |$)" "$LOG" && ok || bad "rc=$RC out=$OUT log=$(cat "$LOG")" + +t "a console lookup failure on a finished build fails the job, with no aws call" +: > "$LOG" +OUT=$(CONSOLE_FAIL='/versions/\?version=' publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::.*lookup of the version failed" <<<"$OUT" && ! grep -q "^aws " "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "an artifact lookup failure on a finished build fails the job, with no aws call" +: > "$LOG" +OUT=$(CONSOLE_FAIL='/artifacts/\?sdk_version=' publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::.*lookup of the artifact failed" <<<"$OUT" && ! grep -q "^aws " "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "a registered artifact that differs from the release takes the full path and fails there" +: > "$LOG"; jq '.[0].signature = "OTHERSIG"' "$CONSOLE/artifacts.json" > "$TMP/a.json" && mv "$TMP/a.json" "$CONSOLE/artifacts.json" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "differs" <<<"$OUT" && grep -qE -- "-p extension\.zip( |$)" "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "a re-run after only the zip uploaded uploads the signature" +fresh; cp "$ASSETS/extension.zip" "$S3/bundles_duplo.demo_1.0.0_sdk-1.0.6_extension.zip" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && [ -f "$S3/bundles_duplo.demo_1.0.0_sdk-1.0.6_extension.zip.sig" ] && ok || bad "rc=$RC out=$OUT" + +t "different bytes already at a key fail the job" +fresh; printf 'OTHER' > "$S3/bundles_duplo.demo_1.0.0_sdk-1.0.6_extension.zip" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "different bytes" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" + +t "a version created by a concurrent run is looked up again" +fresh; OUT=$(RACE=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && [ -f "$CONSOLE/artifacts.json" ] && ok || bad "rc=$RC out=$OUT" + +t "an existing artifact with a different sha256 fails the job" +fresh; echo '[{"uuid":"v-1"}]' > "$CONSOLE/versions.json" +echo '[{"sdk_version":"1.0.6","s3_path":"s3://duplo-helpdesk-channels/bundles/duplo.demo/1.0.0/sdk-1.0.6/extension.zip","sha256":"0000","signature":"SIGBYTES"}]' \ + > "$CONSOLE/artifacts.json" +OUT=$(publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "differs" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" + +t "a missing GitHub digest falls back to hashing the downloaded asset" +fresh; OUT=$(GH_DIGEST="" publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC = 0 ] && jq -e --arg s "$ZSHA" '.[0].sha256 == $s' "$CONSOLE/artifacts.json" >/dev/null && ok || bad "rc=$RC out=$OUT" + +t "a release digest that does not match the downloaded zip fails the job before any upload" +fresh; OUT=$(GH_DIGEST="sha256:deadbeef" publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "deadbeef" <<<"$OUT" && grep -q "$ZSHA" <<<"$OUT" && ! grep -q "put-object" "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "a failed release-digest lookup fails the job before any upload" +fresh; OUT=$(GH_API_RC=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::" <<<"$OUT" && ! grep -q "put-object" "$LOG" && ok || bad "rc=$RC out=$OUT" + +t "an artifact lookup failure on a new build fails the job" +fresh; OUT=$(CONSOLE_FAIL='/artifacts/\?sdk_version=' publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "::error::.*lookup of the artifact failed" <<<"$OUT" && [ ! -f "$CONSOLE/artifacts.json" ] && ok || bad "rc=$RC out=$OUT" + +t "a version lookup that still comes back empty after the POST fails the job" +fresh; OUT=$(VERSION_VANISHES=1 publish_build demo-v1.0.0-sdk-1.0.6 duplo.demo 1.0.0 1.0.6 extensions/demo 2>&1); RC=$? +[ $RC != 0 ] && grep -q "no version uuid" <<<"$OUT" && ok || bad "rc=$RC out=$OUT" + +echo; echo "passed $PASS, failed $FAIL"; [ "$FAIL" -eq 0 ] diff --git a/tests/test-release-extensions.sh b/tests/test-release-extensions.sh new file mode 100755 index 0000000..46d9126 --- /dev/null +++ b/tests/test-release-extensions.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# Tests for scripts/release-extensions.sh — the release job's publish step. +# Runs in a throwaway git repo with a stub `gh` on PATH; never touches GitHub. Usage: ./tests/test-release-extensions.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +ROOT="$PWD" + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT + +# Fakes for gh (release lifecycle, plus the asset download and digest lookup publish_build needs), aws and curl, +# shared with tests/test-publish.sh via tests/_publish_stubs.sh. `api` answers the release's immutable flag from +# $GH_IMMUTABLE, unless the --jq expression is publish_build's digest lookup. `release view ... --json assets` +# answers the asset list a resumed publish reads back, from $GH_ASSETS (default both the zip and its signature), or +# fails outright per $GH_VIEW_RC. Every call is appended to $LOG, aliased below to $GH_LOG so existing assertions +# against $GH_LOG keep working. +mkdir -p "$TMP/bin" +export PATH="$TMP/bin:$PATH" +# shellcheck source=tests/_publish_stubs.sh +. "$ROOT/tests/_publish_stubs.sh" + +# Stub signer the release script finds through EXTENSION_SIGN: writes ZIP.sig unless FAKE_SIGN_RC says to refuse. +cat > "$TMP/bin/fake-sign" <<'EOF' +#!/usr/bin/env bash +[ "${FAKE_SIGN_RC:-0}" = 0 ] || { echo "::error::refused"; exit 1; } +printf 'sig-of-%s' "$(basename "$2")" > "$2.sig" +EOF +chmod +x "$TMP/bin/fake-sign" +export EXTENSION_SIGN="$TMP/bin/fake-sign" + +# Test allowlist: duplo.demo may publish from duplocloud/demo. Never the real .github/extension-publishers.json, +# which ships empty on purpose. +jq -n '{schemaVersion:1, publishers:[{manifestId:"duplo.demo", repository:"duplocloud/demo", consoleExtension:"00000000-0000-4000-8000-000000000001"}]}' > "$TMP/pub.json" + +# new_repo: a git repo holding extensions/demo at version 0.1.0, committed. Declares a resources entry, so only the +# tests that remove it see the no-resource refusal. +new_repo() { + REPO="$TMP/repo.$RANDOM"; mkdir -p "$REPO/extensions/demo" + git -C "$REPO" init -q -b main + git -C "$REPO" config user.email t@example.com; git -C "$REPO" config user.name t + printf '{"id":"duplo.demo","name":"Demo","version":"%s","resources":[{"subType":"demo"}]}\n' "${1:-0.1.0}" \ + > "$REPO/extensions/demo/manifest.json" + echo one > "$REPO/extensions/demo/code.txt" + git -C "$REPO" add -A; git -C "$REPO" commit -qm init + export GH_LOG="$REPO.gh.log" LOG="$REPO.gh.log"; : > "$GH_LOG" +} + +# build_zip [sdkVersion]: the dist/extension.zip build-extension.sh would leave, with sdkVersion stamped. +# An empty argument leaves sdkVersion out of the bundle manifest. +build_zip() { + local d="$REPO/extensions/demo" pkg; pkg="$(mktemp -d)" + if [ -n "${1-}" ]; then jq --arg v "$1" '.sdkVersion = $v' "$d/manifest.json" > "$pkg/manifest.json" + else cp "$d/manifest.json" "$pkg/manifest.json"; fi + mkdir -p "$d/dist"; rm -f "$d/dist/extension.zip" + ( cd "$pkg" && zip -qr "$d/dist/extension.zip" . ); rm -rf "$pkg" +} + +commit_change() { echo "$1" > "$REPO/extensions/demo/code.txt"; git -C "$REPO" commit -qam "$1"; } + +# The script resolves the repo from its own path, as it does when devkit copies it into an extension repo, so its +# sourced helpers (_publishers.sh, _publish.sh) have to travel with it. +run() { + mkdir -p "$REPO/scripts"; cp "$ROOT/scripts/release-extensions.sh" "$REPO/scripts/" 2>/dev/null + cp "$ROOT"/scripts/_*.sh "$REPO/scripts/" 2>/dev/null + GITHUB_SHA="$(git -C "$REPO" rev-parse HEAD)" "$REPO/scripts/release-extensions.sh" 2>&1 +} + +echo "release-extensions:" + +t "publishes a new version under [slug]-v[version]-sdk-[sdkVersion], targeted at the built commit" +new_repo; build_zip 1.0.6; SHA="$(git -C "$REPO" rev-parse HEAD)" +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q -- "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip --target $SHA" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "skips when the -sdk tag exists and the extension is unchanged since it" +new_repo; build_zip 1.0.6; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "already released" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails, naming the version bump, when the extension changed since that version's -sdk release" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; commit_change two; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "manifest.version" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails when the extension changed since that version's legacy (no -sdk) release" +new_repo; git -C "$REPO" tag demo-v0.1.0; commit_change two; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "demo-v0.1.0 " <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "publishes the -sdk build of an unchanged version that has only a legacy release" +new_repo; git -C "$REPO" tag demo-v0.1.0; build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "publishes a build for a new SDK of an unchanged version" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.7 +OUT="$(run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.7" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails when the bundle manifest carries no sdkVersion" +new_repo; build_zip "" +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "sdkVersion" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "warns when the published release landed mutable" +new_repo; build_zip 1.0.6 +OUT="$(GH_IMMUTABLE=false run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning::.*immutable" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "does not warn when the published release is immutable" +new_repo; build_zip 1.0.6 +OUT="$(GH_IMMUTABLE=true run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "::warning::" <<<"$OUT"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "fails when gh release create fails" +new_repo; build_zip 1.0.6 +OUT="$(GH_CREATE_RC=1 run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC out=$OUT"; fi + +t "still publishes the other extensions when one fails" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; commit_change two; build_zip 1.0.6 +mkdir -p "$REPO/extensions/other/dist" +printf '{"id":"duplo.other","name":"Other","version":"0.2.0","resources":[{"subType":"demo"}]}\n' > "$REPO/extensions/other/manifest.json" +git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm other +( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" + cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "release create other-v0.2.0-sdk-1.0.6" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "outside Duplo's orgs, releases the zip only and never signs" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY_OWNER=acme run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip --target" "$GH_LOG" \ + && ! grep -q 'extension.zip.sig' "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "in a Duplo org, attaches the signature beside the zip" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip extensions/demo/dist/extension.zip.sig --target" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "in a Duplo org with no signing key, fails before releasing" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY_OWNER=duplocloud run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "EXTENSION_SIGNING_KEY" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "a refused signature fails before releasing" +new_repo; build_zip 1.0.6 +OUT="$(FAKE_SIGN_RC=1 GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG"; then ok; else bad "rc=$RC log=$(cat "$GH_LOG")"; fi + +t "a Duplo-org repository outside the allowlist signs and releases, then skips publishing with a notice" +new_repo; build_zip 1.0.6 +OUT="$(GITHUB_REPOSITORY=duplocloud/not-listed GITHUB_REPOSITORY_OWNER=duplocloud EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::notice::.*allowlist" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" \ + && grep -q "release create demo-v0.1.0-sdk-1.0.6 extensions/demo/dist/extension.zip extensions/demo/dist/extension.zip.sig --target" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "a failed gh release view fails that extension without stopping the run" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +mkdir -p "$REPO/extensions/other/dist" +printf '{"id":"duplo.other","name":"Other","version":"0.2.0","resources":[{"subType":"demo"}]}\n' > "$REPO/extensions/other/manifest.json" +git -C "$REPO" add extensions/other/manifest.json; git -C "$REPO" commit -qm other +( p="$(mktemp -d)"; jq '.sdkVersion="1.0.6"' "$REPO/extensions/other/manifest.json" > "$p/manifest.json" + cd "$p" && zip -qr "$REPO/extensions/other/dist/extension.zip" . ) +OUT="$(GH_VIEW_RC=1 PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + CONSOLE_API_KEY=secret-key EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "::error::.*gh release view demo-v0.1.0-sdk-1.0.6 failed" <<<"$OUT" \ + && grep -q "release create other-v0.2.0-sdk-1.0.6" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "an allowlisted extension with no CONSOLE_API_KEY fails before signing or releasing" +new_repo; build_zip 1.0.6 +OUT="$(PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + CONSOLE_API_KEY= EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "::error::.*extensions/demo.*CONSOLE_API_KEY" <<<"$OUT" && ! grep -q "release create" "$GH_LOG" \ + && [ ! -f "$REPO/extensions/demo/dist/extension.zip.sig" ] +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "an allowlisted, already released extension with no CONSOLE_API_KEY fails before reading its release" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +OUT="$(PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + CONSOLE_API_KEY= EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" != 0 ] && grep -q "::error::.*CONSOLE_API_KEY" <<<"$OUT" && ! grep -q "release view" "$GH_LOG" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "an existing signed release resumes publishing without re-signing or re-releasing" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +# The release's own assets (never the local dist/ bundle) are what a resumed publish reads, so plant them separately. +mkdir -p "$TMP/assets.19" "$TMP/s3.19" "$TMP/console.19" +printf 'ZIPBYTES' > "$TMP/assets.19/extension.zip"; printf 'SIGBYTES' > "$TMP/assets.19/extension.zip.sig" +ZSHA19="$(shasum -a 256 "$TMP/assets.19/extension.zip" | cut -d' ' -f1)" +OUT="$(ASSETS="$TMP/assets.19" S3="$TMP/s3.19" CONSOLE="$TMP/console.19" ZSHA="$ZSHA19" CONSOLE_API_KEY=secret-key \ + PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && ! grep -q "release create" "$GH_LOG" && [ ! -f "$REPO/extensions/demo/dist/extension.zip.sig" ] \ + && grep -q "registered demo-v0.1.0-sdk-1.0.6" <<<"$OUT" \ + && jq -e --arg s "$ZSHA19" '.[0] | .sha256 == $s and .signature == "SIGBYTES"' "$TMP/console.19/artifacts.json" >/dev/null +then ok; else bad "rc=$RC out=$OUT"; fi + +t "a release cut before signing warns with the bump and does not fail" +new_repo; git -C "$REPO" tag demo-v0.1.0-sdk-1.0.6; build_zip 1.0.6 +OUT="$(GH_ASSETS=extension.zip PUBLISHERS_FILE="$TMP/pub.json" GITHUB_REPOSITORY=duplocloud/demo GITHUB_REPOSITORY_OWNER=duplocloud \ + CONSOLE_API_KEY=secret-key EXTENSION_SIGNING_KEY=k EXTENSION_SIGNING_CERT=c run)"; RC=$? +if [ "$RC" = 0 ] && grep -q "::warning::.*no extension.zip.sig.*Bump manifest.version" <<<"$OUT" && ! grep -q "would publish" <<<"$OUT" +then ok; else bad "rc=$RC out=$OUT"; fi + +t "fails, naming the extension, when the manifest declares no resource" +new_repo; jq 'del(.resources)' "$REPO/extensions/demo/manifest.json" > "$TMP/m.json" && mv "$TMP/m.json" "$REPO/extensions/demo/manifest.json" +build_zip 1.0.6 +OUT="$(run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "::error::.*extensions/demo.*resource" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +t "fails, naming the extension, when the bundle is over the size limit" +new_repo; build_zip 1.0.6 +OUT="$(EXTENSION_MAX_BYTES=1 run)"; RC=$? +if [ "$RC" != 0 ] && ! grep -q "release create" "$GH_LOG" && grep -q "::error::.*extensions/demo.*over 1 bytes" <<<"$OUT" +then ok; else bad "rc=$RC log=$(cat "$GH_LOG") out=$OUT"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/test-sdk-digest.sh b/tests/test-sdk-digest.sh new file mode 100755 index 0000000..87a88b9 --- /dev/null +++ b/tests/test-sdk-digest.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tests for scripts/_sdk_digest.sh — the sdkDigest build-extension.sh stamps into a bundle's manifest. +# Usage: ./tests/test-sdk-digest.sh +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +PASS=0; FAIL=0 +t() { printf ' %s … ' "$1"; } +ok() { echo "ok"; PASS=$((PASS+1)); } +bad() { echo "FAIL: $1"; FAIL=$((FAIL+1)); } + +TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT +. ./scripts/_sdk_digest.sh || { echo "scripts/_sdk_digest.sh not found"; exit 1; } + +# The definition, computed independently: sha256 over " \n" per file, sorted by path bytewise. +expected() { + python3 - "$1" <<'PY' +import hashlib, os, sys +root = sys.argv[1]; lines = [] +for d, _, fs in os.walk(root): + for f in fs: + p = os.path.join(d, f); rel = os.path.relpath(p, root) + lines.append((rel.encode(), hashlib.sha256(open(p, "rb").read()).hexdigest())) +lines.sort() +print(hashlib.sha256(b"".join(h.encode() + b" " + r + b"\n" for r, h in lines)).hexdigest()) +PY +} + +mkfeed() { + FEED="$TMP/feed.$RANDOM"; mkdir -p "$FEED/sub" + echo a > "$FEED/Duplocloud.AiHelpdesk.Sdk.1.0.6.nupkg"; echo b > "$FEED/B.nupkg"; echo c > "$FEED/sub/c.nupkg" +} + +echo "sdk digest:" + +t "matches the definition: sha256 of sorted ' ' lines" +mkfeed +if [ "$(sdk_digest "$FEED")" = "$(expected "$FEED")" ]; then ok; else bad "$(sdk_digest "$FEED") != $(expected "$FEED")"; fi + +t "changes when one package's bytes change" +mkfeed; before="$(sdk_digest "$FEED")"; echo changed > "$FEED/B.nupkg" +if [ "$(sdk_digest "$FEED")" != "$before" ]; then ok; else bad "unchanged: $before"; fi + +t "changes when a package is added" +mkfeed; before="$(sdk_digest "$FEED")"; echo d > "$FEED/D.nupkg" +if [ "$(sdk_digest "$FEED")" != "$before" ]; then ok; else bad "unchanged: $before"; fi + +t "does not depend on where the feed sits or the caller's cwd" +mkfeed; a="$(sdk_digest "$FEED")"; cp -R "$FEED" "$TMP/moved"; b="$( cd / && sdk_digest "$TMP/moved" )" +if [ "$a" = "$b" ]; then ok; else bad "$a != $b"; fi + +t "fails on an empty feed rather than stamping the digest of nothing" +mkdir -p "$TMP/empty" +if ! sdk_digest "$TMP/empty" >/dev/null 2>&1; then ok; else bad "returned $(sdk_digest "$TMP/empty")"; fi + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/tests/vectors/README.md b/tests/vectors/README.md new file mode 100644 index 0000000..6fe535d --- /dev/null +++ b/tests/vectors/README.md @@ -0,0 +1,98 @@ +# Extension signing test vectors + +**These keys are test-only.** `root.pem`, `publisher.pem` and `other.pem` are throwaway EC P-256 +private keys. They are committed on purpose so that two independent verifiers can be checked +against the same inputs. No console, portal or license trusts them. Never use them for anything +else. + +The .NET helpdesk copies this directory as-is into its own test suite. For every entry in +`cases.json`, its verifier must produce the same outcome as the console's +`verify_artifact_signature` (`apps/extensions/signing/verify.py`). + +## Files + +| File | Contents | +|---|---| +| `root.pem` / `root.pub.pem` | Test root key, private and public. Its kid is the only key of `root_keys` in `cases.json` | +| `publisher.pem` | Test publisher private key. Its root-signed certificate rides in each `.sig` header (`cert`) | +| `other.pem` | An unrelated key. It signs the `bad-signature` and `unknown-root` vectors | +| `extension.zip` | A tiny zip holding `manifest.json`: `{"id":"io.example.hello","version":"1.0.0"}` | +| `*.zip.sig` | One compact JWS per file, with no trailing newline. Each case's `sig` names the file it uses | +| `cases.json` | `root_keys` (a map from root kid to PEM file name in this directory) and `cases` | + +## How to run a case + +Each case is: + +``` +{name, sig, sha256, manifest_id, version, sdk_version, trusted_publishers, now, expect} +``` + +Call the verifier once per case with these inputs: + +- **Signature:** the contents of the file named by `sig`. +- **Root keys:** loaded from `root_keys`. +- **`sha256`, `manifest_id`, `version`, `sdk_version`:** the case's own values. They are the + *observed* values: what the portal would have computed from the zip and read from its + manifest. Feed them **directly** into the claim comparison. Do **not** re-hash + `extension.zip` or re-read its manifest for the case run. Several cases deliberately supply + observed values that differ from the zip. +- **`trusted_publishers`:** the license's `trusted_pub`. `null` means skip the trust check, as + the console does. A list (including an empty list) means the certificate's `sub` must be in + it. +- **`now`:** the current time, as integer epoch seconds. Use it for the certificate's + `nbf`/`exp` check instead of the clock. + +`expect` is `ok` or the error code the verifier must report. + +`extension.zip` is only the source of the valid cases' `sha256`, and a sample for a separate +end-to-end test (hash the zip, read the manifest, then verify `extension.zip.sig`). + +## Check order and codes + +The steps run in this order. The first step that fails decides the code, so a token with several +defects reports the earliest one. + +1. **Signature header** (read unverified, only to route). + - `typ` must be `duplo-ext-sig+jwt`, else `bad_typ`. + - `alg` must be `ES256`, else `malformed`. This covers `none` and `HS256`. + - `cert` must be a non-empty string, else `malformed`. + - A header that can't be read at all is `malformed`. +2. **Certificate.** + - `typ` must be `duplo-ext-cert+jwt`, else `bad_typ`. + - Its header `kid` must be in `root_keys`, else `unknown_root`. + - It must verify as ES256 under that root key, else `bad_cert`. + - `iss` must be `duplocloud-console`. `sub` and `kid` must be strings, `ns` a list of strings + and `nbf`/`exp` integers. `jwk` must be an EC P-256 key with 32-byte canonical base64url + coordinates. Any failure is `bad_cert`. + - Validity has zero leeway: `now < nbf` or `now >= exp` is `cert_expired`. +3. **Trust** (only when `trusted_publishers` is not null): the certificate's `sub` must be in + it, else `untrusted_publisher`. +4. **Signature and key binding.** + - The signature must verify as ES256 under the certificate's `jwk`, else `bad_signature`. + Any other decode failure is `malformed`. + - The signature header `kid` must then equal the certificate's `kid`, else `kid_mismatch`. + - The claims `manifest_id`, `version`, `sdk_version` and `sha256` must be non-empty strings, + with `sha256` as 64 lowercase hex characters, else `malformed`. +5. **Claims.** `sha256` is compared first: a mismatch is `hash_mismatch`, and the observed value + is compared case-insensitively. Then `manifest_id`, `version` and `sdk_version` are compared: + a mismatch is `manifest_mismatch`. +6. **Namespace.** `manifest_id` must equal one of the certificate's `ns` prefixes or start with + `prefix + "."`, else `outside_namespace`. + +All codes: `malformed`, `bad_typ`, `unknown_root`, `bad_cert`, `cert_expired`, +`untrusted_publisher`, `kid_mismatch`, `bad_signature`, `hash_mismatch`, `manifest_mismatch`, +`outside_namespace`. + +## Regenerating + +```bash +make manage ARGS='make_signing_vectors' +``` + +The command reuses the existing private keys, so the kids, the certificate claims and the zip +stay the same. ECDSA signatures are randomised, so the `.sig` files change on every run. Before +writing `cases.json`, the command checks every case against the console verifier. + +The committed files are the source of truth. Only regenerate when the format changes, and copy +the result to the helpdesk again. diff --git a/tests/vectors/SOURCE b/tests/vectors/SOURCE new file mode 100644 index 0000000..2670bfb --- /dev/null +++ b/tests/vectors/SOURCE @@ -0,0 +1 @@ +duplocloud-internal/console apps/extensions/tests/vectors at 412ed03bd41c523e4edcb0be53666841ec9792c6. Test-only keys. diff --git a/tests/vectors/alg-hs256.zip.sig b/tests/vectors/alg-hs256.zip.sig new file mode 100644 index 0000000..79b583a --- /dev/null +++ b/tests/vectors/alg-hs256.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiAiSFMyNTYiLCAia2lkIjogIjEzZTkwN2E0NTMwYjUzNjAiLCAidHlwIjogImR1cGxvLWV4dC1zaWcrand0IiwgImNlcnQiOiAiZXlKaGJHY2lPaUpGVXpJMU5pSXNJbXRwWkNJNkltRTNaRFV3TUdObFlURXlPVGd4TVRFaUxDSjBlWEFpT2lKa2RYQnNieTFsZUhRdFkyVnlkQ3RxZDNRaWZRLmV5SnBjM01pT2lKa2RYQnNiMk5zYjNWa0xXTnZibk52YkdVaUxDSnpkV0lpT2lJd01EQXdNREF3TUMwd01EQXdMVFF3TURBdE9EQXdNQzB3TURBd01EQXdNREF3TURFaUxDSnVZVzFsSWpvaVJYaGhiWEJzWlNCUWRXSnNhWE5vWlhJaUxDSnJhV1FpT2lJeE0yVTVNRGRoTkRVek1HSTFNell3SWl3aWFuZHJJanA3SW10MGVTSTZJa1ZESWl3aVkzSjJJam9pVUMweU5UWWlMQ0o0SWpvaU9WQndjV2xKUVVWTlIwTldSbWw2TVVaUVgzSm9PUzFIVEZaNWVVZG5aWEJJVjB3eVNUTXlSMFpSVVNJc0lua2lPaUoyZW1sU0xYUm1hVmx5VFVkbmFuZGpkVE5xVFRoNGIxTmlZakl0VGxwd2JURktNV2RpVTFObkxXWkJJbjBzSW01eklqcGJJbWx2TG1WNFlXMXdiR1VpWFN3aWFXRjBJam94TnpZM01qSTFOakF3TENKdVltWWlPakUzTmpjeU1qVTJNREFzSW1WNGNDSTZNakE0TWpjMU9EUXdNSDAuX0QtUkl5Rzh5UVBITGpXb2RCMXFxcEhXYzFQempSNkw5QkV5MUpQNk4yR245NWlBTTRkdl85VmVLR2gyS3EtMFhqSXpaUnVuNmd3bGdScllud19RalEifQ.eyJtYW5pZmVzdF9pZCI6ICJpby5leGFtcGxlLmhlbGxvIiwgInZlcnNpb24iOiAiMS4wLjAiLCAic2RrX3ZlcnNpb24iOiAiMS4wLjAiLCAic2hhMjU2IjogIjQ0NTQ3MmZjYmQxMTVlMzNkNjMyNDgzOGQ0MmI5ZjU2N2I1MTY3YzQzNDkyNTBmODVhNjU1NWIyNDI0YTk0MGMiLCAiaWF0IjogMTc2NzIyNTYwMH0.EC8rPsZnPXi1NIrpbWb-cZd6s8JrSMaeR6ZjGH3YymA \ No newline at end of file diff --git a/tests/vectors/alg-none.zip.sig b/tests/vectors/alg-none.zip.sig new file mode 100644 index 0000000..e455638 --- /dev/null +++ b/tests/vectors/alg-none.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiAibm9uZSIsICJraWQiOiAiMTNlOTA3YTQ1MzBiNTM2MCIsICJ0eXAiOiAiZHVwbG8tZXh0LXNpZytqd3QiLCAiY2VydCI6ICJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SW1FM1pEVXdNR05sWVRFeU9UZ3hNVEVpTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5fRC1SSXlHOHlRUEhMaldvZEIxcXFwSFdjMVB6alI2TDlCRXkxSlA2TjJHbjk1aUFNNGR2XzlWZUtHaDJLcS0wWGpJelpSdW42Z3dsZ1JyWW53X1FqUSJ9.eyJtYW5pZmVzdF9pZCI6ICJpby5leGFtcGxlLmhlbGxvIiwgInZlcnNpb24iOiAiMS4wLjAiLCAic2RrX3ZlcnNpb24iOiAiMS4wLjAiLCAic2hhMjU2IjogIjQ0NTQ3MmZjYmQxMTVlMzNkNjMyNDgzOGQ0MmI5ZjU2N2I1MTY3YzQzNDkyNTBmODVhNjU1NWIyNDI0YTk0MGMiLCAiaWF0IjogMTc2NzIyNTYwMH0. \ No newline at end of file diff --git a/tests/vectors/bad-signature.zip.sig b/tests/vectors/bad-signature.zip.sig new file mode 100644 index 0000000..6a33f02 --- /dev/null +++ b/tests/vectors/bad-signature.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImNlcnQiOiJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SW1FM1pEVXdNR05sWVRFeU9UZ3hNVEVpTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5fRC1SSXlHOHlRUEhMaldvZEIxcXFwSFdjMVB6alI2TDlCRXkxSlA2TjJHbjk1aUFNNGR2XzlWZUtHaDJLcS0wWGpJelpSdW42Z3dsZ1JyWW53X1FqUSIsImtpZCI6IjEzZTkwN2E0NTMwYjUzNjAiLCJ0eXAiOiJkdXBsby1leHQtc2lnK2p3dCJ9.eyJtYW5pZmVzdF9pZCI6ImlvLmV4YW1wbGUuaGVsbG8iLCJ2ZXJzaW9uIjoiMS4wLjAiLCJzZGtfdmVyc2lvbiI6IjEuMC4wIiwic2hhMjU2IjoiNDQ1NDcyZmNiZDExNWUzM2Q2MzI0ODM4ZDQyYjlmNTY3YjUxNjdjNDM0OTI1MGY4NWE2NTU1YjI0MjRhOTQwYyIsImlhdCI6MTc2NzIyNTYwMH0.xNhBD0Oco9ghRlHP4QeKHOwdLpT7DEM8K9wckkUufiskjeXZjxxVw7Ne3xcadnTkFL1-TucngEhNmNawI5dYmA \ No newline at end of file diff --git a/tests/vectors/cases.json b/tests/vectors/cases.json new file mode 100644 index 0000000..2bb87e1 --- /dev/null +++ b/tests/vectors/cases.json @@ -0,0 +1,200 @@ +{ + "root_keys": { + "a7d500cea1298111": "root.pub.pem" + }, + "cases": [ + { + "name": "valid", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "ok" + }, + { + "name": "valid-trust-check-skipped", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": null, + "now": 1798761600, + "expect": "ok" + }, + { + "name": "tampered-hash", + "sig": "extension.zip.sig", + "sha256": "8370c756d9c473c4d0c0fc89841fd8bfe37cb50fdc107e09d87583819d987946", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "hash_mismatch" + }, + { + "name": "wrong-version", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.1", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "manifest_mismatch" + }, + { + "name": "wrong-namespace", + "sig": "outside-namespace.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "com.other.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "outside_namespace" + }, + { + "name": "expired", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 2082758400, + "expect": "cert_expired" + }, + { + "name": "not-yet-valid", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1767225599, + "expect": "cert_expired" + }, + { + "name": "untrusted-publisher", + "sig": "extension.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000002" + ], + "now": 1798761600, + "expect": "untrusted_publisher" + }, + { + "name": "bad-signature", + "sig": "bad-signature.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "bad_signature" + }, + { + "name": "kid-mismatch", + "sig": "kid-mismatch.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "kid_mismatch" + }, + { + "name": "unknown-root", + "sig": "unknown-root.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "unknown_root" + }, + { + "name": "cert-as-signature", + "sig": "cert-as-signature.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "bad_typ" + }, + { + "name": "alg-none", + "sig": "alg-none.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "malformed" + }, + { + "name": "alg-hs256", + "sig": "alg-hs256.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "malformed" + }, + { + "name": "missing-cert-header", + "sig": "missing-cert-header.zip.sig", + "sha256": "445472fcbd115e33d6324838d42b9f567b5167c4349250f85a6555b2424a940c", + "manifest_id": "io.example.hello", + "version": "1.0.0", + "sdk_version": "1.0.0", + "trusted_publishers": [ + "00000000-0000-4000-8000-000000000001" + ], + "now": 1798761600, + "expect": "malformed" + } + ] +} diff --git a/tests/vectors/cert-as-signature.zip.sig b/tests/vectors/cert-as-signature.zip.sig new file mode 100644 index 0000000..9b4bc27 --- /dev/null +++ b/tests/vectors/cert-as-signature.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImtpZCI6ImE3ZDUwMGNlYTEyOTgxMTEiLCJ0eXAiOiJkdXBsby1leHQtY2VydCtqd3QifQ.eyJpc3MiOiJkdXBsb2Nsb3VkLWNvbnNvbGUiLCJzdWIiOiIwMDAwMDAwMC0wMDAwLTQwMDAtODAwMC0wMDAwMDAwMDAwMDEiLCJuYW1lIjoiRXhhbXBsZSBQdWJsaXNoZXIiLCJraWQiOiIxM2U5MDdhNDUzMGI1MzYwIiwiandrIjp7Imt0eSI6IkVDIiwiY3J2IjoiUC0yNTYiLCJ4IjoiOVBwcWlJQUVNR0NWRml6MUZQX3JoOS1HTFZ5eUdnZXBIV0wySTMyR0ZRUSIsInkiOiJ2emlSLXRmaVlyTUdnandjdTNqTTh4b1NiYjItTlpwbTFKMWdiU1NnLWZBIn0sIm5zIjpbImlvLmV4YW1wbGUiXSwiaWF0IjoxNzY3MjI1NjAwLCJuYmYiOjE3NjcyMjU2MDAsImV4cCI6MjA4Mjc1ODQwMH0._D-RIyG8yQPHLjWodB1qqpHWc1PzjR6L9BEy1JP6N2Gn95iAM4dv_9VeKGh2Kq-0XjIzZRun6gwlgRrYnw_QjQ \ No newline at end of file diff --git a/tests/vectors/extension.zip b/tests/vectors/extension.zip new file mode 100644 index 0000000..fd56c9b Binary files /dev/null and b/tests/vectors/extension.zip differ diff --git a/tests/vectors/extension.zip.sig b/tests/vectors/extension.zip.sig new file mode 100644 index 0000000..ba979f9 --- /dev/null +++ b/tests/vectors/extension.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImNlcnQiOiJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SW1FM1pEVXdNR05sWVRFeU9UZ3hNVEVpTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5fRC1SSXlHOHlRUEhMaldvZEIxcXFwSFdjMVB6alI2TDlCRXkxSlA2TjJHbjk1aUFNNGR2XzlWZUtHaDJLcS0wWGpJelpSdW42Z3dsZ1JyWW53X1FqUSIsImtpZCI6IjEzZTkwN2E0NTMwYjUzNjAiLCJ0eXAiOiJkdXBsby1leHQtc2lnK2p3dCJ9.eyJtYW5pZmVzdF9pZCI6ImlvLmV4YW1wbGUuaGVsbG8iLCJ2ZXJzaW9uIjoiMS4wLjAiLCJzZGtfdmVyc2lvbiI6IjEuMC4wIiwic2hhMjU2IjoiNDQ1NDcyZmNiZDExNWUzM2Q2MzI0ODM4ZDQyYjlmNTY3YjUxNjdjNDM0OTI1MGY4NWE2NTU1YjI0MjRhOTQwYyIsImlhdCI6MTc2NzIyNTYwMH0.BeeC1bx3en8h_gOvK59rCZKY2U9TzFhh5yKCNPtBc9GG7tcxvctRBuguNrcfnqzwCX9tF-4vC0PK9V0KlHIn7g \ No newline at end of file diff --git a/tests/vectors/kid-mismatch.zip.sig b/tests/vectors/kid-mismatch.zip.sig new file mode 100644 index 0000000..6f398f6 --- /dev/null +++ b/tests/vectors/kid-mismatch.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImNlcnQiOiJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SW1FM1pEVXdNR05sWVRFeU9UZ3hNVEVpTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5fRC1SSXlHOHlRUEhMaldvZEIxcXFwSFdjMVB6alI2TDlCRXkxSlA2TjJHbjk1aUFNNGR2XzlWZUtHaDJLcS0wWGpJelpSdW42Z3dsZ1JyWW53X1FqUSIsImtpZCI6IjAwMDAwMDAwMDAwMDAwMDAiLCJ0eXAiOiJkdXBsby1leHQtc2lnK2p3dCJ9.eyJtYW5pZmVzdF9pZCI6ImlvLmV4YW1wbGUuaGVsbG8iLCJ2ZXJzaW9uIjoiMS4wLjAiLCJzZGtfdmVyc2lvbiI6IjEuMC4wIiwic2hhMjU2IjoiNDQ1NDcyZmNiZDExNWUzM2Q2MzI0ODM4ZDQyYjlmNTY3YjUxNjdjNDM0OTI1MGY4NWE2NTU1YjI0MjRhOTQwYyIsImlhdCI6MTc2NzIyNTYwMH0.eGS3jCQC-Uw_c1gwgEhr01sziYkUODzOCJJ0HkpF0HDEbF9WI0a64VSrotdMwok3F9GuxLdjSF3BUBIYINDyNA \ No newline at end of file diff --git a/tests/vectors/missing-cert-header.zip.sig b/tests/vectors/missing-cert-header.zip.sig new file mode 100644 index 0000000..75579e1 --- /dev/null +++ b/tests/vectors/missing-cert-header.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImtpZCI6IjEzZTkwN2E0NTMwYjUzNjAiLCJ0eXAiOiJkdXBsby1leHQtc2lnK2p3dCJ9.eyJtYW5pZmVzdF9pZCI6ImlvLmV4YW1wbGUuaGVsbG8iLCJ2ZXJzaW9uIjoiMS4wLjAiLCJzZGtfdmVyc2lvbiI6IjEuMC4wIiwic2hhMjU2IjoiNDQ1NDcyZmNiZDExNWUzM2Q2MzI0ODM4ZDQyYjlmNTY3YjUxNjdjNDM0OTI1MGY4NWE2NTU1YjI0MjRhOTQwYyIsImlhdCI6MTc2NzIyNTYwMH0.px7QDJRmboKXFwj--JvHebnqqj2A6guZrctskpGPXDHdqZg4VHYRSn1Rp3lskSaENDrVgwu_hi86nOHN4VtvOg \ No newline at end of file diff --git a/tests/vectors/other.pem b/tests/vectors/other.pem new file mode 100644 index 0000000..14a4599 --- /dev/null +++ b/tests/vectors/other.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgaSmJFAN649JaBp4n +PwryCUeZQSh2q2v53HEWOMizj/ShRANCAATLub3gRHIv9zaD1eD3WRc2Z1mJ1NlF +Q67kB9ZrbaBNuHzjPgxkMY5Q/TfqY9CrxBwQleQaP3KefXb20sLLafgN +-----END PRIVATE KEY----- diff --git a/tests/vectors/outside-namespace.zip.sig b/tests/vectors/outside-namespace.zip.sig new file mode 100644 index 0000000..fd7f85f --- /dev/null +++ b/tests/vectors/outside-namespace.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImNlcnQiOiJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SW1FM1pEVXdNR05sWVRFeU9UZ3hNVEVpTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5fRC1SSXlHOHlRUEhMaldvZEIxcXFwSFdjMVB6alI2TDlCRXkxSlA2TjJHbjk1aUFNNGR2XzlWZUtHaDJLcS0wWGpJelpSdW42Z3dsZ1JyWW53X1FqUSIsImtpZCI6IjEzZTkwN2E0NTMwYjUzNjAiLCJ0eXAiOiJkdXBsby1leHQtc2lnK2p3dCJ9.eyJtYW5pZmVzdF9pZCI6ImNvbS5vdGhlci5oZWxsbyIsInZlcnNpb24iOiIxLjAuMCIsInNka192ZXJzaW9uIjoiMS4wLjAiLCJzaGEyNTYiOiI0NDU0NzJmY2JkMTE1ZTMzZDYzMjQ4MzhkNDJiOWY1NjdiNTE2N2M0MzQ5MjUwZjg1YTY1NTViMjQyNGE5NDBjIiwiaWF0IjoxNzY3MjI1NjAwfQ.el-ZypkmjlpSJ3JQyMcMGExp58LvFz2ih5aGFQjh-ZJ0LCJjliQlYezvqMB4fD34YkmuC2dShlCJVCMEzX63FQ \ No newline at end of file diff --git a/tests/vectors/publisher.pem b/tests/vectors/publisher.pem new file mode 100644 index 0000000..60f1dfb --- /dev/null +++ b/tests/vectors/publisher.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg8qjZ1UnY1RkSNtRp +r5f7Hxyv2v8aMFvnlBQy5hsf/3mhRANCAAT0+mqIgAQwYJUWLPUU/+uH34YtXLIa +B6kdYvYjfYYVBL84kfrX4mKzBoI8HLt4zPMaEm29vjWaZtSdYG0koPnw +-----END PRIVATE KEY----- diff --git a/tests/vectors/root.pem b/tests/vectors/root.pem new file mode 100644 index 0000000..e705bc7 --- /dev/null +++ b/tests/vectors/root.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg3JE1sso6VRs8S6EB +4V6acuOt1hoXq31Aa7Ji+kGaw1qhRANCAARiHh+F54bW+g6FK/W9KQkvcj9jqvgR +aCz/DuJsj4aWkCHhdPvHdiJqkrhbjinCxeQNyv4htDDP8MY9gsMK28nB +-----END PRIVATE KEY----- diff --git a/tests/vectors/root.pub.pem b/tests/vectors/root.pub.pem new file mode 100644 index 0000000..384bf90 --- /dev/null +++ b/tests/vectors/root.pub.pem @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYh4fheeG1voOhSv1vSkJL3I/Y6r4 +EWgs/w7ibI+GlpAh4XT7x3YiapK4W44pwsXkDcr+IbQwz/DGPYLDCtvJwQ== +-----END PUBLIC KEY----- diff --git a/tests/vectors/unknown-root.zip.sig b/tests/vectors/unknown-root.zip.sig new file mode 100644 index 0000000..c50fc7b --- /dev/null +++ b/tests/vectors/unknown-root.zip.sig @@ -0,0 +1 @@ +eyJhbGciOiJFUzI1NiIsImNlcnQiOiJleUpoYkdjaU9pSkZVekkxTmlJc0ltdHBaQ0k2SWpGaU5qTTJZek0zTVRZeU5tVTJOek1pTENKMGVYQWlPaUprZFhCc2J5MWxlSFF0WTJWeWRDdHFkM1FpZlEuZXlKcGMzTWlPaUprZFhCc2IyTnNiM1ZrTFdOdmJuTnZiR1VpTENKemRXSWlPaUl3TURBd01EQXdNQzB3TURBd0xUUXdNREF0T0RBd01DMHdNREF3TURBd01EQXdNREVpTENKdVlXMWxJam9pUlhoaGJYQnNaU0JRZFdKc2FYTm9aWElpTENKcmFXUWlPaUl4TTJVNU1EZGhORFV6TUdJMU16WXdJaXdpYW5kcklqcDdJbXQwZVNJNklrVkRJaXdpWTNKMklqb2lVQzB5TlRZaUxDSjRJam9pT1ZCd2NXbEpRVVZOUjBOV1JtbDZNVVpRWDNKb09TMUhURlo1ZVVkblpYQklWMHd5U1RNeVIwWlJVU0lzSW5raU9pSjJlbWxTTFhSbWFWbHlUVWRuYW5kamRUTnFUVGg0YjFOaVlqSXRUbHB3YlRGS01XZGlVMU5uTFdaQkluMHNJbTV6SWpwYkltbHZMbVY0WVcxd2JHVWlYU3dpYVdGMElqb3hOelkzTWpJMU5qQXdMQ0p1WW1ZaU9qRTNOamN5TWpVMk1EQXNJbVY0Y0NJNk1qQTRNamMxT0RRd01IMC5oM2hURDNPaFRzSzdQeVdDZkItQjRhRlhSR1hFVkl1STFFZ1R2d184cEtWRGliX1U1anh6RnVpal95Tm9Lall3U2V6ZTRFbUdnbDJlZEdJX29QLXBMdyIsImtpZCI6IjEzZTkwN2E0NTMwYjUzNjAiLCJ0eXAiOiJkdXBsby1leHQtc2lnK2p3dCJ9.eyJtYW5pZmVzdF9pZCI6ImlvLmV4YW1wbGUuaGVsbG8iLCJ2ZXJzaW9uIjoiMS4wLjAiLCJzZGtfdmVyc2lvbiI6IjEuMC4wIiwic2hhMjU2IjoiNDQ1NDcyZmNiZDExNWUzM2Q2MzI0ODM4ZDQyYjlmNTY3YjUxNjdjNDM0OTI1MGY4NWE2NTU1YjI0MjRhOTQwYyIsImlhdCI6MTc2NzIyNTYwMH0.grs8ZB5PQHvuRs1eXX6jjoR6xKjId8scvEOwXrvzFB-wFtcRWMOM_xS-59I_qKngOQoMtKxWMvTbo247mKrItQ \ No newline at end of file