From 272d975063d8e30246efd3af9e5189012efb4600 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Thu, 30 Jul 2026 15:08:34 +0200 Subject: [PATCH 1/4] STAC-25461 report release-branch build failures to Slack via Cerberus --- .github/workflows/build-binaries.yml | 28 +++++++ .github/workflows/build-deb.yml | 28 +++++++ .github/workflows/cerberus-notify.yml | 95 +++++++++++++++++++++++ .github/workflows/lint-and-unit-tests.yml | 29 +++++++ 4 files changed, 180 insertions(+) create mode 100644 .github/workflows/cerberus-notify.yml diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index d6d0dca547ae..fd594159295b 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -41,6 +41,12 @@ name: Binary builds on: pull_request: + # The release branch, so a failure there reaches Slack via Cerberus + # (STAC-25461). Moves with the release; GitLab hardcoded it the same way + # (`only: - stackstate-7.78.2`). + push: + branches: + - stackstate-7.78.2 workflow_dispatch: permissions: @@ -255,3 +261,25 @@ jobs: docker run --rm \ --entrypoint /opt/stackstate-agent/bin/stackstate-cluster-agent/stackstate-cluster-agent \ "${LOCAL_IMAGE}" version + + # Terminal job: report a failed build of the release branch to the team's Slack + # CI channel (STAC-25461, replacing GitLab's `notify-on-master-fail`). PRs never + # reach it -- the `push` filter above only carries the release branch, so the + # event check is the whole gate. `needs` lists every job, not just the leaves, so + # an early failure that skips its dependents is still reported. + cerberus-notify: + name: Report failure to Slack (Cerberus) + needs: + - godeps-cache + - build-agent + - build-cluster-agent + - build-cluster-agent-image + if: >- + always() + && github.event_name == 'push' + && contains(needs.*.result, 'failure') + uses: ./.github/workflows/cerberus-notify.yml + with: + suite: binary-builds + secrets: + CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 72daaf7183bc..807e7fea52d3 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -67,6 +67,12 @@ name: DEB package build on: pull_request: + # The release branch, so a failure there reaches Slack via Cerberus + # (STAC-25461). Moves with the release; GitLab hardcoded it the same way + # (`only: - stackstate-7.78.2`). + push: + branches: + - stackstate-7.78.2 workflow_dispatch: permissions: @@ -328,3 +334,25 @@ jobs: run: | set -eo pipefail docker run --rm --entrypoint /opt/stackstate-agent/bin/agent/agent "${LOCAL_IMAGE}" version + + # Terminal job: report a failed build of the release branch to the team's Slack + # CI channel (STAC-25461, replacing GitLab's `notify-on-master-fail`). PRs never + # reach it -- the `push` filter above only carries the release branch, so the + # event check is the whole gate. `needs` lists every job, not just the leaves, so + # an early failure that skips its dependents is still reported. + cerberus-notify: + name: Report failure to Slack (Cerberus) + needs: + - godeps-cache + - build-deb + - test-deb-renaming + - build-agent-image + if: >- + always() + && github.event_name == 'push' + && contains(needs.*.result, 'failure') + uses: ./.github/workflows/cerberus-notify.yml + with: + suite: deb-package + secrets: + CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} diff --git a/.github/workflows/cerberus-notify.yml b/.github/workflows/cerberus-notify.yml new file mode 100644 index 000000000000..f47c3db856bb --- /dev/null +++ b/.github/workflows/cerberus-notify.yml @@ -0,0 +1,95 @@ +name: Cerberus notify + +# Ported from the `notify-on-master-fail` job in .gitlab-ci.yml as part of the +# GitLab -> GitHub migration (STAC-25142 / STAC-25461). +# +# Called by each pipeline workflow as a terminal job so a failed build of the +# release branch reaches the team's Slack CI channel. Cerberus is the internal +# notify/block Lambda (source: https://gitlab.com/stackvista/devops/cerberus); +# the calling convention here follows `cerberus-block-on-master-fail` in +# StackVista/stackstate's .github/workflows/ci.yml, with `platform: github` in +# the context so Cerberus builds GitHub pipeline/commit URLs. +# +# `action: notify`, never `action: block`. Policy for migrated repos is notify by +# default; blocking locks the branch (`lock_branch`) on every failure of the +# release branch, additionally requires the Cerberus GitHub App to be installed +# on the repo, and mutates Pulumi-managed branch protection out from under it. +# +# The repo's own .cerberus/cerberus_notify_failure.sh is not reused: it is built +# around GitLab's CI_* variables and predates the `platform` field. +# +# Prerequisite: the CERBERUS_LAMBDA_URL secret and the SLACK_CI_REPORT_CHANNEL +# variable must reach this repo at REPO level, provisioned through pulumi-infra. +# Repo-level, not org-level: org secrets default to visibility=private, which +# excludes this PUBLIC repo (the same constraint STAC-25350 hit for REGISTRY_*). +# Until they are provisioned this workflow warns and exits 0 rather than adding a +# second red job to an already-failed run -- the annotation is the signal. + +on: + workflow_call: + inputs: + suite: + description: Suite label shown in the Slack message, e.g. lint-and-unit-tests. + required: true + type: string + secrets: + CERBERUS_LAMBDA_URL: + required: true + +# Nothing here reads the repository; the payload is built entirely from the +# github context. +permissions: {} + +jobs: + notify: + name: Notify Slack via Cerberus + runs-on: docker-public + timeout-minutes: 5 + steps: + - name: Post the failure to Cerberus + env: + CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} + SLACK_CHANNEL: ${{ vars.SLACK_CI_REPORT_CHANNEL }} + REPOSITORY: ${{ github.repository }} + BRANCH: ${{ github.ref_name }} + PIPELINE: ${{ github.run_id }} + COMMIT_SHA: ${{ github.sha }} + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + SUITE: ${{ inputs.suite }} + run: | + set -euo pipefail + + if [ -z "${CERBERUS_LAMBDA_URL}" ]; then + echo "::warning title=Cerberus not configured::CERBERUS_LAMBDA_URL is unset on this repo, so the ${SUITE} failure was not reported to Slack. Provision it at repo level via pulumi-infra (STAC-25461)." + exit 0 + fi + + COMMIT_TITLE=$(printf '%s' "${COMMIT_MESSAGE}" | head -n1) + + curl --verbose --fail \ + -X POST "${CERBERUS_LAMBDA_URL}" \ + -H "Content-Type: application/json" \ + -d "$(jq -n \ + --arg repo "${REPOSITORY}" \ + --arg branch "${BRANCH}" \ + --arg pipeline "${PIPELINE}" \ + --arg sha "${COMMIT_SHA}" \ + --arg title "${COMMIT_TITLE}" \ + --arg suite "${SUITE}" \ + --arg channel "${SLACK_CHANNEL}" \ + '{ + action: "notify", + context: { + platform: "github", + "project.id": $repo, + "project.slug": $repo, + "project.name": "StackState Agent", + branch: $branch, + pipeline: $pipeline, + "commit.sha": $sha, + "commit.title": $title, + suite: $suite, + channel: $channel + } + }' + )" diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index a9c9a58a7c82..e408122a16b1 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -30,6 +30,12 @@ name: Lint and unit tests on: pull_request: + # The release branch, so a failure there reaches Slack via Cerberus + # (STAC-25461). Moves with the release; GitLab hardcoded it the same way + # (`only: - stackstate-7.78.2`). + push: + branches: + - stackstate-7.78.2 workflow_dispatch: inputs: force_mod_tidy: @@ -262,3 +268,26 @@ jobs: go build -o "$(go env GOTOOLDIR)/covdata" cmd/covdata export STS_UT_BUILD_EXCLUDE="oracle,trivy,trivy_no_javadb,nvml,jetson,bundle_installer,systemd" inv -e test --coverage --race --profile --cpus 4 --build-exclude="${STS_UT_BUILD_EXCLUDE}" --timeout=600 + + # Terminal job: report a failed build of the release branch to the team's Slack + # CI channel (STAC-25461, replacing GitLab's `notify-on-master-fail`). PRs never + # reach it -- the `push` filter above only carries the release branch, so the + # event check is the whole gate. `needs` lists every job, not just the leaves, so + # an early failure that skips its dependents is still reported. + cerberus-notify: + name: Report failure to Slack (Cerberus) + needs: + - godeps-cache + - filename-linting + - mod-tidy + - unbranded-unit-tests + - branded-unit-tests + if: >- + always() + && github.event_name == 'push' + && contains(needs.*.result, 'failure') + uses: ./.github/workflows/cerberus-notify.yml + with: + suite: lint-and-unit-tests + secrets: + CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} From 46a089ea971027bfc1398dc241fdf2456edbe0bd Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Thu, 30 Jul 2026 15:48:08 +0200 Subject: [PATCH 2/4] STAC-25461 read SLACK_CI_REPORT_CHANNEL as a secret, not a variable --- .github/workflows/build-binaries.yml | 1 + .github/workflows/build-deb.yml | 1 + .github/workflows/cerberus-notify.yml | 19 +++++++++++-------- .github/workflows/lint-and-unit-tests.yml | 1 + 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index fd594159295b..3c47045d5789 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -283,3 +283,4 @@ jobs: suite: binary-builds secrets: CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} + SLACK_CI_REPORT_CHANNEL: ${{ secrets.SLACK_CI_REPORT_CHANNEL }} diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 807e7fea52d3..29b4d47ace8b 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -356,3 +356,4 @@ jobs: suite: deb-package secrets: CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} + SLACK_CI_REPORT_CHANNEL: ${{ secrets.SLACK_CI_REPORT_CHANNEL }} diff --git a/.github/workflows/cerberus-notify.yml b/.github/workflows/cerberus-notify.yml index f47c3db856bb..71b8ec7de24e 100644 --- a/.github/workflows/cerberus-notify.yml +++ b/.github/workflows/cerberus-notify.yml @@ -18,12 +18,13 @@ name: Cerberus notify # The repo's own .cerberus/cerberus_notify_failure.sh is not reused: it is built # around GitLab's CI_* variables and predates the `platform` field. # -# Prerequisite: the CERBERUS_LAMBDA_URL secret and the SLACK_CI_REPORT_CHANNEL -# variable must reach this repo at REPO level, provisioned through pulumi-infra. -# Repo-level, not org-level: org secrets default to visibility=private, which -# excludes this PUBLIC repo (the same constraint STAC-25350 hit for REGISTRY_*). -# Until they are provisioned this workflow warns and exits 0 rather than adding a -# second red job to an already-failed run -- the annotation is the signal. +# Prerequisite: CERBERUS_LAMBDA_URL and SLACK_CI_REPORT_CHANNEL are org secrets that +# default to visibility=private, which excludes this PUBLIC repo. StackVista/pulumi-infra +# PR #257 flips both to visibility=all (the same fix STAC-25350 applied to REGISTRY_*). +# Until it lands this workflow warns and exits 0 rather than adding a second red job to +# an already-failed run -- the annotation is the signal. +# +# Both are secrets, so SLACK_CI_REPORT_CHANNEL is read from `secrets`, not `vars`. on: workflow_call: @@ -35,6 +36,8 @@ on: secrets: CERBERUS_LAMBDA_URL: required: true + SLACK_CI_REPORT_CHANNEL: + required: false # Nothing here reads the repository; the payload is built entirely from the # github context. @@ -49,7 +52,7 @@ jobs: - name: Post the failure to Cerberus env: CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} - SLACK_CHANNEL: ${{ vars.SLACK_CI_REPORT_CHANNEL }} + SLACK_CHANNEL: ${{ secrets.SLACK_CI_REPORT_CHANNEL }} REPOSITORY: ${{ github.repository }} BRANCH: ${{ github.ref_name }} PIPELINE: ${{ github.run_id }} @@ -60,7 +63,7 @@ jobs: set -euo pipefail if [ -z "${CERBERUS_LAMBDA_URL}" ]; then - echo "::warning title=Cerberus not configured::CERBERUS_LAMBDA_URL is unset on this repo, so the ${SUITE} failure was not reported to Slack. Provision it at repo level via pulumi-infra (STAC-25461)." + echo "::warning title=Cerberus not configured::CERBERUS_LAMBDA_URL is not visible to this repo, so the ${SUITE} failure was not reported to Slack. Needs pulumi-infra PR #257 (STAC-25461)." exit 0 fi diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index e408122a16b1..56dcf50a2e67 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -291,3 +291,4 @@ jobs: suite: lint-and-unit-tests secrets: CERBERUS_LAMBDA_URL: ${{ secrets.CERBERUS_LAMBDA_URL }} + SLACK_CI_REPORT_CHANNEL: ${{ secrets.SLACK_CI_REPORT_CHANNEL }} From 5980757704c53bfacc9bc757be557245f374d28c Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Thu, 30 Jul 2026 16:53:18 +0200 Subject: [PATCH 3/4] STAC-25461 point the Cerberus prerequisite at the repo-level secrets PR --- .github/workflows/cerberus-notify.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cerberus-notify.yml b/.github/workflows/cerberus-notify.yml index 71b8ec7de24e..ec1563ade951 100644 --- a/.github/workflows/cerberus-notify.yml +++ b/.github/workflows/cerberus-notify.yml @@ -18,11 +18,12 @@ name: Cerberus notify # The repo's own .cerberus/cerberus_notify_failure.sh is not reused: it is built # around GitLab's CI_* variables and predates the `platform` field. # -# Prerequisite: CERBERUS_LAMBDA_URL and SLACK_CI_REPORT_CHANNEL are org secrets that -# default to visibility=private, which excludes this PUBLIC repo. StackVista/pulumi-infra -# PR #257 flips both to visibility=all (the same fix STAC-25350 applied to REGISTRY_*). -# Until it lands this workflow warns and exits 0 rather than adding a second red job to -# an already-failed run -- the annotation is the signal. +# Prerequisite: CERBERUS_LAMBDA_URL and SLACK_CI_REPORT_CHANNEL must reach this repo as +# REPO-level secrets (StackVista/pulumi-infra PR #258). The org-level copies are +# visibility=private, which excludes this PUBLIC repo, and widening them is not an option: +# the Cerberus endpoint is unauthenticated, so the URL is the whole capability. Until #258 +# applies, this workflow warns and exits 0 rather than adding a second red job to an +# already-failed run -- the annotation is the signal. # # Both are secrets, so SLACK_CI_REPORT_CHANNEL is read from `secrets`, not `vars`. @@ -63,7 +64,7 @@ jobs: set -euo pipefail if [ -z "${CERBERUS_LAMBDA_URL}" ]; then - echo "::warning title=Cerberus not configured::CERBERUS_LAMBDA_URL is not visible to this repo, so the ${SUITE} failure was not reported to Slack. Needs pulumi-infra PR #257 (STAC-25461)." + echo "::warning title=Cerberus not configured::CERBERUS_LAMBDA_URL is not visible to this repo, so the ${SUITE} failure was not reported to Slack. Needs pulumi-infra PR #258 (STAC-25461)." exit 0 fi From 72c17a1412c7641ba57e793661f4ad5e4279a7f0 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Fri, 31 Jul 2026 09:41:20 +0200 Subject: [PATCH 4/4] STAC-25461 make Cerberus secrets optional so the missing-secret path warns instead of failing --- .github/workflows/cerberus-notify.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cerberus-notify.yml b/.github/workflows/cerberus-notify.yml index ec1563ade951..ff5f891b7322 100644 --- a/.github/workflows/cerberus-notify.yml +++ b/.github/workflows/cerberus-notify.yml @@ -35,8 +35,12 @@ on: required: true type: string secrets: + # Both are `required: false`. A caller passing `${{ secrets.X }}` for a secret the + # repo does not hold yields an empty string, which GitHub rejects as "not provided" + # against a required secret and fails the call before the run step's guard can + # warn -- the failure mode this workflow exists to avoid. CERBERUS_LAMBDA_URL: - required: true + required: false SLACK_CI_REPORT_CHANNEL: required: false