diff --git a/.github/scripts/agent-godeps-cache-metadata.sh b/.github/scripts/agent-godeps-cache-metadata.sh index 00eedbf97c02..77bb7bb5872e 100755 --- a/.github/scripts/agent-godeps-cache-metadata.sh +++ b/.github/scripts/agent-godeps-cache-metadata.sh @@ -18,11 +18,19 @@ agent_godeps_compute_metadata() { : "${QUAY_REGISTRY:?QUAY_REGISTRY is required}" # quay.io : "${REGISTRY_HOST:?REGISTRY_HOST is required}" # registry.tooling.stackstate.io (quay proxy) : "${BASE_IMAGE_TAG:?BASE_IMAGE_TAG is required}" # datadog_build tag the cache derives FROM + : "${BASE_IMAGE_NAME:?BASE_IMAGE_NAME is required}" # datadog_build_linux_x64 | datadog_build_linux_arm64 + : "${ARCH:?ARCH is required}" # amd64 | arm64 # Hash inputs: every module manifest (go.work + go.work.sum + modules.yml + all - # nested go.mod/go.sum) plus the base image tag and the two files that define the + # nested go.mod/go.sum) plus the base image and the two files that define the # cache mechanism itself (Dockerfile + this script), so changing how the cache is # built also rotates the tag. + # + # The base image NAME is hashed, not just its tag: the amd64 and arm64 + # datadog_build images share tag 7af9194f, so hashing the tag alone would give + # both arches the same cache tag and let one arch's image satisfy the other's + # existence check. ARCH is in the tag as well, so the collision is impossible + # by construction and the arch is readable off the ref. local godeps_hash godeps_hash="$( { @@ -32,12 +40,12 @@ agent_godeps_compute_metadata() { '.github/scripts/agent-godeps-cache-metadata.sh' \ | LC_ALL=C sort -z \ | xargs -0 sha256sum - printf 'base:%s\n' "${BASE_IMAGE_TAG}" + printf 'base:%s:%s\n' "${BASE_IMAGE_NAME}" "${BASE_IMAGE_TAG}" } | sha256sum | cut -c1-16 )" local image_repo="stackstate/stackstate-agent-godeps-cache" - local image_tag="godeps-${godeps_hash}" + local image_tag="godeps-${ARCH}-${godeps_hash}" # shellcheck disable=SC2034 # consumed by the sourcing workflow step ci_image_push="${QUAY_REGISTRY}/${image_repo}:${image_tag}" # shellcheck disable=SC2034 # consumed by the sourcing workflow step diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 07e61013b91d..a8cf8d6aa4e8 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -67,8 +67,8 @@ env: CONDA_ENV: ddpy3 jobs: - godeps-cache: - name: Go dependency cache image + godeps-cache-amd64: + name: Go dependency cache image (amd64) # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: ./.github/workflows/godeps-cache.yml @@ -76,17 +76,47 @@ jobs: # (STAC-25494). Covers a ~7m build plus runner scheduling; on timeout this builds # it itself, so a failed sibling costs latency, not correctness. with: + arch: amd64 build_delay_seconds: 900 secrets: REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + godeps-cache-arm64: + name: Go dependency cache image (arm64) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + # Designated builder of the arm64 cache image, hence delay 0. Lint and unit tests + # plays that role for amd64 but has no arm64 lane, so if this workflow also waited + # nothing would ever build the arm64 image inside the delay window and both arm64 + # consumers would build it at once -- the duplicate-build case STAC-25494 fixed. + with: + arch: arm64 + build_delay_seconds: 0 + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + build-agent: - name: Build agent binary (branded / StackState) + name: Build agent binary (branded / StackState, ${{ matrix.arch }}) # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: godeps-cache - runs-on: xlarge-public + needs: + - godeps-cache-amd64 + - godeps-cache-arm64 + strategy: + # An arm64 regression must not abort the amd64 leg: amd64 is the lane that has + # been green for weeks, and during arm64 bring-up its result is the signal. + fail-fast: false + matrix: + include: + - arch: amd64 + runner: xlarge-public + cache_image: ${{ needs.godeps-cache-amd64.outputs.ci_image }} + - arch: arm64 + runner: arm64-xlarge-public + cache_image: ${{ needs.godeps-cache-arm64.outputs.ci_image }} + runs-on: ${{ matrix.runner }} timeout-minutes: 120 container: # Warm Go module cache image (godeps-cache job); pulled via the same @@ -94,7 +124,7 @@ jobs: # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref # is effectively digest-pinned; it cannot be a static digest because it is # computed per run -- hence the narrow unpinned-images ignore below. - image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + image: ${{ matrix.cache_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -140,10 +170,24 @@ jobs: ls -la bin/agent build-cluster-agent: - name: Build cluster-agent binary (branded / StackState) + name: Build cluster-agent binary (branded / StackState, ${{ matrix.arch }}) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: godeps-cache - runs-on: xlarge-public + needs: + - godeps-cache-amd64 + - godeps-cache-arm64 + strategy: + # An arm64 regression must not abort the amd64 leg: amd64 is the lane that has + # been green for weeks, and during arm64 bring-up its result is the signal. + fail-fast: false + matrix: + include: + - arch: amd64 + runner: xlarge-public + cache_image: ${{ needs.godeps-cache-amd64.outputs.ci_image }} + - arch: arm64 + runner: arm64-xlarge-public + cache_image: ${{ needs.godeps-cache-arm64.outputs.ci_image }} + runs-on: ${{ matrix.runner }} timeout-minutes: 120 container: # Warm Go module cache image (godeps-cache job); pulled via the same @@ -151,7 +195,7 @@ jobs: # a sha256 content hash of the module graph (see godeps-cache.yml), so this ref # is effectively digest-pinned; it cannot be a static digest because it is # computed per run -- hence the narrow unpinned-images ignore below. - image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + image: ${{ matrix.cache_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -194,7 +238,10 @@ jobs: - name: Upload cluster-agent build artifacts uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: cluster-agent-binary + # Per-arch name: matrix legs upload concurrently and actions/upload-artifact + # v4 rejects a second upload to an existing artifact name, so a shared name + # would fail the arm64 leg outright. + name: cluster-agent-binary-${{ matrix.arch }} # Paths match the GitLab build_cluster_agent artifacts; the cluster-agent # image phase (phase 5) consumes the binary + Dockerfile manifest. path: | @@ -205,7 +252,7 @@ jobs: if-no-files-found: error build-cluster-agent-image: - name: Build cluster-agent container image (docker build, no push on PR) + name: Build cluster-agent container image (docker build, no push on PR, ${{ matrix.arch }}) # Ported from pre_release_cluster_agent_image. Lives in this workflow rather # than build-deb.yml so it can `needs:` the job that produces its input -- # cross-workflow artifact hand-off would need run-id plumbing for no gain. @@ -213,12 +260,25 @@ jobs: # quay.io/stackstate, which the PR lane of a PUBLIC repo must not do. # Publishing lands in phase 4, gated on the branch. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + # `needs` on a matrix job waits for every leg, so the arm64 image build also + # waits out the amd64 binary. Actions has no per-leg dependency; the cost is + # latency on an already-parallel lane, not correctness. needs: build-cluster-agent + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: docker-public + - arch: arm64 + runner: arm64-xlarge-public # DinD class: docker CLI in the runner image, dockerd in a native sidecar. - runs-on: docker-public + # Native per arch -- the binary and the Dockerfile's base stages are both + # arch-specific, so this cannot be cross-built without QEMU. + runs-on: ${{ matrix.runner }} timeout-minutes: 45 env: - LOCAL_IMAGE: stackstate-cluster-agent:ci + LOCAL_IMAGE: stackstate-cluster-agent:ci-${{ matrix.arch }} steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -228,7 +288,7 @@ jobs: - name: Download cluster-agent binary uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: cluster-agent-binary + name: cluster-agent-binary-${{ matrix.arch }} - name: Log in to the registry proxy env: @@ -275,7 +335,8 @@ jobs: cerberus-notify: name: Report failure to Slack (Cerberus) needs: - - godeps-cache + - godeps-cache-amd64 + - godeps-cache-arm64 - build-agent - build-cluster-agent - build-cluster-agent-image diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 615bd8d08a42..bfdc7c240b5a 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -14,9 +14,20 @@ name: DEB package build # it are different trust boundaries and belong in different workflows. Publishing # lands in phase 4, gated on the branch, not on the PR. # -# Only the amd64 pipeline is ported. GitLab fans .gitlab-ci-agent.yml out twice -# from .gitlab-ci.yml (agent-x86 / agent-arm); the arm64 half needs an arm64 XL -# runner class, which does not exist yet (only `arm64-docker`). Tracked separately. +# Both architectures are built. GitLab fans .gitlab-ci-agent.yml out twice from +# .gitlab-ci.yml (agent-x86 / agent-arm) with per-arch variables; here that fan-out +# is an `arch` matrix (STAC-25498), unblocked by the arm64-xlarge-public ARC scale +# set landing on argocd-apps main (STAC-25492). +# +# Of the GitLab per-arch variables only ARCH carries over. PKG_ARCH_SUFFIX is the +# same amd64/arm64 value and is used for exactly one thing -- selecting the built +# .deb (.gitlab-ci-agent.yml:695) -- which `matrix.arch` now does. PACKAGE_ARCH / +# DD_PKG_ARCH / STS_PKG_ARCH are passed to build_deb but nothing in the StackState +# omnibus path reads them; the amd64 lane has been green without them, and omnibus +# derives the package architecture from the (native) build host. DOCKER_ARCH is +# deliberately NOT ported: its only consumer is publish_image.sh, which belongs to +# the publishing lane. Note for STAC-25457 -- on arm64 DOCKER_ARCH is `aarch64`, +# NOT `arm64`, so it cannot simply be aliased to `matrix.arch`. # # Deliberate differences from the GitLab job: # * `deps_deb` is NOT ported. Its only downstream-consumed artifact was @@ -95,8 +106,8 @@ env: MAJOR_VERSION: '3' jobs: - godeps-cache: - name: Go dependency cache image + godeps-cache-amd64: + name: Go dependency cache image (amd64) # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: ./.github/workflows/godeps-cache.yml @@ -104,17 +115,45 @@ jobs: # (STAC-25494). Covers a ~7m build plus runner scheduling; on timeout this builds # it itself, so a failed sibling costs latency, not correctness. with: + arch: amd64 + build_delay_seconds: 900 + secrets: + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} + + godeps-cache-arm64: + name: Go dependency cache image (arm64) + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/godeps-cache.yml + # Same stagger, different designated builder: Binary builds owns the arm64 image + # (delay 0) the way Lint and unit tests owns the amd64 one, so this side waits. + with: + arch: arm64 build_delay_seconds: 900 secrets: REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} build-deb: - name: Build DEB package (omnibus, branded / StackState) + name: Build DEB package (omnibus, branded / StackState, ${{ matrix.arch }}) # Same-repo PRs only: the registry-proxy secret is not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - needs: godeps-cache - runs-on: xlarge-public + needs: + - godeps-cache-amd64 + - godeps-cache-arm64 + strategy: + # An arm64 regression must not abort the amd64 leg: amd64 is the lane that has + # been green for weeks, and during arm64 bring-up its result is the signal. + fail-fast: false + matrix: + include: + - arch: amd64 + runner: xlarge-public + cache_image: ${{ needs.godeps-cache-amd64.outputs.ci_image }} + - arch: arm64 + runner: arm64-xlarge-public + cache_image: ${{ needs.godeps-cache-arm64.outputs.ci_image }} + runs-on: ${{ matrix.runner }} # Cold omnibus build: no git cache, so every embedded software (openssl, cpython, # openscap, sqlite, ...) compiles from source. Generous until we have a real number. timeout-minutes: 300 @@ -125,7 +164,7 @@ jobs: # the module graph (see godeps-cache.yml), so this ref is effectively # digest-pinned; it cannot be a static digest because it is computed per run -- # hence the narrow unpinned-images ignore below. - image: ${{ needs.godeps-cache.outputs.ci_image }} # zizmor: ignore[unpinned-images] + image: ${{ matrix.cache_image }} # zizmor: ignore[unpinned-images] credentials: username: ${{ vars.REGISTRY_USER }} password: ${{ secrets.REGISTRY_PASSWORD }} @@ -235,7 +274,10 @@ jobs: - name: Upload DEB package and image manifests uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: deb-package + # Per-arch name: matrix legs upload concurrently and actions/upload-artifact + # v4 rejects a second upload to an existing artifact name, so a shared name + # would fail the arm64 leg outright. + name: deb-package-${{ matrix.arch }} # Paths match the GitLab build_deb artifacts; the branding gate below and # the image phase (phase 5) consume them. path: | @@ -250,10 +292,16 @@ jobs: if-no-files-found: error test-deb-renaming: - name: DEB branding verification (no DataDog references) + name: DEB branding verification (no DataDog references, ${{ matrix.arch }}) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository needs: build-deb - # Cheap gate: unpacks the .deb and greps it. Does not need the XL class. + strategy: + fail-fast: false + matrix: + arch: [amd64, arm64] + # Cheap gate: unpacks the .deb and greps it. Does not need the XL class -- and + # stays on amd64 for BOTH legs: `ar` + grep read the arm64 .deb just fine, so + # burning a scarce arm64 XL runner on a grep would buy nothing. runs-on: docker-public timeout-minutes: 30 container: @@ -271,7 +319,7 @@ jobs: - name: Download DEB package uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: deb-package + name: deb-package-${{ matrix.arch }} - name: Verify package carries no DataDog branding run: | @@ -289,16 +337,27 @@ jobs: ./test/renaming/test_deb.sh "${debs[0]}" build-agent-image: - name: Build agent container image (docker build, no push on PR) + name: Build agent container image (docker build, no push on PR, ${{ matrix.arch }}) if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository needs: build-deb + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: docker-public + - arch: arm64 + runner: arm64-xlarge-public # DinD class: docker CLI in the runner image, dockerd in a native sidecar. - runs-on: docker-public + # Native per arch -- the .deb and the Dockerfile's base stages are both + # arch-specific, so this cannot be cross-built without QEMU. + runs-on: ${{ matrix.runner }} timeout-minutes: 60 env: - # ARCH in the agent-x86 trigger block of .gitlab-ci.yml; also the .deb suffix. - ARCH: amd64 - LOCAL_IMAGE: stackstate-agent:ci + # ARCH in the agent-x86 / agent-arm trigger blocks of .gitlab-ci.yml; also the + # .deb suffix. + ARCH: ${{ matrix.arch }} + LOCAL_IMAGE: stackstate-agent:ci-${{ matrix.arch }} steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -308,7 +367,7 @@ jobs: - name: Download DEB package uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: deb-package + name: deb-package-${{ matrix.arch }} - name: Log in to the registry proxy env: @@ -348,7 +407,8 @@ jobs: cerberus-notify: name: Report failure to Slack (Cerberus) needs: - - godeps-cache + - godeps-cache-amd64 + - godeps-cache-arm64 - build-deb - test-deb-renaming - build-agent-image diff --git a/.github/workflows/godeps-cache.yml b/.github/workflows/godeps-cache.yml index 175e5d01cda6..900042de58f8 100644 --- a/.github/workflows/godeps-cache.yml +++ b/.github/workflows/godeps-cache.yml @@ -22,12 +22,23 @@ name: Go dependency cache image # images, and stackstate-agent is PUBLIC, so any org member's same-repo branch can use # whatever push credential this workflow carries. Scoping it to a rebuildable cache # keeps a compromised branch away from the images the CI fleet boots from. +# +# One call per architecture (STAC-25498). Each arch gets its own base image and its own +# content-addressed tag, and is built NATIVELY on a runner of that architecture -- no +# QEMU/binfmt emulation, which would make the module extraction in the Dockerfile +# unusably slow. The two arches are independent images, not a multi-arch manifest: +# consumers reference them from `container.image`, which resolves against the runner's +# own architecture anyway. on: workflow_call: inputs: + arch: + description: Target architecture, amd64 or arm64. Selects the datadog_build base image and the runner that builds the cache. + type: string + required: true base_image_tag: - description: Tag of the datadog_build_linux_x64 base image the cache derives FROM. + description: Tag of the datadog_build base image the cache derives FROM. Both arch variants carry the same tag. type: string default: "7af9194f" build_delay_seconds: @@ -58,7 +69,10 @@ permissions: jobs: metadata: - name: Compute cache image tag and look up existing image + name: Compute ${{ inputs.arch }} cache image tag and look up existing image + # Stays on amd64 regardless of `arch`: this job only hashes files and calls + # `docker manifest inspect`, both architecture-independent. Only the build below + # has to run natively. runs-on: docker-public timeout-minutes: 15 outputs: @@ -70,6 +84,8 @@ jobs: QUAY_USER: ${{ vars.QUAY_USER }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} + BASE_IMAGE_NAME: ${{ inputs.arch == 'arm64' && 'datadog_build_linux_arm64' || 'datadog_build_linux_x64' }} + ARCH: ${{ inputs.arch }} steps: - name: Check out repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -105,10 +121,11 @@ jobs: echo "Cache image exists: ${image_exists}" build: - name: Build and publish missing cache image + name: Build and publish missing ${{ inputs.arch }} cache image if: ${{ needs.metadata.outputs.image_exists != 'true' }} needs: metadata - runs-on: docker-public + # Native build: an arm64 cache image must be produced on an arm64 runner. + runs-on: ${{ inputs.arch == 'arm64' && 'arm64-xlarge-public' || 'docker-public' }} timeout-minutes: 60 # Consumer workflows call this in parallel on the same commit; without a shared gate # they all see image_exists=false and build the same content-addressed tag at once. @@ -127,6 +144,8 @@ jobs: QUAY_USER: ${{ vars.QUAY_USER }} QUAY_PASSWORD: ${{ secrets.QUAY_PASSWORD }} BASE_IMAGE_TAG: ${{ inputs.base_image_tag }} + BASE_IMAGE_NAME: ${{ inputs.arch == 'arm64' && 'datadog_build_linux_arm64' || 'datadog_build_linux_x64' }} + ARCH: ${{ inputs.arch }} BUILD_DELAY_SECONDS: ${{ inputs.build_delay_seconds }} steps: - name: Check out repository @@ -140,7 +159,7 @@ jobs: source .github/scripts/agent-godeps-cache-metadata.sh agent_godeps_compute_metadata - base_image="${REGISTRY_HOST}/quay/stackstate/datadog_build_linux_x64:${BASE_IMAGE_TAG}" + base_image="${REGISTRY_HOST}/quay/stackstate/${BASE_IMAGE_NAME}:${BASE_IMAGE_TAG}" # Log into both registries: pull the base via the registry.tooling proxy, # push the result to quay.io/stackstate. diff --git a/.github/workflows/lint-and-unit-tests.yml b/.github/workflows/lint-and-unit-tests.yml index 3706d2278c96..33f0a930d73c 100644 --- a/.github/workflows/lint-and-unit-tests.yml +++ b/.github/workflows/lint-and-unit-tests.yml @@ -62,13 +62,16 @@ env: jobs: godeps-cache: - name: Go dependency cache image + name: Go dependency cache image (amd64) # Same-repo PRs only: the registry/quay secrets are not exposed to fork PRs. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: ./.github/workflows/godeps-cache.yml - # Designated builder of the cache image (STAC-25494): no wait, so this workflow -- - # the longest of the three -- is never held up. The other two wait it out. + # Designated builder of the amd64 cache image (STAC-25494): no wait, so this + # workflow -- the longest of the three -- is never held up. The other two wait it + # out. amd64 only: lint and unit tests deliberately gain no arm64 lane + # (STAC-25498), since each suite runs once on the path to master. with: + arch: amd64 build_delay_seconds: 0 secrets: REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}