From 017cfdb11a6147b7aba23baa1fea07201f811c7a Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Fri, 14 Aug 2026 01:27:11 +0800 Subject: [PATCH 1/2] Added experimental support for multi-agent image Signed-off-by: michaelawyu --- .github/helpers/multi-agent/.dockerignore | 6 + .github/helpers/multi-agent/Makefile | 75 +++++++ .../build-publish-multi-agent-mcr.yml | 187 ++++++++++++++++++ docker/multi-agent.Dockerfile | 75 +++++++ 4 files changed, 343 insertions(+) create mode 100644 .github/helpers/multi-agent/.dockerignore create mode 100644 .github/helpers/multi-agent/Makefile create mode 100644 .github/workflows/build-publish-multi-agent-mcr.yml create mode 100644 docker/multi-agent.Dockerfile diff --git a/.github/helpers/multi-agent/.dockerignore b/.github/helpers/multi-agent/.dockerignore new file mode 100644 index 000000000..62481ed5f --- /dev/null +++ b/.github/helpers/multi-agent/.dockerignore @@ -0,0 +1,6 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ +# Ignore git metadata. +**/.git diff --git a/.github/helpers/multi-agent/Makefile b/.github/helpers/multi-agent/Makefile new file mode 100644 index 000000000..260380d63 --- /dev/null +++ b/.github/helpers/multi-agent/Makefile @@ -0,0 +1,75 @@ +REGISTRY ?= ghcr.io +ifndef TAG +$(error TAG is required; set it explicitly) +endif +MULTI_AGENT_IMAGE_VERSION ?= $(TAG) + +MULTI_AGENT_IMAGE_NAME ?= multi-agent + +TARGET_OS ?= linux +TARGET_ARCH ?= amd64 +AUTO_DETECT_ARCH ?= TRUE + +# Auto-detect system architecture if it is allowed and the necessary commands are available on the system. +ifeq ($(AUTO_DETECT_ARCH), TRUE) +ARCH_CMD_INSTALLED := $(shell command -v arch 2>/dev/null) +ifdef ARCH_CMD_INSTALLED +TARGET_ARCH := $(shell arch) +# The arch command may return arch strings that are aliases of expected TARGET_ARCH values; +# do the mapping here. +ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),x86_64)) + TARGET_ARCH := amd64 +else ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),aarch64 arm)) + TARGET_ARCH := arm64 +endif +endif +endif + +# Note (chenyu1): switch to the `plain` progress type to see the full outputs in the docker build +# progress. +BUILDKIT_PROGRESS_TYPE ?= auto + +## -------------------------------------- +## Images +## -------------------------------------- + +OUTPUT_TYPE ?= type=registry +BUILDX_BUILDER_NAME ?= img-builder +QEMU_VERSION ?= 7.2.0-1 +BUILDKIT_VERSION ?= v0.18.1 + +# By default, docker buildx create will pull image moby/buildkit:buildx-stable-1 and hit the too many requests error +# +# Note (chenyu1): the step below sets up emulation for building/running non-native binaries on the host. The original +# setup assumes that the Makefile is always run on an x86_64 platform, and adds support for non-x86_64 hosts. Here +# we keep the original setup if the build target is x86_64 platforms (default) for compatibility reasons, but will switch to +# a more general setup for non-x86_64 hosts. +# +# On some systems the emulation setup might not work at all (e.g., macOS on Apple Silicon -> Rosetta 2 will be used +# by Docker Desktop as the default emulation option for AMD64 on ARM64 container compatibility). +.PHONY: docker-buildx-builder +docker-buildx-builder: + $(info Auto-detected system architecture: $(TARGET_ARCH)) + @if ! docker buildx ls | grep $(BUILDX_BUILDER_NAME); then \ + if [ "$(TARGET_ARCH)" = "amd64" ] ; then \ + echo "The target is an x86_64 platform; setting up emulation for other known architectures"; \ + docker run --rm --privileged mcr.microsoft.com/mirror/docker/multiarch/qemu-user-static:$(QEMU_VERSION) --reset -p yes; \ + else \ + echo "Setting up emulation for known architectures"; \ + docker run --rm --privileged tonistiigi/binfmt --install all; \ + fi ;\ + docker buildx create --driver-opt image=mcr.microsoft.com/oss/v2/moby/buildkit:$(BUILDKIT_VERSION) --name $(BUILDX_BUILDER_NAME) --use; \ + docker buildx inspect $(BUILDX_BUILDER_NAME) --bootstrap; \ + fi + +.PHONY: docker-build-multi-agent +docker-build-multi-agent: docker-buildx-builder ## Build multi-agent image + docker buildx build \ + --file fleet/docker/multi-agent.Dockerfile \ + --tag $(REGISTRY)/$(MULTI_AGENT_IMAGE_NAME):$(MULTI_AGENT_IMAGE_VERSION) \ + --output=$(OUTPUT_TYPE) \ + --platform=$(TARGET_OS)/$(TARGET_ARCH) \ + --pull \ + --progress=$(BUILDKIT_PROGRESS_TYPE) \ + --build-arg GOARCH=$(TARGET_ARCH) \ + --build-arg GOOS=$(TARGET_OS) . diff --git a/.github/workflows/build-publish-multi-agent-mcr.yml b/.github/workflows/build-publish-multi-agent-mcr.yml new file mode 100644 index 000000000..84e63de6d --- /dev/null +++ b/.github/workflows/build-publish-multi-agent-mcr.yml @@ -0,0 +1,187 @@ +# This Github Action will build and publish images to Azure Container Registry (ACR), from where the published images will be +# automatically pushed to the trusted registry, Microsoft Container Registry (MCR). + +# TO-DO (chenyu1): evaluate if we need to hide arch-specific images in ACR. + +name: Building and Pushing Multi-Agent Image to MCR +on: + workflow_dispatch: + inputs: + fleetReleaseTag: + description: 'Release tag for the fleet agents, defaults to the latest one' + type: string + fleetNetworkingReleaseTag: + description: 'Release tag for the fleet networking agents, defaults to the latest one' + type: string + +permissions: + id-token: write + contents: read + +env: + # `public` indicates images to MCR will be publicly available, and will be removed in the final MCR images + REGISTRY_REPO: public/aks/fleet + +jobs: + prepare-variables: + runs-on: ubuntu-latest + outputs: + fleet_release_tag: ${{ steps.vars.outputs.fleet_release_tag }} + fleet_networking_release_tag: ${{ steps.vars.outputs.fleet_networking_release_tag }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: 'Set output variables (fleet-related)' + id: vars + run: | + # Set the Fleet release tag. + FLEET_RELEASE_TAG=${{ inputs.fleetReleaseTag }} + if [ -z "$FLEET_RELEASE_TAG" ]; then + FLEET_RELEASE_TAG=$(git describe --tags "$(git rev-list --tags --max-count=1)") + echo "No user input exists for the fleet release tag; fall back to the latest tag ($FLEET_RELEASE_TAG)." + fi + echo "fleet_release_tag=$FLEET_RELEASE_TAG" >> "$GITHUB_OUTPUT" + echo "Using Fleet release tag: $FLEET_RELEASE_TAG" + + # Set the Fleet networking release tag. + FLEET_NETWORKING_RELEASE_TAG=${{ inputs.fleetNetworkingReleaseTag }} + if [ -z "$FLEET_NETWORKING_RELEASE_TAG" ]; then + FLEET_NETWORKING_RELEASE_TAG=$(curl "https://api.github.com/repos/Azure/fleet-networking/tags" | jq -r '.[0].name') + echo "No user input exists for the fleet networking release tag; fall back to the latest tag ($FLEET_NETWORKING_RELEASE_TAG)." + fi + echo "fleet_networking_release_tag=$FLEET_NETWORKING_RELEASE_TAG" >> "$GITHUB_OUTPUT" + echo "Using Fleet Networking release tag: $FLEET_NETWORKING_RELEASE_TAG" + + # NOTE: As exporting a variable from a secret is not possible, the shared variable registry obtained + # from AZURE_REGISTRY secret is not exported from here. + + publish-images-amd64: + runs-on: + labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu"] + needs: prepare-variables + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.prepare-variables.outputs.fleet_release_tag }} + path: fleet + - uses: actions/checkout@v6 + with: + repository: Azure/fleet-networking + ref: ${{ needs.prepare-variables.outputs.fleet_networking_release_tag }} + path: fleet-networking + - name: 'Copy the multi-agent Makefile and .dockerignore to the workspace root' + run: | + cp fleet/.github/helpers/multi-agent/Makefile . + cp fleet/.github/helpers/multi-agent/.dockerignore . + - name: 'Login the ACR' + run: | + az login --identity + az acr login -n ${{ secrets.AZURE_REGISTRY }} + - name: Build and publish multi-agent + run: | + make docker-build-multi-agent + env: + TAG: ${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 + REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}} + + publish-images-arm64: + runs-on: + labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu-arm64"] + needs: prepare-variables + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.prepare-variables.outputs.fleet_release_tag }} + path: fleet + - uses: actions/checkout@v6 + with: + repository: Azure/fleet-networking + ref: ${{ needs.prepare-variables.outputs.fleet_networking_release_tag }} + path: fleet-networking + - name: 'Copy the multi-agent Makefile and .dockerignore to the workspace root' + run: | + cp fleet/.github/helpers/multi-agent/Makefile . + cp fleet/.github/helpers/multi-agent/.dockerignore . + - name: 'Install the Azure CLI' + # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have Azure CLI installed by default; + # install it manually here. + run: + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + - name: 'Set up build dependencies' + # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have the common build + # tools (e.g., make) installed by default; install them manually. + run: | + sudo apt-get update + sudo apt-get install -y build-essential acl + - name: 'Set up Docker' + # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have Docker installed by default, + # and cannot have Docker installed via the docker/setup-docker-action Github Action, hence the manual setup + # steps here. + run: | + sudo apt-get update + sudo apt-get -y install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + - name: 'Enable Docker access' + # Note (chenyu1): there are situations where the newgrp command will not take effect; set access + # to the docker daemon directly just in case. + run: | + sudo groupadd docker || true + echo "Adding $USER to the docker group" + sudo usermod -aG docker "$USER" + newgrp docker + sudo setfacl --modify "user:$USER:rw" /var/run/docker.sock + - name: 'Login the ACR' + # Note (chenyu1): must not use root privileges; the system seems to have some trouble + # retrieving credentials when sudo is used. + run: | + az login --identity + az acr login -n ${{ secrets.AZURE_REGISTRY }} + - name: 'Verify Docker CLI' + run: | + docker version + docker info + - name: Build and publish multi-agent + run: | + make docker-build-multi-agent + env: + TAG: ${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64 + REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}} + TARGET_ARCH: arm64 + + create-image-manifest-bundle: + runs-on: + # Use the x86_64 1ES pool to run this job; in theory it can be run on the ARM64 1ES pool as well. + labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu"] + needs: [prepare-variables, publish-images-amd64, publish-images-arm64] + steps: + - name: 'Wait until images are processed' + # Note (chenyu1): as we are pulling from ACR rather than MCR, the images should be available almost + # immediately after the push is done; the delay is added here as a precaution. + run: | + echo "Waiting for 3 minutes to ensure that images are fully processed" + sleep 180 + - name: 'Login the ACR' + run: | + az login --identity + az acr login -n ${{ secrets.AZURE_REGISTRY }} + - name: 'Pull the multi-agent images from ACR' + # Note (chenyu1): must set the target platform explictly. + run: | + docker pull --platform linux/amd64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 + docker pull --platform linux/arm64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64 + - name: 'Create and push multi-arch image manifests for the multi-agent image' + # Note (chenyu1): use `docker buildx imagetools create`, otherwise attestations cannot be perserved. + run: | + docker buildx imagetools create \ + -t ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }} \ + ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 \ + ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64 diff --git a/docker/multi-agent.Dockerfile b/docker/multi-agent.Dockerfile new file mode 100644 index 000000000..c2196d2ce --- /dev/null +++ b/docker/multi-agent.Dockerfile @@ -0,0 +1,75 @@ +# Build all Fleet required binaries. +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.26.5 AS builder + +ARG GOOS=linux +ARG GOARCH=amd64 + +# Build the Fleet hub/member agent and refresh token binaries. +WORKDIR /core-workspace + +# Copy the go.mod and go.sum files. +COPY fleet/go.mod go.mod +COPY fleet/go.sum go.sum +# Cache dependencies before building and copying the source code to avoid re-downloading upon retries. +# It also ensures that source code changes do not invalidate the downloaded Go dependencies layer. +RUN go mod download + +# Copy the Go source code. +COPY fleet/cmd/ cmd/ +COPY fleet/apis/ apis/ +COPY fleet/pkg/ pkg/ + +# Build the hub agent. +RUN echo "Building the hub agent binary for GOOS=$GOOS GOARCH=$GOARCH" +RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GOEXPERIMENT=systemcrypto GO111MODULE=on go build -o hubagent cmd/hubagent/main.go + +# Build the member agent. +RUN echo "Building the member agent binary for GOOS=$GOOS GOARCH=$GOARCH" +RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GOEXPERIMENT=systemcrypto GO111MODULE=on go build -o memberagent cmd/memberagent/main.go + +# Build the refresh token binary. +RUN echo "Building the refresh token binary for GOOS=$GOOS GOARCH=$GOARCH" +RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GOEXPERIMENT=systemcrypto GO111MODULE=on go build -o refreshtoken cmd/authtoken/main.go + +# Build the Fleet networking agent binaries. +WORKDIR /networking-workspace + +# Copy the go.mod and go.sum files. +COPY fleet-networking/go.mod go.mod +COPY fleet-networking/go.sum go.sum +# Cache dependencies before building and copying the source code to avoid re-downloading upon retries. +# It also ensures that source code changes do not invalidate the downloaded Go dependencies layer. +RUN go mod download + +# Copy the Go source code. +COPY fleet-networking/cmd/ cmd/ +COPY fleet-networking/pkg/ pkg/ +COPY fleet-networking/api/ api/ + +# Build the hub networking agent. +RUN echo "Building the hub networking agent binary for GOOS=$GOOS GOARCH=$GOARCH" +RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GO111MODULE=on go build -o hub-net-controller-manager cmd/hub-net-controller-manager/main.go + +# Build the member networking agent. +RUN echo "Building the member networking agent binary for GOOS=$GOOS GOARCH=$GOARCH" +RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GO111MODULE=on go build -o member-net-controller-manager cmd/member-net-controller-manager/main.go + +# Build the MCS networking agent. +RUN echo "Building the MCS networking agent binary for GOOS=$GOOS GOARCH=$GOARCH" +RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GO111MODULE=on go build -o mcs-controller-manager cmd/mcs-controller-manager/main.go + +# Use Azure Linux distroless base image to package the binaries. +# For more details, refer to https://mcr.microsoft.com/en-us/artifact/mar/azurelinux/distroless/base/about. +FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 +WORKDIR / +COPY --from=builder /core-workspace/hubagent . +COPY --from=builder /core-workspace/memberagent . +COPY --from=builder /core-workspace/refreshtoken . +COPY --from=builder /networking-workspace/hub-net-controller-manager . +COPY --from=builder /networking-workspace/member-net-controller-manager . +COPY --from=builder /networking-workspace/mcs-controller-manager . + +USER 65532:65532 + +# No default entrypoint is set for the unified image. +ENTRYPOINT [] \ No newline at end of file From 67d92fcb511d7b2585663f4671a9e3eab21097bf Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Fri, 14 Aug 2026 01:43:51 +0800 Subject: [PATCH 2/2] Minor fixes Signed-off-by: michaelawyu --- .../build-publish-multi-agent-mcr.yml | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-publish-multi-agent-mcr.yml b/.github/workflows/build-publish-multi-agent-mcr.yml index 84e63de6d..92b0e0dd3 100644 --- a/.github/workflows/build-publish-multi-agent-mcr.yml +++ b/.github/workflows/build-publish-multi-agent-mcr.yml @@ -163,25 +163,25 @@ jobs: labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu"] needs: [prepare-variables, publish-images-amd64, publish-images-arm64] steps: - - name: 'Wait until images are processed' - # Note (chenyu1): as we are pulling from ACR rather than MCR, the images should be available almost - # immediately after the push is done; the delay is added here as a precaution. - run: | - echo "Waiting for 3 minutes to ensure that images are fully processed" - sleep 180 - - name: 'Login the ACR' - run: | - az login --identity - az acr login -n ${{ secrets.AZURE_REGISTRY }} - - name: 'Pull the multi-agent images from ACR' - # Note (chenyu1): must set the target platform explictly. - run: | - docker pull --platform linux/amd64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 - docker pull --platform linux/arm64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64 - - name: 'Create and push multi-arch image manifests for the multi-agent image' - # Note (chenyu1): use `docker buildx imagetools create`, otherwise attestations cannot be perserved. - run: | - docker buildx imagetools create \ - -t ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }} \ - ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 \ - ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64 + - name: 'Wait until images are processed' + # Note (chenyu1): as we are pulling from ACR rather than MCR, the images should be available almost + # immediately after the push is done; the delay is added here as a precaution. + run: | + echo "Waiting for 3 minutes to ensure that images are fully processed" + sleep 180 + - name: 'Login the ACR' + run: | + az login --identity + az acr login -n ${{ secrets.AZURE_REGISTRY }} + - name: 'Pull the multi-agent images from ACR' + # Note (chenyu1): must set the target platform explicitly. + run: | + docker pull --platform linux/amd64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 + docker pull --platform linux/arm64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64 + - name: 'Create and push multi-arch image manifests for the multi-agent image' + # Note (chenyu1): use `docker buildx imagetools create`, otherwise attestations cannot be preserved. + run: | + docker buildx imagetools create \ + -t ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }} \ + ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 \ + ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64