feat: switch to multi-agent image (1/)

.github/helpers/multi-agent/.dockerignore

@@ -0,0 +1,6 @@
+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore build and test binaries.
+bin/
+testbin/
+# Ignore git metadata.
+**/.git

.github/helpers/multi-agent/Makefile

@@ -0,0 +1,75 @@
+REGISTRY ?= ghcr.io
+ifndef TAG
+$(error TAG is required; set it explicitly)
+endif
+MULTI_AGENT_IMAGE_VERSION ?= $(TAG)
+
+MULTI_AGENT_IMAGE_NAME ?= multi-agent
+
+TARGET_OS ?= linux
+TARGET_ARCH ?= amd64
+AUTO_DETECT_ARCH ?= TRUE
+
+# Auto-detect system architecture if it is allowed and the necessary commands are available on the system.
+ifeq ($(AUTO_DETECT_ARCH), TRUE)
+ARCH_CMD_INSTALLED := $(shell command -v arch 2>/dev/null)
+ifdef ARCH_CMD_INSTALLED
+TARGET_ARCH := $(shell arch)
+# The arch command may return arch strings that are aliases of expected TARGET_ARCH values;
+# do the mapping here.
+ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),x86_64))
+	TARGET_ARCH := amd64
+else ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),aarch64 arm))
+	TARGET_ARCH := arm64
+endif
+endif
+endif
+
+# Note (chenyu1): switch to the `plain` progress type to see the full outputs in the docker build
+# progress.
+BUILDKIT_PROGRESS_TYPE ?= auto
+
+## --------------------------------------
+## Images
+## --------------------------------------
+
+OUTPUT_TYPE ?= type=registry
+BUILDX_BUILDER_NAME ?= img-builder
+QEMU_VERSION ?= 7.2.0-1
+BUILDKIT_VERSION ?= v0.18.1
+
+# By default, docker buildx create will pull image moby/buildkit:buildx-stable-1 and hit the too many requests error
+#
+# Note (chenyu1): the step below sets up emulation for building/running non-native binaries on the host. The original
+# setup assumes that the Makefile is always run on an x86_64 platform, and adds support for non-x86_64 hosts. Here
+# we keep the original setup if the build target is x86_64 platforms (default) for compatibility reasons, but will switch to
+# a more general setup for non-x86_64 hosts.
+#
+# On some systems the emulation setup might not work at all (e.g., macOS on Apple Silicon -> Rosetta 2 will be used
+# by Docker Desktop as the default emulation option for AMD64 on ARM64 container compatibility).
+.PHONY: docker-buildx-builder
+docker-buildx-builder:
+	$(info Auto-detected system architecture: $(TARGET_ARCH))
+	@if ! docker buildx ls | grep $(BUILDX_BUILDER_NAME); then \
+		if [ "$(TARGET_ARCH)" = "amd64" ] ; then \
+			echo "The target is an x86_64 platform; setting up emulation for other known architectures"; \
+			docker run --rm --privileged mcr.microsoft.com/mirror/docker/multiarch/qemu-user-static:$(QEMU_VERSION) --reset -p yes; \
+		else \
+			echo "Setting up emulation for known architectures"; \
+			docker run --rm --privileged tonistiigi/binfmt --install all; \
+		fi ;\
+		docker buildx create --driver-opt image=mcr.microsoft.com/oss/v2/moby/buildkit:$(BUILDKIT_VERSION) --name $(BUILDX_BUILDER_NAME) --use; \
+		docker buildx inspect $(BUILDX_BUILDER_NAME) --bootstrap; \
+	fi
+
+.PHONY: docker-build-multi-agent
+docker-build-multi-agent: docker-buildx-builder ## Build multi-agent image
+	docker buildx build \
+		--file fleet/docker/multi-agent.Dockerfile \
+		--tag $(REGISTRY)/$(MULTI_AGENT_IMAGE_NAME):$(MULTI_AGENT_IMAGE_VERSION) \
+		--output=$(OUTPUT_TYPE) \
+		--platform=$(TARGET_OS)/$(TARGET_ARCH) \
+		--pull \
+		--progress=$(BUILDKIT_PROGRESS_TYPE) \
+		--build-arg GOARCH=$(TARGET_ARCH) \
+		--build-arg GOOS=$(TARGET_OS) .

.github/workflows/build-publish-multi-agent-mcr.yml

@@ -0,0 +1,187 @@
+# This Github Action will build and publish images to Azure Container Registry (ACR), from where the published images will be
+# automatically pushed to the trusted registry, Microsoft Container Registry (MCR).
+
+# TO-DO (chenyu1): evaluate if we need to hide arch-specific images in ACR.
+
+name: Building and Pushing Multi-Agent Image to MCR
+on:
+  workflow_dispatch:
+    inputs:
+      fleetReleaseTag:
+        description: 'Release tag for the fleet agents, defaults to the latest one'
+        type: string
+      fleetNetworkingReleaseTag:
+        description: 'Release tag for the fleet networking agents, defaults to the latest one'
+        type: string
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  # `public` indicates images to MCR will be publicly available, and will be removed in the final MCR images
+  REGISTRY_REPO: public/aks/fleet
+
+jobs:
+  prepare-variables:
+    runs-on: ubuntu-latest
+    outputs:
+      fleet_release_tag: ${{ steps.vars.outputs.fleet_release_tag }}
+      fleet_networking_release_tag: ${{ steps.vars.outputs.fleet_networking_release_tag }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - name: 'Set output variables (fleet-related)'
+        id: vars
+        run: |
+          # Set the Fleet release tag.
+          FLEET_RELEASE_TAG=${{ inputs.fleetReleaseTag }}
+          if [ -z "$FLEET_RELEASE_TAG" ]; then
+            FLEET_RELEASE_TAG=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+            echo "No user input exists for the fleet release tag; fall back to the latest tag ($FLEET_RELEASE_TAG)."
+          fi
+          echo "fleet_release_tag=$FLEET_RELEASE_TAG" >> "$GITHUB_OUTPUT"
+          echo "Using Fleet release tag: $FLEET_RELEASE_TAG"
+
+          # Set the Fleet networking release tag.
+          FLEET_NETWORKING_RELEASE_TAG=${{ inputs.fleetNetworkingReleaseTag }}
+          if [ -z "$FLEET_NETWORKING_RELEASE_TAG" ]; then
+            FLEET_NETWORKING_RELEASE_TAG=$(curl "https://api.github.com/repos/Azure/fleet-networking/tags" | jq -r '.[0].name')
+            echo "No user input exists for the fleet networking release tag; fall back to the latest tag ($FLEET_NETWORKING_RELEASE_TAG)."
+          fi
+          echo "fleet_networking_release_tag=$FLEET_NETWORKING_RELEASE_TAG" >> "$GITHUB_OUTPUT"
+          echo "Using Fleet Networking release tag: $FLEET_NETWORKING_RELEASE_TAG"
+
+          # NOTE: As exporting a variable from a secret is not possible, the shared variable registry obtained
+          # from AZURE_REGISTRY secret is not exported from here.
+
+  publish-images-amd64:
+    runs-on:
+      labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu"]
+    needs: prepare-variables
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.prepare-variables.outputs.fleet_release_tag }}
+          path: fleet
+      - uses: actions/checkout@v6
+        with:
+          repository: Azure/fleet-networking
+          ref: ${{ needs.prepare-variables.outputs.fleet_networking_release_tag }}
+          path: fleet-networking
+      - name: 'Copy the multi-agent Makefile and .dockerignore to the workspace root'
+        run: |
+          cp fleet/.github/helpers/multi-agent/Makefile .
+          cp fleet/.github/helpers/multi-agent/.dockerignore .
+      - name: 'Login the ACR'
+        run: |
+          az login --identity 
+          az acr login -n ${{ secrets.AZURE_REGISTRY }}
+      - name: Build and publish multi-agent
+        run: |
+          make docker-build-multi-agent
+        env:
+          TAG: ${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64
+          REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}
+
+  publish-images-arm64:
+    runs-on:
+      labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu-arm64"]
+    needs: prepare-variables
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.prepare-variables.outputs.fleet_release_tag }}
+          path: fleet
+      - uses: actions/checkout@v6
+        with:
+          repository: Azure/fleet-networking
+          ref: ${{ needs.prepare-variables.outputs.fleet_networking_release_tag }}
+          path: fleet-networking
+      - name: 'Copy the multi-agent Makefile and .dockerignore to the workspace root'
+        run: |
+          cp fleet/.github/helpers/multi-agent/Makefile .
+          cp fleet/.github/helpers/multi-agent/.dockerignore .
+      - name: 'Install the Azure CLI'
+        # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have Azure CLI installed by default;
+        # install it manually here.
+        run:
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+      - name: 'Set up build dependencies'
+        # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have the common build
+        # tools (e.g., make) installed by default; install them manually.
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential acl
+      - name: 'Set up Docker'
+        # Note (chenyu1): the self-hosted 1ES ARM64 pool, for some reason, does not have Docker installed by default,
+        # and cannot have Docker installed via the docker/setup-docker-action Github Action, hence the manual setup
+        # steps here.
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+            $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+      - name: 'Enable Docker access'
+        # Note (chenyu1): there are situations where the newgrp command will not take effect; set access
+        # to the docker daemon directly just in case.
+        run: |
+          sudo groupadd docker || true
+          echo "Adding $USER to the docker group"
+          sudo usermod -aG docker "$USER"
+          newgrp docker
+          sudo setfacl --modify "user:$USER:rw" /var/run/docker.sock
+      - name: 'Login the ACR'
+        # Note (chenyu1): must not use root privileges; the system seems to have some trouble
+        # retrieving credentials when sudo is used.
+        run: |
+          az login --identity
+          az acr login -n ${{ secrets.AZURE_REGISTRY }}
+      - name: 'Verify Docker CLI'
+        run: |
+          docker version
+          docker info
+      - name: Build and publish multi-agent
+        run: |
+          make docker-build-multi-agent
+        env:
+          TAG: ${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64
+          REGISTRY: ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}
+          TARGET_ARCH: arm64
+
+  create-image-manifest-bundle:
+    runs-on:
+      # Use the x86_64 1ES pool to run this job; in theory it can be run on the ARM64 1ES pool as well.
+      labels: [self-hosted, "1ES.Pool=1es-aks-fleet-pool-ubuntu"]
+    needs: [prepare-variables, publish-images-amd64, publish-images-arm64]
+    steps:
+      - name: 'Wait until images are processed'
+        # Note (chenyu1): as we are pulling from ACR rather than MCR, the images should be available almost
+        # immediately after the push is done; the delay is added here as a precaution.
+        run: |
+          echo "Waiting for 3 minutes to ensure that images are fully processed"
+          sleep 180
+      - name: 'Login the ACR'
+        run: |
+          az login --identity 
+          az acr login -n ${{ secrets.AZURE_REGISTRY }}
+      - name: 'Pull the multi-agent images from ACR'
+        # Note (chenyu1): must set the target platform explicitly.
+        run: |
+          docker pull --platform linux/amd64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64
+          docker pull --platform linux/arm64 ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64
+      - name: 'Create and push multi-arch image manifests for the multi-agent image'
+        # Note (chenyu1): use `docker buildx imagetools create`, otherwise attestations cannot be preserved.
+        run: |
+          docker buildx imagetools create \
+            -t ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }} \
+            ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-amd64 \
+            ${{ secrets.AZURE_REGISTRY }}/${{ env.REGISTRY_REPO}}/multi-agent:${{ needs.prepare-variables.outputs.fleet_release_tag }}-arm64

docker/multi-agent.Dockerfile

@@ -0,0 +1,75 @@
+# Build all Fleet required binaries.
+FROM mcr.microsoft.com/oss/go/microsoft/golang:1.26.5 AS builder
+
+ARG GOOS=linux
+ARG GOARCH=amd64
+
+# Build the Fleet hub/member agent and refresh token binaries.
+WORKDIR /core-workspace
+
+# Copy the go.mod and go.sum files.
+COPY fleet/go.mod go.mod
+COPY fleet/go.sum go.sum
+# Cache dependencies before building and copying the source code to avoid re-downloading upon retries.
+# It also ensures that source code changes do not invalidate the downloaded Go dependencies layer.
+RUN go mod download
+
+# Copy the Go source code.
+COPY fleet/cmd/ cmd/
+COPY fleet/apis/ apis/
+COPY fleet/pkg/ pkg/
+
+# Build the hub agent.
+RUN echo "Building the hub agent binary for GOOS=$GOOS GOARCH=$GOARCH"
+RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GOEXPERIMENT=systemcrypto GO111MODULE=on go build -o hubagent cmd/hubagent/main.go
+
+# Build the member agent.
+RUN echo "Building the member agent binary for GOOS=$GOOS GOARCH=$GOARCH"
+RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GOEXPERIMENT=systemcrypto GO111MODULE=on go build -o memberagent cmd/memberagent/main.go
+
+# Build the refresh token binary.
+RUN echo "Building the refresh token binary for GOOS=$GOOS GOARCH=$GOARCH"
+RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GOEXPERIMENT=systemcrypto GO111MODULE=on go build -o refreshtoken cmd/authtoken/main.go
+
+# Build the Fleet networking agent binaries.
+WORKDIR /networking-workspace
+
+# Copy the go.mod and go.sum files.
+COPY fleet-networking/go.mod go.mod
+COPY fleet-networking/go.sum go.sum
+# Cache dependencies before building and copying the source code to avoid re-downloading upon retries.
+# It also ensures that source code changes do not invalidate the downloaded Go dependencies layer.
+RUN go mod download
+
+# Copy the Go source code.
+COPY fleet-networking/cmd/ cmd/
+COPY fleet-networking/pkg/ pkg/
+COPY fleet-networking/api/ api/
+
+# Build the hub networking agent.
+RUN echo "Building the hub networking agent binary for GOOS=$GOOS GOARCH=$GOARCH"
+RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GO111MODULE=on go build -o hub-net-controller-manager cmd/hub-net-controller-manager/main.go
+
+# Build the member networking agent.
+RUN echo "Building the member networking agent binary for GOOS=$GOOS GOARCH=$GOARCH"
+RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GO111MODULE=on go build -o member-net-controller-manager cmd/member-net-controller-manager/main.go
+
+# Build the MCS networking agent.
+RUN echo "Building the MCS networking agent binary for GOOS=$GOOS GOARCH=$GOARCH"
+RUN CGO_ENABLED=1 GOOS=$GOOS GOARCH=$GOARCH GO111MODULE=on go build -o mcs-controller-manager cmd/mcs-controller-manager/main.go
+
+# Use Azure Linux distroless base image to package the binaries.
+# For more details, refer to https://mcr.microsoft.com/en-us/artifact/mar/azurelinux/distroless/base/about.
+FROM mcr.microsoft.com/azurelinux/distroless/base:3.0
+WORKDIR /
+COPY --from=builder /core-workspace/hubagent .
+COPY --from=builder /core-workspace/memberagent .
+COPY --from=builder /core-workspace/refreshtoken .
+COPY --from=builder /networking-workspace/hub-net-controller-manager .
+COPY --from=builder /networking-workspace/member-net-controller-manager .
+COPY --from=builder /networking-workspace/mcs-controller-manager .
+
+USER 65532:65532
+
+# No default entrypoint is set for the unified image.
+ENTRYPOINT []