From 745c025198b5600568105448f666ab63b708be67 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 22 Sep 2026 16:31:19 -0700 Subject: [PATCH 1/2] Measure Linux CI disk usage and clean SDK staging Isolate UseDotNet download and extraction files in a dedicated Linux temp directory, then remove it before tests and packaging. Record disk usage around installation, cleanup, and the build to quantify savings and identify remaining pressure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-pr.yml | 25 ++++++++++++++++++++++++- eng/pipelines/linux-disk-usage.sh | 15 +++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 eng/pipelines/linux-disk-usage.sh diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 407d0044ed..e3010f1cd1 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -163,24 +163,45 @@ stages: steps: - checkout: self clean: true + - bash: | + set -euo pipefail + bash eng/pipelines/linux-disk-usage.sh "Before SDK installation" + mkdir -p "$(Agent.TempDirectory)/scaffolding-sdk-staging" + displayName: Measure Linux disk usage before SDK installation + # UseDotNet downloads and extracts into Agent.TempDirectory before copying into .dotnet. - task: UseDotNet@2 displayName: Install .NET 8 SDK inputs: packageType: sdk version: 8.x installationPath: $(Build.SourcesDirectory)/.dotnet + env: + AGENT_TEMPDIRECTORY: $(Agent.TempDirectory)/scaffolding-sdk-staging - task: UseDotNet@2 displayName: Install .NET 9 SDK inputs: packageType: sdk version: 9.x installationPath: $(Build.SourcesDirectory)/.dotnet + env: + AGENT_TEMPDIRECTORY: $(Agent.TempDirectory)/scaffolding-sdk-staging - task: UseDotNet@2 displayName: Install .NET 10 SDK inputs: packageType: sdk version: 10.x installationPath: $(Build.SourcesDirectory)/.dotnet + env: + AGENT_TEMPDIRECTORY: $(Agent.TempDirectory)/scaffolding-sdk-staging + - bash: bash eng/pipelines/linux-disk-usage.sh "After SDK installation" + displayName: Measure Linux disk usage after SDK installation + condition: succeededOrFailed() + - bash: | + set -euo pipefail + rm -rf -- "$(Agent.TempDirectory)/scaffolding-sdk-staging" + bash eng/pipelines/linux-disk-usage.sh "After SDK staging cleanup" + displayName: Clean Linux SDK download and extraction staging + condition: succeededOrFailed() - script: eng/common/build.sh --configuration $(_BuildConfig) --prepareMachine @@ -188,6 +209,9 @@ stages: name: Build displayName: Build condition: succeeded() + - bash: bash eng/pipelines/linux-disk-usage.sh "After build" + displayName: Measure Linux disk usage after build + condition: succeededOrFailed() - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - stage: integration @@ -198,4 +222,3 @@ stages: parameters: buildConfig: Release poolVmImage: windows-latest - diff --git a/eng/pipelines/linux-disk-usage.sh b/eng/pipelines/linux-disk-usage.sh new file mode 100644 index 0000000000..89ca3b2f20 --- /dev/null +++ b/eng/pipelines/linux-disk-usage.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "Disk usage: $1" +df -h / "$BUILD_SOURCESDIRECTORY" "$AGENT_TEMPDIRECTORY" +for path in \ + "$BUILD_SOURCESDIRECTORY/.dotnet" \ + "$BUILD_SOURCESDIRECTORY/artifacts" \ + "${NUGET_PACKAGES:-$HOME/.nuget/packages}" \ + "$AGENT_TEMPDIRECTORY" \ + "$AGENT_TEMPDIRECTORY/scaffolding-sdk-staging"; do + if [[ -e "$path" ]]; then + du -sh "$path" + fi +done From b672725b1b393b2a48ea98aa256711350f78bbb0 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 23 Sep 2026 08:01:25 -0700 Subject: [PATCH 2/2] Clean only SDK staging produced by Linux UseDotNet tasks Use the observed agent temp location and compare entries before and after SDK installation. Remove only newly created GUID-named SDK archives and extraction directories, leaving other temp entries and installed SDKs intact. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2ff0cd05-fdd3-47bb-8b0c-983417c3829c --- azure-pipelines-pr.yml | 10 ++----- eng/pipelines/linux-disk-usage.sh | 3 +- eng/pipelines/linux-sdk-staging.sh | 45 ++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 eng/pipelines/linux-sdk-staging.sh diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index e3010f1cd1..cdf052e09b 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -166,7 +166,7 @@ stages: - bash: | set -euo pipefail bash eng/pipelines/linux-disk-usage.sh "Before SDK installation" - mkdir -p "$(Agent.TempDirectory)/scaffolding-sdk-staging" + bash eng/pipelines/linux-sdk-staging.sh snapshot displayName: Measure Linux disk usage before SDK installation # UseDotNet downloads and extracts into Agent.TempDirectory before copying into .dotnet. - task: UseDotNet@2 @@ -175,30 +175,24 @@ stages: packageType: sdk version: 8.x installationPath: $(Build.SourcesDirectory)/.dotnet - env: - AGENT_TEMPDIRECTORY: $(Agent.TempDirectory)/scaffolding-sdk-staging - task: UseDotNet@2 displayName: Install .NET 9 SDK inputs: packageType: sdk version: 9.x installationPath: $(Build.SourcesDirectory)/.dotnet - env: - AGENT_TEMPDIRECTORY: $(Agent.TempDirectory)/scaffolding-sdk-staging - task: UseDotNet@2 displayName: Install .NET 10 SDK inputs: packageType: sdk version: 10.x installationPath: $(Build.SourcesDirectory)/.dotnet - env: - AGENT_TEMPDIRECTORY: $(Agent.TempDirectory)/scaffolding-sdk-staging - bash: bash eng/pipelines/linux-disk-usage.sh "After SDK installation" displayName: Measure Linux disk usage after SDK installation condition: succeededOrFailed() - bash: | set -euo pipefail - rm -rf -- "$(Agent.TempDirectory)/scaffolding-sdk-staging" + bash eng/pipelines/linux-sdk-staging.sh clean bash eng/pipelines/linux-disk-usage.sh "After SDK staging cleanup" displayName: Clean Linux SDK download and extraction staging condition: succeededOrFailed() diff --git a/eng/pipelines/linux-disk-usage.sh b/eng/pipelines/linux-disk-usage.sh index 89ca3b2f20..3b7786a018 100644 --- a/eng/pipelines/linux-disk-usage.sh +++ b/eng/pipelines/linux-disk-usage.sh @@ -7,8 +7,7 @@ for path in \ "$BUILD_SOURCESDIRECTORY/.dotnet" \ "$BUILD_SOURCESDIRECTORY/artifacts" \ "${NUGET_PACKAGES:-$HOME/.nuget/packages}" \ - "$AGENT_TEMPDIRECTORY" \ - "$AGENT_TEMPDIRECTORY/scaffolding-sdk-staging"; do + "$AGENT_TEMPDIRECTORY"; do if [[ -e "$path" ]]; then du -sh "$path" fi diff --git a/eng/pipelines/linux-sdk-staging.sh b/eng/pipelines/linux-sdk-staging.sh new file mode 100644 index 0000000000..cce67564d4 --- /dev/null +++ b/eng/pipelines/linux-sdk-staging.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +set -euo pipefail + +snapshot="$PIPELINE_WORKSPACE/scaffolding-sdk-temp-before.txt" + +case "${1:-}" in + snapshot) + find "$AGENT_TEMPDIRECTORY" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort > "$snapshot" + ;; + clean) + if [[ ! -f "$snapshot" ]]; then + echo "Missing SDK staging snapshot: $snapshot" >&2 + exit 1 + fi + + after="$(mktemp "$PIPELINE_WORKSPACE/scaffolding-sdk-temp-after.XXXXXX")" + new_entries="$(mktemp "$PIPELINE_WORKSPACE/scaffolding-sdk-temp-new.XXXXXX")" + trap 'rm -f -- "$after" "$new_entries"' EXIT + find "$AGENT_TEMPDIRECTORY" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort > "$after" + comm -13 "$snapshot" "$after" > "$new_entries" + + while IFS= read -r name; do + [[ "$name" =~ ^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$ ]] || continue + path="$AGENT_TEMPDIRECTORY/$name" + if [[ -d "$path" ]]; then + [[ -d "$path/sdk" && -d "$path/shared/Microsoft.NETCore.App" ]] || continue + sdk_dir="$(find "$path/sdk" -mindepth 1 -maxdepth 1 -type d \( -name '8.*' -o -name '9.*' -o -name '10.*' \) -print -quit)" + [[ -n "$sdk_dir" ]] || continue + elif [[ -f "$path" ]]; then + [[ "$(file -b --mime-type "$path")" == application/gzip ]] || continue + tar --force-local -tzf "$path" | awk '/(^|\/)sdk\/(8|9|10)\./ { found = 1 } END { exit !found }' || continue + else + continue + fi + + du -sh -- "$path" + rm -rf -- "$path" + done < "$new_entries" + rm -f -- "$snapshot" + ;; + *) + echo "Usage: $0 snapshot|clean" >&2 + exit 1 + ;; +esac