From 9cf09a90788202fb2e944ee237288996e02cd585 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 31 Jul 2026 01:22:19 -0400 Subject: [PATCH 1/3] ci: activate dependency-aware package builds and tests --- .github/workflows/ci.yml | 436 ++++++++++++++++++++++++++++++------ ci/tools/compute_ci_plan.py | 147 ++++++++++++ 2 files changed, 515 insertions(+), 68 deletions(-) create mode 100644 ci/tools/compute_ci_plan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aadf222306..0be7f9445b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,24 +72,31 @@ jobs: echo "skip=${skip}" >> "$GITHUB_OUTPUT" echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" - # Detect which top-level modules were touched by the PR so downstream build - # and test jobs can avoid rebuilding/retesting modules unaffected by the - # change. See issue #299. + # Detect which packages were touched by the PR so downstream build and test + # jobs can avoid rebuilding/retesting packages unaffected by the change. + # See issue #299. # # Dependency graph (verified in pyproject.toml files): # cuda_pathfinder -> (no internal deps) # cuda_bindings -> cuda_pathfinder # cuda_core -> cuda_pathfinder, cuda_bindings - # cuda_python -> cuda_bindings (meta package) + # cuda_python -> cuda_pathfinder, cuda_bindings, cuda_core (meta package) # # A change to cuda_pathfinder (or shared infra) forces a rebuild of every # downstream module. A change to cuda_bindings forces rebuild of cuda_core. - # A change to cuda_core alone skips rebuilding/retesting cuda_bindings. + # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and + # cuda_pathfinder, but still retests the downstream cuda-python metapackage. + # CI/planner changes are shared by design, so this implementation runs the + # full pipeline; exercise selective cases in follow-up package-only PRs. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed # paths" baseline for those events. detect-changes: runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pull-requests: read outputs: bindings: ${{ steps.compose.outputs.bindings }} core: ${{ steps.compose.outputs.core }} @@ -100,10 +107,14 @@ jobs: build_bindings: ${{ steps.compose.outputs.build_bindings }} build_core: ${{ steps.compose.outputs.build_core }} build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} + build_python: ${{ steps.compose.outputs.build_python }} test_bindings: ${{ steps.compose.outputs.test_bindings }} test_core: ${{ steps.compose.outputs.test_core }} test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} pr_merge_base: ${{ steps.filter.outputs.merge_base }} + test_python: ${{ steps.compose.outputs.test_python }} + baseline_run_id: ${{ steps.compose.outputs.baseline_run_id }} + baseline_sha: ${{ steps.compose.outputs.baseline_sha }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -136,52 +147,177 @@ jobs: # off by `if:`, so `BASE_REF` is never consumed there. BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} run: | - # Diff against the merge base with the PR's actual target branch. - # Uses merge-base so diverged branches only show files changed on - # the PR side, not upstream commits. + set -euo pipefail if [[ -z "${BASE_REF}" ]]; then echo "Could not resolve PR base branch from get-pr-info output" >&2 exit 1 fi + + # Diff against the merge base with the PR's actual target branch. + # Disabling rename detection reports both sides of a cross-package + # move, which prevents the source package from being skipped. base=$(git merge-base HEAD "origin/${BASE_REF}") - changed=$(git diff --name-only "$base"...HEAD) + git diff --no-renames --name-only -z "$base"...HEAD > changed-paths + python ci/tools/compute_ci_plan.py changed-paths >> "$GITHUB_OUTPUT" + echo "merge_base=${base}" >> "$GITHUB_OUTPUT" + + { + echo "### Selective CI changed paths" + echo + tr '\0' '\n' < changed-paths | sed 's/^/- `/' | sed 's/$/`/' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Resolve reusable base artifacts + id: baseline + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + env: + BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + + unavailable() { + echo "available=false" >> "$GITHUB_OUTPUT" + echo "No complete reusable artifact set was found; this run will build and test everything." >> "$GITHUB_STEP_SUMMARY" + exit 0 + } + + if [[ -z "${BASE_REF}" ]]; then + unavailable + fi + + merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + if ! runs=$(gh run list \ + --repo "${{ github.repository }}" \ + --branch "${BASE_REF}" \ + --commit "${merge_base}" \ + --event push \ + --workflow ci.yml \ + --status success \ + --limit 1 \ + --json databaseId,headSha,createdAt); then + unavailable + fi + + # Reuse only artifacts produced from the exact commit used as the + # PR diff base. Using the latest base-branch run is unsafe for a PR + # that was opened before newer changes landed on that branch. + run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") + baseline_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") + if [[ -z "${run_id}" || "${baseline_sha}" != "${merge_base}" ]]; then + unavailable + fi + + if ! artifact_names=$(gh api \ + "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts?per_page=100" \ + --paginate \ + --jq '.artifacts[] | select(.expired == false) | .name'); then + unavailable + fi - has_match() { - grep -qE "$1" <<< "$changed" && echo true || echo false + has_artifact() { + grep -Fxq "$1" <<< "$artifact_names" } + missing=() + for name in cuda-pathfinder-wheel cuda-python-wheel; do + has_artifact "$name" || missing+=("$name") + done + + cuda_version=$(yq '.cuda.build.version' ci/versions.yml) + if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then + unavailable + fi + if [[ -z "${python_versions}" ]]; then + unavailable + fi + while IFS= read -r python_version; do + python=${python_version//./} + for platform in linux-64 linux-aarch64 win-64; do + binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${baseline_sha}" + core="cuda-core-python${python}-${platform}-${baseline_sha}" + has_artifact "$binding" || missing+=("$binding") + has_artifact "$core" || missing+=("$core") + done + done <<< "${python_versions}" + + if (( ${#missing[@]} != 0 )); then + printf 'Missing reusable artifact: %s\n' "${missing[@]}" >&2 + unavailable + fi + { - echo "bindings=$(has_match '^cuda_bindings/')" - echo "core=$(has_match '^cuda_core/')" - echo "pathfinder=$(has_match '^cuda_pathfinder/')" - echo "python_meta=$(has_match '^cuda_python/')" - echo "test_helpers=$(has_match '^cuda_python_test_helpers/')" - echo "shared=$(has_match '^(\.github/|ci/|scripts/|toolshed/|conftest\.py$|pyproject\.toml$|pixi\.(toml|lock)$|pytest\.ini$|ruff\.toml$)')" - echo "merge_base=${base}" + echo "available=true" + echo "run_id=${run_id}" + echo "sha=${baseline_sha}" } >> "$GITHUB_OUTPUT" + { + echo + echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." + } >> "$GITHUB_STEP_SUMMARY" - name: Compose gating outputs id: compose env: IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BINDINGS: ${{ steps.filter.outputs.bindings || 'false' }} - CORE: ${{ steps.filter.outputs.core || 'false' }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder || 'false' }} - PYTHON_META: ${{ steps.filter.outputs.python_meta || 'false' }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers || 'false' }} - SHARED: ${{ steps.filter.outputs.shared || 'false' }} + BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available || 'false' }} + BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} + BASELINE_SHA: ${{ steps.baseline.outputs.sha }} + BINDINGS: ${{ steps.filter.outputs.bindings_source }} + CORE: ${{ steps.filter.outputs.core_source }} + PATHFINDER: ${{ steps.filter.outputs.pathfinder_source }} + PYTHON_META: ${{ steps.filter.outputs.python_source }} + TEST_HELPERS: ${{ steps.filter.outputs.test_helpers }} + SHARED: ${{ steps.filter.outputs.shared }} + BUILD_BINDINGS: ${{ steps.filter.outputs.build_bindings }} + BUILD_CORE: ${{ steps.filter.outputs.build_core }} + BUILD_PATHFINDER: ${{ steps.filter.outputs.build_pathfinder }} + BUILD_PYTHON: ${{ steps.filter.outputs.build_python }} + TEST_BINDINGS: ${{ steps.filter.outputs.test_bindings }} + TEST_CORE: ${{ steps.filter.outputs.test_core }} + TEST_PATHFINDER: ${{ steps.filter.outputs.test_pathfinder }} + TEST_PYTHON: ${{ steps.filter.outputs.test_python }} run: | set -euxo pipefail - # Non-PR events (push to main, tag push, schedule, workflow_dispatch) - # always exercise the full pipeline because there is no baseline for - # a meaningful diff. - if [[ "${IS_PR}" != "true" ]]; then + planner_valid=true + if [[ "${IS_PR}" == "true" ]]; then + for value in \ + "${BINDINGS}" "${CORE}" "${PATHFINDER}" "${PYTHON_META}" \ + "${TEST_HELPERS}" "${SHARED}" \ + "${BUILD_BINDINGS}" "${BUILD_CORE}" "${BUILD_PATHFINDER}" "${BUILD_PYTHON}" \ + "${TEST_BINDINGS}" "${TEST_CORE}" "${TEST_PATHFINDER}" "${TEST_PYTHON}"; do + if [[ "${value}" != "true" && "${value}" != "false" ]]; then + planner_valid=false + fi + done + if [[ "${BASELINE_AVAILABLE}" == "true" && + ( -z "${BASELINE_RUN_ID}" || -z "${BASELINE_SHA}" ) ]]; then + planner_valid=false + fi + fi + + # Non-PR events produce the complete trusted artifact set. PRs also + # run everything when the trusted base artifact inventory is absent + # or the planner did not emit a complete boolean result. + if [[ "${IS_PR}" != "true" || + "${BASELINE_AVAILABLE}" != "true" || + "${planner_valid}" != "true" ]]; then bindings=true core=true pathfinder=true python_meta=true test_helpers=true shared=true + build_bindings=true + build_core=true + build_pathfinder=true + build_python=true + test_bindings=true + test_core=true + test_pathfinder=true + test_python=true + baseline_run_id="" + baseline_sha="" else bindings="${BINDINGS}" core="${CORE}" @@ -189,32 +325,18 @@ jobs: python_meta="${PYTHON_META}" test_helpers="${TEST_HELPERS}" shared="${SHARED}" + build_bindings="${BUILD_BINDINGS}" + build_core="${BUILD_CORE}" + build_pathfinder="${BUILD_PATHFINDER}" + build_python="${BUILD_PYTHON}" + test_bindings="${TEST_BINDINGS}" + test_core="${TEST_CORE}" + test_pathfinder="${TEST_PATHFINDER}" + test_python="${TEST_PYTHON}" + baseline_run_id="${BASELINE_RUN_ID}" + baseline_sha="${BASELINE_SHA}" fi - or_flag() { - for v in "$@"; do - if [[ "${v}" == "true" ]]; then - echo "true" - return - fi - done - echo "false" - } - - # Build gating: pathfinder change forces rebuild of bindings and - # core; bindings change forces rebuild of core. shared changes force - # a full rebuild. - build_pathfinder="$(or_flag "${shared}" "${pathfinder}")" - build_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}")" - build_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}")" - - # Test gating: tests for a module must run whenever that module, any - # of its runtime dependencies, the shared test helper package, or - # shared infra changes. pathfinder tests are cheap and always run. - test_pathfinder=true - test_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${test_helpers}")" - test_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}" "${test_helpers}")" - { echo "bindings=${bindings}" echo "core=${core}" @@ -225,11 +347,27 @@ jobs: echo "build_bindings=${build_bindings}" echo "build_core=${build_core}" echo "build_pathfinder=${build_pathfinder}" + echo "build_python=${build_python}" echo "test_bindings=${test_bindings}" echo "test_core=${test_core}" echo "test_pathfinder=${test_pathfinder}" + echo "test_python=${test_python}" + echo "baseline_run_id=${baseline_run_id}" + echo "baseline_sha=${baseline_sha}" } >> "$GITHUB_OUTPUT" + { + echo + echo "### Effective package plan" + echo + echo "| Package | Build | Test |" + echo "| --- | --- | --- |" + echo "| cuda-pathfinder | ${build_pathfinder} | ${test_pathfinder} |" + echo "| cuda-bindings | ${build_bindings} | ${test_bindings} |" + echo "| cuda-core | ${build_core} | ${test_core} |" + echo "| cuda-python | ${build_python} | ${test_python} |" + } >> "$GITHUB_STEP_SUMMARY" + api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- @@ -314,15 +452,17 @@ jobs: merge-base: ${{ needs.detect-changes.outputs.pr_merge_base }} # NOTE: Build jobs are intentionally split by platform rather than using a single - # matrix. This allows each test job to depend only on its corresponding build, - # so faster platforms can proceed through build & test without waiting for slower - # ones. Keep these job definitions textually identical except for: + # matrix. This lets each test job consume its platform-specific artifacts as + # soon as they are ready. ARM64 and Windows tests also wait for linux-64, + # which produces the universal pathfinder and cuda-python wheels. Keep these + # job definitions textually identical except for: # - host-platform value # - if: condition (build-linux-64 omits doc-only check since it's needed for docs) build-linux-64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: @@ -330,50 +470,105 @@ jobs: - linux-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-linux-aarch64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - linux-aarch64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-windows: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - win-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by @@ -385,26 +580,57 @@ jobs: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 name: Test sdist linux-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-linux.yml with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 + - build-windows name: Test sdist win-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-windows.yml with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -418,8 +644,14 @@ jobs: host-platform: - linux-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars @@ -433,7 +665,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -443,13 +678,20 @@ jobs: host-platform: - linux-aarch64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-linux-aarch64 secrets: inherit uses: ./.github/workflows/test-wheel-linux.yml @@ -458,7 +700,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -468,13 +713,20 @@ jobs: host-platform: - win-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-windows secrets: inherit uses: ./.github/workflows/test-wheel-windows.yml @@ -483,7 +735,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} doc: name: Docs @@ -538,13 +793,19 @@ jobs: if: always() runs-on: ubuntu-latest needs: + - ci-vars - should-skip - detect-changes + - build-linux-64 + - build-linux-aarch64 + - build-windows - test-sdist-linux - test-sdist-windows - test-linux-64 - test-linux-aarch64 - test-windows + - api-check-core-vs-release + - api-check-core-vs-base - doc - precommit-windows steps: @@ -565,6 +826,16 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" + build_selected="${{ needs.detect-changes.outputs.build_pathfinder == 'true' || + needs.detect-changes.outputs.build_bindings == 'true' || + needs.detect-changes.outputs.build_core == 'true' || + needs.detect-changes.outputs.build_python == 'true' }}" + test_selected="${{ needs.detect-changes.outputs.test_pathfinder == 'true' || + needs.detect-changes.outputs.test_bindings == 'true' || + needs.detect-changes.outputs.test_core == 'true' || + needs.detect-changes.outputs.test_python == 'true' }}" + core_changed="${{ needs.detect-changes.outputs.core == 'true' }}" + is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { name=$1; expected=$2; result=$3 @@ -575,18 +846,47 @@ jobs: fi } - # always expected to succeed (even in [doc-only] mode) - check_result "should-skip" "success" "${{ needs.should-skip.result }}" - check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" - check_result "doc" "success" "${{ needs.doc.result }}" + # Control jobs, the universal linux build, docs, and Windows + # pre-commit checks always run. + check_result "ci-vars" "success" "${{ needs.ci-vars.result }}" + check_result "should-skip" "success" "${{ needs.should-skip.result }}" + check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" + check_result "build-linux-64" "success" "${{ needs.build-linux-64.result }}" + check_result "doc" "success" "${{ needs.doc.result }}" check_result "precommit-windows" "success" "${{ needs.precommit-windows.result }}" - # [doc-only] flips these from 'success' to 'skipped' - if [[ "$doc_only" == "true" ]]; then expected="skipped"; else expected="success"; fi + # Platform builds run whenever any package needs to be built or tested. + expected="skipped" + if [[ "$doc_only" != "true" && + ( "$build_selected" == "true" || "$test_selected" == "true" ) ]]; then + expected="success" + fi + check_result "build-linux-aarch64" "$expected" "${{ needs.build-linux-aarch64.result }}" + check_result "build-windows" "$expected" "${{ needs.build-windows.result }}" + + # Sdist and wheel tests are independently gated by the effective plan. + expected="skipped" + if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then + expected="success" + fi check_result "test-sdist-linux" "$expected" "${{ needs.test-sdist-linux.result }}" check_result "test-sdist-windows" "$expected" "${{ needs.test-sdist-windows.result }}" + + expected="skipped" + if [[ "$doc_only" != "true" && "$test_selected" == "true" ]]; then + expected="success" + fi check_result "test-linux-64" "$expected" "${{ needs.test-linux-64.result }}" check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" + # API compatibility checks only run for cuda_core source changes. + expected="skipped" + if [[ "$core_changed" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" + + expected="skipped" + if [[ "$is_pr" == "true" && "$core_changed" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" + [[ "$status" == "success" ]] diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py new file mode 100644 index 00000000000..7095514b73e --- /dev/null +++ b/ci/tools/compute_ci_plan.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Compute the package build and test closure for a set of changed paths.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +PACKAGES = { + "cuda_pathfinder": "pathfinder", + "cuda_bindings": "bindings", + "cuda_core": "core", + "cuda_python": "python", +} + +SHARED_PREFIXES = ( + ".github/", + "ci/", + "scripts/", + "toolshed/", +) + +SHARED_FILES = { + ".pre-commit-config.yaml", + "conftest.py", + "pixi.lock", + "pixi.toml", + "pytest.ini", + "ruff.toml", +} + +KNOWN_REPOSITORY_FILES = { + ".git-blame-ignore-revs", + ".gitignore", + "AGENTS.md", + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "README.md", + "SECURITY.md", +} + + +def _bool(value: bool) -> str: + return str(value).lower() + + +def _read_paths(path: Path) -> list[str]: + return [value.decode("utf-8", errors="surrogateescape") for value in path.read_bytes().split(b"\0") if value] + + +def compute_plan(paths: list[str]) -> dict[str, bool]: + source = dict.fromkeys(PACKAGES.values(), False) + tests = dict.fromkeys(PACKAGES.values(), False) + docs = False + test_helpers = False + shared = False + unknown = False + + for path in paths: + package_dir, separator, relative_path = path.partition("/") + package = PACKAGES.get(package_dir) + if package is not None and separator: + if relative_path.startswith("docs/"): + docs = True + elif relative_path.startswith(("tests/", "examples/")): + tests[package] = True + else: + source[package] = True + continue + + if path.startswith("cuda_python_test_helpers/"): + test_helpers = True + elif path.startswith("benchmarks/cuda_bindings/"): + tests["bindings"] = True + elif path.startswith(SHARED_PREFIXES) or path in SHARED_FILES: + shared = True + elif path in KNOWN_REPOSITORY_FILES: + # Repository policy and prose files do not affect package artifacts. + continue + else: + unknown = True + + full = shared or unknown + + build_pathfinder = full or source["pathfinder"] + # Development cuda-python wheels exactly pin cuda-bindings, so a + # metapackage change needs a matching bindings artifact for its smoke test. + build_bindings = full or source["pathfinder"] or source["bindings"] or source["python"] + build_core = full or source["pathfinder"] or source["bindings"] or source["core"] + # A core-only change can reuse the baseline cuda-python wheel: rebuilding + # it would also require rebuilding the exact-version cuda-bindings pin. + build_python = full or source["pathfinder"] or source["bindings"] or source["python"] + + test_pathfinder = full or source["pathfinder"] or tests["pathfinder"] + test_bindings = full or source["pathfinder"] or source["bindings"] or tests["bindings"] or test_helpers + test_core = full or source["pathfinder"] or source["bindings"] or source["core"] or tests["core"] or test_helpers + test_python = ( + full or source["pathfinder"] or source["bindings"] or source["core"] or source["python"] or tests["python"] + ) + + return { + "shared": shared, + "unknown": unknown, + "docs": docs, + "test_helpers": test_helpers, + "pathfinder_source": source["pathfinder"], + "bindings_source": source["bindings"], + "core_source": source["core"], + "python_source": source["python"], + "pathfinder_tests": tests["pathfinder"], + "bindings_tests": tests["bindings"], + "core_tests": tests["core"], + "python_tests": tests["python"], + "build_pathfinder": build_pathfinder, + "build_bindings": build_bindings, + "build_core": build_core, + "build_python": build_python, + "test_pathfinder": test_pathfinder, + "test_bindings": test_bindings, + "test_core": test_core, + "test_python": test_python, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "paths_file", + type=Path, + help="NUL-separated changed-path list produced by git diff --name-only -z", + ) + args = parser.parse_args() + + plan = compute_plan(_read_paths(args.paths_file)) + for key, value in plan.items(): + print(f"{key}={_bool(value)}") + + +if __name__ == "__main__": + main() From da7ebee702759b45c0a81fa0c1a9f7f233fbf874 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Thu, 13 Aug 2026 22:45:31 -0400 Subject: [PATCH 2/3] ci: use paths-filter for selective CI planning --- .github/workflows/ci.yml | 307 ++++++++++++++++++------------------ ci/tools/compute_ci_plan.py | 147 ----------------- 2 files changed, 156 insertions(+), 298 deletions(-) delete mode 100644 ci/tools/compute_ci_plan.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0be7f9445b5..16ab65b3db2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,7 @@ jobs: # downstream module. A change to cuda_bindings forces rebuild of cuda_core. # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and # cuda_pathfinder, but still retests the downstream cuda-python metapackage. - # CI/planner changes are shared by design, so this implementation runs the + # CI/filter changes are shared by design, so this implementation runs the # full pipeline; exercise selective cases in follow-up package-only PRs. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed @@ -98,43 +98,98 @@ jobs: contents: read pull-requests: read outputs: - bindings: ${{ steps.compose.outputs.bindings }} - core: ${{ steps.compose.outputs.core }} - pathfinder: ${{ steps.compose.outputs.pathfinder }} - python_meta: ${{ steps.compose.outputs.python_meta }} - test_helpers: ${{ steps.compose.outputs.test_helpers }} - shared: ${{ steps.compose.outputs.shared }} - build_bindings: ${{ steps.compose.outputs.build_bindings }} - build_core: ${{ steps.compose.outputs.build_core }} - build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} - build_python: ${{ steps.compose.outputs.build_python }} - test_bindings: ${{ steps.compose.outputs.test_bindings }} - test_core: ${{ steps.compose.outputs.test_core }} - test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} - pr_merge_base: ${{ steps.filter.outputs.merge_base }} - test_python: ${{ steps.compose.outputs.test_python }} - baseline_run_id: ${{ steps.compose.outputs.baseline_run_id }} - baseline_sha: ${{ steps.compose.outputs.baseline_sha }} + # Missing base artifacts or a skipped path filter fail open to the full pipeline. + core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.core_source == 'true' }} + build_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' }} + build_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + build_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' }} + build_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + test_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.pathfinder_tests == 'true' }} + test_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.bindings_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + test_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.core_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + pr_merge_base: ${{ steps.merge-base.outputs.sha }} + test_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.python_source == 'true' || + steps.filter.outputs.python_tests == 'true' }} + baseline_run_id: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.run_id || '' }} + baseline_sha: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.sha || '' }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # Treeless clone: commit graph is needed for `git merge-base` and - # `git diff --name-only` below, but historical blobs aren't. + # Treeless clone: the commit graph is needed to resolve the PR merge + # base and classify its changed paths, but historical blobs aren't. fetch-depth: 0 filter: blob:none # copy-pr-bot pushes every PR (whether it targets main or a backport # branch such as 12.9.x) to pull-request/, so the base branch # cannot be inferred from github.ref_name. Look it up via the - # upstream PR metadata so the diff below is rooted at the right place. + # upstream PR metadata so change detection is rooted at the right place. - name: Resolve PR base branch id: pr-info if: ${{ startsWith(github.ref_name, 'pull-request/') }} uses: nv-gha-runners/get-pr-info@main - - name: Detect changed paths - id: filter + - name: Resolve PR merge base + id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: # GitHub Actions evaluates step-level `env:` expressions eagerly — @@ -153,25 +208,83 @@ jobs: exit 1 fi - # Diff against the merge base with the PR's actual target branch. - # Disabling rename detection reports both sides of a cross-package - # move, which prevents the source package from being skipped. base=$(git merge-base HEAD "origin/${BASE_REF}") - git diff --no-renames --name-only -z "$base"...HEAD > changed-paths - python ci/tools/compute_ci_plan.py changed-paths >> "$GITHUB_OUTPUT" - echo "merge_base=${base}" >> "$GITHUB_OUTPUT" + echo "sha=${base}" >> "$GITHUB_OUTPUT" - { - echo "### Selective CI changed paths" - echo - tr '\0' '\n' < changed-paths | sed 's/^/- `/' | sed 's/$/`/' - } >> "$GITHUB_STEP_SUMMARY" + - name: Classify changed paths + id: filter + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + with: + # The workflow runs on copy-pr-bot push branches, so compare the + # checked-out PR head against the base resolved from PR metadata. + base: ${{ steps.merge-base.outputs.sha }} + ref: ${{ github.sha }} + token: '' + predicate-quantifier: some-with-excludes + filters: | + pathfinder_source: + - 'cuda_pathfinder/**' + - '!cuda_pathfinder/docs/**' + - '!cuda_pathfinder/tests/**' + - '!cuda_pathfinder/examples/**' + bindings_source: + - 'cuda_bindings/**' + - '!cuda_bindings/docs/**' + - '!cuda_bindings/tests/**' + - '!cuda_bindings/examples/**' + core_source: + - 'cuda_core/**' + - '!cuda_core/docs/**' + - '!cuda_core/tests/**' + - '!cuda_core/examples/**' + python_source: + - 'cuda_python/**' + - '!cuda_python/docs/**' + - '!cuda_python/tests/**' + - '!cuda_python/examples/**' + pathfinder_tests: + - 'cuda_pathfinder/tests/**' + - 'cuda_pathfinder/examples/**' + bindings_tests: + - 'cuda_bindings/tests/**' + - 'cuda_bindings/examples/**' + - 'benchmarks/cuda_bindings/**' + core_tests: + - 'cuda_core/tests/**' + - 'cuda_core/examples/**' + python_tests: + - 'cuda_python/tests/**' + - 'cuda_python/examples/**' + test_helpers: + - 'cuda_python_test_helpers/**' + # Shared infrastructure and unknown paths run the full pipeline. + # Exclude only paths whose narrower behavior is defined above or + # repository policy/prose files known not to affect artifacts. + force_all: + - '**' + - '!cuda_pathfinder/**' + - '!cuda_bindings/**' + - '!cuda_core/**' + - '!cuda_python/**' + - '!cuda_python_test_helpers/**' + - '!benchmarks/cuda_bindings/**' + - '!.git-blame-ignore-revs' + - '!.gitignore' + - '!AGENTS.md' + - '!CHANGELOG.md' + - '!CODE_OF_CONDUCT.md' + - '!CONTRIBUTING.md' + - '!LICENSE' + - '!README.md' + - '!SECURITY.md' - name: Resolve reusable base artifacts id: baseline if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} GH_TOKEN: ${{ github.token }} run: | set -uo pipefail @@ -186,7 +299,10 @@ jobs: unavailable fi - merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + merge_base="${MERGE_BASE}" + if [[ -z "${merge_base}" ]]; then + unavailable + fi if ! runs=$(gh run list \ --repo "${{ github.repository }}" \ --branch "${BASE_REF}" \ @@ -256,118 +372,6 @@ jobs: echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." } >> "$GITHUB_STEP_SUMMARY" - - name: Compose gating outputs - id: compose - env: - IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BASELINE_AVAILABLE: ${{ steps.baseline.outputs.available || 'false' }} - BASELINE_RUN_ID: ${{ steps.baseline.outputs.run_id }} - BASELINE_SHA: ${{ steps.baseline.outputs.sha }} - BINDINGS: ${{ steps.filter.outputs.bindings_source }} - CORE: ${{ steps.filter.outputs.core_source }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder_source }} - PYTHON_META: ${{ steps.filter.outputs.python_source }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers }} - SHARED: ${{ steps.filter.outputs.shared }} - BUILD_BINDINGS: ${{ steps.filter.outputs.build_bindings }} - BUILD_CORE: ${{ steps.filter.outputs.build_core }} - BUILD_PATHFINDER: ${{ steps.filter.outputs.build_pathfinder }} - BUILD_PYTHON: ${{ steps.filter.outputs.build_python }} - TEST_BINDINGS: ${{ steps.filter.outputs.test_bindings }} - TEST_CORE: ${{ steps.filter.outputs.test_core }} - TEST_PATHFINDER: ${{ steps.filter.outputs.test_pathfinder }} - TEST_PYTHON: ${{ steps.filter.outputs.test_python }} - run: | - set -euxo pipefail - planner_valid=true - if [[ "${IS_PR}" == "true" ]]; then - for value in \ - "${BINDINGS}" "${CORE}" "${PATHFINDER}" "${PYTHON_META}" \ - "${TEST_HELPERS}" "${SHARED}" \ - "${BUILD_BINDINGS}" "${BUILD_CORE}" "${BUILD_PATHFINDER}" "${BUILD_PYTHON}" \ - "${TEST_BINDINGS}" "${TEST_CORE}" "${TEST_PATHFINDER}" "${TEST_PYTHON}"; do - if [[ "${value}" != "true" && "${value}" != "false" ]]; then - planner_valid=false - fi - done - if [[ "${BASELINE_AVAILABLE}" == "true" && - ( -z "${BASELINE_RUN_ID}" || -z "${BASELINE_SHA}" ) ]]; then - planner_valid=false - fi - fi - - # Non-PR events produce the complete trusted artifact set. PRs also - # run everything when the trusted base artifact inventory is absent - # or the planner did not emit a complete boolean result. - if [[ "${IS_PR}" != "true" || - "${BASELINE_AVAILABLE}" != "true" || - "${planner_valid}" != "true" ]]; then - bindings=true - core=true - pathfinder=true - python_meta=true - test_helpers=true - shared=true - build_bindings=true - build_core=true - build_pathfinder=true - build_python=true - test_bindings=true - test_core=true - test_pathfinder=true - test_python=true - baseline_run_id="" - baseline_sha="" - else - bindings="${BINDINGS}" - core="${CORE}" - pathfinder="${PATHFINDER}" - python_meta="${PYTHON_META}" - test_helpers="${TEST_HELPERS}" - shared="${SHARED}" - build_bindings="${BUILD_BINDINGS}" - build_core="${BUILD_CORE}" - build_pathfinder="${BUILD_PATHFINDER}" - build_python="${BUILD_PYTHON}" - test_bindings="${TEST_BINDINGS}" - test_core="${TEST_CORE}" - test_pathfinder="${TEST_PATHFINDER}" - test_python="${TEST_PYTHON}" - baseline_run_id="${BASELINE_RUN_ID}" - baseline_sha="${BASELINE_SHA}" - fi - - { - echo "bindings=${bindings}" - echo "core=${core}" - echo "pathfinder=${pathfinder}" - echo "python_meta=${python_meta}" - echo "test_helpers=${test_helpers}" - echo "shared=${shared}" - echo "build_bindings=${build_bindings}" - echo "build_core=${build_core}" - echo "build_pathfinder=${build_pathfinder}" - echo "build_python=${build_python}" - echo "test_bindings=${test_bindings}" - echo "test_core=${test_core}" - echo "test_pathfinder=${test_pathfinder}" - echo "test_python=${test_python}" - echo "baseline_run_id=${baseline_run_id}" - echo "baseline_sha=${baseline_sha}" - } >> "$GITHUB_OUTPUT" - - { - echo - echo "### Effective package plan" - echo - echo "| Package | Build | Test |" - echo "| --- | --- | --- |" - echo "| cuda-pathfinder | ${build_pathfinder} | ${test_pathfinder} |" - echo "| cuda-bindings | ${build_bindings} | ${test_bindings} |" - echo "| cuda-core | ${build_core} | ${test_core} |" - echo "| cuda-python | ${build_python} | ${test_python} |" - } >> "$GITHUB_STEP_SUMMARY" - api-check-core-vs-release: name: API check (cuda_core vs. latest release) if: >- @@ -834,7 +838,7 @@ jobs: needs.detect-changes.outputs.test_bindings == 'true' || needs.detect-changes.outputs.test_core == 'true' || needs.detect-changes.outputs.test_python == 'true' }}" - core_changed="${{ needs.detect-changes.outputs.core == 'true' }}" + run_core_api_check="${{ needs.detect-changes.outputs.core == 'true' }}" is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { @@ -880,13 +884,14 @@ jobs: check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" - # API compatibility checks only run for cuda_core source changes. + # API compatibility checks run for cuda_core source changes and for + # conservative full runs when reusable base artifacts are unavailable. expected="skipped" - if [[ "$core_changed" == "true" ]]; then expected="success"; fi + if [[ "$run_core_api_check" == "true" ]]; then expected="success"; fi check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" expected="skipped" - if [[ "$is_pr" == "true" && "$core_changed" == "true" ]]; then expected="success"; fi + if [[ "$is_pr" == "true" && "$run_core_api_check" == "true" ]]; then expected="success"; fi check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" [[ "$status" == "success" ]] diff --git a/ci/tools/compute_ci_plan.py b/ci/tools/compute_ci_plan.py deleted file mode 100644 index 7095514b73e..00000000000 --- a/ci/tools/compute_ci_plan.py +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""Compute the package build and test closure for a set of changed paths.""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -PACKAGES = { - "cuda_pathfinder": "pathfinder", - "cuda_bindings": "bindings", - "cuda_core": "core", - "cuda_python": "python", -} - -SHARED_PREFIXES = ( - ".github/", - "ci/", - "scripts/", - "toolshed/", -) - -SHARED_FILES = { - ".pre-commit-config.yaml", - "conftest.py", - "pixi.lock", - "pixi.toml", - "pytest.ini", - "ruff.toml", -} - -KNOWN_REPOSITORY_FILES = { - ".git-blame-ignore-revs", - ".gitignore", - "AGENTS.md", - "CHANGELOG.md", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "README.md", - "SECURITY.md", -} - - -def _bool(value: bool) -> str: - return str(value).lower() - - -def _read_paths(path: Path) -> list[str]: - return [value.decode("utf-8", errors="surrogateescape") for value in path.read_bytes().split(b"\0") if value] - - -def compute_plan(paths: list[str]) -> dict[str, bool]: - source = dict.fromkeys(PACKAGES.values(), False) - tests = dict.fromkeys(PACKAGES.values(), False) - docs = False - test_helpers = False - shared = False - unknown = False - - for path in paths: - package_dir, separator, relative_path = path.partition("/") - package = PACKAGES.get(package_dir) - if package is not None and separator: - if relative_path.startswith("docs/"): - docs = True - elif relative_path.startswith(("tests/", "examples/")): - tests[package] = True - else: - source[package] = True - continue - - if path.startswith("cuda_python_test_helpers/"): - test_helpers = True - elif path.startswith("benchmarks/cuda_bindings/"): - tests["bindings"] = True - elif path.startswith(SHARED_PREFIXES) or path in SHARED_FILES: - shared = True - elif path in KNOWN_REPOSITORY_FILES: - # Repository policy and prose files do not affect package artifacts. - continue - else: - unknown = True - - full = shared or unknown - - build_pathfinder = full or source["pathfinder"] - # Development cuda-python wheels exactly pin cuda-bindings, so a - # metapackage change needs a matching bindings artifact for its smoke test. - build_bindings = full or source["pathfinder"] or source["bindings"] or source["python"] - build_core = full or source["pathfinder"] or source["bindings"] or source["core"] - # A core-only change can reuse the baseline cuda-python wheel: rebuilding - # it would also require rebuilding the exact-version cuda-bindings pin. - build_python = full or source["pathfinder"] or source["bindings"] or source["python"] - - test_pathfinder = full or source["pathfinder"] or tests["pathfinder"] - test_bindings = full or source["pathfinder"] or source["bindings"] or tests["bindings"] or test_helpers - test_core = full or source["pathfinder"] or source["bindings"] or source["core"] or tests["core"] or test_helpers - test_python = ( - full or source["pathfinder"] or source["bindings"] or source["core"] or source["python"] or tests["python"] - ) - - return { - "shared": shared, - "unknown": unknown, - "docs": docs, - "test_helpers": test_helpers, - "pathfinder_source": source["pathfinder"], - "bindings_source": source["bindings"], - "core_source": source["core"], - "python_source": source["python"], - "pathfinder_tests": tests["pathfinder"], - "bindings_tests": tests["bindings"], - "core_tests": tests["core"], - "python_tests": tests["python"], - "build_pathfinder": build_pathfinder, - "build_bindings": build_bindings, - "build_core": build_core, - "build_python": build_python, - "test_pathfinder": test_pathfinder, - "test_bindings": test_bindings, - "test_core": test_core, - "test_python": test_python, - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - "paths_file", - type=Path, - help="NUL-separated changed-path list produced by git diff --name-only -z", - ) - args = parser.parse_args() - - plan = compute_plan(_read_paths(args.paths_file)) - for key, value in plan.items(): - print(f"{key}={_bool(value)}") - - -if __name__ == "__main__": - main() From 42fe412d683de6f3b2cf7becc72e06d0ddca2d57 Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Fri, 14 Aug 2026 00:25:40 -0400 Subject: [PATCH 3/3] ci: refine selective path classification --- .github/workflows/ci.yml | 113 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16ab65b3db2..9120b2e062f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,8 @@ jobs: # downstream module. A change to cuda_bindings forces rebuild of cuda_core. # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and # cuda_pathfinder, but still retests the downstream cuda-python metapackage. - # CI/filter changes are shared by design, so this implementation runs the - # full pipeline; exercise selective cases in follow-up package-only PRs. + # Shared build/orchestration changes run the full pipeline; test-only CI + # infrastructure runs every test suite without rebuilding package wheels. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed # paths" baseline for those events. @@ -102,6 +102,7 @@ jobs: core: >- ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || steps.filter.outputs.core_source == 'true' }} build_pathfinder: >- ${{ steps.baseline.outputs.available != 'true' || @@ -133,12 +134,14 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.pathfinder_tests == 'true' }} test_bindings: >- ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.bindings_tests == 'true' || @@ -147,6 +150,7 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.core_source == 'true' || @@ -157,6 +161,7 @@ jobs: ${{ steps.baseline.outputs.available != 'true' || steps.filter.outputs.changes == '' || steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || steps.filter.outputs.pathfinder_source == 'true' || steps.filter.outputs.bindings_source == 'true' || steps.filter.outputs.core_source == 'true' || @@ -228,39 +233,79 @@ jobs: - '!cuda_pathfinder/docs/**' - '!cuda_pathfinder/tests/**' - '!cuda_pathfinder/examples/**' + - '!cuda_pathfinder/pixi.lock' + - '!cuda_pathfinder/pixi.toml' + - '!cuda_pathfinder/AGENTS.md' + - '!cuda_pathfinder/CLAUDE.md' bindings_source: - 'cuda_bindings/**' - '!cuda_bindings/docs/**' - '!cuda_bindings/tests/**' - '!cuda_bindings/examples/**' + - '!cuda_bindings/pixi.lock' + - '!cuda_bindings/pixi.toml' + - '!cuda_bindings/AGENTS.md' + - '!cuda_bindings/CLAUDE.md' core_source: - 'cuda_core/**' - '!cuda_core/docs/**' - '!cuda_core/tests/**' - '!cuda_core/examples/**' + - '!cuda_core/pixi.lock' + - '!cuda_core/pixi.toml' + - '!cuda_core/pytest.ini' + - '!cuda_core/AGENTS.md' + - '!cuda_core/CLAUDE.md' python_source: - 'cuda_python/**' - '!cuda_python/docs/**' - '!cuda_python/tests/**' - '!cuda_python/examples/**' + - '!cuda_python/AGENTS.md' + - '!cuda_python/CLAUDE.md' + # cuda_python/README.md is a symlink to this packaging input. + - 'README.md' pathfinder_tests: - 'cuda_pathfinder/tests/**' - 'cuda_pathfinder/examples/**' + - '!cuda_pathfinder/**/AGENTS.md' + - '!cuda_pathfinder/**/CLAUDE.md' bindings_tests: - 'cuda_bindings/tests/**' - 'cuda_bindings/examples/**' - 'benchmarks/cuda_bindings/**' + - '!benchmarks/cuda_bindings/pixi.lock' + - '!benchmarks/cuda_bindings/pixi.toml' + - '!cuda_bindings/**/AGENTS.md' + - '!cuda_bindings/**/CLAUDE.md' core_tests: - 'cuda_core/tests/**' - 'cuda_core/examples/**' + - 'cuda_core/pytest.ini' + - '!cuda_core/**/AGENTS.md' + - '!cuda_core/**/CLAUDE.md' python_tests: - 'cuda_python/tests/**' - 'cuda_python/examples/**' + - '!cuda_python/**/AGENTS.md' + - '!cuda_python/**/CLAUDE.md' test_helpers: - - 'cuda_python_test_helpers/**' + - 'cuda_python_test_helpers/cuda_python_test_helpers/**' + # These files configure or implement wheel tests, but do not + # change any package artifact. + all_tests: + - '.github/workflows/test-wheel-linux.yml' + - '.github/workflows/test-wheel-windows.yml' + - 'ci/test-matrix.yml' + - 'ci/tools/configure_driver_mode.ps1' + - 'ci/tools/guess_latest.sh' + - 'ci/tools/install_gpu_driver.ps1' + - 'ci/tools/install_gpu_driver.sh' + - 'ci/tools/run-tests' + - 'ci/tools/setup-sanitizer' # Shared infrastructure and unknown paths run the full pipeline. - # Exclude only paths whose narrower behavior is defined above or - # repository policy/prose files known not to affect artifacts. + # Exclude paths classified above and paths consumed only by an + # independent or unconditional CI job. force_all: - '**' - '!cuda_pathfinder/**' @@ -269,15 +314,67 @@ jobs: - '!cuda_python/**' - '!cuda_python_test_helpers/**' - '!benchmarks/cuda_bindings/**' - - '!.git-blame-ignore-revs' + - '!benchmarks/cuda_core/**' + - '!.agents/**' + - '!.coveragerc' - '!.gitignore' + - '!.pre-commit-config.yaml' + - '!.spdx-ignore' - '!AGENTS.md' - - '!CHANGELOG.md' - - '!CODE_OF_CONDUCT.md' + - '!CLAUDE.md' - '!CONTRIBUTING.md' - '!LICENSE' - '!README.md' - '!SECURITY.md' + - '!context7.json' + - '!greptile.json' + - '!pixi.lock' + - '!pixi.toml' + - '!pytest.ini' + - '!ruff.toml' + - '!toolshed/**' + - '!.github/ISSUE_TEMPLATE/**' + - '!.github/PULL_REQUEST_TEMPLATE.md' + - '!.github/RELEASE-core.md' + - '!.github/actionlint.yaml' + - '!.github/actions/doc_preview/**' + - '!.github/actions/get_pr_number/**' + - '!.github/copy-pr-bot.yaml' + - '!.github/dependabot.yml' + - '!.github/labeler.yml' + - '!.github/workflows/backport.yml' + - '!.github/workflows/bandit.yml' + - '!.github/workflows/build-docs.yml' + - '!.github/workflows/ci-nightly.yml' + - '!.github/workflows/ci-pixi-source-test.yml' + - '!.github/workflows/cleanup-pr-previews.yml' + - '!.github/workflows/coverage.yml' + - '!.github/workflows/pr-auto-label.yml' + - '!.github/workflows/pr-metadata-check.yml' + - '!.github/workflows/release-cuda-pathfinder.yml' + - '!.github/workflows/release-upload.yml' + - '!.github/workflows/release.yml' + - '!.github/workflows/security-suite.yml' + - '!.github/workflows/test-wheel-linux.yml' + - '!.github/workflows/test-wheel-windows.yml' + - '!.github/workflows/triagelabel.yml' + - '!ci/.ci-pipeline-regen.md' + - '!ci/ci-pipeline.svg' + - '!ci/cleanup-pr-previews' + - '!ci/test-matrix.yml' + - '!ci/tools/check_mempool_hygiene.py' + - '!ci/tools/check_pixi_cuda_version.py' + - '!ci/tools/check_release_notes.py' + - '!ci/tools/configure_driver_mode.ps1' + - '!ci/tools/download-wheels' + - '!ci/tools/guess_latest.sh' + - '!ci/tools/install_gpu_driver.ps1' + - '!ci/tools/install_gpu_driver.sh' + - '!ci/tools/run-tests' + - '!ci/tools/run_pytest_with_stack.py' + - '!ci/tools/setup-sanitizer' + - '!ci/tools/tests/**' + - '!ci/tools/validate-release-wheels' - name: Resolve reusable base artifacts id: baseline