diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aadf222306..9120b2e062f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,58 +72,129 @@ jobs: echo "skip=${skip}" >> "$GITHUB_OUTPUT" echo "doc_only=${doc_only}" >> "$GITHUB_OUTPUT" - # Detect which top-level modules were touched by the PR so downstream build - # and test jobs can avoid rebuilding/retesting modules unaffected by the - # change. See issue #299. + # Detect which packages were touched by the PR so downstream build and test + # jobs can avoid rebuilding/retesting packages unaffected by the change. + # See issue #299. # # Dependency graph (verified in pyproject.toml files): # cuda_pathfinder -> (no internal deps) # cuda_bindings -> cuda_pathfinder # cuda_core -> cuda_pathfinder, cuda_bindings - # cuda_python -> cuda_bindings (meta package) + # cuda_python -> cuda_pathfinder, cuda_bindings, cuda_core (meta package) # # A change to cuda_pathfinder (or shared infra) forces a rebuild of every # downstream module. A change to cuda_bindings forces rebuild of cuda_core. - # A change to cuda_core alone skips rebuilding/retesting cuda_bindings. + # A change to cuda_core alone skips rebuilding/retesting cuda_bindings and + # cuda_pathfinder, but still retests the downstream cuda-python metapackage. + # Shared build/orchestration changes run the full pipeline; test-only CI + # infrastructure runs every test suite without rebuilding package wheels. # On push to main, tag refs, schedule, or workflow_dispatch events we # unconditionally run everything because there is no meaningful "changed # paths" baseline for those events. detect-changes: runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pull-requests: read outputs: - bindings: ${{ steps.compose.outputs.bindings }} - core: ${{ steps.compose.outputs.core }} - pathfinder: ${{ steps.compose.outputs.pathfinder }} - python_meta: ${{ steps.compose.outputs.python_meta }} - test_helpers: ${{ steps.compose.outputs.test_helpers }} - shared: ${{ steps.compose.outputs.shared }} - build_bindings: ${{ steps.compose.outputs.build_bindings }} - build_core: ${{ steps.compose.outputs.build_core }} - build_pathfinder: ${{ steps.compose.outputs.build_pathfinder }} - test_bindings: ${{ steps.compose.outputs.test_bindings }} - test_core: ${{ steps.compose.outputs.test_core }} - test_pathfinder: ${{ steps.compose.outputs.test_pathfinder }} - pr_merge_base: ${{ steps.filter.outputs.merge_base }} + # Missing base artifacts or a skipped path filter fail open to the full pipeline. + core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.core_source == 'true' }} + build_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' }} + build_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + build_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' }} + build_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.python_source == 'true' }} + test_pathfinder: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.pathfinder_tests == 'true' }} + test_bindings: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.bindings_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + test_core: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.core_tests == 'true' || + steps.filter.outputs.test_helpers == 'true' }} + pr_merge_base: ${{ steps.merge-base.outputs.sha }} + test_python: >- + ${{ steps.baseline.outputs.available != 'true' || + steps.filter.outputs.changes == '' || + steps.filter.outputs.force_all == 'true' || + steps.filter.outputs.all_tests == 'true' || + steps.filter.outputs.pathfinder_source == 'true' || + steps.filter.outputs.bindings_source == 'true' || + steps.filter.outputs.core_source == 'true' || + steps.filter.outputs.python_source == 'true' || + steps.filter.outputs.python_tests == 'true' }} + baseline_run_id: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.run_id || '' }} + baseline_sha: >- + ${{ steps.filter.outputs.changes != '' && + steps.baseline.outputs.available == 'true' && + steps.baseline.outputs.sha || '' }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # Treeless clone: commit graph is needed for `git merge-base` and - # `git diff --name-only` below, but historical blobs aren't. + # Treeless clone: the commit graph is needed to resolve the PR merge + # base and classify its changed paths, but historical blobs aren't. fetch-depth: 0 filter: blob:none # copy-pr-bot pushes every PR (whether it targets main or a backport # branch such as 12.9.x) to pull-request/, so the base branch # cannot be inferred from github.ref_name. Look it up via the - # upstream PR metadata so the diff below is rooted at the right place. + # upstream PR metadata so change detection is rooted at the right place. - name: Resolve PR base branch id: pr-info if: ${{ startsWith(github.ref_name, 'pull-request/') }} uses: nv-gha-runners/get-pr-info@main - - name: Detect changed paths - id: filter + - name: Resolve PR merge base + id: merge-base if: ${{ startsWith(github.ref_name, 'pull-request/') }} env: # GitHub Actions evaluates step-level `env:` expressions eagerly — @@ -136,99 +207,267 @@ jobs: # off by `if:`, so `BASE_REF` is never consumed there. BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} run: | - # Diff against the merge base with the PR's actual target branch. - # Uses merge-base so diverged branches only show files changed on - # the PR side, not upstream commits. + set -euo pipefail if [[ -z "${BASE_REF}" ]]; then echo "Could not resolve PR base branch from get-pr-info output" >&2 exit 1 fi + base=$(git merge-base HEAD "origin/${BASE_REF}") - changed=$(git diff --name-only "$base"...HEAD) + echo "sha=${base}" >> "$GITHUB_OUTPUT" - has_match() { - grep -qE "$1" <<< "$changed" && echo true || echo false + - name: Classify changed paths + id: filter + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + with: + # The workflow runs on copy-pr-bot push branches, so compare the + # checked-out PR head against the base resolved from PR metadata. + base: ${{ steps.merge-base.outputs.sha }} + ref: ${{ github.sha }} + token: '' + predicate-quantifier: some-with-excludes + filters: | + pathfinder_source: + - 'cuda_pathfinder/**' + - '!cuda_pathfinder/docs/**' + - '!cuda_pathfinder/tests/**' + - '!cuda_pathfinder/examples/**' + - '!cuda_pathfinder/pixi.lock' + - '!cuda_pathfinder/pixi.toml' + - '!cuda_pathfinder/AGENTS.md' + - '!cuda_pathfinder/CLAUDE.md' + bindings_source: + - 'cuda_bindings/**' + - '!cuda_bindings/docs/**' + - '!cuda_bindings/tests/**' + - '!cuda_bindings/examples/**' + - '!cuda_bindings/pixi.lock' + - '!cuda_bindings/pixi.toml' + - '!cuda_bindings/AGENTS.md' + - '!cuda_bindings/CLAUDE.md' + core_source: + - 'cuda_core/**' + - '!cuda_core/docs/**' + - '!cuda_core/tests/**' + - '!cuda_core/examples/**' + - '!cuda_core/pixi.lock' + - '!cuda_core/pixi.toml' + - '!cuda_core/pytest.ini' + - '!cuda_core/AGENTS.md' + - '!cuda_core/CLAUDE.md' + python_source: + - 'cuda_python/**' + - '!cuda_python/docs/**' + - '!cuda_python/tests/**' + - '!cuda_python/examples/**' + - '!cuda_python/AGENTS.md' + - '!cuda_python/CLAUDE.md' + # cuda_python/README.md is a symlink to this packaging input. + - 'README.md' + pathfinder_tests: + - 'cuda_pathfinder/tests/**' + - 'cuda_pathfinder/examples/**' + - '!cuda_pathfinder/**/AGENTS.md' + - '!cuda_pathfinder/**/CLAUDE.md' + bindings_tests: + - 'cuda_bindings/tests/**' + - 'cuda_bindings/examples/**' + - 'benchmarks/cuda_bindings/**' + - '!benchmarks/cuda_bindings/pixi.lock' + - '!benchmarks/cuda_bindings/pixi.toml' + - '!cuda_bindings/**/AGENTS.md' + - '!cuda_bindings/**/CLAUDE.md' + core_tests: + - 'cuda_core/tests/**' + - 'cuda_core/examples/**' + - 'cuda_core/pytest.ini' + - '!cuda_core/**/AGENTS.md' + - '!cuda_core/**/CLAUDE.md' + python_tests: + - 'cuda_python/tests/**' + - 'cuda_python/examples/**' + - '!cuda_python/**/AGENTS.md' + - '!cuda_python/**/CLAUDE.md' + test_helpers: + - 'cuda_python_test_helpers/cuda_python_test_helpers/**' + # These files configure or implement wheel tests, but do not + # change any package artifact. + all_tests: + - '.github/workflows/test-wheel-linux.yml' + - '.github/workflows/test-wheel-windows.yml' + - 'ci/test-matrix.yml' + - 'ci/tools/configure_driver_mode.ps1' + - 'ci/tools/guess_latest.sh' + - 'ci/tools/install_gpu_driver.ps1' + - 'ci/tools/install_gpu_driver.sh' + - 'ci/tools/run-tests' + - 'ci/tools/setup-sanitizer' + # Shared infrastructure and unknown paths run the full pipeline. + # Exclude paths classified above and paths consumed only by an + # independent or unconditional CI job. + force_all: + - '**' + - '!cuda_pathfinder/**' + - '!cuda_bindings/**' + - '!cuda_core/**' + - '!cuda_python/**' + - '!cuda_python_test_helpers/**' + - '!benchmarks/cuda_bindings/**' + - '!benchmarks/cuda_core/**' + - '!.agents/**' + - '!.coveragerc' + - '!.gitignore' + - '!.pre-commit-config.yaml' + - '!.spdx-ignore' + - '!AGENTS.md' + - '!CLAUDE.md' + - '!CONTRIBUTING.md' + - '!LICENSE' + - '!README.md' + - '!SECURITY.md' + - '!context7.json' + - '!greptile.json' + - '!pixi.lock' + - '!pixi.toml' + - '!pytest.ini' + - '!ruff.toml' + - '!toolshed/**' + - '!.github/ISSUE_TEMPLATE/**' + - '!.github/PULL_REQUEST_TEMPLATE.md' + - '!.github/RELEASE-core.md' + - '!.github/actionlint.yaml' + - '!.github/actions/doc_preview/**' + - '!.github/actions/get_pr_number/**' + - '!.github/copy-pr-bot.yaml' + - '!.github/dependabot.yml' + - '!.github/labeler.yml' + - '!.github/workflows/backport.yml' + - '!.github/workflows/bandit.yml' + - '!.github/workflows/build-docs.yml' + - '!.github/workflows/ci-nightly.yml' + - '!.github/workflows/ci-pixi-source-test.yml' + - '!.github/workflows/cleanup-pr-previews.yml' + - '!.github/workflows/coverage.yml' + - '!.github/workflows/pr-auto-label.yml' + - '!.github/workflows/pr-metadata-check.yml' + - '!.github/workflows/release-cuda-pathfinder.yml' + - '!.github/workflows/release-upload.yml' + - '!.github/workflows/release.yml' + - '!.github/workflows/security-suite.yml' + - '!.github/workflows/test-wheel-linux.yml' + - '!.github/workflows/test-wheel-windows.yml' + - '!.github/workflows/triagelabel.yml' + - '!ci/.ci-pipeline-regen.md' + - '!ci/ci-pipeline.svg' + - '!ci/cleanup-pr-previews' + - '!ci/test-matrix.yml' + - '!ci/tools/check_mempool_hygiene.py' + - '!ci/tools/check_pixi_cuda_version.py' + - '!ci/tools/check_release_notes.py' + - '!ci/tools/configure_driver_mode.ps1' + - '!ci/tools/download-wheels' + - '!ci/tools/guess_latest.sh' + - '!ci/tools/install_gpu_driver.ps1' + - '!ci/tools/install_gpu_driver.sh' + - '!ci/tools/run-tests' + - '!ci/tools/run_pytest_with_stack.py' + - '!ci/tools/setup-sanitizer' + - '!ci/tools/tests/**' + - '!ci/tools/validate-release-wheels' + + - name: Resolve reusable base artifacts + id: baseline + if: ${{ startsWith(github.ref_name, 'pull-request/') }} + env: + BASE_REF: ${{ steps.pr-info.outputs.pr-info && fromJSON(steps.pr-info.outputs.pr-info).base.ref || '' }} + MERGE_BASE: ${{ steps.merge-base.outputs.sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + + unavailable() { + echo "available=false" >> "$GITHUB_OUTPUT" + echo "No complete reusable artifact set was found; this run will build and test everything." >> "$GITHUB_STEP_SUMMARY" + exit 0 } - { - echo "bindings=$(has_match '^cuda_bindings/')" - echo "core=$(has_match '^cuda_core/')" - echo "pathfinder=$(has_match '^cuda_pathfinder/')" - echo "python_meta=$(has_match '^cuda_python/')" - echo "test_helpers=$(has_match '^cuda_python_test_helpers/')" - echo "shared=$(has_match '^(\.github/|ci/|scripts/|toolshed/|conftest\.py$|pyproject\.toml$|pixi\.(toml|lock)$|pytest\.ini$|ruff\.toml$)')" - echo "merge_base=${base}" - } >> "$GITHUB_OUTPUT" + if [[ -z "${BASE_REF}" ]]; then + unavailable + fi - - name: Compose gating outputs - id: compose - env: - IS_PR: ${{ startsWith(github.ref_name, 'pull-request/') }} - BINDINGS: ${{ steps.filter.outputs.bindings || 'false' }} - CORE: ${{ steps.filter.outputs.core || 'false' }} - PATHFINDER: ${{ steps.filter.outputs.pathfinder || 'false' }} - PYTHON_META: ${{ steps.filter.outputs.python_meta || 'false' }} - TEST_HELPERS: ${{ steps.filter.outputs.test_helpers || 'false' }} - SHARED: ${{ steps.filter.outputs.shared || 'false' }} - run: | - set -euxo pipefail - # Non-PR events (push to main, tag push, schedule, workflow_dispatch) - # always exercise the full pipeline because there is no baseline for - # a meaningful diff. - if [[ "${IS_PR}" != "true" ]]; then - bindings=true - core=true - pathfinder=true - python_meta=true - test_helpers=true - shared=true - else - bindings="${BINDINGS}" - core="${CORE}" - pathfinder="${PATHFINDER}" - python_meta="${PYTHON_META}" - test_helpers="${TEST_HELPERS}" - shared="${SHARED}" + merge_base="${MERGE_BASE}" + if [[ -z "${merge_base}" ]]; then + unavailable + fi + if ! runs=$(gh run list \ + --repo "${{ github.repository }}" \ + --branch "${BASE_REF}" \ + --commit "${merge_base}" \ + --event push \ + --workflow ci.yml \ + --status success \ + --limit 1 \ + --json databaseId,headSha,createdAt); then + unavailable fi - or_flag() { - for v in "$@"; do - if [[ "${v}" == "true" ]]; then - echo "true" - return - fi - done - echo "false" + # Reuse only artifacts produced from the exact commit used as the + # PR diff base. Using the latest base-branch run is unsafe for a PR + # that was opened before newer changes landed on that branch. + run_id=$(jq -r '.[0].databaseId // empty' <<< "$runs") + baseline_sha=$(jq -r '.[0].headSha // empty' <<< "$runs") + if [[ -z "${run_id}" || "${baseline_sha}" != "${merge_base}" ]]; then + unavailable + fi + + if ! artifact_names=$(gh api \ + "repos/${{ github.repository }}/actions/runs/${run_id}/artifacts?per_page=100" \ + --paginate \ + --jq '.artifacts[] | select(.expired == false) | .name'); then + unavailable + fi + + has_artifact() { + grep -Fxq "$1" <<< "$artifact_names" } - # Build gating: pathfinder change forces rebuild of bindings and - # core; bindings change forces rebuild of core. shared changes force - # a full rebuild. - build_pathfinder="$(or_flag "${shared}" "${pathfinder}")" - build_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}")" - build_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}")" + missing=() + for name in cuda-pathfinder-wheel cuda-python-wheel; do + has_artifact "$name" || missing+=("$name") + done + + cuda_version=$(yq '.cuda.build.version' ci/versions.yml) + if ! python_versions=$(yq -r '.jobs.build.strategy.matrix."python-version"[]' .github/workflows/build-wheel.yml); then + unavailable + fi + if [[ -z "${python_versions}" ]]; then + unavailable + fi + while IFS= read -r python_version; do + python=${python_version//./} + for platform in linux-64 linux-aarch64 win-64; do + binding="cuda-bindings-python${python}-cuda${cuda_version}-${platform}-${baseline_sha}" + core="cuda-core-python${python}-${platform}-${baseline_sha}" + has_artifact "$binding" || missing+=("$binding") + has_artifact "$core" || missing+=("$core") + done + done <<< "${python_versions}" - # Test gating: tests for a module must run whenever that module, any - # of its runtime dependencies, the shared test helper package, or - # shared infra changes. pathfinder tests are cheap and always run. - test_pathfinder=true - test_bindings="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${test_helpers}")" - test_core="$(or_flag "${shared}" "${pathfinder}" "${bindings}" "${core}" "${test_helpers}")" + if (( ${#missing[@]} != 0 )); then + printf 'Missing reusable artifact: %s\n' "${missing[@]}" >&2 + unavailable + fi { - echo "bindings=${bindings}" - echo "core=${core}" - echo "pathfinder=${pathfinder}" - echo "python_meta=${python_meta}" - echo "test_helpers=${test_helpers}" - echo "shared=${shared}" - echo "build_bindings=${build_bindings}" - echo "build_core=${build_core}" - echo "build_pathfinder=${build_pathfinder}" - echo "test_bindings=${test_bindings}" - echo "test_core=${test_core}" - echo "test_pathfinder=${test_pathfinder}" + echo "available=true" + echo "run_id=${run_id}" + echo "sha=${baseline_sha}" } >> "$GITHUB_OUTPUT" + { + echo + echo "Reusable artifacts: run \`${run_id}\` at \`${baseline_sha}\` on \`${BASE_REF}\`." + } >> "$GITHUB_STEP_SUMMARY" api-check-core-vs-release: name: API check (cuda_core vs. latest release) @@ -314,15 +553,17 @@ jobs: merge-base: ${{ needs.detect-changes.outputs.pr_merge_base }} # NOTE: Build jobs are intentionally split by platform rather than using a single - # matrix. This allows each test job to depend only on its corresponding build, - # so faster platforms can proceed through build & test without waiting for slower - # ones. Keep these job definitions textually identical except for: + # matrix. This lets each test job consume its platform-specific artifacts as + # soon as they are ready. ARM64 and Windows tests also wait for linux-64, + # which produces the universal pathfinder and cuda-python wheels. Keep these + # job definitions textually identical except for: # - host-platform value # - if: condition (build-linux-64 omits doc-only check since it's needed for docs) build-linux-64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: @@ -330,50 +571,105 @@ jobs: - linux-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-linux-aarch64: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - linux-aarch64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # See build-linux-64 for why build jobs are split by platform. build-windows: needs: - ci-vars - should-skip + - detect-changes strategy: fail-fast: false matrix: host-platform: - win-64 name: Build ${{ matrix.host-platform }}, CUDA ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python) || + fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/build-wheel.yml with: host-platform: ${{ matrix.host-platform }} cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} prev-cuda-version: ${{ needs.ci-vars.outputs.CUDA_PREV_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} + test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + baseline-run-id: ${{ needs.detect-changes.outputs.baseline_run_id }} + baseline-sha: ${{ needs.detect-changes.outputs.baseline_sha }} # NOTE: test-sdist jobs are split by platform (mirroring build-* and test-wheel-*) # so platform-specific sources (e.g. cuda_bindings/*_windows.pyx selected by @@ -385,26 +681,57 @@ jobs: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 name: Test sdist linux-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-linux.yml with: host-platform: linux-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # See test-sdist-linux for why sdist test jobs are split by platform. test-sdist-windows: needs: - ci-vars - should-skip + - detect-changes + - build-linux-64 + - build-windows name: Test sdist win-64 - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.skip) && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.skip) && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.build_pathfinder) || + fromJSON(needs.detect-changes.outputs.build_bindings) || + fromJSON(needs.detect-changes.outputs.build_core) || + fromJSON(needs.detect-changes.outputs.build_python)) }} + permissions: + actions: read + contents: read secrets: inherit uses: ./.github/workflows/test-sdist-windows.yml with: host-platform: win-64 cuda-version: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} + build-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.build_pathfinder) }} + build-bindings: ${{ fromJSON(needs.detect-changes.outputs.build_bindings) }} + build-core: ${{ fromJSON(needs.detect-changes.outputs.build_core) }} + build-python: ${{ fromJSON(needs.detect-changes.outputs.build_python) }} # NOTE: Test jobs are split by platform for the same reason as build jobs (see # build-linux-64). Keep these job definitions textually identical except for: @@ -418,8 +745,14 @@ jobs: host-platform: - linux-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars @@ -433,7 +766,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-linux-aarch64: @@ -443,13 +779,20 @@ jobs: host-platform: - linux-aarch64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-linux-aarch64 secrets: inherit uses: ./.github/workflows/test-wheel-linux.yml @@ -458,7 +801,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} # See test-linux-64 for why test jobs are split by platform. test-windows: @@ -468,13 +814,20 @@ jobs: host-platform: - win-64 name: Test ${{ matrix.host-platform }} - if: ${{ github.repository_owner == 'nvidia' && !fromJSON(needs.should-skip.outputs.doc-only) }} + if: ${{ github.repository_owner == 'nvidia' && + !fromJSON(needs.should-skip.outputs.doc-only) && + (fromJSON(needs.detect-changes.outputs.test_pathfinder) || + fromJSON(needs.detect-changes.outputs.test_bindings) || + fromJSON(needs.detect-changes.outputs.test_core) || + fromJSON(needs.detect-changes.outputs.test_python)) }} permissions: + actions: read contents: read # This is required for actions/checkout needs: - ci-vars - should-skip - detect-changes + - build-linux-64 - build-windows secrets: inherit uses: ./.github/workflows/test-wheel-windows.yml @@ -483,7 +836,10 @@ jobs: host-platform: ${{ matrix.host-platform }} build-ctk-ver: ${{ needs.ci-vars.outputs.CUDA_BUILD_VER }} nruns: ${{ (github.event_name == 'schedule' && 5) || 1}} + test-pathfinder: ${{ fromJSON(needs.detect-changes.outputs.test_pathfinder) }} test-bindings: ${{ fromJSON(needs.detect-changes.outputs.test_bindings) }} + test-core: ${{ fromJSON(needs.detect-changes.outputs.test_core) }} + test-python: ${{ fromJSON(needs.detect-changes.outputs.test_python) }} doc: name: Docs @@ -538,13 +894,19 @@ jobs: if: always() runs-on: ubuntu-latest needs: + - ci-vars - should-skip - detect-changes + - build-linux-64 + - build-linux-aarch64 + - build-windows - test-sdist-linux - test-sdist-windows - test-linux-64 - test-linux-aarch64 - test-windows + - api-check-core-vs-release + - api-check-core-vs-base - doc - precommit-windows steps: @@ -565,6 +927,16 @@ jobs: fi doc_only="${{ needs.should-skip.outputs.doc-only }}" + build_selected="${{ needs.detect-changes.outputs.build_pathfinder == 'true' || + needs.detect-changes.outputs.build_bindings == 'true' || + needs.detect-changes.outputs.build_core == 'true' || + needs.detect-changes.outputs.build_python == 'true' }}" + test_selected="${{ needs.detect-changes.outputs.test_pathfinder == 'true' || + needs.detect-changes.outputs.test_bindings == 'true' || + needs.detect-changes.outputs.test_core == 'true' || + needs.detect-changes.outputs.test_python == 'true' }}" + run_core_api_check="${{ needs.detect-changes.outputs.core == 'true' }}" + is_pr="${{ startsWith(github.ref_name, 'pull-request/') }}" status="success" check_result() { name=$1; expected=$2; result=$3 @@ -575,18 +947,48 @@ jobs: fi } - # always expected to succeed (even in [doc-only] mode) - check_result "should-skip" "success" "${{ needs.should-skip.result }}" - check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" - check_result "doc" "success" "${{ needs.doc.result }}" + # Control jobs, the universal linux build, docs, and Windows + # pre-commit checks always run. + check_result "ci-vars" "success" "${{ needs.ci-vars.result }}" + check_result "should-skip" "success" "${{ needs.should-skip.result }}" + check_result "detect-changes" "success" "${{ needs.detect-changes.result }}" + check_result "build-linux-64" "success" "${{ needs.build-linux-64.result }}" + check_result "doc" "success" "${{ needs.doc.result }}" check_result "precommit-windows" "success" "${{ needs.precommit-windows.result }}" - # [doc-only] flips these from 'success' to 'skipped' - if [[ "$doc_only" == "true" ]]; then expected="skipped"; else expected="success"; fi + # Platform builds run whenever any package needs to be built or tested. + expected="skipped" + if [[ "$doc_only" != "true" && + ( "$build_selected" == "true" || "$test_selected" == "true" ) ]]; then + expected="success" + fi + check_result "build-linux-aarch64" "$expected" "${{ needs.build-linux-aarch64.result }}" + check_result "build-windows" "$expected" "${{ needs.build-windows.result }}" + + # Sdist and wheel tests are independently gated by the effective plan. + expected="skipped" + if [[ "$doc_only" != "true" && "$build_selected" == "true" ]]; then + expected="success" + fi check_result "test-sdist-linux" "$expected" "${{ needs.test-sdist-linux.result }}" check_result "test-sdist-windows" "$expected" "${{ needs.test-sdist-windows.result }}" + + expected="skipped" + if [[ "$doc_only" != "true" && "$test_selected" == "true" ]]; then + expected="success" + fi check_result "test-linux-64" "$expected" "${{ needs.test-linux-64.result }}" check_result "test-linux-aarch64" "$expected" "${{ needs.test-linux-aarch64.result }}" check_result "test-windows" "$expected" "${{ needs.test-windows.result }}" + # API compatibility checks run for cuda_core source changes and for + # conservative full runs when reusable base artifacts are unavailable. + expected="skipped" + if [[ "$run_core_api_check" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-release" "$expected" "${{ needs.api-check-core-vs-release.result }}" + + expected="skipped" + if [[ "$is_pr" == "true" && "$run_core_api_check" == "true" ]]; then expected="success"; fi + check_result "api-check-core-vs-base" "$expected" "${{ needs.api-check-core-vs-base.result }}" + [[ "$status" == "success" ]]