diff --git a/.github/workflows/fork-ci-mirror-fast.yml b/.github/workflows/fork-ci-mirror-fast.yml new file mode 100644 index 00000000000..c4d8c0a9e02 --- /dev/null +++ b/.github/workflows/fork-ci-mirror-fast.yml @@ -0,0 +1,84 @@ +name: Fork CI Mirror - CPU-only lint/style checks + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + issue_comment: + types: [created] + +jobs: + check-trigger: + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.trigger-check.outputs.should_run }} + steps: + - name: Check if PR and /run-ci-mirror comment + id: trigger-check + if: github.event_name == 'issue_comment' && github.event.issue.pull_request + run: | + if [[ "${{ github.event.comment.body }}" == *"/run-ci-mirror"* ]]; then + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "should_run=false" >> $GITHUB_OUTPUT + fi + + - name: Auto-run on PR event + id: auto-trigger + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + run: | + echo "should_run=true" >> $GITHUB_OUTPUT + + linting: + runs-on: ubuntu-latest + needs: check-trigger + if: | + needs.check-trigger.outputs.should_run == 'true' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v1 + with: + version: 0.7.2 + + - name: Install linting tools + run: | + # Install the lint tools directly rather than `uv sync`, which resolves + # the full lockfile and tries to BUILD git-sourced CUDA-only deps + # (flash_mla) that cannot compile on a CPU GitHub-hosted runner. + # Versions pinned to match the pyproject.toml `linting` group + mypy + # (invoked by autoformat.sh). + uv venv + uv pip install "ruff~=0.9.0" "black==24.4.2" "isort==5.13.2" \ + "flake8==7.1.0" "pylint==3.2.6" "mypy" + + - name: Get base ref + id: get-base + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_REF="${{ github.base_ref }}" + else + BASE_REF="main" + fi + echo "base_ref=${BASE_REF}" >> $GITHUB_OUTPUT + + - name: Run linting + run: | + export PATH=".venv/bin:$PATH" + export GITLAB_ENDPOINT=github.com + export CI_PROJECT_NAMESPACE=NVIDIA + export BASE_REF="${{ steps.get-base.outputs.base_ref }}" + export CHECK_ONLY=true + export SKIP_DOCS=false + # autoformat.sh diffs changed files against `autoformatter-remote/$BASE_REF`, + # adding that remote with `|| true`. Pre-register it pointing at THIS fork so + # the mirror lints only what this branch changed relative to the fork's own + # base — not the (possibly drifted) upstream NVIDIA main. + git remote add autoformatter-remote \ + "https://github.com/${{ github.repository }}.git" || true + git fetch autoformatter-remote "${BASE_REF}" + bash tools/autoformat.sh diff --git a/CI_MIRROR_GAPS.md b/CI_MIRROR_GAPS.md new file mode 100644 index 00000000000..abbe44269d6 --- /dev/null +++ b/CI_MIRROR_GAPS.md @@ -0,0 +1,58 @@ +# Fork CI Mirror — CPU-Only Gap Documentation + +This document outlines the scope of CPU-feasible CI checks mirrored in this fork's `fork-ci-mirror-fast.yml` workflow, and the permanent gaps due to lack of GPU/self-hosted runners. + +## Mirrored Checks + +### Linting & Code Style (`fork-ci-mirror-fast.yml`) +- **Tool**: `uv sync --locked --only-group linting` + `bash tools/autoformat.sh` (CHECK_ONLY mode) +- **Runtime**: `ubuntu-latest` (GitHub-hosted Linux runner, ~2–3 minutes) +- **Purpose**: Detects code formatting, import sorting (isort), linting (pylint/flake8), and style violations +- **Trigger**: + - Auto-runs on all `pull_request` events (opened, synchronize, reopened) + - Auto-runs on `workflow_dispatch` (manual trigger) + - Can also be triggered via PR comment with `/run-ci-mirror` + +## Permanent Gaps — GPU/Distributed Training Infrastructure + +The following critical CI jobs **cannot** be mirrored on GitHub-hosted runners and remain an acknowledged permanent limitation: + +### 1. Container Build (`cicd-container-build`) +- **Requires**: NVIDIA self-hosted GPU runner (`nvidia-ci-aws-gpu-x8`) +- **Purpose**: Builds Docker container with CUDA/NVIDIA dependencies +- **Gap Reason**: Requires Docker registry push permissions and AWS ECR access; GPU not needed but NVIDIA self-hosted runner required for auth + +### 2. Unit Tests (`cicd-unit-tests-latest`) +- **Requires**: NVIDIA self-hosted GPU runner (`nvidia-ci-aws-gpu-x8`) +- **Purpose**: Validates CUDA kernels, distributed training ops, model correctness +- **Gap Reason**: Tests depend on NVIDIA GPU hardware; no substitute on CPU + +### 3. Integration Tests (`cicd-integration-tests-latest`) +- **Requires**: NVIDIA self-hosted GPU runner (`nvidia-ci-aws-gpu-x8`) +- **Purpose**: Validates distributed training across multiple GPUs/nodes, DDP, FSDP workflows +- **Gap Reason**: Tests depend on Slurm cluster with H100 GPUs; no CPU equivalent + +### 4. Installation Tests (`install-test.yml`) +- **Requires**: NVIDIA self-hosted CPU runner (`linux-amd64-cpu16`) + NGC PyTorch container +- **Purpose**: Validates pip/UV installs on NGC PyTorch images with CUDA libraries present +- **Gap Reason**: Requires NVIDIA self-hosted runner and NGC private container access + +### 5. Copyright Header Checks (`copyright-check.yml`) +- **Requires**: NVIDIA-NeMo/FW-CI-templates shared workflow (private org access) +- **Purpose**: Validates NVIDIA copyright headers on new files +- **Gap Reason**: Template workflow is in private NVIDIA org; fork cannot access without auth + +--- + +## Summary + +| Check | Mirrored | Reason | +|-------|----------|--------| +| **Linting** | ✅ Yes | Runs on `ubuntu-latest` with standard build tools | +| **Container Build** | ❌ No | Requires NVIDIA self-hosted runner + AWS ECR auth | +| **Unit Tests** | ❌ No | GPU hardware required | +| **Integration Tests** | ❌ No | Distributed GPU training cluster required | +| **Install Tests** | ❌ No | NVIDIA self-hosted runner + NGC container auth | +| **Copyright Checks** | ❌ No | NVIDIA private workflow dependency | + +This fork CI mirror is intended **solely for pre-validation before upstreaming PRs**. It catches common lint/style issues early, but full validation still requires running tests on NVIDIA's CI infrastructure before merging upstream.