From 8f4d95be34b4baaebf689029a3f6abba5559bf08 Mon Sep 17 00:00:00 2001 From: Srijan Upadhyay Date: Wed, 8 Jul 2026 14:03:58 +0530 Subject: [PATCH 1/4] Add CPU-only fork CI mirror for lint/style checks --- .github/workflows/fork-ci-mirror-fast.yml | 70 +++++++++++++++++++++++ CI_MIRROR_GAPS.md | 58 +++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 .github/workflows/fork-ci-mirror-fast.yml create mode 100644 CI_MIRROR_GAPS.md diff --git a/.github/workflows/fork-ci-mirror-fast.yml b/.github/workflows/fork-ci-mirror-fast.yml new file mode 100644 index 00000000000..6076b687e16 --- /dev/null +++ b/.github/workflows/fork-ci-mirror-fast.yml @@ -0,0 +1,70 @@ +name: Fork CI Mirror - CPU-only lint/style checks + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + issue_comment: + types: [created] + +jobs: + check-trigger: + runs-on: ubuntu-latest + outputs: + should_run: ${{ steps.trigger-check.outputs.should_run }} + steps: + - name: Check if PR and /run-ci-mirror comment + id: trigger-check + if: github.event_name == 'issue_comment' && github.event.issue.pull_request + run: | + if [[ "${{ github.event.comment.body }}" == *"/run-ci-mirror"* ]]; then + echo "should_run=true" >> $GITHUB_OUTPUT + else + echo "should_run=false" >> $GITHUB_OUTPUT + fi + + - name: Auto-run on PR event + id: auto-trigger + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + run: | + echo "should_run=true" >> $GITHUB_OUTPUT + + linting: + runs-on: ubuntu-latest + needs: check-trigger + if: | + needs.check-trigger.outputs.should_run == 'true' || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v1 + with: + version: 0.7.2 + + - name: Install linting tools + run: | + uv sync --locked --only-group linting + + - name: Get base ref + id: get-base + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + BASE_REF="${{ github.base_ref }}" + else + BASE_REF="main" + fi + echo "base_ref=${BASE_REF}" >> $GITHUB_OUTPUT + + - name: Run linting + run: | + export PATH=".venv/bin:$PATH" + export GITLAB_ENDPOINT=github.com + export CI_PROJECT_NAMESPACE=NVIDIA + export BASE_REF="${{ steps.get-base.outputs.base_ref }}" + export CHECK_ONLY=true + export SKIP_DOCS=false + bash tools/autoformat.sh diff --git a/CI_MIRROR_GAPS.md b/CI_MIRROR_GAPS.md new file mode 100644 index 00000000000..abbe44269d6 --- /dev/null +++ b/CI_MIRROR_GAPS.md @@ -0,0 +1,58 @@ +# Fork CI Mirror — CPU-Only Gap Documentation + +This document outlines the scope of CPU-feasible CI checks mirrored in this fork's `fork-ci-mirror-fast.yml` workflow, and the permanent gaps due to lack of GPU/self-hosted runners. + +## Mirrored Checks + +### Linting & Code Style (`fork-ci-mirror-fast.yml`) +- **Tool**: `uv sync --locked --only-group linting` + `bash tools/autoformat.sh` (CHECK_ONLY mode) +- **Runtime**: `ubuntu-latest` (GitHub-hosted Linux runner, ~2–3 minutes) +- **Purpose**: Detects code formatting, import sorting (isort), linting (pylint/flake8), and style violations +- **Trigger**: + - Auto-runs on all `pull_request` events (opened, synchronize, reopened) + - Auto-runs on `workflow_dispatch` (manual trigger) + - Can also be triggered via PR comment with `/run-ci-mirror` + +## Permanent Gaps — GPU/Distributed Training Infrastructure + +The following critical CI jobs **cannot** be mirrored on GitHub-hosted runners and remain an acknowledged permanent limitation: + +### 1. Container Build (`cicd-container-build`) +- **Requires**: NVIDIA self-hosted GPU runner (`nvidia-ci-aws-gpu-x8`) +- **Purpose**: Builds Docker container with CUDA/NVIDIA dependencies +- **Gap Reason**: Requires Docker registry push permissions and AWS ECR access; GPU not needed but NVIDIA self-hosted runner required for auth + +### 2. Unit Tests (`cicd-unit-tests-latest`) +- **Requires**: NVIDIA self-hosted GPU runner (`nvidia-ci-aws-gpu-x8`) +- **Purpose**: Validates CUDA kernels, distributed training ops, model correctness +- **Gap Reason**: Tests depend on NVIDIA GPU hardware; no substitute on CPU + +### 3. Integration Tests (`cicd-integration-tests-latest`) +- **Requires**: NVIDIA self-hosted GPU runner (`nvidia-ci-aws-gpu-x8`) +- **Purpose**: Validates distributed training across multiple GPUs/nodes, DDP, FSDP workflows +- **Gap Reason**: Tests depend on Slurm cluster with H100 GPUs; no CPU equivalent + +### 4. Installation Tests (`install-test.yml`) +- **Requires**: NVIDIA self-hosted CPU runner (`linux-amd64-cpu16`) + NGC PyTorch container +- **Purpose**: Validates pip/UV installs on NGC PyTorch images with CUDA libraries present +- **Gap Reason**: Requires NVIDIA self-hosted runner and NGC private container access + +### 5. Copyright Header Checks (`copyright-check.yml`) +- **Requires**: NVIDIA-NeMo/FW-CI-templates shared workflow (private org access) +- **Purpose**: Validates NVIDIA copyright headers on new files +- **Gap Reason**: Template workflow is in private NVIDIA org; fork cannot access without auth + +--- + +## Summary + +| Check | Mirrored | Reason | +|-------|----------|--------| +| **Linting** | ✅ Yes | Runs on `ubuntu-latest` with standard build tools | +| **Container Build** | ❌ No | Requires NVIDIA self-hosted runner + AWS ECR auth | +| **Unit Tests** | ❌ No | GPU hardware required | +| **Integration Tests** | ❌ No | Distributed GPU training cluster required | +| **Install Tests** | ❌ No | NVIDIA self-hosted runner + NGC container auth | +| **Copyright Checks** | ❌ No | NVIDIA private workflow dependency | + +This fork CI mirror is intended **solely for pre-validation before upstreaming PRs**. It catches common lint/style issues early, but full validation still requires running tests on NVIDIA's CI infrastructure before merging upstream. From e3e842c98ff5e78c4367a4519b53aac4f61587bc Mon Sep 17 00:00:00 2001 From: Srijan Upadhyay Date: Wed, 8 Jul 2026 14:31:11 +0530 Subject: [PATCH 2/4] Install lint tools without building CUDA-only project deps --- .github/workflows/fork-ci-mirror-fast.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fork-ci-mirror-fast.yml b/.github/workflows/fork-ci-mirror-fast.yml index 6076b687e16..9e5efff86e7 100644 --- a/.github/workflows/fork-ci-mirror-fast.yml +++ b/.github/workflows/fork-ci-mirror-fast.yml @@ -47,7 +47,10 @@ jobs: - name: Install linting tools run: | - uv sync --locked --only-group linting + # --no-install-project: install only the linting group (ruff/black/isort/ + # flake8/pylint); do NOT build megatron-core, which drags in CUDA-only + # packages (flash_mla) that cannot build on a CPU GitHub-hosted runner. + uv sync --locked --only-group linting --no-install-project - name: Get base ref id: get-base From 174e29cfa1ded556800d198879a057dbbca4587d Mon Sep 17 00:00:00 2001 From: Srijan Upadhyay Date: Wed, 8 Jul 2026 14:33:06 +0530 Subject: [PATCH 3/4] Install lint tools directly to avoid building CUDA-only git deps --- .github/workflows/fork-ci-mirror-fast.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fork-ci-mirror-fast.yml b/.github/workflows/fork-ci-mirror-fast.yml index 9e5efff86e7..785fd5316e4 100644 --- a/.github/workflows/fork-ci-mirror-fast.yml +++ b/.github/workflows/fork-ci-mirror-fast.yml @@ -47,10 +47,14 @@ jobs: - name: Install linting tools run: | - # --no-install-project: install only the linting group (ruff/black/isort/ - # flake8/pylint); do NOT build megatron-core, which drags in CUDA-only - # packages (flash_mla) that cannot build on a CPU GitHub-hosted runner. - uv sync --locked --only-group linting --no-install-project + # Install the lint tools directly rather than `uv sync`, which resolves + # the full lockfile and tries to BUILD git-sourced CUDA-only deps + # (flash_mla) that cannot compile on a CPU GitHub-hosted runner. + # Versions pinned to match the pyproject.toml `linting` group + mypy + # (invoked by autoformat.sh). + uv venv + uv pip install "ruff~=0.9.0" "black==24.4.2" "isort==5.13.2" \ + "flake8==7.1.0" "pylint==3.2.6" "mypy" - name: Get base ref id: get-base From 10850cca522bc658279af5983de230d6c963dd9d Mon Sep 17 00:00:00 2001 From: Srijan Upadhyay Date: Wed, 8 Jul 2026 14:35:37 +0530 Subject: [PATCH 4/4] Diff lint changeset against fork base, not upstream main --- .github/workflows/fork-ci-mirror-fast.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/fork-ci-mirror-fast.yml b/.github/workflows/fork-ci-mirror-fast.yml index 785fd5316e4..c4d8c0a9e02 100644 --- a/.github/workflows/fork-ci-mirror-fast.yml +++ b/.github/workflows/fork-ci-mirror-fast.yml @@ -74,4 +74,11 @@ jobs: export BASE_REF="${{ steps.get-base.outputs.base_ref }}" export CHECK_ONLY=true export SKIP_DOCS=false + # autoformat.sh diffs changed files against `autoformatter-remote/$BASE_REF`, + # adding that remote with `|| true`. Pre-register it pointing at THIS fork so + # the mirror lints only what this branch changed relative to the fork's own + # base — not the (possibly drifted) upstream NVIDIA main. + git remote add autoformatter-remote \ + "https://github.com/${{ github.repository }}.git" || true + git fetch autoformatter-remote "${BASE_REF}" bash tools/autoformat.sh