diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..3bb574da8a --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,334 @@ +base: + schema: 2 + name: mi355x-glm52-atom-agentx-disagg-mtp + slurm: + time_limit: "08:00:00" + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + enable_multiple_frontends: false + args: + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + prefill-policy: round_robin + decode-policy: round_robin + atom-pd-rank-mapping-policy: none + engine: atom + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + <<: &environment + PYTHONUNBUFFERED: "1" + PYTHONHASHSEED: "0" + # srtctl's pre-download sees only HF_HOME, so the node-local cache is mounted at $HF_HOME/hub. + HF_HOME: /mnt/hf_home + HF_HUB_CACHE: /mnt/hf_home/hub + AITER_LOG_LEVEL: WARNING + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_USE_FLYDSL_MOE_SORTING: "1" + ATOM_MLA_PAGE_SIZE: "1" + ATOM_ONLINE_QUANT_STREAMING: "0" + ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB: "2047" + ATOM_USE_TRITON_MLA: "0" + TOPK_FORCE_PATH: one + MAX_JOBS: "16" + HSA_ENABLE_IPC_MODE_LEGACY: "1" + MC_DISABLE_HIP: "1" + # Rail-isolated RDMA: each transfer uses the same rail on both ends. + MC_GID_INDEX: "@fabric.rdma-gid-index" + ATOM_MOONCAKE_MATCHED_RAILS: auto + NCCL_IB_DISABLE: "1" + VLLM_PP_LAYER_PARTITION: "20,20,20,18" + OFFLOAD_PROFILE: "1" + OFFLOAD_MIN_LOAD_TOKENS: "0" + args: + <<: &server + served-model-name: amd/GLM-5.2-MXFP4 + trust-remote-code: true + kv_cache_dtype: fp8 + block-size: 16 + # model.layers.78 is the MTP head, shipped in BF16; excluding it keeps + # online ptpc_fp8 quantization off the draft while experts stay MXFP4. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' + level: 3 + cudagraph-mode: FULL + method: mtp + timeout-keep-alive: 900 + pipeline-parallel-size: 4 + enforce-eager: true + gpu-memory-utilization: 0.85 + max-num-seqs: 512 + max-num-batched-tokens: 8192 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: *environment + args: *server + sbatch_directives: + cpus-per-task: "128" + mem: "0" + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + extra_mount: + # The node-local hub cache the single-node GLM-5.2 points already fill. + - /var/lib/hf-hub-cache:/mnt/hf_home/hub + health_check: + max_attempts: 1080 + interval_seconds: 5 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "atom:" + AIPERF_APPLY_CHAT_TEMPLATE: "true" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + +override_1p1d_mcstore: + host_setup: + nodes: all + timeout_seconds: 1800 + commands: + # Reserves the Store owners' 2 MiB pages on the head node, where srtctl starts the prefill. + - HEAD_NODE_HUGEPAGES_PER_NUMA=393216 IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" + services: + - &master + name: mc-master + type: generic + placement: + node: prefill + start: before_workers + critical: true + inherit_discovery_env: false + # numa_exec.py binds the process's CPUs and memory to that NUMA node. + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26051, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26080, --metrics_port=26090, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + env: &store_env + # A Mooncake process that sees a GPU probes every buffer through HIP. + HIP_VISIBLE_DEVICES: "-1" + CUDA_VISIBLE_DEVICES: "-1" + # Peers with different QP counts cannot connect. + MC_NUM_QP_PER_EP: "1" + MC_GID_INDEX: "@fabric.rdma-gid-index" + readiness: + port: 26051 + timeout_seconds: 300 + - &owner + <<: *master + name: mc-owner-numa0 + # Each owner registers on rdma0-3, so every stage reaches it on its own rail. + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, "--device_names=rdma0,rdma1,rdma2,rdma3", --global_segment_size=824633720832, --local_buffer_size=0, --threads=4] + env: + <<: *store_env + # Back the segment with the reserved 2 MiB pages; the NIC cannot register 64 GiB regions of 4 KiB pages. + MC_STORE_USE_HUGEPAGE: "1" + MC_MAX_MR_SIZE: "68719476736" + MC_TCP_BIND_ADDRESS: "{node_ip}" + # The remapped container root lacks host CAP_IPC_LOCK, so registering the segment counts against RLIMIT_MEMLOCK. + preamble: ulimit -l unlimited + readiness: + log: + pattern: Starting real client service + timeout_seconds: 1800 + interval_seconds: 5 + - <<: *owner + name: mc-owner-numa1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, "--device_names=rdma0,rdma1,rdma2,rdma3", --global_segment_size=824633720832, --local_buffer_size=0, --threads=4] + roles: + prefill: + env: + <<: &store_prefill_env + # More than one load worker hung the PP prefill pipeline at C96. + OFFLOAD_LOAD_WORKERS: "1" + MC_NUM_QP_PER_EP: "1" + # Registers the worker's HBM transfer pool in 1 GiB regions. + MC_MAX_MR_SIZE: "1073741824" + # Every owner sits on rdma0-3 and this fabric is rail-isolated, so each stage must reach an owner through the NIC of its own name. + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + args: + num-speculative-tokens: 3 + # One pool shared by all four stages, as in ATOM's CI; each client still uses its GPU's NIC. + extra-kv-connectors: + - kv_connector: mooncake_store + kv_role: offload + kv_connector_extra_config: + mooncake_store.protocol: rdma + mooncake_store.master: "127.0.0.1:26051" + mooncake_store.metadata: "http://127.0.0.1:26080/metadata" + decode: + nodes: colocate + env: + AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + # The P/D transfer must match the prefill's QP count, one with the Store. + MC_NUM_QP_PER_EP: "1" + args: + <<: &dcp_decode + decode-context-parallel-size: 4 + gpu-memory-utilization: 0.94 + max-num-seqs: 512 + max-num-batched-tokens: 2048 + cudagraph-capture-sizes: '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' + num-speculative-tokens: 3 + +override_1p2d_mcstore_c56: + resources: &spread + # A node per decode; srtctl otherwise packs two TP4 decodes onto one node. + spread_workers: true + frontend: + args: &cache_aware + policy: cache_aware + prefill-policy: cache_aware + decode-policy: cache_aware + cache-threshold: 0.8 + balance-abs-threshold: 2 + balance-rel-threshold: 2.0 + eviction-interval: 300 + host_setup: &prefill_budget_host_setup + nodes: all + timeout_seconds: 1800 + commands: + # Reserves the Store owners' 2 MiB pages on the head node, where srtctl starts the prefill. + - HEAD_NODE_HUGEPAGES_PER_NUMA=357376 IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" + services: &prefill_budget_services + - <<: *master + name: mc-master-rdma0 + - <<: *master + name: mc-master-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26151 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26251 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26351 + timeout_seconds: 300 + - <<: *owner + name: mc-owner-rdma0 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + roles: + prefill: + env: *store_prefill_env + args: + num-speculative-tokens: 4 + # Each stage's Store client uses its GPU's NIC and that NIC's pool. + extra-kv-connectors: &per_nic_store_connector + - kv_connector: mooncake_store + kv_role: offload + kv_connector_extra_config: + mooncake_store.protocol: rdma + mooncake_store.pools: + rdma0: {master: "127.0.0.1:26051", metadata: "http://127.0.0.1:26080/metadata"} + rdma1: {master: "127.0.0.1:26151", metadata: "http://127.0.0.1:26180/metadata"} + rdma2: {master: "127.0.0.1:26251", metadata: "http://127.0.0.1:26280/metadata"} + rdma3: {master: "127.0.0.1:26351", metadata: "http://127.0.0.1:26380/metadata"} + decode: + nodes: 2 + workers: 2 + env: + AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + MC_NUM_QP_PER_EP: "1" + args: + <<: *dcp_decode + num-speculative-tokens: 4 + +override_1p3d_mcstore_c12: + resources: *spread + frontend: + args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services + roles: + prefill: + env: *store_prefill_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *per_nic_store_connector + decode: + nodes: 3 + workers: 3 + env: &tp4_decode_env + PYTHONNOUSERSITE: "1" + MC_NUM_QP_PER_EP: "1" + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 24 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24]' + disable-uvicorn-access-log: true + +override_1p3d_mcstore_c16: + resources: *spread + frontend: + args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services + roles: + prefill: + env: *store_prefill_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *per_nic_store_connector + decode: + nodes: 3 + workers: 3 + env: *tp4_decode_env + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 32 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32]' + disable-uvicorn-access-log: true + +override_1p3d_mcstore_c24: + resources: *spread + frontend: + args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services + roles: + prefill: + env: *store_prefill_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *per_nic_store_connector + decode: + nodes: 3 + workers: 3 + env: *tp4_decode_env + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 48 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48]' + disable-uvicorn-access-log: true diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 49c87bb99a..95be5bceab 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1171,6 +1171,59 @@ glm5.2-fp4-mi355x-atom-agentic-mtp-mono: search-space: - { tp: 4, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: agentic.yaml } +glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: + image: rocm/atom-dev:nightly_202610131249@sha256:d5b3fcd99bd44e3489740d1b026037f49f2e3f6160dc70b70c3fd23a125a4905 + model: amd/GLM-5.2-MXFP4 + model-prefix: glm5.2 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "6fd1bb1be97c79b571eb86674786d0d888353b73" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx + scenarios: + agentic-coding: + # 1.0 gives each 4-GPU server 1499 GB: 1P1D's 2 x 768 GiB owners hold 825 GB per NUMA node of its prefill's and colocated decode's shares; 1P2D and 1P3D owners hold the prefill's 1499 GB. + - dram-utilization: 1.0 + search-space: + - spec-decoding: "mtp" + conc-list: [ 56, 72, 96, 112, 128 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 56 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p2d_mcstore_c56 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 2, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 12 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c12 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 16 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c16 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 24 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c24 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + # Upstream ROCm nightly rather than the deepseekv41-flash-0909 release tag: that # tag predates vllm-project/vllm#56503, which moves the mHC delayed pre block off # the eager Torch reference and onto AITER. MI355X run 34710937012 passed the diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index fd4a72bae0..b3d95e9b59 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11670,3 +11670,14 @@ - "Retire all MiniMax-M3 and DeepSeek-V4-Pro AgentX serving configurations across NVIDIA and AMD, deleting their unused recipes, setup assets, and launch-policy rows." - "Keep MiniMax-M3.1-Flash and DeepSeek-V4.1-Flash benchmark coverage unchanged. Preserve SPEED-Bench collectors, historical golden AL curves, their B300 checkpoint mappings, and the independent CollectiveX communication workload." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4664 + +- config-keys: + - glm5.2-fp4-mi355x-atom-disagg-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation (AToMesh router, Mooncake transfer, MTP) on rocm/atom-dev:nightly_202610131249." + - "Target: MXFP4 routed experts, the other linear layers quantized online to FP8 (ptpc_fp8), the MTP draft in BF16 as shipped; FP8 KV cache." + - "One PP4 prefill with a Mooncake Store prefix tier; 1P1D with a DCP4 decode at concurrency 56-128, 1P2D with DCP4 decodes at 56, 1P3D with TP4 decodes at 12-24." + - "Patch srt-slurm so ATOM's -tp leaves out the pipeline stages (NVIDIA/srt-slurm#604)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 diff --git a/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh b/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh index 2c281bf90b..e167f3d718 100755 --- a/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh +++ b/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh @@ -59,3 +59,27 @@ if (( after_free > reserved )); then echo "Host preparation could not reclaim all unused hugepages" >&2 exit 1 fi + +# A recipe whose head-node service registers NUMA-bound 2 MiB HugeTLB memory with +# the NIC (a Mooncake Store owner) sets HEAD_NODE_HUGEPAGES_PER_NUMA; unset, no +# pages are reserved. The container cannot grow the pool, so it is grown here, 2% +# above the pages host services hold, and each NUMA node must end with its share +# free. The recipe's teardown releases them (HUGEPAGES_TARGET=0 hugepage_sanity.sh). +pages_per_numa="${HEAD_NODE_HUGEPAGES_PER_NUMA:-}" +if [[ -n "$pages_per_numa" ]]; then + launch_ip="${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}" + if [[ " $(hostname -I) " == *" $launch_ip "* ]]; then + numa_nodes=(/sys/devices/system/node/node[0-9]*) + echo "[hugepage] $(hostname -s) is the head node; reserving $pages_per_numa 2 MiB pages on each of ${#numa_nodes[@]} NUMA nodes" + held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' "$meminfo") + HUGEPAGES_TARGET=$(( held + pages_per_numa * ${#numa_nodes[@]} * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 \ + bash "$(dirname "${BASH_SOURCE[0]}")/hugepage_sanity.sh" + for numa_node in "${numa_nodes[@]}"; do + numa_free=$(cat "$numa_node/hugepages/hugepages-2048kB/free_hugepages") + echo "[hugepage] ${numa_node##*/}: $numa_free free 2 MiB pages, $pages_per_numa needed" + if (( numa_free < pages_per_numa )); then + exit 1 + fi + done + fi +fi diff --git a/inferencex-e2e/runners/srt-slurm/patches/604-atom-tp-from-pp.patch b/inferencex-e2e/runners/srt-slurm/patches/604-atom-tp-from-pp.patch new file mode 100644 index 0000000000..33a6019d1f --- /dev/null +++ b/inferencex-e2e/runners/srt-slurm/patches/604-atom-tp-from-pp.patch @@ -0,0 +1,60 @@ +From e23ca1bed9aee9a59905fb30b27f09b3ef9e9224 Mon Sep 17 00:00:00 2001 +From: Jasen2201 +Date: Wed, 14 Oct 2026 04:15:36 +0000 +Subject: [PATCH] fix(atom): size -tp from the worker's GPUs over its PP and + PCP stages + +ATOM runs a worker on tp x pp x pcp GPUs, but srtctl always passed +`-tp `. A 4-GPU prefill worker with +`pipeline-parallel-size: 4` therefore launched as +`-tp 4 --pipeline-parallel-size 4`, and ATOM sized its world to 16 GPUs. + +`-tp` is now the worker's GPU count divided by `pipeline-parallel-size` +and `prefill-context-parallel-size`. Decode context parallelism reuses the +TP GPUs and does not change it. A split that does not divide the worker's +GPUs fails before launch. + +Co-Authored-By: Claude Opus 5.5 +Signed-off-by: Jasen2201 +--- + src/srtctl/backends/atom.py | 20 +++++++++++++++++++- + 1 file changed, 19 insertions(+), 1 deletion(-) + +diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py +index edf9a9b7..4e5b836e 100644 +--- a/src/srtctl/backends/atom.py ++++ b/src/srtctl/backends/atom.py +@@ -155,7 +155,7 @@ class AtomBackend(Backend): + "--server-port", + str(process.http_port), + "-tp", +- str(len(process.gpu_indices)), ++ str(_tensor_parallel_size(len(process.gpu_indices), config)), + ] + ) + kv_transfer_config = self._kv_transfer_config(process, worker_ip, extra_connectors) +@@ -183,3 +183,21 @@ def _config_to_cli_args(config: dict[str, Any]) -> list[str]: + + def _canonical_arg_key(key: str) -> str: + return key.lstrip("-").replace("_", "-") ++ ++ ++# ATOM sizes a worker as tp x pp x pcp GPUs; decode context parallelism reuses the TP GPUs. ++_GPU_SPLITTING_ARGS = ("pipeline-parallel-size", "prefill-context-parallel-size") ++ ++ ++def _tensor_parallel_size(gpu_count: int, config: Mapping[str, Any]) -> int: ++ """The worker's GPUs left for tensor parallelism after its pipeline and prefill-context stages.""" ++ stages = 1 ++ for key, value in config.items(): ++ if _canonical_arg_key(key) in _GPU_SPLITTING_ARGS: ++ stages *= int(value) ++ if stages < 1 or gpu_count % stages: ++ raise ValueError( ++ f"ATOM worker has {gpu_count} GPU(s), which its pipeline x prefill-context parallelism of {stages} " ++ "does not divide" ++ ) ++ return gpu_count // stages +-- +2.43.0 + diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index d9052d1aa8..a6b80e31c3 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,3 +9,4 @@ Each patch backports an upstream fix that the shared submodule pin does not incl | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exclusive node's exporter (DCGM or AMD) reports every GPU, which the collector otherwise flags as `unexpected_device`. The pin includes #573, but upstream merged a different design: it keeps every row and excuses idle worker-node GPUs only after a DCGM worker GPU UUID probe. [`infx/results/power/multinode.py`](../../../infx/results/power/multinode.py) does not apply that excusal yet, so keep this filter until it does. | +| `604-atom-tp-from-pp.patch` | [NVIDIA/srt-slurm#604](https://github.com/NVIDIA/srt-slurm/pull/604) | ATOM's `-tp` is the worker's GPUs divided by its pipeline and prefill-context stages, so a 4-GPU PP4 prefill launches with `-tp 1` instead of `-tp 4` |