diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh index 8e62295ab2..046a22fbea 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh @@ -4,8 +4,9 @@ set -eo pipefail # DeepSeek-V4.1-Flash on MI300X (gfx942): native DSpark, GPU-resident KV. A copy # of the validated MI355X arm; gfx942 has no FP4 MFMA, so the MXFP4 experts run # through whichever AITER MoE kernel vLLM's selector supports on this arch, and -# the Engram tables stay row-sharded on GPU per the upstream AMD defaults -# (192 GB x TP8 holds the 511 GB checkpoint with room for KV). +# the Engram tables are row-sharded on GPU at TP8 and offloaded to pinned +# host memory below TP8 (192 GB x TP8 holds the 511 GB checkpoint with +# room for KV; TP4 and TP2 do not). # https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION @@ -58,6 +59,25 @@ while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2 CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) done +# vllm-project/vllm#57491 widened the two is_cuda() gates to is_cuda_alike(), so +# this image resolves an Engram config on gfx942 and, because cpu_offload now +# defaults on through VLLM_PLE_CPU_OFFLOAD, an explicit value is required rather +# than the default. Keep the 183 GiB of tables resident at TP8 so that curve +# stays comparable with the validated concurrency 1-32 run; offload below TP8, +# where half or more of the checkpoint already fills a 192 GB card. +if (( TP < 8 )); then + ENGRAM_CONFIG='{"cpu_offload":true}' +else + ENGRAM_CONFIG='{"cpu_offload":false}' +fi + +# The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits +# buffer at startup: 16 GiB at 8192, 8 GiB at 4096. 8192 leaves TP4 ~64 GiB of +# KV per rank after the ~81 GiB of offloaded weights. There is no TP2 arm here: +# run 35671005506 measured "Available KV cache memory: -13.51 GiB" at TP2 even +# with the buffer halved, so a 192 GB card cannot serve this checkpoint at TP2. +BATCHED_TOKENS=8192 + # Use the runner-specific port assigned by launch_mi300x-amd.sh. export AIPERF_SERVER_URL="http://localhost:${PORT}" export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" @@ -81,6 +101,7 @@ VLLM_CMD=( --tokenizer-mode deepseek_v41 --tool-call-parser deepseek_v41 --enable-auto-tool-choice --reasoning-parser deepseek_v41 + --engram-config "$ENGRAM_CONFIG" # aiter: auto selection picked the unfused Triton MoE (TRITON_UNFUSED) on # gfx942 in run 35306398350 and still segfaulted at piecewise capture, so # the MoE kernel was not the culprit; keep the upstream recipe's name, @@ -91,9 +112,15 @@ VLLM_CMD=( --max-model-len 1048576 --max-num-seqs "$MAX_NUM_SEQS" --max-cudagraph-capture-size "$CAPTURE_SIZE" - # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits - # buffer at startup (16 GiB at 8192); 16384 would cost 32 GiB of a 192 GB card. - --max-num-batched-tokens 8192 + --max-num-batched-tokens "$BATCHED_TOKENS" + # vllm-project/vllm#56227 turned SWA bounded replay on by default between the + # eed1f3d0 pin and this one. It relies on a window clamp that landed in the + # FlashInfer and FlashMLA kernels; the ROCm sparse SWA path only gained the + # replay_start kwarg, which crashed every gfx950 point with + # HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a replay + # start. gfx942 runs the same ROCm sparse path. Drop this once ROCm clamps + # too; prefix caching stays on. + --no-swa-bounded-replay --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' --disable-uvicorn-access-log ) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index bae2efc100..c7f5d254b0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1168,7 +1168,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: # golden AL 3.51 for throughput, real block verification for evals, adaptive # verification off on ROCm. dsv41flash-fp4-mi300x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657 + image: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi300x-amd @@ -1179,8 +1179,20 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. + # The recipe sets engram cpu_offload per TP: off at TP8 so this curve stays + # comparable with the validated concurrency 1-32 run, on below TP8 where + # the tables would not fit beside the checkpoint share on a 192 GB card. + # vllm-project/vllm#57491 widened the two is_cuda() gates to + # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + # TP4 halves the GPUs per server. With the Engram tables in pinned host + # memory the resident weights are ~81 GiB per rank. + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + # No TP2 arm on this SKU. Measured in run 35671005506: at TP2 the resident + # weights are ~145 GiB per rank, and even with the indexer buffer halved + # to 4096 batched tokens the engine reported "Available KV cache memory: + # -13.51 GiB" and refused to start. A 192 GB card cannot hold this + # checkpoint at TP2 with a usable KV pool; MI325X does it on 256 GB. # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 # GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d061c54cae..445c3adc51 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8597,3 +8597,28 @@ - "Use lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 with bundled DSpark K=6, full-node CPU/DRAM allocation, and per-point request/CUDA-graph limits." - "Map the exact DeepSeek-V4-Pro-0813 recipe alias to the pre-staged NScale checkpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3257 + +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add TP4 and TP2 arms with Engram CPU offload to the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the 183 GiB of tables GPU-resident. With the tables in pinned host memory the resident weights are ~81 GiB per rank at TP4 and ~145 GiB per rank at TP2, against 192 GB per card" + - "Cap the scheduler at TP2 only: halve the sparse-attention indexer buffer to 4096 batched tokens (8 GiB rather than 16 GiB of a [batched-tokens, 1M] fp8 logits allocation), scale max-num-seqs to 2*CONC clamped to [16, 256] and cap cudagraph capture at 512. Same cap set the Blackwell TP2 arms validated on B200, GB200 and GB300. TP2 concurrency stops at 8 because ~145 GiB of weights leaves the smallest KV pool of any arm here" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the ROCm 10.0 nightly channel build of the same commit the MI355X arm moved to in SemiAnalysisAI/InferenceX#3326 (digest sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1, published 2026-09-21T05:50:03Z); it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" + - "为 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 与 TP2 臂,同时保留 183 GiB Engram 表驻留 GPU 的现有 TP8 臂;表移至固定页主机内存后,TP4 每 rank 常驻权重约 81 GiB,TP2 约 145 GiB,而每张卡为 192 GB" + - "仅在 TP2 收紧调度器:将稀疏注意力 indexer 缓冲区减半至 4096 batched tokens([batched-tokens, 1M] fp8 logits 分配由 16 GiB 降至 8 GiB),max-num-seqs 取 2*CONC 并夹在 [16, 256],cudagraph capture 上限 512。该组参数与 Blackwell TP2 臂在 B200、GB200、GB300 上验证过的一致。TP2 并发止于 8,因为约 145 GiB 权重使其 KV 池为本配方中最小" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984(与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建);该镜像包含 vllm-project/vllm#57491,将两处 is_cuda() 判定放宽为 is_cuda_alike(),使 engram_config 在 gfx942 上可解析;由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启,配方按 TP 显式设置该值" + - "新增 --no-swa-bounded-replay:vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启,其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失,曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃;gfx942 使用同一路径。prefix caching 保持开启" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336 +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Drop the TP2 arm from the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe: measured infeasible. In Actions run 35671005506 the TP2 concurrency-1 point reported \"Available KV cache memory: -13.51 GiB\" and the engine refused to start with \"No available memory for the cache blocks\", even with the sparse-attention indexer buffer already halved to 4096 batched tokens. At TP2 the Engram-offloaded resident weights are ~145 GiB per rank, which a 192 GB card cannot hold alongside a usable 1M-context KV pool. Raising gpu-memory-utilization to 0.95 recovers ~9 GiB and halving the buffer again ~4 GiB, which still leaves effectively no KV, so the arm is removed rather than tuned. MI325X runs the same TP2 configuration on 256 GB per card" + - "The TP8 and TP4 arms are unaffected and stay. TP4 passed at concurrency 1 and 32 in the same run with the tables offloaded, at ~81 GiB per rank" + - "从 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方中移除 TP2 臂:经实测不可行。在 Actions 运行 35671005506 中,TP2 并发 1 数据点报告 \"Available KV cache memory: -13.51 GiB\",引擎以 \"No available memory for the cache blocks\" 拒绝启动,且此时稀疏注意力 indexer 缓冲区已减半至 4096 batched tokens。TP2 下经 Engram 卸载后每 rank 常驻权重约 145 GiB,192 GB 的卡无法在容纳该权重的同时保留可用的 1M 上下文 KV 池。将 gpu-memory-utilization 提高到 0.95 可回收约 9 GiB,再次将缓冲区减半可回收约 4 GiB,仍几乎不剩 KV,因此选择移除该臂而非调参。MI325X 以每卡 256 GB 运行相同的 TP2 配置" + - "TP8 与 TP4 臂不受影响并保留。TP4 在同一运行中以每 rank 约 81 GiB 在并发 1 与 32 通过" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336