diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh index 8e62295ab2..12d3c30efd 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh @@ -4,7 +4,8 @@ set -eo pipefail # DeepSeek-V4.1-Flash on MI300X (gfx942): native DSpark, GPU-resident KV. A copy # of the validated MI355X arm; gfx942 has no FP4 MFMA, so the MXFP4 experts run # through whichever AITER MoE kernel vLLM's selector supports on this arch, and -# the Engram tables stay row-sharded on GPU per the upstream AMD defaults +# the Engram tables are row-sharded on GPU at TP8 and offloaded to pinned +# host memory at TP4 # (192 GB x TP8 holds the 511 GB checkpoint with room for KV). # https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml source "$(dirname "$0")/../../benchmark_lib.sh" @@ -58,6 +59,18 @@ while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2 CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) done +# vllm-project/vllm#57491 widened the two is_cuda() gates to is_cuda_alike(), so +# this image resolves an Engram config on gfx942 and, because cpu_offload now +# defaults on through VLLM_PLE_CPU_OFFLOAD, an explicit value is required rather +# than the default. Keep the tables resident at TP8 so that curve stays +# comparable with the validated concurrency 1-32 run; offload at TP4, where half +# the checkpoint plus the tables does not fit a 192 GB card. +if (( TP == 4 )); then + ENGRAM_CONFIG='{"cpu_offload":true}' +else + ENGRAM_CONFIG='{"cpu_offload":false}' +fi + # Use the runner-specific port assigned by launch_mi300x-amd.sh. export AIPERF_SERVER_URL="http://localhost:${PORT}" export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" @@ -81,6 +94,7 @@ VLLM_CMD=( --tokenizer-mode deepseek_v41 --tool-call-parser deepseek_v41 --enable-auto-tool-choice --reasoning-parser deepseek_v41 + --engram-config "$ENGRAM_CONFIG" # aiter: auto selection picked the unfused Triton MoE (TRITON_UNFUSED) on # gfx942 in run 35306398350 and still segfaulted at piecewise capture, so # the MoE kernel was not the culprit; keep the upstream recipe's name, @@ -94,6 +108,14 @@ VLLM_CMD=( # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits # buffer at startup (16 GiB at 8192); 16384 would cost 32 GiB of a 192 GB card. --max-num-batched-tokens 8192 + # vllm-project/vllm#56227 turned SWA bounded replay on by default between the + # eed1f3d0 pin and this one. It relies on a window clamp that landed in the + # FlashInfer and FlashMLA kernels; the ROCm sparse SWA path only gained the + # replay_start kwarg, which crashed every gfx950 point with + # HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a replay + # start. gfx942 runs the same ROCm sparse path. Drop this once ROCm clamps + # too; prefix caching stays on. + --no-swa-bounded-replay --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' --disable-uvicorn-access-log ) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 322dcd0310..b71472c25f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1168,7 +1168,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: # golden AL 3.51 for throughput, real block verification for evals, adaptive # verification off on ROCm. dsv41flash-fp4-mi300x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657 + image: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi300x-amd @@ -1179,8 +1179,16 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. + # The recipe now sets engram cpu_offload per TP: off at TP8 so this + # curve stays comparable with the validated concurrency 1-32 run, on at + # TP4 where the tables would not fit beside half the checkpoint on a + # 192 GB card. vllm-project/vllm#57491 widened the two is_cuda() gates to + # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + # TP4 halves the GPUs per server. With the Engram tables in pinned host + # memory the resident weights are ~81 GiB per rank, leaving ~60 GiB of + # KV per GPU after the indexer's 16 GiB logits buffer. + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 # GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9103a6cc53..c6995e3648 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8566,3 +8566,16 @@ description: - "Update the vLLM ROCm image from nightly-d9105ea8 to v0.29.0; benchmark configuration unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3063 + +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add a TP4 arm with Engram CPU offload to the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the tables GPU-resident; with the tables in pinned host memory the resident weights are ~81 GiB per rank, leaving ~60 GiB of KV per GPU on a 192 GB card after the indexer's 16 GiB logits buffer" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the ROCm 10.0 nightly channel build of the same commit the MI355X arm moved to in SemiAnalysisAI/InferenceX#3326 (digest sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1, published 2026-09-21T05:50:03Z); it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" + - "为 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 臂,同时保留 Engram 表驻留 GPU 的现有 TP8 臂;表移至固定页主机内存后每 rank 常驻权重约 81 GiB,在 192 GB 卡上扣除 indexer 的 16 GiB logits 缓冲区后每张 GPU 约剩 60 GiB KV" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984(与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建);该镜像包含 vllm-project/vllm#57491,将两处 is_cuda() 判定放宽为 is_cuda_alike(),使 engram_config 在 gfx942 上可解析;由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启,配方按 TP 显式设置该值" + - "新增 --no-swa-bounded-replay:vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启,其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失,曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃;gfx942 使用同一路径。prefix caching 保持开启" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3332