From 0e01a6ebd19cae398a2143bc44a022f6c0bf5437 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:09:48 -0400 Subject: [PATCH 1/2] =?UTF-8?q?dsv41flash-fp4-mi300x-vllm-agentic-dspark:?= =?UTF-8?q?=20add=20TP4=20arm=20with=20Engram=20host=20offload=20/=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=20Engram=20=E4=B8=BB=E6=9C=BA=E5=8D=B8?= =?UTF-8?q?=E8=BD=BD=E7=9A=84=20TP4=20=E8=87=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a TP4 search-space row beside the existing TP8 arm and repin the image from the expired nightly-eed1f3d0 tag to nightly-3df4ae15, the same commit the MI355X arm moved to. That image carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942; cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD, so the script sets it explicitly per TP: off at TP8 to keep that curve comparable, on at TP4 where the tables do not fit beside half the checkpoint. Also add --no-swa-bounded-replay. vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path lacks, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT. gfx942 runs the same path. Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic/dsv41flash_fp4_mi300x_mtp.sh | 24 ++++++++++++++++++- configs/amd-master.yaml | 12 ++++++++-- perf-changelog.yaml | 13 ++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh index 8e62295ab2..12d3c30efd 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh @@ -4,7 +4,8 @@ set -eo pipefail # DeepSeek-V4.1-Flash on MI300X (gfx942): native DSpark, GPU-resident KV. A copy # of the validated MI355X arm; gfx942 has no FP4 MFMA, so the MXFP4 experts run # through whichever AITER MoE kernel vLLM's selector supports on this arch, and -# the Engram tables stay row-sharded on GPU per the upstream AMD defaults +# the Engram tables are row-sharded on GPU at TP8 and offloaded to pinned +# host memory at TP4 # (192 GB x TP8 holds the 511 GB checkpoint with room for KV). # https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml source "$(dirname "$0")/../../benchmark_lib.sh" @@ -58,6 +59,18 @@ while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2 CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) done +# vllm-project/vllm#57491 widened the two is_cuda() gates to is_cuda_alike(), so +# this image resolves an Engram config on gfx942 and, because cpu_offload now +# defaults on through VLLM_PLE_CPU_OFFLOAD, an explicit value is required rather +# than the default. Keep the tables resident at TP8 so that curve stays +# comparable with the validated concurrency 1-32 run; offload at TP4, where half +# the checkpoint plus the tables does not fit a 192 GB card. +if (( TP == 4 )); then + ENGRAM_CONFIG='{"cpu_offload":true}' +else + ENGRAM_CONFIG='{"cpu_offload":false}' +fi + # Use the runner-specific port assigned by launch_mi300x-amd.sh. export AIPERF_SERVER_URL="http://localhost:${PORT}" export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" @@ -81,6 +94,7 @@ VLLM_CMD=( --tokenizer-mode deepseek_v41 --tool-call-parser deepseek_v41 --enable-auto-tool-choice --reasoning-parser deepseek_v41 + --engram-config "$ENGRAM_CONFIG" # aiter: auto selection picked the unfused Triton MoE (TRITON_UNFUSED) on # gfx942 in run 35306398350 and still segfaulted at piecewise capture, so # the MoE kernel was not the culprit; keep the upstream recipe's name, @@ -94,6 +108,14 @@ VLLM_CMD=( # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits # buffer at startup (16 GiB at 8192); 16384 would cost 32 GiB of a 192 GB card. --max-num-batched-tokens 8192 + # vllm-project/vllm#56227 turned SWA bounded replay on by default between the + # eed1f3d0 pin and this one. It relies on a window clamp that landed in the + # FlashInfer and FlashMLA kernels; the ROCm sparse SWA path only gained the + # replay_start kwarg, which crashed every gfx950 point with + # HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a replay + # start. gfx942 runs the same ROCm sparse path. Drop this once ROCm clamps + # too; prefix caching stays on. + --no-swa-bounded-replay --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' --disable-uvicorn-access-log ) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9b4fcadeb7..31fc16d913 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1170,7 +1170,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: # golden AL 3.51 for throughput, real block verification for evals, adaptive # verification off on ROCm. dsv41flash-fp4-mi300x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657 + image: vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi300x-amd @@ -1181,8 +1181,16 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. + # The recipe now sets engram cpu_offload per TP: off at TP8 so this + # curve stays comparable with the validated concurrency 1-32 run, on at + # TP4 where the tables would not fit beside half the checkpoint on a + # 192 GB card. vllm-project/vllm#57491 widened the two is_cuda() gates to + # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + # TP4 halves the GPUs per server. With the Engram tables in pinned host + # memory the resident weights are ~81 GiB per rank, leaving ~60 GiB of + # KV per GPU after the indexer's 16 GiB logits buffer. + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 # GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5b4309f5f..a36c195030 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8455,3 +8455,16 @@ description: - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add a TP4 arm with Engram CPU offload to the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the tables GPU-resident; with the tables in pinned host memory the resident weights are ~81 GiB per rank, leaving ~60 GiB of KV per GPU on a 192 GB card after the indexer's 16 GiB logits buffer" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the same commit the MI355X arm moved to; it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" + - "为 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 臂，同时保留 Engram 表驻留 GPU 的现有 TP8 臂；表移至固定页主机内存后每 rank 常驻权重约 81 GiB，在 192 GB 卡上扣除 indexer 的 16 GiB logits 缓冲区后每张 GPU 约剩 60 GiB KV" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂相同的 commit）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" + - "新增 --no-swa-bounded-replay：vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启，其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失，曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃；gfx942 使用同一路径。prefix caching 保持开启" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3332 From 9b5f964d7e10a03c191345d82be18d5c6b99e764 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:18:33 -0400 Subject: [PATCH 2/2] =?UTF-8?q?Repin=20dsv41flash-mi300x-tp4-engram-offloa?= =?UTF-8?q?d=20to=20the=20ROCm=2010.0=20nightly=20channel=20/=20=E5=B0=86?= =?UTF-8?q?=E9=95=9C=E5=83=8F=E9=87=8D=E6=96=B0=E5=9B=BA=E5=AE=9A=E5=88=B0?= =?UTF-8?q?=20ROCm=2010.0=20nightly=20=E6=B8=A0=E9=81=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match SemiAnalysisAI/InferenceX#3326, which moved MI355X to nightly-rocm100-3df4ae15 on the ROCm 10.0 channel. Same vLLM commit, different ROCm runtime. 与 SemiAnalysisAI/InferenceX#3326 保持一致：相同的 vLLM commit，ROCm 10.0 运行时。 Co-Authored-By: Claude Opus 5 (1M context) --- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index eef9a80e0c..693170c160 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1168,7 +1168,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: # golden AL 3.51 for throughput, real block verification for evals, adaptive # verification off on ROCm. dsv41flash-fp4-mi300x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984 + image: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi300x-amd diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c4988146ea..ad09b62f0f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8538,9 +8538,9 @@ - agentic-coding description: - "Add a TP4 arm with Engram CPU offload to the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the tables GPU-resident; with the tables in pinned host memory the resident weights are ~81 GiB per rank, leaving ~60 GiB of KV per GPU on a 192 GB card after the indexer's 16 GiB logits buffer" - - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the same commit the MI355X arm moved to; it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the ROCm 10.0 nightly channel build of the same commit the MI355X arm moved to in SemiAnalysisAI/InferenceX#3326 (digest sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1, published 2026-09-21T05:50:03Z); it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" - "为 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 臂，同时保留 Engram 表驻留 GPU 的现有 TP8 臂；表移至固定页主机内存后每 rank 常驻权重约 81 GiB，在 192 GB 卡上扣除 indexer 的 16 GiB logits 缓冲区后每张 GPU 约剩 60 GiB KV" - - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂相同的 commit）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" - "新增 --no-swa-bounded-replay：vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启，其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失，曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃；gfx942 使用同一路径。prefix caching 保持开启" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3332