From 5032af0ef6b8f969642628151ee2c4e00430fab8 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:33:35 -0400 Subject: [PATCH 1/4] =?UTF-8?q?[Klaud=20Cold]=20Add=20MI300X=20TP4=20and?= =?UTF-8?q?=20TP2=20DeepSeek-V4.1-Flash=20vLLM=20AgentX=20arms=20with=20En?= =?UTF-8?q?gram=20host=20offload=20/=20=E6=96=B0=E5=A2=9E=20Engram=20?= =?UTF-8?q?=E4=B8=BB=E6=9C=BA=E5=8D=B8=E8=BD=BD=E7=9A=84=20MI300X=20TP4=20?= =?UTF-8?q?=E4=B8=8E=20TP2=20DeepSeek-V4.1-Flash=20vLLM=20AgentX=20?= =?UTF-8?q?=E8=87=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic/dsv41flash_fp4_mi300x_mtp.sh | 44 ++++++++++++++++--- configs/amd-master.yaml | 14 +++++- perf-changelog.yaml | 14 ++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh index 8e62295ab2..9024767ea7 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh @@ -4,8 +4,9 @@ set -eo pipefail # DeepSeek-V4.1-Flash on MI300X (gfx942): native DSpark, GPU-resident KV. A copy # of the validated MI355X arm; gfx942 has no FP4 MFMA, so the MXFP4 experts run # through whichever AITER MoE kernel vLLM's selector supports on this arch, and -# the Engram tables stay row-sharded on GPU per the upstream AMD defaults -# (192 GB x TP8 holds the 511 GB checkpoint with room for KV). +# the Engram tables are row-sharded on GPU at TP8 and offloaded to pinned +# host memory below TP8 (192 GB x TP8 holds the 511 GB checkpoint with +# room for KV; TP4 and TP2 do not). # https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION @@ -58,6 +59,32 @@ while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2 CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) done +# vllm-project/vllm#57491 widened the two is_cuda() gates to is_cuda_alike(), so +# this image resolves an Engram config on gfx942 and, because cpu_offload now +# defaults on through VLLM_PLE_CPU_OFFLOAD, an explicit value is required rather +# than the default. Keep the 183 GiB of tables resident at TP8 so that curve +# stays comparable with the validated concurrency 1-32 run; offload below TP8, +# where half or more of the checkpoint already fills a 192 GB card. +if (( TP < 8 )); then + ENGRAM_CONFIG='{"cpu_offload":true}' +else + ENGRAM_CONFIG='{"cpu_offload":false}' +fi + +# The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits +# buffer at startup: 32 GiB at 16384, 16 GiB at 8192, 8 GiB at 4096. TP2 leaves +# ~145 GiB of offloaded weights on every rank, so halve the buffer and cap the +# scheduler there to keep a usable KV pool. Same shape as the Blackwell TP2 +# arms, which validated this cap set on B200, GB200 and GB300. +BATCHED_TOKENS=8192 +if (( TP == 2 )); then + BATCHED_TOKENS=4096 + MAX_NUM_SEQS=$((2 * CONC)) + if (( MAX_NUM_SEQS > 256 )); then MAX_NUM_SEQS=256; fi + if (( MAX_NUM_SEQS < 16 )); then MAX_NUM_SEQS=16; fi + if (( CAPTURE_SIZE > 512 )); then CAPTURE_SIZE=512; fi +fi + # Use the runner-specific port assigned by launch_mi300x-amd.sh. export AIPERF_SERVER_URL="http://localhost:${PORT}" export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" @@ -81,6 +108,7 @@ VLLM_CMD=( --tokenizer-mode deepseek_v41 --tool-call-parser deepseek_v41 --enable-auto-tool-choice --reasoning-parser deepseek_v41 + --engram-config "$ENGRAM_CONFIG" # aiter: auto selection picked the unfused Triton MoE (TRITON_UNFUSED) on # gfx942 in run 35306398350 and still segfaulted at piecewise capture, so # the MoE kernel was not the culprit; keep the upstream recipe's name, @@ -91,9 +119,15 @@ VLLM_CMD=( --max-model-len 1048576 --max-num-seqs "$MAX_NUM_SEQS" --max-cudagraph-capture-size "$CAPTURE_SIZE" - # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits - # buffer at startup (16 GiB at 8192); 16384 would cost 32 GiB of a 192 GB card. - --max-num-batched-tokens 8192 + --max-num-batched-tokens "$BATCHED_TOKENS" + # vllm-project/vllm#56227 turned SWA bounded replay on by default between the + # eed1f3d0 pin and this one. It relies on a window clamp that landed in the + # FlashInfer and FlashMLA kernels; the ROCm sparse SWA path only gained the + # replay_start kwarg, which crashed every gfx950 point with + # HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a replay + # start. gfx942 runs the same ROCm sparse path. Drop this once ROCm clamps + # too; prefix caching stays on. + --no-swa-bounded-replay --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' --disable-uvicorn-access-log ) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 6bb5359a78..327c1399f0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1168,7 +1168,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: # golden AL 3.51 for throughput, real block verification for evals, adaptive # verification off on ROCm. dsv41flash-fp4-mi300x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657 + image: vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi300x-amd @@ -1179,8 +1179,18 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.60 search-space: - # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. + # The recipe sets engram cpu_offload per TP: off at TP8 so this curve stays + # comparable with the validated concurrency 1-32 run, on below TP8 where + # the tables would not fit beside the checkpoint share on a 192 GB card. + # vllm-project/vllm#57491 widened the two is_cuda() gates to + # is_cuda_alike(), so engram_config resolves on gfx942 in this image. - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + # TP4 halves the GPUs per server. With the Engram tables in pinned host + # memory the resident weights are ~81 GiB per rank. + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + # TP2 quarters them. Resident weights rise to ~145 GiB per rank, so the + # recipe halves the indexer buffer and caps the scheduler here. + - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8] } # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 # GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9419a68f0b..f76733fcea 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8531,3 +8531,17 @@ - "为 GB300 vLLM DeepSeek-V4.1-Flash AgentX 配方在现有 TP4 臂旁新增 TP2 臂，Engram 表继续通过 --engram-config cpu_offload 放在固定页主机 DRAM；每张 277 GiB GPU 的权重升至约 175 GiB" - "在 dsv41flash_fp4_vllm_mtp.sh 中将 B200 TP2 的上限推广到所有 TP2 臂：--max-num-batched-tokens 4096（上游 16384 时 indexer 的 [batched-tokens, 1M] fp8 缓冲区达 32 GiB）、--max-num-seqs 为并发的两倍（16-256）、CUDA graph 捕获上限 512，为每张 GPU 留出约 36 GiB KV；TP4 与 TP8 臂沿用上游默认值" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3321 +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Add TP4 and TP2 arms with Engram CPU offload to the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the 183 GiB of tables GPU-resident. With the tables in pinned host memory the resident weights are ~81 GiB per rank at TP4 and ~145 GiB per rank at TP2, against 192 GB per card" + - "Cap the scheduler at TP2 only: halve the sparse-attention indexer buffer to 4096 batched tokens (8 GiB rather than 16 GiB of a [batched-tokens, 1M] fp8 logits allocation), scale max-num-seqs to 2*CONC clamped to [16, 256] and cap cudagraph capture at 512. Same cap set the Blackwell TP2 arms validated on B200, GB200 and GB300. TP2 concurrency stops at 8 because ~145 GiB of weights leaves the smallest KV pool of any arm here" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the same commit the MI355X arm moved to; it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" + - "为 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 与 TP2 臂，同时保留 183 GiB Engram 表驻留 GPU 的现有 TP8 臂；表移至固定页主机内存后，TP4 每 rank 常驻权重约 81 GiB，TP2 约 145 GiB，而每张卡为 192 GB" + - "仅在 TP2 收紧调度器：将稀疏注意力 indexer 缓冲区减半至 4096 batched tokens（[batched-tokens, 1M] fp8 logits 分配由 16 GiB 降至 8 GiB），max-num-seqs 取 2*CONC 并夹在 [16, 256]，cudagraph capture 上限 512。该组参数与 Blackwell TP2 臂在 B200、GB200、GB300 上验证过的一致。TP2 并发止于 8，因为约 145 GiB 权重使其 KV 池为本配方中最小" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂相同的 commit）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" + - "新增 --no-swa-bounded-replay：vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启，其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失，曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃；gfx942 使用同一路径。prefix caching 保持开启" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336 From f3933917ded8d1ddde13fa69e3650b1aebd8d530 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:18:03 -0400 Subject: [PATCH 2/4] =?UTF-8?q?Repin=20dsv41flash-mi300x-tp2-tp4-engram-of?= =?UTF-8?q?fload=20to=20the=20ROCm=2010.0=20nightly=20channel=20/=20?= =?UTF-8?q?=E5=B0=86=E9=95=9C=E5=83=8F=E9=87=8D=E6=96=B0=E5=9B=BA=E5=AE=9A?= =?UTF-8?q?=E5=88=B0=20ROCm=2010.0=20nightly=20=E6=B8=A0=E9=81=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match SemiAnalysisAI/InferenceX#3326, which moved MI355X to nightly-rocm100-3df4ae15 on the ROCm 10.0 channel. Same vLLM commit, different ROCm runtime. 与 SemiAnalysisAI/InferenceX#3326 保持一致：相同的 vLLM commit，ROCm 10.0 运行时。 Co-Authored-By: Claude Opus 5 (1M context) --- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 327c1399f0..d685b6c614 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1168,7 +1168,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: # golden AL 3.51 for throughput, real block verification for evals, adaptive # verification off on ROCm. dsv41flash-fp4-mi300x-vllm-agentic-dspark: - image: vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984 + image: vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi300x-amd diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f76733fcea..19c0f2dc9d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8538,10 +8538,10 @@ description: - "Add TP4 and TP2 arms with Engram CPU offload to the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe, keeping the existing TP8 arm with the 183 GiB of tables GPU-resident. With the tables in pinned host memory the resident weights are ~81 GiB per rank at TP4 and ~145 GiB per rank at TP2, against 192 GB per card" - "Cap the scheduler at TP2 only: halve the sparse-attention indexer buffer to 4096 batched tokens (8 GiB rather than 16 GiB of a [batched-tokens, 1M] fp8 logits allocation), scale max-num-seqs to 2*CONC clamped to [16, 256] and cap cudagraph capture at 512. Same cap set the Blackwell TP2 arms validated on B200, GB200 and GB300. TP2 concurrency stops at 8 because ~145 GiB of weights leaves the smallest KV pool of any arm here" - - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the same commit the MI355X arm moved to; it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" + - "Repin the image from the expired nightly-eed1f3d0 tag to vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984, the ROCm 10.0 nightly channel build of the same commit the MI355X arm moved to in SemiAnalysisAI/InferenceX#3326 (digest sha256:eccb72b74b8c04ce7406d9212e200b129f58e24be642795a857f755a94fdb3a1, published 2026-09-21T05:50:03Z); it carries vllm-project/vllm#57491, which widened the two is_cuda() gates to is_cuda_alike() so engram_config resolves on gfx942, and because cpu_offload now defaults on via VLLM_PLE_CPU_OFFLOAD the recipe sets it explicitly per TP" - "Add --no-swa-bounded-replay: vllm-project/vllm#56227 turned SWA bounded replay on by default between the two pins and relies on a window clamp the ROCm sparse SWA path does not have, which crashed every gfx950 point with HSA_STATUS_ERROR_MEMORY_FAULT; gfx942 runs the same path. Prefix caching stays on" - "为 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方新增启用 Engram CPU 卸载的 TP4 与 TP2 臂，同时保留 183 GiB Engram 表驻留 GPU 的现有 TP8 臂；表移至固定页主机内存后，TP4 每 rank 常驻权重约 81 GiB，TP2 约 145 GiB，而每张卡为 192 GB" - "仅在 TP2 收紧调度器：将稀疏注意力 indexer 缓冲区减半至 4096 batched tokens（[batched-tokens, 1M] fp8 logits 分配由 16 GiB 降至 8 GiB），max-num-seqs 取 2*CONC 并夹在 [16, 256]，cudagraph capture 上限 512。该组参数与 Blackwell TP2 臂在 B200、GB200、GB300 上验证过的一致。TP2 并发止于 8，因为约 145 GiB 权重使其 KV 池为本配方中最小" - - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂相同的 commit）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" + - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" - "新增 --no-swa-bounded-replay：vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启，其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失，曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃；gfx942 使用同一路径。prefix caching 保持开启" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336 From d9843b32aa44a1c05f680cdefc151c4e82ffbc75 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:12:24 -0400 Subject: [PATCH 3/4] =?UTF-8?q?Drop=20the=20MI300X=20TP2=20arm:=20measured?= =?UTF-8?q?=20infeasible=20on=20a=20192=20GB=20card=20/=20=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=20MI300X=20TP2=20=E8=87=82=EF=BC=9A=E5=9C=A8=20192=20?= =?UTF-8?q?GB=20=E5=8D=A1=E4=B8=8A=E5=AE=9E=E6=B5=8B=E4=B8=8D=E5=8F=AF?= =?UTF-8?q?=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 35671005506 reported "Available KV cache memory: -13.51 GiB" at TP2 concurrency 1 and the engine refused to start, even with the indexer buffer already halved to 4096 batched tokens. TP8 and TP4 are unaffected; TP4 passed at concurrency 1 and 32 in the same run. 运行 35671005506 在 TP2 并发 1 下报告 "Available KV cache memory: -13.51 GiB" 并拒绝启动，此时 indexer 缓冲区已减半至 4096。TP8 与 TP4 不受影响。 Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic/dsv41flash_fp4_mi300x_mtp.sh | 15 ++++----------- configs/amd-master.yaml | 8 +++++--- perf-changelog.yaml | 10 ++++++++++ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh index 9024767ea7..046a22fbea 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh @@ -72,18 +72,11 @@ else fi # The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits -# buffer at startup: 32 GiB at 16384, 16 GiB at 8192, 8 GiB at 4096. TP2 leaves -# ~145 GiB of offloaded weights on every rank, so halve the buffer and cap the -# scheduler there to keep a usable KV pool. Same shape as the Blackwell TP2 -# arms, which validated this cap set on B200, GB200 and GB300. +# buffer at startup: 16 GiB at 8192, 8 GiB at 4096. 8192 leaves TP4 ~64 GiB of +# KV per rank after the ~81 GiB of offloaded weights. There is no TP2 arm here: +# run 35671005506 measured "Available KV cache memory: -13.51 GiB" at TP2 even +# with the buffer halved, so a 192 GB card cannot serve this checkpoint at TP2. BATCHED_TOKENS=8192 -if (( TP == 2 )); then - BATCHED_TOKENS=4096 - MAX_NUM_SEQS=$((2 * CONC)) - if (( MAX_NUM_SEQS > 256 )); then MAX_NUM_SEQS=256; fi - if (( MAX_NUM_SEQS < 16 )); then MAX_NUM_SEQS=16; fi - if (( CAPTURE_SIZE > 512 )); then CAPTURE_SIZE=512; fi -fi # Use the runner-specific port assigned by launch_mi300x-amd.sh. export AIPERF_SERVER_URL="http://localhost:${PORT}" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 324653c347..c7f5d254b0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1188,9 +1188,11 @@ dsv41flash-fp4-mi300x-vllm-agentic-dspark: # TP4 halves the GPUs per server. With the Engram tables in pinned host # memory the resident weights are ~81 GiB per rank. - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } - # TP2 quarters them. Resident weights rise to ~145 GiB per rank, so the - # recipe halves the indexer buffer and caps the scheduler here. - - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8] } + # No TP2 arm on this SKU. Measured in run 35671005506: at TP2 the resident + # weights are ~145 GiB per rank, and even with the indexer buffer halved + # to 4096 batched tokens the engine reported "Available KV cache memory: + # -13.51 GiB" and refused to start. A 192 GB card cannot hold this + # checkpoint at TP2 with a usable KV pool; MI325X does it on 256 GB. # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 # GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4db254d64c..096fc232fd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8602,3 +8602,13 @@ - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" - "新增 --no-swa-bounded-replay：vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启，其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失，曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃；gfx942 使用同一路径。prefix caching 保持开启" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336 +- config-keys: + - dsv41flash-fp4-mi300x-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Drop the TP2 arm from the MI300X vLLM DeepSeek-V4.1-Flash AgentX recipe: measured infeasible. In Actions run 35671005506 the TP2 concurrency-1 point reported \"Available KV cache memory: -13.51 GiB\" and the engine refused to start with \"No available memory for the cache blocks\", even with the sparse-attention indexer buffer already halved to 4096 batched tokens. At TP2 the Engram-offloaded resident weights are ~145 GiB per rank, which a 192 GB card cannot hold alongside a usable 1M-context KV pool. Raising gpu-memory-utilization to 0.95 recovers ~9 GiB and halving the buffer again ~4 GiB, which still leaves effectively no KV, so the arm is removed rather than tuned. MI325X runs the same TP2 configuration on 256 GB per card" + - "The TP8 and TP4 arms are unaffected and stay. TP4 passed at concurrency 1 and 32 in the same run with the tables offloaded, at ~81 GiB per rank" + - "从 MI300X vLLM DeepSeek-V4.1-Flash AgentX 配方中移除 TP2 臂：经实测不可行。在 Actions 运行 35671005506 中，TP2 并发 1 数据点报告 \"Available KV cache memory: -13.51 GiB\"，引擎以 \"No available memory for the cache blocks\" 拒绝启动，且此时稀疏注意力 indexer 缓冲区已减半至 4096 batched tokens。TP2 下经 Engram 卸载后每 rank 常驻权重约 145 GiB，192 GB 的卡无法在容纳该权重的同时保留可用的 1M 上下文 KV 池。将 gpu-memory-utilization 提高到 0.95 可回收约 9 GiB，再次将缓冲区减半可回收约 4 GiB，仍几乎不剩 KV，因此选择移除该臂而非调参。MI325X 以每卡 256 GB 运行相同的 TP2 配置" + - "TP8 与 TP4 臂不受影响并保留。TP4 在同一运行中以每 rank 约 81 GiB 在并发 1 与 32 通过" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3336 From 9c62c833d7496575c628df41b2505992eea3f765 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 22 Sep 2026 08:21:53 -0400 Subject: [PATCH 4/4] chore: refresh PR #3336 for sweep reuse [skip-sweep] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync with origin/main after the green sweep run 35678619120; the reuse gate authorizes that run on this head. 在绿色 sweep 运行 35678619120 之后与 origin/main 同步；reuse gate 在此 head 上授权该运行。 Co-Authored-By: Claude Opus 5 (1M context)