From bc2444ed3e261e0c4c654682e7132e89bfb0bef1 Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Wed, 14 Oct 2026 22:25:49 -0500 Subject: [PATCH 1/3] feat(agentx): bump Kimi-K3 FP4 MI355X ATOM image and widen the DSpark draft MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move kimik3-fp4-mi355x-atom-agentic-mtp to rocm/atom-dev:nightly_202610131505 and run a 4-token DSpark draft from concurrency 14 up. Concurrency 48, 56 and 72 previously served without a draft model and now run the same draft with ReplaySSM. Concurrency 1 and 4 keep the base 7-token draft. 将 kimik3-fp4-mi355x-atom-agentic-mtp 升级到镜像 rocm/atom-dev:nightly_202610131505，并从并发 14 起使用 4 个草稿 token 的 DSpark。并发 48、56、72 原先不启用草稿模型，现在同样启用该草稿并开启 ReplaySSM。并发 1 和 4 保持基础配置的 7 个草稿 token。 Co-Authored-By: Claude Opus 5 --- .../kimik3/atom/mi355x-fp4-mtp/agentic.yaml | 23 ++++++++----------- inferencex-e2e/configs/amd-master.yaml | 6 ++--- inferencex-e2e/perf-changelog.yaml | 10 ++++++++ 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml index 3ee414f245..1884b3a997 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/kimik3/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + container_image: rocm/atom-dev@sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884 enable_multiple_frontends: false observability: enabled: false @@ -74,8 +74,6 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' -# One variant per point. Full graphs cover every verify batch up to 2x CONC -# times the verify window (1 + draft tokens). override_tp8_c1: roles: agg: @@ -99,7 +97,7 @@ override_tp8_dcp8_c14_lmcache: agg: args: decode-context-parallel-size: 8 - num-speculative-tokens: 3 + num-speculative-tokens: 4 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]' extra-kv-connectors: &lmcache_offload - kv_connector: lmcache_offload @@ -135,7 +133,7 @@ override_tp8_dcp8_c16_lmcache: agg: args: decode-context-parallel-size: 8 - num-speculative-tokens: 3 + num-speculative-tokens: 4 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128]' extra-kv-connectors: *lmcache_offload env: @@ -155,15 +153,14 @@ override_tp8_dcp8_c48_lmcache: agg: args: decode-context-parallel-size: 8 - method: null - draft-model: null - num-speculative-tokens: null + num-speculative-tokens: 4 max-num-seqs: 96 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96]' extra-kv-connectors: *lmcache_offload env: <<: *lmcache_env LMCACHE_MAX_LOCAL_CPU_SIZE: '128' + ATOM_ENABLE_REPLAYSSM: '1' ATOM_PREFILL_DECODE_INTERVAL: '4' ATOM_PREFILL_DELAYER_MAX_QUEUE_MS: '5000' benchmark: @@ -178,15 +175,14 @@ override_tp8_dcp8_c56_lmcache: agg: args: decode-context-parallel-size: 8 - method: null - draft-model: null - num-speculative-tokens: null + num-speculative-tokens: 4 max-num-seqs: 112 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112]' extra-kv-connectors: *lmcache_offload env: <<: *lmcache_env LMCACHE_MAX_LOCAL_CPU_SIZE: '192' + ATOM_ENABLE_REPLAYSSM: '1' AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' ATOM_PREFILL_DECODE_INTERVAL: '4' ATOM_PREFILL_DELAYER_MAX_QUEUE_MS: '5000' @@ -202,15 +198,14 @@ override_tp8_dcp8_c72_lmcache: agg: args: decode-context-parallel-size: 8 - method: null - draft-model: null - num-speculative-tokens: null + num-speculative-tokens: 4 max-num-seqs: 144 cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144]' extra-kv-connectors: *lmcache_offload env: <<: *lmcache_env LMCACHE_MAX_LOCAL_CPU_SIZE: '192' + ATOM_ENABLE_REPLAYSSM: '1' AITER_REUSE_IDENTICAL_COMM_GROUPS: '1' ATOM_PREFILL_DECODE_INTERVAL: '4' ATOM_PREFILL_DELAYER_MAX_QUEUE_MS: '5000' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 24f60ae790..81a43bc38e 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -696,7 +696,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # speculative decoding. Acceptance is pinned to the committed golden curve in # infx/golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml # at both draft lengths used here: 7 draft tokens -> AL 3.84 at concurrency 1 and 4, -# 3 draft tokens -> AL 3.00 at concurrency 14 and 16. +# 4 draft tokens -> AL 3.36 from concurrency 14 up. # Companion to kimik3-fp4-mi355x-vllm-agentic-mtp: same checkpoint, same runner. # TP8 only -- the 1.56 TB MXFP4 checkpoint is ~195 GB/GPU and TP4 cannot load, # which is also why there is no DP-attention arm. @@ -706,7 +706,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # contexts, so dcp-size 8 shards the KV read across # all 8 GPUs and the LMCache DRAM tier backs the # paged KV. -# throughput (48, 56, 72) same shape, no draft model. +# throughput (48, 56, 72) same shape, with the draft model. # Two utilization blocks because the per-rank LMCache pool differs, and the # launcher divides the aggregate budget by TP: # 0.343 -> 1028 GB aggregate -> 128 GB/rank (conc 14 through 48). @@ -714,7 +714,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: # wider in-flight window needs the deeper pool to keep the paged KV # resident. kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202609251613 + image: rocm/atom-dev:nightly_202610131505 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index de1606360c..f4aeda6a47 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9832,3 +9832,13 @@ - "Append the GB300 DeepSeek-V4.1-Flash Dynamo + vLLM AgentX 8P1D point at concurrency 4096 to the existing curve." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4071 append-only: true + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission to image rocm/atom-dev:nightly_202610131505." + - "Widen the DSpark draft from 3 to 4 tokens at concurrency 14 and 16, which moves the golden acceptance length from 3.00 to 3.36." + - "Run the same 4-token DSpark draft with ReplaySSM at concurrency 48, 56 and 72, which previously served without a draft model. Concurrency 1 and 4 keep the 7-token draft." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING From 6224baff5b7fac460cc9d60b852ed2c611d0343c Mon Sep 17 00:00:00 2001 From: Guanbao Yu Date: Wed, 14 Oct 2026 22:27:19 -0500 Subject: [PATCH 2/3] chore(changelog): point the Kimi-K3 ATOM AgentX entry at its PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 填入 Kimi-K3 ATOM AgentX 变更记录的 PR 链接。 Co-Authored-By: Claude Opus 5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f4aeda6a47..9e2de45cca 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9841,4 +9841,4 @@ - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission to image rocm/atom-dev:nightly_202610131505." - "Widen the DSpark draft from 3 to 4 tokens at concurrency 14 and 16, which moves the golden acceptance length from 3.00 to 3.36." - "Run the same 4-token DSpark draft with ReplaySSM at concurrency 48, 56 and 72, which previously served without a draft model. Concurrency 1 and 4 keep the 7-token draft." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PENDING + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4085 From 4d7b7b8f7a73c73e1a5aa7e5c4335fafd544cc61 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Thu, 15 Oct 2026 10:27:15 -0700 Subject: [PATCH 3/3] perf(amd): Kimi-K3 MI355X ATOM AgentX on nightly_202610131505 with a 4-token DSpark draft from c14 up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move the MI355X Kimi-K3 FP4 ATOM AgentX submission from image rocm/atom-dev:nightly_202609251613 (ATOM 0.1.7.dev36+g68e0df5e7) to rocm/atom-dev:nightly_202610131505 (ATOM 0.1.8.dev44+g661ef7661), and pin the recipe frontend to that tag's digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884. - Keep the published concurrency set [1, 4, 14, 16, 48, 56, 72]; no points are added or removed. - Widen the DSpark draft from 3 to 4 tokens at concurrency 14 and 16, which moves the golden acceptance length from 3.00 to 3.36. - Serve concurrency 48, 56 and 72 with the same 4-token DSpark draft and ReplaySSM (ATOM_ENABLE_REPLAYSSM=1) instead of no draft model. Concurrency 1 and 4 keep the 7-token draft at AL 3.84. max-num-seqs, the CUDA-graph ladders, dcp-size, LMCache sizing and the prefill-delayer settings are unchanged. - The new image carries the Kimi-K3 prefill gated RMSNorm pipeline (ROCm/ATOM#2458) and moves the LMCache wheel that the in-process lmcache_offload connector imports from 0.5.6.dev98 to 0.5.6.dev139 (ROCm/ATOM#2438). - No change to the Inferact/Kimi-K3-DSpark draft's precision: the checkpoint is BF16 throughout, and at ATOM 661ef7661 online_quant_config still excludes every draft linear (layers.*, context_proj), so its weights and activations stay BF16 at every drafted point, including the newly drafted concurrency 48, 56 and 72, and it shares the target's FP8 KV cache. - 将 MI355X Kimi-K3 FP4 ATOM AgentX 提交的镜像由 rocm/atom-dev:nightly_202609251613（ATOM 0.1.7.dev36+g68e0df5e7）改为 rocm/atom-dev:nightly_202610131505（ATOM 0.1.8.dev44+g661ef7661），并将配方 frontend 固定到该 tag 的 digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884。 - 发布的并发集合 [1, 4, 14, 16, 48, 56, 72] 保持不变，没有新增或删除任何点。 - 并发 14 和 16 的 DSpark 草稿由 3 个 token 扩到 4 个，golden 接受长度由 3.00 变为 3.36。 - 并发 48、56 和 72 改为使用同样的 4 token DSpark 草稿并开启 ReplaySSM（ATOM_ENABLE_REPLAYSSM=1），此前不使用草稿模型。并发 1 和 4 保持 7 token 草稿（AL 3.84）。max-num-seqs、CUDA graph 尺寸列表、dcp-size、LMCache 容量和 prefill-delayer 设置均不变。 - 新镜像包含 Kimi-K3 prefill 门控 RMSNorm 流水线（ROCm/ATOM#2458），并将进程内 lmcache_offload 连接器所导入的 LMCache wheel 由 0.5.6.dev98 升级到 0.5.6.dev139（ROCm/ATOM#2438）。 - Inferact/Kimi-K3-DSpark 草稿的精度无变化：该 checkpoint 全部为 BF16，且在 ATOM 661ef7661 中 online_quant_config 仍排除所有草稿线性层（layers.*、context_proj），因此在所有使用草稿的点上（包括新启用草稿的并发 48、56 和 72），其权重和激活均保持 BF16，并与目标模型共享 FP8 KV cache。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e2c86de1e2..6d516f6c96 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9849,7 +9849,10 @@ scenario-type: - agentic-coding description: - - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission to image rocm/atom-dev:nightly_202610131505." + - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission from image rocm/atom-dev:nightly_202609251613 (ATOM 0.1.7.dev36+g68e0df5e7) to rocm/atom-dev:nightly_202610131505 (ATOM 0.1.8.dev44+g661ef7661), and pin the recipe frontend to that tag's digest sha256:b78300a6caf6c1fec3b1f27b4f7333326c9d34911bfc7ae229d13d2ebbc43884." + - "Keep the published concurrency set [1, 4, 14, 16, 48, 56, 72]; no points are added or removed." - "Widen the DSpark draft from 3 to 4 tokens at concurrency 14 and 16, which moves the golden acceptance length from 3.00 to 3.36." - - "Run the same 4-token DSpark draft with ReplaySSM at concurrency 48, 56 and 72, which previously served without a draft model. Concurrency 1 and 4 keep the 7-token draft." + - "Serve concurrency 48, 56 and 72 with the same 4-token DSpark draft and ReplaySSM (ATOM_ENABLE_REPLAYSSM=1) instead of no draft model. Concurrency 1 and 4 keep the 7-token draft at AL 3.84. max-num-seqs, the CUDA-graph ladders, dcp-size, LMCache sizing and the prefill-delayer settings are unchanged." + - "The new image carries the Kimi-K3 prefill gated RMSNorm pipeline (ROCm/ATOM#2458) and moves the LMCache wheel that the in-process lmcache_offload connector imports from 0.5.6.dev98 to 0.5.6.dev139 (ROCm/ATOM#2438)." + - "No change to the Inferact/Kimi-K3-DSpark draft's precision: the checkpoint is BF16 throughout, and at ATOM 661ef7661 online_quant_config still excludes every draft linear (layers.*, context_proj), so its weights and activations stay BF16 at every drafted point, including the newly drafted concurrency 48, 56 and 72, and it shares the target's FP8 KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4085