From 2bf45f6e01d3bcbe679ca3df343cce12779091d0 Mon Sep 17 00:00:00 2001 From: zhuyuhua-v Date: Mon, 19 Oct 2026 08:50:10 +0000 Subject: [PATCH 1/4] feat(agentx): retune GLM5.2 FP4 MI355X ATOM recipe on _1018 docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track the GLM-5.2 standalone recipe of ROCm/ATOM PR 2632 on rocm/atom-dev:nightly_202610181506. - TP4 C1/C2 run the GLM-5.2 mono decode at 3 draft tokens (AL 2.99); every other point sets ATOM_MONO_ENABLE=0, which ATOM defaults on. - C1 joins the TP4 arm; C4/C8 stay at MTP5 and C10 at MTP4. - Restore the TP4+DCP4 arm at C16-C48 with ATOM_DCP_PREFILL_QREP=1 and the LMCache DRAM tier at 256 GiB per rank via extra-kv-connectors (MTP4, MTP3 at C48). - dram-utilization 0.171 -> 0.74 so TOTAL_CPU_DRAM_GB (1109 GB) covers the 4 x 256 GiB = 1099.5 GB LMCache tier. 中文：按 ROCm/ATOM PR 2632 的 GLM-5.2 单机配方更新 MI355X ATOM AgentX 配置，镜像换为 rocm/atom-dev:nightly_202610181506。TP4 C1/C2 使用 GLM-5.2 mono decode，3 个 draft token（AL 2.99），其余点显式设置 ATOM_MONO_ENABLE=0（ATOM 默认开启）；TP4 新增 C1，C4/C8 保持 MTP5， C10 保持 MTP4。恢复 TP4+DCP4 臂（C16-C48），开启 ATOM_DCP_PREFILL_QREP=1，并通过 extra-kv-connectors 接入每 rank 256 GiB 的 LMCache DRAM 层（MTP4，C48 为 MTP3）。dram-utilization 由 0.171 调为 0.74，使 TOTAL_CPU_DRAM_GB（1109 GB）覆盖 4 x 256 GiB = 1099.5 GB 的 LMCache 层。 Signed-off-by: zhuyuhua-v --- .../glm5.2/atom/mi355x-fp4-mtp/agentic.yaml | 124 +++++++++++++++++- inferencex-e2e/configs/amd-master.yaml | 24 ++-- inferencex-e2e/perf-changelog.yaml | 12 ++ 3 files changed, 145 insertions(+), 15 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 853e2fdfdc..7e7efe9ca2 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + container_image: rocm/atom-dev@sha256:1bb1909ada9e3d178e315a3f89d041c05c7e90e7cc32de8d507ff5ab4cd4702c enable_multiple_frontends: false observability: enabled: false @@ -33,13 +33,32 @@ base: PYTHONNOUSERSITE: '1' AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: '1' + # GLM-5.2 mono decode is on by default in ATOM; only TP4 C1/C2 run it. + ATOM_MONO_ENABLE: '0' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' -# One variant per point. Admission is 2x CONC with graphs captured up to it. The -# draft is five tokens through concurrency 8 and four above it. +# One variant per point. Admission is 2x CONC. TP4 C1/C2 run mono decode at +# three draft tokens; C4/C8 draft five and C10 four on the production path. The +# DCP4 arm shards the decode KV, adds the LMCache DRAM tier at 256 GiB per rank +# (1024 GiB, 1099.5 GB per server, within the 1109 GB TP4 share at +# dram-utilization 0.74) and drafts four tokens, three at C48. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' + benchmark: + env: + CONC: '1' + override_tp4_c2: roles: agg: @@ -47,7 +66,9 @@ override_tp4_c2: args: max-num-seqs: 4 cudagraph-capture-sizes: '[1,2,4]' - num-speculative-tokens: 5 + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' benchmark: env: CONC: '2' @@ -88,6 +109,101 @@ override_tp4_c10: env: CONC: '10' +override_tp4_dcp4_c16_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32]' + num-speculative-tokens: 4 + extra-kv-connectors: &lmcache_offload + - kv_connector: lmcache_offload + kv_role: offload + env: &dcp_lmcache_env + # DCP sparse prefill builds the group's query heads locally. + ATOM_DCP_PREFILL_QREP: '1' + PYTHONHASHSEED: '0' + LMCACHE_LOCAL_CPU: 'True' + LMCACHE_NUMA_MODE: auto + LMCACHE_MAX_LOCAL_CPU_SIZE: '256' + LMCACHE_CHUNK_SIZE: '256' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c24_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 48 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48]' + num-speculative-tokens: 4 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c32_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 64 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64]' + num-speculative-tokens: 4 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c40_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 80 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80]' + num-speculative-tokens: 4 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c48_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 96 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96]' + num-speculative-tokens: 3 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + override_tp8_c1: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index a8ea888081..635a135b9e 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1462,14 +1462,14 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative -# decoding, matching the standalone recipe (ROCm/ATOM PR 2345). Small -# concurrency (C2-C10) is TP4-only on the GPU prefix cache with no KV offload; -# large concurrency (C16-C48) adds DCP4 to shard the decode KV and keeps the -# LMCache DRAM offload. The script resolves the draft depth and forced golden AL -# per concurrency point (K5/3.61 for C2-C8, K4/3.33 for C10-C40, K3/2.99 for -# C48). The TP8 arm stays GPU-resident with no KV offload. +# decoding, matching the standalone recipe (ROCm/ATOM PR 2632). Small +# concurrency (C1-C10) is TP4-only on the GPU prefix cache with no KV offload; +# C1/C2 run the GLM-5.2 mono decode at K3. Large concurrency (C16-C48) adds DCP4 +# to shard the decode KV and the LMCache DRAM tier. The recipe sets the draft +# depth per point (K3 for C1/C2 and C48, K5 for C4/C8, K4 for C10-C40). The TP8 +# arm stays GPU-resident with no KV offload. glm5.2-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202609211553 + image: rocm/atom-dev:nightly_202610181506 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds @@ -1479,11 +1479,13 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4-mtp scenarios: agentic-coding: - # dram-utilization 0.171 -> TOTAL_CPU_DRAM_GB=256 per TP4 rank (recipe's - # LMCACHE_MAX_LOCAL_CPU_SIZE=256): min(3079397,2861022)MiB*0.171*4/8 = 256. - - dram-utilization: 0.171 + # dram-utilization 0.74 -> TOTAL_CPU_DRAM_GB=1109 per TP4 server + # (2861022 MiB * 0.74 * 4/8); the recipe's LMCache tier is 256 GiB x 4 ranks + # = 1099.5 GB. + - dram-utilization: 0.74 search-space: - - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 4, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [16, 24, 32, 40, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } dsv4-fp4-mi355x-sglang-agentic-mtp: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b88d216b21..f06fcd353f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10139,3 +10139,15 @@ - "Re-sweep Muse Spark Open B200 AgentX on the full uncapped trace corpus (semianalysis_cc_traces_weka_062126) after #4287 dropped the Muse Spark 256k exception; image and search space are unchanged." - "Set HF_HUB_OFFLINE=1 so every rank resolves the DFlash2 draft from the shared HF cache, matching MI355X; concurrent launches otherwise hit the Hub 429 API rate limit." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4291 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Track the updated GLM-5.2 standalone recipe (ROCm/ATOM PR 2632) on the MI355X ATOM AgentX config, on rocm/atom-dev:nightly_202610181506 (was nightly_202609211553)." + - "TP4 C1 and C2 run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) at 3 draft tokens, golden AL 2.99; C1 joins the TP4 arm. Every other point sets ATOM_MONO_ENABLE=0, which ATOM otherwise defaults on." + - "TP4 C4 and C8 stay at 5 draft tokens (AL 3.61) and C10 at 4 (AL 3.33) on the production path; the TP8 arm at C1, C2 and C4 is unchanged." + - "Restore the TP4+DCP4 arm at C16, C24, C32 and C40 (4 draft tokens, AL 3.33) and C48 (3 draft tokens, AL 2.99) with ATOM_DCP_PREFILL_QREP=1 and the LMCache DRAM tier (0.5.6.dev139+gf22dec28) at 256 GiB per rank through extra-kv-connectors." + - "dram-utilization 0.171 -> 0.74, so TOTAL_CPU_DRAM_GB is 1109 GB per TP4 server, covering the 4 x 256 GiB = 1099.5 GB LMCache tier." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From c919625c6e80d05b9a6a5602f9831327f8444d61 Mon Sep 17 00:00:00 2001 From: zhuyuhua-v Date: Tue, 20 Oct 2026 22:35:13 -0500 Subject: [PATCH 2/4] feat(agentx): cap GLM5.2 FP4 MI355X ATOM TP4 C1/C2 at max-num-seqs CONC Track ROCm/ATOM PR 2632: TP4 C1 and C2 start the server with max-num-seqs 1 and 2 (exactly CONC) and capture graphs only up to it. --- .../glm5.2/atom/mi355x-fp4-mtp/agentic.yaml | 11 ++++++----- inferencex-e2e/perf-changelog.yaml | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 7e7efe9ca2..65ed6c5782 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -40,7 +40,8 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' -# One variant per point. Admission is 2x CONC. TP4 C1/C2 run mono decode at +# One variant per point. Admission is 2x CONC, except TP4 C1/C2, which admit +# exactly CONC with graphs captured up to it. TP4 C1/C2 run mono decode at # three draft tokens; C4/C8 draft five and C10 four on the production path. The # DCP4 arm shards the decode KV, adds the LMCache DRAM tier at 256 GiB per rank # (1024 GiB, 1099.5 GB per server, within the 1109 GB TP4 share at @@ -50,8 +51,8 @@ override_tp4_c1: agg: gpus: 4 args: - max-num-seqs: 2 - cudagraph-capture-sizes: '[1,2]' + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' num-speculative-tokens: 3 env: ATOM_MONO_ENABLE: '1' @@ -64,8 +65,8 @@ override_tp4_c2: agg: gpus: 4 args: - max-num-seqs: 4 - cudagraph-capture-sizes: '[1,2,4]' + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' num-speculative-tokens: 3 env: ATOM_MONO_ENABLE: '1' diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e5bb3b7346..440a914409 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10499,6 +10499,7 @@ description: - "Track the updated GLM-5.2 standalone recipe (ROCm/ATOM PR 2632) on the MI355X ATOM AgentX config, on rocm/atom-dev:nightly_202610181506 (was nightly_202609211553)." - "TP4 C1 and C2 run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) at 3 draft tokens, golden AL 2.99; C1 joins the TP4 arm. Every other point sets ATOM_MONO_ENABLE=0, which ATOM otherwise defaults on." + - "TP4 C1 and C2 cap the server at exactly the concurrency (max-num-seqs 1 and 2, CUDA graphs captured at [1] and [1,2]); every other point admits 2x the concurrency." - "TP4 C4 and C8 stay at 5 draft tokens (AL 3.61) and C10 at 4 (AL 3.33) on the production path; the TP8 arm at C1, C2 and C4 is unchanged." - "Restore the TP4+DCP4 arm at C16, C24, C32 and C40 (4 draft tokens, AL 3.33) and C48 (3 draft tokens, AL 2.99) with ATOM_DCP_PREFILL_QREP=1 and the LMCache DRAM tier (0.5.6.dev139+gf22dec28) at 256 GiB per rank through extra-kv-connectors." - "dram-utilization 0.171 -> 0.74, so TOTAL_CPU_DRAM_GB is 1109 GB per TP4 server, covering the 4 x 256 GiB = 1099.5 GB LMCache tier." From 4cbffd76bb16247420c3b9d8916175e5e5b16c58 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 25 Oct 2026 22:09:58 -0700 Subject: [PATCH 3/4] docs(changelog): rewrite GLM-5.2 FP4 MI355X ATOM block from the diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Image rocm/atom-dev:nightly_202609211553 (ATOM 0.1.7.dev517+g52a6a96fd) -> nightly_202610181506 (ATOM 0.1.8.dev77+gdf86c4829); the atomesh router container_image digest moves from f00a9dc5 (nightly_202609161445) to 1bb1909a, the same nightly as the workers. - Settings follow the GLM-5.2 recipe in ROCm/ATOM PR 2632 (open; recipe document only, measured on df86c4829). - Add TP4 C1 (TP4 now C1-C10) and restore the TP4+DCP4 arm at C16-C48 with the LMCache DRAM tier (0.5.6.dev139+gf22dec28), which #3463 dropped with the other search-space entries without an srt-recipe; TP8 C1-C4 unchanged. - dram-utilization 0.171 -> 0.74, so TOTAL_CPU_DRAM_GB is 1109 GB per TP4 server, covering the 4 x 256 GiB LMCache tier. - TP4 C1 and C2 run GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) at 3 draft tokens (golden AL 2.99), max-num-seqs equal to the concurrency, graphs [1] and [1,2]; C2 was 5 draft tokens, max-num-seqs 4, graphs [1,2,4]. - Every other point sets ATOM_MONO_ENABLE=0, which the new ATOM defaults on; TP4 C4/C8 (5 drafts, AL 3.61), C10 (4, AL 3.33) and TP8 keep their settings. - DCP4 points: decode-context-parallel-size 4, ATOM_DCP_PREFILL_QREP=1, LMCache 256 GiB per rank via extra-kv-connectors lmcache_offload, 2x admission, 4 draft tokens (AL 3.33), 3 at C48 (AL 2.99). - Relevant ATOM changes: GLM-5.2 mono decode (#2479, #2483, #2535); DCP prefill QREP and QREP with MTP (#2437, #2290); DCP prefill fixes with MTP (#2430, #2496, #2506, #2348); MTP verify/draft-graph fixes (#2451, #2367, #2566, #2495); lmcache_offload fixes (#2305, #2339, #2547) and the LMCache pin (#2438). - Draft data types unchanged: the BF16 MTP layer 78 stays BF16 (excluded from ptpc_fp8 by model.layers.78.*), with the target's FP8 KV and FP4 indexer cache; mono decode covers only target layers 0-77. - 镜像 rocm/atom-dev:nightly_202609211553（ATOM 0.1.7.dev517+g52a6a96fd）-> nightly_202610181506（ATOM 0.1.8.dev77+gdf86c4829）；atomesh router 的 container_image digest 从 f00a9dc5（nightly_202609161445）改为 1bb1909a，与 worker 使用同一 nightly。 - 设置与 ROCm/ATOM PR 2632 中的 GLM-5.2 recipe 一致（该 PR 尚未合并，只改 recipe 文档，测量基于 df86c4829）。 - 新增 TP4 C1（TP4 覆盖 C1-C10），并恢复带 LMCache DRAM 层（0.5.6.dev139+gf22dec28）的 TP4+DCP4 C16-C48 分支；该分支曾在 #3463 中随其他没有 srt-recipe 的搜索空间条目一起被删除。TP8 C1-C4 不变。 - dram-utilization 0.171 -> 0.74，每个 TP4 服务的 TOTAL_CPU_DRAM_GB 为 1109 GB，可容纳 4 x 256 GiB 的 LMCache 层。 - TP4 C1 和 C2 以 3 个 draft token（golden AL 2.99）运行 GLM-5.2 mono decode（ATOM_MONO_ENABLE=1），max-num-seqs 等于并发数，graph 为 [1] 和 [1,2]；C2 原为 5 个 draft token、max-num-seqs 4、graph [1,2,4]。 - 其他点位都设置 ATOM_MONO_ENABLE=0（新 ATOM 默认开启）；TP4 C4/C8（5 个 draft，AL 3.61）、C10（4 个，AL 3.33）和 TP8 设置不变。 - DCP4 点位：decode-context-parallel-size 4，ATOM_DCP_PREFILL_QREP=1，通过 extra-kv-connectors lmcache_offload 每个 rank 使用 256 GiB LMCache，准入为 2 倍并发，4 个 draft token（AL 3.33），C48 为 3 个（AL 2.99）。 - 相关 ATOM 改动：GLM-5.2 mono decode（#2479、#2483、#2535）；DCP prefill QREP 及 QREP 与 MTP 同时使用（#2437、#2290）；DCP prefill 与 MTP 的修复（#2430、#2496、#2506、#2348）；MTP verify/draft graph 修复（#2451、#2367、#2566、#2495）；lmcache_offload 修复（#2305、#2339、#2547）和 LMCache 版本固定（#2438）。 - Draft 数据类型不变：BF16 的 MTP 第 78 层保持 BF16（被 model.layers.78.* 排除在 ptpc_fp8 之外），与 target 共用 FP8 KV cache 和 FP4 indexer cache；mono decode 只覆盖 target 第 0-77 层。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3de1176786..d4fc3aa118 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11216,10 +11216,13 @@ scenario-type: - agentic-coding description: - - "Track the updated GLM-5.2 standalone recipe (ROCm/ATOM PR 2632) on the MI355X ATOM AgentX config, on rocm/atom-dev:nightly_202610181506 (was nightly_202609211553)." - - "TP4 C1 and C2 run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) at 3 draft tokens, golden AL 2.99; C1 joins the TP4 arm. Every other point sets ATOM_MONO_ENABLE=0, which ATOM otherwise defaults on." - - "TP4 C1 and C2 cap the server at exactly the concurrency (max-num-seqs 1 and 2, CUDA graphs captured at [1] and [1,2]); every other point admits 2x the concurrency." - - "TP4 C4 and C8 stay at 5 draft tokens (AL 3.61) and C10 at 4 (AL 3.33) on the production path; the TP8 arm at C1, C2 and C4 is unchanged." - - "Restore the TP4+DCP4 arm at C16, C24, C32 and C40 (4 draft tokens, AL 3.33) and C48 (3 draft tokens, AL 2.99) with ATOM_DCP_PREFILL_QREP=1 and the LMCache DRAM tier (0.5.6.dev139+gf22dec28) at 256 GiB per rank through extra-kv-connectors." + - "Image rocm/atom-dev:nightly_202609211553 (ATOM 0.1.7.dev517+g52a6a96fd) -> nightly_202610181506 (ATOM 0.1.8.dev77+gdf86c4829). The atomesh router container_image digest moves from f00a9dc5 (nightly_202609161445) to 1bb1909a, the same nightly as the workers." + - "Settings follow the GLM-5.2 recipe in ROCm/ATOM PR 2632 (open; recipe document only, measured on df86c4829)." + - "Add TP4 C1, so the TP4 arm covers C1-C10. Restore the TP4+DCP4 arm at C16, C24, C32, C40 and C48 with the LMCache DRAM tier (0.5.6.dev139+gf22dec28), which #3463 dropped along with the other search-space entries without an srt-recipe. TP8 C1, C2 and C4 are unchanged." - "dram-utilization 0.171 -> 0.74, so TOTAL_CPU_DRAM_GB is 1109 GB per TP4 server, covering the 4 x 256 GiB = 1099.5 GB LMCache tier." + - "TP4 C1 and C2 run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) at 3 draft tokens (golden AL 2.99), with max-num-seqs equal to the concurrency and CUDA graphs at [1] and [1,2]. C2 was 5 draft tokens, max-num-seqs 4 and graphs [1,2,4]." + - "Every other point sets ATOM_MONO_ENABLE=0. The new ATOM turns mono on by default and would also take TP4 C4-C10. TP4 C4 and C8 (5 draft tokens, AL 3.61), C10 (4, AL 3.33) and TP8 keep their settings." + - "The DCP4 points set decode-context-parallel-size 4, ATOM_DCP_PREFILL_QREP=1, and LMCache at 256 GiB per rank (chunk 256, OFFLOAD_MIN_LOAD_TOKENS 8192, PYTHONHASHSEED 0) through extra-kv-connectors lmcache_offload. They admit 2x the concurrency with graphs up to it and draft 4 tokens (AL 3.33), 3 at C48 (AL 2.99)." + - "ATOM changes in the new image that matter here: GLM-5.2 mono decode and ATOM_MONO_ENABLE (ROCm/ATOM #2479, #2483, #2535). ATOM_DCP_PREFILL_QREP (#2437) and QREP together with MTP (#2290); the old image turned QREP off whenever MTP was on. DCP prefill fixes with the MTP draft (#2430, #2496, #2506, #2348). MTP verify and draft-graph fixes (#2451, #2367, #2566, #2495). lmcache_offload lookup and save fixes (#2305, #2339, #2547) and the LMCache 0.5.6.dev139+gf22dec28 pin (#2438)." + - "Draft data types unchanged: the MTP draft (model.layers.78 of amd/GLM-5.2-MXFP4, which ships all-BF16 including its routed experts) stays BF16 because online_quant_config excludes model.layers.78.* from ptpc_fp8. It uses the FP8 KV cache and FP4 indexer cache like the target. Mono decode covers only target layers 0-77." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4308 From 6535bafacafedbb5cef20436cbbf5e1078aff50b Mon Sep 17 00:00:00 2001 From: zhuyuhua-v Date: Mon, 26 Oct 2026 02:21:39 -0500 Subject: [PATCH 4/4] eval(agentx): cover GLM-5.2 mono decode with a TP4 C2 GSM8K The default agentic selector evals only the highest concurrency, DCP4 C48, where ATOM_MONO_ENABLE is 0. TP4 C1/C2 are the points that turn the new mono-decode kernel on, so add an evals-only key whose highest point is C2. Co-authored-by: Cursor --- inferencex-e2e/configs/amd-master.yaml | 20 ++++++++++++++++++++ inferencex-e2e/perf-changelog.yaml | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 9e3cb0f6d6..9b91d3e458 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1488,6 +1488,26 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 4, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [16, 24, 32, 40, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } +# GSM8K for the GLM-5.2 mono decode. The agentic selector evals one point per +# group, at the highest concurrency, which for the key above is DCP4 C48 with +# ATOM_MONO_ENABLE=0. This key is only TP4 C1/C2, so its eval is C2, where +# override_tp4_c2 sets ATOM_MONO_ENABLE=1. The changelog entry is evals-only; +# throughput stays on glm5.2-fp4-mi355x-atom-agentic-mtp. +glm5.2-fp4-mi355x-atom-agentic-mtp-mono: + image: rocm/atom-dev:nightly_202610181506 + model: amd/GLM-5.2-MXFP4 + model-prefix: glm5.2 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom + multinode: false + srt-recipe-dir: glm5.2/atom/mi355x-fp4-mtp + scenarios: + agentic-coding: + - dram-utilization: 0.74 + search-space: + - { tp: 4, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: agentic.yaml } + dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261019 model: deepseek-ai/DeepSeek-V4-Pro-0813 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d4fc3aa118..cc22de4e54 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11226,3 +11226,12 @@ - "ATOM changes in the new image that matter here: GLM-5.2 mono decode and ATOM_MONO_ENABLE (ROCm/ATOM #2479, #2483, #2535). ATOM_DCP_PREFILL_QREP (#2437) and QREP together with MTP (#2290); the old image turned QREP off whenever MTP was on. DCP prefill fixes with the MTP draft (#2430, #2496, #2506, #2348). MTP verify and draft-graph fixes (#2451, #2367, #2566, #2495). lmcache_offload lookup and save fixes (#2305, #2339, #2547) and the LMCache 0.5.6.dev139+gf22dec28 pin (#2438)." - "Draft data types unchanged: the MTP draft (model.layers.78 of amd/GLM-5.2-MXFP4, which ships all-BF16 including its routed experts) stays BF16 because online_quant_config excludes model.layers.78.* from ptpc_fp8. It uses the FP8 KV cache and FP4 indexer cache like the target. Mono decode covers only target layers 0-77." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4308 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp-mono + scenario-type: + - agentic-coding + evals-only: true + description: + - "GSM8K on the GLM-5.2 mono-decode path. The default agentic selector evals one point per group, at the highest concurrency: DCP4 C48, where ATOM_MONO_ENABLE is 0. TP4 C1 and C2 are the points that set ATOM_MONO_ENABLE=1, so this key contains only those two and the eval lands on C2. Throughput stays on glm5.2-fp4-mi355x-atom-agentic-mtp." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4308