diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml index 853e2fdfdc..65ed6c5782 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4-mtp/agentic.yaml @@ -6,7 +6,7 @@ base: gpus_per_node: 8 frontend: type: atomesh - container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + container_image: rocm/atom-dev@sha256:1bb1909ada9e3d178e315a3f89d041c05c7e90e7cc32de8d507ff5ab4cd4702c enable_multiple_frontends: false observability: enabled: false @@ -33,21 +33,43 @@ base: PYTHONNOUSERSITE: '1' AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: '1' + # GLM-5.2 mono decode is on by default in ATOM; only TP4 C1/C2 run it. + ATOM_MONO_ENABLE: '0' benchmark: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' AIPERF_APPLY_CHAT_TEMPLATE: 'true' -# One variant per point. Admission is 2x CONC with graphs captured up to it. The -# draft is five tokens through concurrency 8 and four above it. +# One variant per point. Admission is 2x CONC, except TP4 C1/C2, which admit +# exactly CONC with graphs captured up to it. TP4 C1/C2 run mono decode at +# three draft tokens; C4/C8 draft five and C10 four on the production path. The +# DCP4 arm shards the decode KV, adds the LMCache DRAM tier at 256 GiB per rank +# (1024 GiB, 1099.5 GB per server, within the 1109 GB TP4 share at +# dram-utilization 0.74) and drafts four tokens, three at C48. +override_tp4_c1: + roles: + agg: + gpus: 4 + args: + max-num-seqs: 1 + cudagraph-capture-sizes: '[1]' + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' + benchmark: + env: + CONC: '1' + override_tp4_c2: roles: agg: gpus: 4 args: - max-num-seqs: 4 - cudagraph-capture-sizes: '[1,2,4]' - num-speculative-tokens: 5 + max-num-seqs: 2 + cudagraph-capture-sizes: '[1,2]' + num-speculative-tokens: 3 + env: + ATOM_MONO_ENABLE: '1' benchmark: env: CONC: '2' @@ -88,6 +110,101 @@ override_tp4_c10: env: CONC: '10' +override_tp4_dcp4_c16_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 32 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32]' + num-speculative-tokens: 4 + extra-kv-connectors: &lmcache_offload + - kv_connector: lmcache_offload + kv_role: offload + env: &dcp_lmcache_env + # DCP sparse prefill builds the group's query heads locally. + ATOM_DCP_PREFILL_QREP: '1' + PYTHONHASHSEED: '0' + LMCACHE_LOCAL_CPU: 'True' + LMCACHE_NUMA_MODE: auto + LMCACHE_MAX_LOCAL_CPU_SIZE: '256' + LMCACHE_CHUNK_SIZE: '256' + OFFLOAD_MIN_LOAD_TOKENS: '8192' + benchmark: + env: + CONC: '16' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c24_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 48 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48]' + num-speculative-tokens: 4 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '24' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c32_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 64 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64]' + num-speculative-tokens: 4 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c40_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 80 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80]' + num-speculative-tokens: 4 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + +override_tp4_dcp4_c48_lmcache: + roles: + agg: + gpus: 4 + args: + decode-context-parallel-size: 4 + max-num-seqs: 96 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96]' + num-speculative-tokens: 3 + extra-kv-connectors: *lmcache_offload + env: *dcp_lmcache_env + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1109' + override_tp8_c1: roles: agg: diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index d75eb9218e..99774cd09a 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1490,14 +1490,14 @@ glm5.2-fp8-mi355x-sglang-agentic-mtp: - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [8, 12, 16], spec-decoding: mtp, srt-recipe: agentic.yaml } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative -# decoding, matching the standalone recipe (ROCm/ATOM PR 2345). Small -# concurrency (C2-C10) is TP4-only on the GPU prefix cache with no KV offload; -# large concurrency (C16-C48) adds DCP4 to shard the decode KV and keeps the -# LMCache DRAM offload. The script resolves the draft depth and forced golden AL -# per concurrency point (K5/3.61 for C2-C8, K4/3.33 for C10-C40, K3/2.99 for -# C48). The TP8 arm stays GPU-resident with no KV offload. +# decoding, matching the standalone recipe (ROCm/ATOM PR 2632). Small +# concurrency (C1-C10) is TP4-only on the GPU prefix cache with no KV offload; +# C1/C2 run the GLM-5.2 mono decode at K3. Large concurrency (C16-C48) adds DCP4 +# to shard the decode KV and the LMCache DRAM tier. The recipe sets the draft +# depth per point (K3 for C1/C2 and C48, K5 for C4/C8, K4 for C10-C40). The TP8 +# arm stays GPU-resident with no KV offload. glm5.2-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202609211553 + image: rocm/atom-dev:nightly_202610181506 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds @@ -1507,13 +1507,35 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4-mtp scenarios: agentic-coding: - # dram-utilization 0.171 -> TOTAL_CPU_DRAM_GB=256 per TP4 rank (recipe's - # LMCACHE_MAX_LOCAL_CPU_SIZE=256): min(3079397,2861022)MiB*0.171*4/8 = 256. - - dram-utilization: 0.171 + # dram-utilization 0.74 -> TOTAL_CPU_DRAM_GB=1109 per TP4 server + # (2861022 MiB * 0.74 * 4/8); the recipe's LMCache tier is 256 GiB x 4 ranks + # = 1099.5 GB. + - dram-utilization: 0.74 search-space: - - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } + - { tp: 4, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [16, 24, 32, 40, 48], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } +# GSM8K for the GLM-5.2 mono decode. The agentic selector evals one point per +# group, at the highest concurrency, which for the key above is DCP4 C48 with +# ATOM_MONO_ENABLE=0. This key is only TP4 C1/C2, so its eval is C2, where +# override_tp4_c2 sets ATOM_MONO_ENABLE=1. The changelog entry is evals-only; +# throughput stays on glm5.2-fp4-mi355x-atom-agentic-mtp. +glm5.2-fp4-mi355x-atom-agentic-mtp-mono: + image: rocm/atom-dev:nightly_202610181506 + model: amd/GLM-5.2-MXFP4 + model-prefix: glm5.2 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom + multinode: false + srt-recipe-dir: glm5.2/atom/mi355x-fp4-mtp + scenarios: + agentic-coding: + - dram-utilization: 0.74 + search-space: + - { tp: 4, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: agentic.yaml } + dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.22-rocm720-mi35x-20261019 model: deepseek-ai/DeepSeek-V4-Pro-0813 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index fd894cc02f..bd36ce984f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11413,3 +11413,19 @@ - Restore the MI355X MiniMax-M3 FP4 vLLM AgentX TP2/TP4 grid from run 34609443618 attempt 2 using SRT recipes. Update nightly-2a02f6efe319c885e3ccbcecde402e0028f9ec1e to nightly-df6603f0583375f244d1ac6d36265462b7490392 (2026-10-22, digest sha256:81899892d73777a4ae81e16f39e749169687f29249505eb4c56da83bd097d41c). - Enable upstream MiniMax-M3 monokernel at TP4 with Model Runner V2, AITER MoE, FP8 KV/indexer and block size 128; keep TP2 on ordinary decode. Preserve EAGLE3-GQA with three draft tokens and use automatic golden acceptance for throughput with real verification for evals. pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4418 + +- config-keys: + - glm5.2-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Image rocm/atom-dev:nightly_202609211553 (ATOM 0.1.7.dev517+g52a6a96fd) -> nightly_202610181506 (ATOM 0.1.8.dev77+gdf86c4829). The atomesh router container_image digest moves from f00a9dc5 (nightly_202609161445) to 1bb1909a, the same nightly as the workers." + - "Settings follow the GLM-5.2 recipe in ROCm/ATOM PR 2632 (open; recipe document only, measured on df86c4829)." + - "Add TP4 C1, so the TP4 arm covers C1-C10. Restore the TP4+DCP4 arm at C16, C24, C32, C40 and C48 with the LMCache DRAM tier (0.5.6.dev139+gf22dec28), which #3463 dropped along with the other search-space entries without an srt-recipe. TP8 C1, C2 and C4 are unchanged." + - "dram-utilization 0.171 -> 0.74, so TOTAL_CPU_DRAM_GB is 1109 GB per TP4 server, covering the 4 x 256 GiB = 1099.5 GB LMCache tier." + - "TP4 C1 and C2 run the GLM-5.2 mono decode (ATOM_MONO_ENABLE=1) at 3 draft tokens (golden AL 2.99), with max-num-seqs equal to the concurrency and CUDA graphs at [1] and [1,2]. C2 was 5 draft tokens, max-num-seqs 4 and graphs [1,2,4]." + - "Every other point sets ATOM_MONO_ENABLE=0. The new ATOM turns mono on by default and would also take TP4 C4-C10. TP4 C4 and C8 (5 draft tokens, AL 3.61), C10 (4, AL 3.33) and TP8 keep their settings." + - "The DCP4 points set decode-context-parallel-size 4, ATOM_DCP_PREFILL_QREP=1, and LMCache at 256 GiB per rank (chunk 256, OFFLOAD_MIN_LOAD_TOKENS 8192, PYTHONHASHSEED 0) through extra-kv-connectors lmcache_offload. They admit 2x the concurrency with graphs up to it and draft 4 tokens (AL 3.33), 3 at C48 (AL 2.99)." + - "ATOM changes in the new image that matter here: GLM-5.2 mono decode and ATOM_MONO_ENABLE (ROCm/ATOM #2479, #2483, #2535). ATOM_DCP_PREFILL_QREP (#2437) and QREP together with MTP (#2290); the old image turned QREP off whenever MTP was on. DCP prefill fixes with the MTP draft (#2430, #2496, #2506, #2348). MTP verify and draft-graph fixes (#2451, #2367, #2566, #2495). lmcache_offload lookup and save fixes (#2305, #2339, #2547) and the LMCache 0.5.6.dev139+gf22dec28 pin (#2438)." + - "Draft data types unchanged: the MTP draft (model.layers.78 of amd/GLM-5.2-MXFP4, which ships all-BF16 including its routed experts) stays BF16 because online_quant_config excludes model.layers.78.* from ptpc_fp8. It uses the FP8 KV cache and FP4 indexer cache like the target. Mono decode covers only target layers 0-77." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4308