From 8df9856e3b1d0796a8a7c0803a9a053325d23aad Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 05:03:01 +0000 Subject: [PATCH 01/16] feat(agentx): add GLM-5.2 MI355X ATOM PD disaggregation points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: eight AgentX points with a CPP4 (PP4 x TP1) ATOM prefill behind AToMesh and Mooncake RDMA to the decodes. 1P1D on one node at C56-C112 keeps the prefill's prefix tier in a Mooncake Store of four per-NIC pools on reserved 2 MiB HugeTLB pages; 1P2D at C56 and 1P3D at C12-C24 use LMCache in the prefill. host_setup drops the page cache and compacts memory before every point. Ship NVIDIA/srt-slurm#604 as runners/srt-slurm/patches/atom-tp-from-pp.patch so srtctl launches the PP4 prefill with -tp 1. 新增 glm5.2-fp4-mi355x-atom-disagg-agentic-mtp：八个 AgentX 测试点。ATOM prefill 为 CPP4（PP4 x TP1），经 AToMesh 路由，并通过 Mooncake RDMA 把 KV 传给 decode。单节点 1P1D（C56-C112）把 prefill 的前缀缓存层放在 Mooncake Store 中：四个按网卡划分的池，使用预留的 2 MiB HugeTLB 大页；1P2D（C56）和 1P3D（C12-C24）在 prefill 中使用 LMCache。每个测试点开始前，host_setup 都会 清理页缓存并整理内存。 以 runners/srt-slurm/patches/atom-tp-from-pp.patch 的形式附带 NVIDIA/srt-slurm#604，使 srtctl 以 -tp 1 启动 PP4 prefill。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 335 ++++++++++++++++++ inferencex-e2e/configs/amd-master.yaml | 88 +++++ inferencex-e2e/perf-changelog.yaml | 12 + .../runners/srt-slurm/patches/README.md | 1 + .../srt-slurm/patches/atom-tp-from-pp.patch | 60 ++++ 5 files changed, 496 insertions(+) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml create mode 100644 inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml new file mode 100644 index 0000000000..70a67211cc --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -0,0 +1,335 @@ +base: + schema: 2 + name: mi355x-glm52-atom-agentx-disagg-mtp + slurm: + time_limit: "08:00:00" + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + enable_multiple_frontends: false + args: + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + prefill-policy: round_robin + decode-policy: round_robin + atom-pd-rank-mapping-policy: none + engine: atom + # A recipe host_setup replaces the cluster default, so run that first (NIC + # checks, GPU drain, release of hugepages a previous job left). Then reclaim + # memory on every node: drop the clean page cache and compact what is free. + host_setup: + nodes: all + timeout_seconds: 1800 + commands: + - &cluster_setup IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + - &reclaim_memory sync && echo 1 | sudo -n tee /proc/sys/vm/drop_caches >/dev/null && echo 1 | sudo -n tee /proc/sys/vm/compact_memory >/dev/null + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: + <<: &environment + PYTHONUNBUFFERED: "1" + PYTHONHASHSEED: "0" + AITER_LOG_LEVEL: WARNING + AITER_QUICK_REDUCE_QUANTIZATION: INT4 + AITER_USE_FLYDSL_MOE_SORTING: "1" + ATOM_MLA_PAGE_SIZE: "1" + ATOM_ONLINE_QUANT_STREAMING: "0" + ATOM_SPARSE_INDEXER_LOGITS_BUDGET_MB: "2047" + ATOM_USE_TRITON_MLA: "0" + TOPK_FORCE_PATH: one + MAX_JOBS: "16" + HSA_ENABLE_IPC_MODE_LEGACY: "1" + MC_DISABLE_HIP: "1" + # Rail-isolated RDMA: each transfer uses the same rail on both ends. + MC_GID_INDEX: "@fabric.rdma-gid-index" + ATOM_MOONCAKE_MATCHED_RAILS: auto + NCCL_IB_DISABLE: "1" + VLLM_PP_LAYER_PARTITION: "20,20,20,18" + OFFLOAD_PROFILE: "1" + OFFLOAD_MIN_LOAD_TOKENS: "0" + args: + <<: &server + served-model-name: amd/GLM-5.2-MXFP4 + trust-remote-code: true + kv_cache_dtype: fp8 + block-size: 16 + # model.layers.78 is the MTP head, shipped in BF16; excluding it keeps + # online ptpc_fp8 quantization off the draft while experts stay MXFP4. + online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' + level: 3 + cudagraph-mode: FULL + method: mtp + timeout-keep-alive: 900 + pipeline-parallel-size: 4 + enforce-eager: true + gpu-memory-utilization: 0.85 + max-num-seqs: 512 + max-num-batched-tokens: 8192 + decode: + nodes: 1 + workers: 1 + gpus: 4 + env: *environment + args: *server + sbatch_directives: + cpus-per-task: "128" + mem: "0" + srun_options: + mem: "0" + container-writable: "" + container-remap-root: "" + health_check: + max_attempts: 1080 + interval_seconds: 5 + benchmark: + env: + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "atom:" + AIPERF_APPLY_CHAT_TEMPLATE: "true" + TOKENIZERS_PARALLELISM: "false" + TRANSFORMERS_VERBOSITY: error + +# One variant per point. A DCP4 decode admits 512 and captures every batch up +# to 256; a 1P3D decode admits twice CONC and captures up to that. + +override_1p1d_mcstore_c56: &mcstore_1p1d + host_setup: + nodes: all + timeout_seconds: 1800 + commands: + - *cluster_setup + - *reclaim_memory + # Owners bind NUMA0 (rdma0-1) and NUMA1 (rdma2-3), 178,688 2 MiB pages + # each, so each NUMA node needs 357,376 free. Reserve both nodes' share + # with 2 GiB spare each on the prefill node, and fail the node here when + # the kernel grants fewer. + - >- + if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then + HUGEPAGES_TARGET=716800 HUGEPAGE_GROW_TOLERANCE_PCT=0 + bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; + for node in 0 1; do + free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 357376"; + [[ $free -ge 357376 ]] || exit 1; + done; fi + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" + # Pool k is rdma k: its master listens on 26051 + 100k (HTTP metadata on + # 26080 + 100k), and its owner registers a 349 GiB segment on rdma k. + # numa_exec.py (shipped in the image) binds each process's CPUs and memory + # to one NUMA node. + services: + - &master + name: mc-master-rdma0 + type: generic + placement: + node: prefill + start: before_workers + critical: true + inherit_discovery_env: false + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26051, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26080, --metrics_port=26090, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + env: &store_env + # A Mooncake process that sees a GPU probes every buffer through HIP. + HIP_VISIBLE_DEVICES: "-1" + CUDA_VISIBLE_DEVICES: "-1" + # Peers with different QP counts cannot connect. + MC_NUM_QP_PER_EP: "1" + MC_GID_INDEX: "@fabric.rdma-gid-index" + readiness: + port: 26051 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26151 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26251 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26351 + timeout_seconds: 300 + - &owner + <<: *master + name: mc-owner-rdma0 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + env: + <<: *store_env + # Map the segment from the reserved 2 MiB pages, never 4 KiB ones the + # NIC cannot register in 64 GiB regions. + MC_STORE_USE_HUGEPAGE: "1" + MC_MAX_MR_SIZE: "68719476736" + MC_TCP_BIND_ADDRESS: "{node_ip}" + readiness: + log: + pattern: Starting real client service + timeout_seconds: 1800 + interval_seconds: 5 + - <<: *owner + name: mc-owner-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + roles: + prefill: + env: + # More than one load worker hung the PP prefill pipeline at C96. + OFFLOAD_LOAD_WORKERS: "1" + MC_NUM_QP_PER_EP: "1" + # Registers the worker's HBM transfer pool in 1 GiB regions. + MC_MAX_MR_SIZE: "1073741824" + args: + num-speculative-tokens: 3 + # Each stage's Store client uses its GPU's NIC and that NIC's pool. + extra-kv-connectors: + - kv_connector: mooncake_store + kv_role: offload + kv_connector_extra_config: + mooncake_store.protocol: rdma + mooncake_store.pools: + rdma0: {master: "127.0.0.1:26051", metadata: "http://127.0.0.1:26080/metadata"} + rdma1: {master: "127.0.0.1:26151", metadata: "http://127.0.0.1:26180/metadata"} + rdma2: {master: "127.0.0.1:26251", metadata: "http://127.0.0.1:26280/metadata"} + rdma3: {master: "127.0.0.1:26351", metadata: "http://127.0.0.1:26380/metadata"} + decode: + nodes: colocate + env: + AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + # The P/D transfer must match the prefill's QP count, one with the Store. + MC_NUM_QP_PER_EP: "1" + args: + <<: &dcp_decode + decode-context-parallel-size: 4 + gpu-memory-utilization: 0.94 + max-num-seqs: 512 + max-num-batched-tokens: 2048 + cudagraph-capture-sizes: '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' + num-speculative-tokens: 3 + +override_1p1d_mcstore_c72: *mcstore_1p1d + +override_1p1d_mcstore_c96: *mcstore_1p1d + +override_1p1d_mcstore_c112: *mcstore_1p1d + +override_1p2d_lmcache_c56: + resources: &spread + # A node per decode; srtctl otherwise packs two TP4 decodes onto one node. + spread_workers: true + frontend: + args: &cache_aware + policy: cache_aware + prefill-policy: cache_aware + decode-policy: cache_aware + cache-threshold: 0.8 + balance-abs-threshold: 2 + balance-rel-threshold: 2.0 + eviction-interval: 300 + roles: + prefill: + env: &lmcache_env + LMCACHE_LOCAL_CPU: "True" + LMCACHE_NUMA_MODE: auto + LMCACHE_MAX_LOCAL_CPU_SIZE: "256" + LMCACHE_CHUNK_SIZE: "256" + # Every PP stage runs a lookup server, so each stage refreshes its own LRU. + LMCACHE_LOOKUP_SERVER_WORKER_IDS: "0,1,2,3" + args: + num-speculative-tokens: 4 + extra-kv-connectors: &lmcache_connector + - kv_connector: lmcache_offload + kv_role: offload + decode: + nodes: 2 + workers: 2 + env: + AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + args: + <<: *dcp_decode + num-speculative-tokens: 4 + +override_1p3d_lmcache_c12: + resources: *spread + frontend: + args: *cache_aware + roles: + prefill: + env: *lmcache_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *lmcache_connector + decode: + nodes: 3 + workers: 3 + env: + PYTHONNOUSERSITE: "1" + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 24 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24]' + disable-uvicorn-access-log: true + +override_1p3d_lmcache_c16: + resources: *spread + frontend: + args: *cache_aware + roles: + prefill: + env: *lmcache_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *lmcache_connector + decode: + nodes: 3 + workers: 3 + env: + PYTHONNOUSERSITE: "1" + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 32 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32]' + disable-uvicorn-access-log: true + +override_1p3d_lmcache_c24: + resources: *spread + frontend: + args: *cache_aware + roles: + prefill: + env: *lmcache_env + args: + num-speculative-tokens: 5 + extra-kv-connectors: *lmcache_connector + decode: + nodes: 3 + workers: 3 + env: + PYTHONNOUSERSITE: "1" + args: + num-speculative-tokens: 5 + gpu-memory-utilization: 0.95 + max-num-seqs: 48 + max-num-batched-tokens: 16384 + cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32,36,40,44,48]' + disable-uvicorn-access-log: true diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 24f60ae790..39a8d7bb57 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1484,6 +1484,94 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } +# GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM prefill/decode +# disaggregation behind AToMesh, with Mooncake RDMA moving KV from prefill to +# decode and MTP drafts. The prefill is one CPP4 worker: PP4 x TP1 on four GPUs. +# (1) 1P1D on one node, a DCP4 decode on GPUs 4-7, MTP3, concurrency 56-112, +# with the prefill's prefix tier in a Mooncake Store of four per-NIC pools +# on 2 MiB HugeTLB pages. +# (2) LMCache in the prefill: 1P2D with two DCP4 decodes, MTP4, concurrency 56; +# 1P3D with three TP4 decodes, MTP5, concurrency 12-24. Each decode has a +# node of its own. +# dram-utilization is over the prefill's four GPUs, half of the node's capped +# 3 TB: 1.0 gives the Store's 4 x 349 GiB (1499 GB), and 0.733 gives LMCache's +# 4 x 256 GiB (1099 GB). srtctl sizes the PP4 prefill as -tp 1 through +# runners/srt-slurm/patches/atom-tp-from-pp.patch. +glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: + image: rocm/atom-dev:nightly_202610131249@sha256:d5b3fcd99bd44e3489740d1b026037f49f2e3f6160dc70b70c3fd23a125a4905 + model: amd/GLM-5.2-MXFP4 + model-prefix: glm5.2 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "6fd1bb1be97c79b571eb86674786d0d888353b73" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx + scenarios: + agentic-coding: + - dram-utilization: 1.0 + search-space: + - spec-decoding: "mtp" + conc-list: [ 56 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c56 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 72 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c72 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 96 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c96 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 112 ] + kv-offloading: dram + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c112 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - dram-utilization: 0.733 + search-space: + - spec-decoding: "mtp" + conc-list: [ 56 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p2d_lmcache_c56 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 2, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 12 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c12 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 16 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c16 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + - spec-decoding: "mtp" + conc-list: [ 24 ] + kv-offloading: dram + kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } + srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c24 + prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } + decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } + dsv4-fp4-mi355x-sglang-agentic-mtp: image: lmsysorg/sglang-rocm:v0.5.21-rocm720-mi35x-20261007 model: deepseek-ai/DeepSeek-V4-Pro-0813 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 74c014908a..54832f1d5f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9718,3 +9718,15 @@ - "Enable INT4 QuickReduce with a 256 MiB size limit and FlyDSL stage-2 FP8 on prefill and decode workers." - "Configure MoRIIO READ and prefill-only SimpleCPUOffloadConnector through native role arguments. Use recipe-scoped draft checkpoint and RDMA device/provider mounts. Use the upstream discovery-template fix from srt-slurm #508 through the shared v2.51.1 dependency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3582 + +- config-keys: + - glm5.2-fp4-mi355x-atom-disagg-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." + - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. host_setup reserves the pages before the job and teardown releases them." + - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." + - "host_setup drops the page cache and compacts free memory on every node before each point." + - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index fcadc7bf88..9207d66b70 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -8,3 +8,4 @@ Each patch is a temporary fix for an open upstream PR. When the PR merges and th | Patch | Upstream PR | Fix | |-------|-------------|-----| +| `atom-tp-from-pp.patch` | [NVIDIA/srt-slurm#604](https://github.com/NVIDIA/srt-slurm/pull/604) | ATOM's `-tp` is the worker's GPUs divided by its pipeline and prefill-context stages, so a 4-GPU PP4 prefill launches with `-tp 1` instead of `-tp 4` | diff --git a/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch b/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch new file mode 100644 index 0000000000..33a6019d1f --- /dev/null +++ b/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch @@ -0,0 +1,60 @@ +From e23ca1bed9aee9a59905fb30b27f09b3ef9e9224 Mon Sep 17 00:00:00 2001 +From: Jasen2201 +Date: Wed, 14 Oct 2026 04:15:36 +0000 +Subject: [PATCH] fix(atom): size -tp from the worker's GPUs over its PP and + PCP stages + +ATOM runs a worker on tp x pp x pcp GPUs, but srtctl always passed +`-tp `. A 4-GPU prefill worker with +`pipeline-parallel-size: 4` therefore launched as +`-tp 4 --pipeline-parallel-size 4`, and ATOM sized its world to 16 GPUs. + +`-tp` is now the worker's GPU count divided by `pipeline-parallel-size` +and `prefill-context-parallel-size`. Decode context parallelism reuses the +TP GPUs and does not change it. A split that does not divide the worker's +GPUs fails before launch. + +Co-Authored-By: Claude Opus 5.5 +Signed-off-by: Jasen2201 +--- + src/srtctl/backends/atom.py | 20 +++++++++++++++++++- + 1 file changed, 19 insertions(+), 1 deletion(-) + +diff --git a/src/srtctl/backends/atom.py b/src/srtctl/backends/atom.py +index edf9a9b7..4e5b836e 100644 +--- a/src/srtctl/backends/atom.py ++++ b/src/srtctl/backends/atom.py +@@ -155,7 +155,7 @@ class AtomBackend(Backend): + "--server-port", + str(process.http_port), + "-tp", +- str(len(process.gpu_indices)), ++ str(_tensor_parallel_size(len(process.gpu_indices), config)), + ] + ) + kv_transfer_config = self._kv_transfer_config(process, worker_ip, extra_connectors) +@@ -183,3 +183,21 @@ def _config_to_cli_args(config: dict[str, Any]) -> list[str]: + + def _canonical_arg_key(key: str) -> str: + return key.lstrip("-").replace("_", "-") ++ ++ ++# ATOM sizes a worker as tp x pp x pcp GPUs; decode context parallelism reuses the TP GPUs. ++_GPU_SPLITTING_ARGS = ("pipeline-parallel-size", "prefill-context-parallel-size") ++ ++ ++def _tensor_parallel_size(gpu_count: int, config: Mapping[str, Any]) -> int: ++ """The worker's GPUs left for tensor parallelism after its pipeline and prefill-context stages.""" ++ stages = 1 ++ for key, value in config.items(): ++ if _canonical_arg_key(key) in _GPU_SPLITTING_ARGS: ++ stages *= int(value) ++ if stages < 1 or gpu_count % stages: ++ raise ValueError( ++ f"ATOM worker has {gpu_count} GPU(s), which its pipeline x prefill-context parallelism of {stages} " ++ "does not divide" ++ ) ++ return gpu_count // stages +-- +2.43.0 + From 267bc6cf448637d30431ef6695a4af00c7dfe0d0 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 05:13:15 +0000 Subject: [PATCH 02/16] chore(changelog): link the GLM-5.2 ATOM PD entry to its PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set the new perf-changelog entry's pr-link to #4045. 将新增的 perf-changelog 条目的 pr-link 设为 #4045。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 54832f1d5f..3b8861632c 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9729,4 +9729,4 @@ - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." - "host_setup drops the page cache and compacts free memory on every node before each point." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 From a377cbcb2275afdd1d4521cea7a6956b83eae4e8 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 07:34:56 +0000 Subject: [PATCH 03/16] fix(agentx): cache the GLM-5.2 checkpoint once and size Store hugepages above held pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Set HF_HOME/HF_HUB_CACHE on the ATOM roles. amd/GLM-5.2-MXFP4 has no staged copy on mi355x-amds, so the job serves hf:; srtctl downloads it once into the shared hub cache only when a role sets HF_HOME. - Follow the UMBP recipe's host_setup for the Store: the cluster setup.sh frees leftover hugepages and drops the page cache, then the prefill node reserves the owners' 714752 2 MiB pages plus 2% on top of the pages host services hold. mi355x-amds nodes hold 5656, which the old fixed target of 716800 left too few for both NUMA nodes. The LMCache points keep the cluster default host setup. - Run the four Store points from one variant, so the planner schedules one GSM8K eval for them instead of four. - Drop header and section comments per AGENTS.md, and rename the srt-slurm patch to 604-atom-tp-from-pp.patch like the other patches. - 为 ATOM 角色设置 HF_HOME/HF_HUB_CACHE。mi355x-amds 上没有预置的 amd/GLM-5.2-MXFP4，任务以 hf: 方式加载；只有角色设置了 HF_HOME，srtctl 才会把模型一次性下载到共享 hub 缓存。 - Store 的 host_setup 参照 UMBP recipe：先由集群 setup.sh 释放残留大页并 清理页缓存，再在 prefill 节点上于主机服务已占用的大页之上，预留 owner 所需 的 714752 个 2 MiB 页并多留 2%。mi355x-amds 节点常驻占用 5656 页，原先固定 的 716800 目标会导致两个 NUMA 节点的空闲页不足。LMCache 测试点沿用集群默认 的 host setup。 - 四个 Store 测试点共用一个变体，规划器只为它们安排一个 GSM8K eval，而不是 四个。 - 按 AGENTS.md 删除头注释与段落注释，并把 srt-slurm 补丁按惯例重命名为 604-atom-tp-from-pp.patch。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 43 +++++-------------- inferencex-e2e/configs/amd-master.yaml | 40 ++--------------- inferencex-e2e/perf-changelog.yaml | 4 +- ...rom-pp.patch => 604-atom-tp-from-pp.patch} | 0 .../runners/srt-slurm/patches/README.md | 2 +- 5 files changed, 18 insertions(+), 71 deletions(-) rename inferencex-e2e/runners/srt-slurm/patches/{atom-tp-from-pp.patch => 604-atom-tp-from-pp.patch} (100%) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 70a67211cc..8ec354facb 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -18,15 +18,6 @@ base: decode-policy: round_robin atom-pd-rank-mapping-policy: none engine: atom - # A recipe host_setup replaces the cluster default, so run that first (NIC - # checks, GPU drain, release of hugepages a previous job left). Then reclaim - # memory on every node: drop the clean page cache and compact what is free. - host_setup: - nodes: all - timeout_seconds: 1800 - commands: - - &cluster_setup IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - - &reclaim_memory sync && echo 1 | sudo -n tee /proc/sys/vm/drop_caches >/dev/null && echo 1 | sudo -n tee /proc/sys/vm/compact_memory >/dev/null roles: prefill: nodes: 1 @@ -36,6 +27,9 @@ base: <<: &environment PYTHONUNBUFFERED: "1" PYTHONHASHSEED: "0" + # srtctl downloads an hf: checkpoint once into the shared hub cache only when HF_HOME is set. + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub AITER_LOG_LEVEL: WARNING AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: "1" @@ -95,23 +89,18 @@ base: TOKENIZERS_PARALLELISM: "false" TRANSFORMERS_VERBOSITY: error -# One variant per point. A DCP4 decode admits 512 and captures every batch up -# to 256; a 1P3D decode admits twice CONC and captures up to that. - -override_1p1d_mcstore_c56: &mcstore_1p1d +override_1p1d_mcstore: host_setup: nodes: all timeout_seconds: 1800 commands: - - *cluster_setup - - *reclaim_memory - # Owners bind NUMA0 (rdma0-1) and NUMA1 (rdma2-3), 178,688 2 MiB pages - # each, so each NUMA node needs 357,376 free. Reserve both nodes' share - # with 2 GiB spare each on the prefill node, and fail the node here when - # the kernel grants fewer. + # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + # Owners take 714752 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. - >- if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then - HUGEPAGES_TARGET=716800 HUGEPAGE_GROW_TOLERANCE_PCT=0 + held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); + HUGEPAGES_TARGET=$(( held + 714752 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; for node in 0 1; do free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); @@ -120,10 +109,6 @@ override_1p1d_mcstore_c56: &mcstore_1p1d done; fi teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" - # Pool k is rdma k: its master listens on 26051 + 100k (HTTP metadata on - # 26080 + 100k), and its owner registers a 349 GiB segment on rdma k. - # numa_exec.py (shipped in the image) binds each process's CPUs and memory - # to one NUMA node. services: - &master name: mc-master-rdma0 @@ -133,6 +118,7 @@ override_1p1d_mcstore_c56: &mcstore_1p1d start: before_workers critical: true inherit_discovery_env: false + # numa_exec.py binds the process's CPUs and memory to that NUMA node. command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26051, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26080, --metrics_port=26090, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] env: &store_env # A Mooncake process that sees a GPU probes every buffer through HIP. @@ -168,8 +154,7 @@ override_1p1d_mcstore_c56: &mcstore_1p1d command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] env: <<: *store_env - # Map the segment from the reserved 2 MiB pages, never 4 KiB ones the - # NIC cannot register in 64 GiB regions. + # Back the segment with the reserved 2 MiB pages; the NIC cannot register 64 GiB regions of 4 KiB pages. MC_STORE_USE_HUGEPAGE: "1" MC_MAX_MR_SIZE: "68719476736" MC_TCP_BIND_ADDRESS: "{node_ip}" @@ -223,12 +208,6 @@ override_1p1d_mcstore_c56: &mcstore_1p1d cudagraph-capture-sizes: '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' num-speculative-tokens: 3 -override_1p1d_mcstore_c72: *mcstore_1p1d - -override_1p1d_mcstore_c96: *mcstore_1p1d - -override_1p1d_mcstore_c112: *mcstore_1p1d - override_1p2d_lmcache_c56: resources: &spread # A node per decode; srtctl otherwise packs two TP4 decodes onto one node. diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 39a8d7bb57..0ab8611b48 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1484,19 +1484,6 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: - { tp: 4, kv-offloading: none, conc-list: [2, 4, 8, 10], spec-decoding: mtp, srt-recipe: agentic.yaml } - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: agentic.yaml } -# GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM prefill/decode -# disaggregation behind AToMesh, with Mooncake RDMA moving KV from prefill to -# decode and MTP drafts. The prefill is one CPP4 worker: PP4 x TP1 on four GPUs. -# (1) 1P1D on one node, a DCP4 decode on GPUs 4-7, MTP3, concurrency 56-112, -# with the prefill's prefix tier in a Mooncake Store of four per-NIC pools -# on 2 MiB HugeTLB pages. -# (2) LMCache in the prefill: 1P2D with two DCP4 decodes, MTP4, concurrency 56; -# 1P3D with three TP4 decodes, MTP5, concurrency 12-24. Each decode has a -# node of its own. -# dram-utilization is over the prefill's four GPUs, half of the node's capped -# 3 TB: 1.0 gives the Store's 4 x 349 GiB (1499 GB), and 0.733 gives LMCache's -# 4 x 256 GiB (1099 GB). srtctl sizes the PP4 prefill as -tp 1 through -# runners/srt-slurm/patches/atom-tp-from-pp.patch. glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: image: rocm/atom-dev:nightly_202610131249@sha256:d5b3fcd99bd44e3489740d1b026037f49f2e3f6160dc70b70c3fd23a125a4905 model: amd/GLM-5.2-MXFP4 @@ -1511,36 +1498,17 @@ glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx scenarios: agentic-coding: + # The Store's 4 x 349 GiB owners: all 1499 GB of the 4-GPU prefill's share of the 3 TB cap. - dram-utilization: 1.0 search-space: - spec-decoding: "mtp" - conc-list: [ 56 ] - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c56 - prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } - decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - - spec-decoding: "mtp" - conc-list: [ 72 ] - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c72 - prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } - decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - - spec-decoding: "mtp" - conc-list: [ 96 ] - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c96 - prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } - decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - - spec-decoding: "mtp" - conc-list: [ 112 ] + conc-list: [ 56, 72, 96, 112 ] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } - srt-recipe: disagg-variants.yaml:override_1p1d_mcstore_c112 + srt-recipe: disagg-variants.yaml:override_1p1d_mcstore prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } + # LMCache's 4 x 256 GiB stages: 1099 GB of that share. - dram-utilization: 0.733 search-space: - spec-decoding: "mtp" diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3b8861632c..b7293859fe 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9725,8 +9725,8 @@ - agentic-coding description: - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. host_setup reserves the pages before the job and teardown releases them." + - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. After the cluster host setup drops the page cache, host_setup reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." - - "host_setup drops the page cache and compacts free memory on every node before each point." + - "Workers load the checkpoint from the cluster's shared Hugging Face hub cache, which srtctl fills once before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 diff --git a/inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch b/inferencex-e2e/runners/srt-slurm/patches/604-atom-tp-from-pp.patch similarity index 100% rename from inferencex-e2e/runners/srt-slurm/patches/atom-tp-from-pp.patch rename to inferencex-e2e/runners/srt-slurm/patches/604-atom-tp-from-pp.patch diff --git a/inferencex-e2e/runners/srt-slurm/patches/README.md b/inferencex-e2e/runners/srt-slurm/patches/README.md index f8b2e0ab4a..cba1bceb2b 100644 --- a/inferencex-e2e/runners/srt-slurm/patches/README.md +++ b/inferencex-e2e/runners/srt-slurm/patches/README.md @@ -9,4 +9,4 @@ Each patch is a temporary fix for an open upstream PR. When the PR merges and th | Patch | Upstream PR | Fix | |-------|-------------|-----| | `573-participating-gpus.patch` | [NVIDIA/srt-slurm#573](https://github.com/NVIDIA/srt-slurm/pull/573) | Record only the GPUs workers occupy on a worker node; an exporter such as AMD's reports every GPU, which the collector otherwise flags as `unexpected_device`. | -| `atom-tp-from-pp.patch` | [NVIDIA/srt-slurm#604](https://github.com/NVIDIA/srt-slurm/pull/604) | ATOM's `-tp` is the worker's GPUs divided by its pipeline and prefill-context stages, so a 4-GPU PP4 prefill launches with `-tp 1` instead of `-tp 4` | +| `604-atom-tp-from-pp.patch` | [NVIDIA/srt-slurm#604](https://github.com/NVIDIA/srt-slurm/pull/604) | ATOM's `-tp` is the worker's GPUs divided by its pipeline and prefill-context stages, so a 4-GPU PP4 prefill launches with `-tp 1` instead of `-tp 4` | From f981a268a7043c4e0583984b92abd193f9859b00 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 08:53:11 +0000 Subject: [PATCH 04/16] fix(agentx): read the GLM-5.2 checkpoint from each node's local hub cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mount /var/lib/hf-hub-cache at /mnt/hf_hub_cache and point HF_HUB_CACHE at it, as the single-node GLM-5.2 points do, so nodes that already hold amd/GLM-5.2-MXFP4 load it from local NVMe instead of downloading it into the shared cache. HF_HOME stays set so srtctl fills the first node's cache before workers start. 像单机 GLM-5.2 测试点一样，把 /var/lib/hf-hub-cache 挂载到 /mnt/hf_hub_cache，并让 HF_HUB_CACHE 指向它。已经有 amd/GLM-5.2-MXFP4 的 节点会直接从本地 NVMe 加载，不再下载到共享缓存。保留 HF_HOME，使 srtctl 在 worker 启动前先补齐第一个节点的缓存。 Co-Authored-By: Claude Opus 5.5 --- .../glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml | 9 ++++++--- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 8ec354facb..ef53bfa899 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -27,9 +27,9 @@ base: <<: &environment PYTHONUNBUFFERED: "1" PYTHONHASHSEED: "0" - # srtctl downloads an hf: checkpoint once into the shared hub cache only when HF_HOME is set. - HF_HOME: /hf_hub_cache - HF_HUB_CACHE: /hf_hub_cache/hub + # srtctl fills the first node's hub cache before workers start only when HF_HOME is set. + HF_HOME: /tmp/hf_home + HF_HUB_CACHE: /mnt/hf_hub_cache AITER_LOG_LEVEL: WARNING AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: "1" @@ -79,6 +79,9 @@ base: mem: "0" container-writable: "" container-remap-root: "" + extra_mount: + # The node-local hub cache the single-node GLM-5.2 points already fill. + - /var/lib/hf-hub-cache:/mnt/hf_hub_cache health_check: max_attempts: 1080 interval_seconds: 5 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b7293859fe..2a42cf83cc 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9727,6 +9727,6 @@ - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. After the cluster host setup drops the page cache, host_setup reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." - - "Workers load the checkpoint from the cluster's shared Hugging Face hub cache, which srtctl fills once before they start." + - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 From 4aaf7c071e69d08c00af8aab17cd604411de7762 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 09:14:26 +0000 Subject: [PATCH 05/16] fix(agentx): raise the Store owners' memlock limit before they register MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On mi355x-amds both Store points failed in the first owner: every ibv_reg_mr of its 349 GiB HugeTLB segment returned ENOMEM at once. The pyxis --container-remap-root user is root only inside its namespace, so the NIC's pin of the segment counts against RLIMIT_MEMLOCK. Run `ulimit -l unlimited` before each owner starts, as the DSv4 SGLang GB200 and GB300 recipes do for their Mooncake stores. 两个 Store 测试点都在 mi355x-amds 上的第一个 owner 处失败：它对 349 GiB HugeTLB 段的每次 ibv_reg_mr 都立刻返回 ENOMEM。pyxis 的 --container-remap-root 用户只在自己的命名空间内是 root，网卡锁定该段时会 计入 RLIMIT_MEMLOCK。现在每个 owner 启动前先执行 `ulimit -l unlimited`， 与 DSv4 SGLang GB200、GB300 配方对 Mooncake store 的做法一致。 Co-Authored-By: Claude Opus 5.5 --- .../glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index ef53bfa899..52470a7ac9 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -161,6 +161,8 @@ override_1p1d_mcstore: MC_STORE_USE_HUGEPAGE: "1" MC_MAX_MR_SIZE: "68719476736" MC_TCP_BIND_ADDRESS: "{node_ip}" + # The remapped container root lacks host CAP_IPC_LOCK, so registering the segment counts against RLIMIT_MEMLOCK. + preamble: ulimit -l unlimited readiness: log: pattern: Starting real client service From 4f75d9416e3301c6fe788417702951d3f97430f3 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Wed, 14 Oct 2026 09:28:56 +0000 Subject: [PATCH 06/16] fix(agentx): role-aware, window-scoped ATOM cache rates with validity (cherry-pick #3964) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry #3964 (2db01926, 0fda3f1e, 0842d829) so this PR's sweep prices the ATOM P/D cache rates with it: count only prefill endpoints, take the CPU rate from atom:prefix_cache_offload_tokens (reported for LMCache and the Mooncake Store alike), and slice counters to the profiling window. Applied as #3964's net diff onto current main; drop it once #3964 merges. 携带 #3964（2db01926、0fda3f1e、0842d829），使本 PR 的 sweep 用它来计算 ATOM P/D 的缓存命中率：只统计 prefill 端点，CPU 命中率取自 atom:prefix_cache_offload_tokens（LMCache 与 Mooncake Store 都会上报）， 并按 profiling 窗口切分计数。以 #3964 相对其基线的净改动应用到当前 main 上； #3964 合入后即可移除。 Co-Authored-By: Claude Opus 5.5 --- .../infx/results/agentic/__init__.py | 11 +- .../infx/results/agentic/artifacts.py | 3 + .../infx/results/agentic/backends/atom.py | 354 +++++++++++-- .../infx/results/agentic/backends/base.py | 1 + .../infx/results/agentic/server_metrics.py | 86 ++- .../infx/tests/bench/test_agentic_command.py | 98 +++- .../agentic/test_atom_cache_accounting.py | 493 ++++++++++++++++++ .../agentic/test_process_agentic_result.py | 338 ++++++++---- 8 files changed, 1247 insertions(+), 137 deletions(-) create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 7cb92a82ff..a378329a28 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -9,7 +9,12 @@ from infx.results.topology import Parallelism, validate_parallelism from .request_metrics import compute_request_metrics -from .server_metrics import compute_server_metrics +from .server_metrics import ( + compute_server_metrics, + deployment_from_env, + endpoint_roles_from_env, + request_scope_from_records, +) def _env_int(env: Mapping[str, str], name: str, default: int = 0) -> int: @@ -167,6 +172,7 @@ def build_result( "records_dropped_total": 0, "records_warmup_dropped": 0, "records_error_dropped": 0, + "records_profiling_error_dropped": 0, "error_categories": {}, } @@ -216,6 +222,9 @@ def build_result( framework=framework, records=records, server_logs=server_logs, + endpoint_roles=endpoint_roles_from_env(env), + request_scope=request_scope_from_records(records, request_accounting), + deployment=deployment_from_env(env), ) if "total_tput_tps" in request_flat and num_gpus > 0: diff --git a/inferencex-e2e/infx/results/agentic/artifacts.py b/inferencex-e2e/infx/results/agentic/artifacts.py index a35ee829ae..ef072d274a 100644 --- a/inferencex-e2e/infx/results/agentic/artifacts.py +++ b/inferencex-e2e/infx/results/agentic/artifacts.py @@ -45,6 +45,7 @@ def load_records_with_accounting( "records_dropped_total": 0, "records_warmup_dropped": 0, "records_error_dropped": 0, + "records_profiling_error_dropped": 0, "error_categories": {}, } with open(path) as f: @@ -61,6 +62,8 @@ def load_records_with_accounting( accounting["records_warmup_dropped"] += 1 if error: accounting["records_error_dropped"] += 1 + if not is_warmup: + accounting["records_profiling_error_dropped"] += 1 category = _error_category(error) categories = accounting["error_categories"] categories[category] = categories.get(category, 0) + 1 diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 947024b111..681d502f7c 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -2,12 +2,25 @@ from __future__ import annotations +import math +from collections.abc import Mapping from typing import Any -from infx.results.agentic.common import gauge_stat, normalize_fraction, rate, sum_stat +from infx.results.agentic.common import gauge_stat, normalize_fraction, series_stat, sum_stat from .base import ServerMetricsBackend, counter_int +# P/D decode workers re-count prefilled prompts in prefix_cache_full_tokens without reuse. +ACCOUNTED_ROLES = frozenset({"prefill", "unified"}) +FULL = "prefix_cache_full_tokens" +CACHED = "prefix_cache_cached_tokens" +OFFLOAD = "prefix_cache_offload_tokens" +LOADED = "lmcache_loaded_tokens" +REQUESTS = "prefix_cache_requests" +_COUNTERS = (FULL, CACHED, OFFLOAD, LOADED, REQUESTS) +VALIDITY_SCHEMA = "atom-cache-validity/1" +PRICING_STATUSES = frozenset({"VALID_COMPLETE", "VALID_ZERO"}) + def _atom_names(suffix: str) -> list[str]: """Accept direct worker metrics and Atomesh's colon-normalized form.""" @@ -19,8 +32,269 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] +def _series_by_endpoint( + section: Mapping[str, Any], stem: str +) -> tuple[dict[str, dict[str, Any]], list[str]]: + """One series per endpoint across all name aliases; a second one is a duplicate.""" + by_endpoint: dict[str, dict[str, Any]] = {} + duplicates: list[str] = [] + for name in _atom_counter_names(stem): + entry = section.get(name) + if not isinstance(entry, dict): + continue + for series in entry.get("series") or []: + url = series.get("endpoint_url") if isinstance(series, dict) else None + if not isinstance(url, str): + continue + if url in by_endpoint: + duplicates.append(url) + by_endpoint[url] = series + return by_endpoint, duplicates + + +def _slices(series: dict[str, Any]) -> list[tuple[int, int, float]] | None: + """Ordered, non-overlapping, finite, non-negative AIPerf timeslices, or None.""" + raw = series.get("timeslices") + if not isinstance(raw, list) or not raw: + return None + out: list[tuple[int, int, float]] = [] + for item in raw: + if not isinstance(item, dict): + return None + lo, hi, value = item.get("start_ns"), item.get("end_ns"), item.get("total") + if not isinstance(lo, int) or not isinstance(hi, int) or hi <= lo: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + if not math.isfinite(value) or value < 0: + return None + out.append((lo, hi, float(value))) + out.sort() + if any(out[i][0] < out[i - 1][1] for i in range(1, len(out))): + return None + return out + + +def _window_parts(slices: list[tuple[int, int, float]], start: int, end: int) -> dict[str, Any]: + """Split slice totals around [start, end]; flag sampling gaps inside the window.""" + parts = {"before": 0.0, "start_straddle": 0.0, "inside": 0.0, "end_straddle": 0.0} + covering = [s for s in slices if s[1] > start and s[0] < end] + gap = ( + not covering + or covering[0][0] > start + or covering[-1][1] < end + or any(covering[i][0] != covering[i - 1][1] for i in range(1, len(covering))) + ) + for lo, hi, value in slices: + if hi <= start: + parts["before"] += value + elif lo >= start and hi <= end: + parts["inside"] += value + else: + if lo < start < hi: + parts["start_straddle"] += value + if lo < end < hi and not lo < start < hi: + parts["end_straddle"] += value + parts["gap"] = gap + return parts + + +def account_cache( + metrics: dict[str, dict[str, Any]], + *, + framework: str, + endpoint_roles: Mapping[str, str] | None, + profiling_window: tuple[int, int] | None, + warmup_metrics: Mapping[str, Any] | None = None, + request_scope: Mapping[str, Any] | None = None, + disagg: bool | None = None, + kv_offloading: str | None = None, +) -> dict[str, Any]: + """Prefix-cache reuse on prefill (or unified) workers inside the profiling window. + + GPU = cached/full and CPU = offload/full on the same endpoints and timeslices. ATOM records + LMCache reuse in prefix_cache_offload_tokens, disjoint from the HBM `cached` counter, so + overall = GPU + CPU. lmcache_loaded_tokens is transfer volume and stays diagnostic only. + `validity` tells consumers whether the rates are a complete, same-scope actual observation + (`pricing_eligible`); warnings alone are not the guard. + """ + result: dict[str, Any] = dict.fromkeys( + ("gpu", "cpu", "overall", "full", "cached", "offload", "loaded", "requests") + ) + reasons: list[str] = [] + validity: dict[str, Any] = { + "schema": VALIDITY_SCHEMA, + "status": None, + "pricing_eligible": False, + "observed_scope": None, + "requested_scope": "profiling_successful_requests", + "cpu_component": None, + "roles_source": None, + "accounted_endpoints": [], + "window": None, + "request_scope": None, + "diagnostics": {}, + "reasons": reasons, + } + result.update(validity=validity, warnings=[]) + + def refuse(status: str, reason: str) -> dict[str, Any]: + validity["status"] = status + reasons.append(reason) + result["warnings"].append(f"atom cache accounting: {reason}") + return result + + series: dict[str, dict[str, dict[str, Any]]] = {} + for stem in _COUNTERS: + series[stem], duplicates = _series_by_endpoint(metrics, stem) + if duplicates: + return refuse("INVALID_COUNTER", f"duplicate {stem} series for {duplicates}") + observed = sorted(series[FULL]) + if not observed: + return refuse("MISSING_OBSERVATION", "no prefix_cache_full_tokens series") + + if endpoint_roles: + unmapped = [url for url in observed if url not in endpoint_roles] + accounted = sorted(u for u, role in endpoint_roles.items() if role in ACCOUNTED_ROLES) + missing = [url for url in accounted if url not in observed] + if unmapped or missing or not accounted: + return refuse( + "ROLE_OR_COVERAGE_UNRESOLVED", + "endpoint roles do not cover the scraped endpoints " + f"(unmapped={unmapped}, missing={missing})", + ) + validity["roles_source"] = "launcher_env" + elif disagg is False and len(observed) == 1: + accounted = observed + validity["roles_source"] = "single_endpoint_non_disagg" + else: + return refuse( + "ROLE_OR_COVERAGE_UNRESOLVED", + f"{framework or 'atom'}: no launcher endpoint roles and not a proven single unified " + "instance (SRT_PREFILL_ENDPOINTS/SRT_DECODE_ENDPOINTS or SRT_AGG_ENDPOINTS needed)", + ) + validity["accounted_endpoints"] = accounted + + for stem in (FULL, CACHED): + if any(url not in series[stem] for url in accounted): + return refuse("MISSING_OBSERVATION", f"{stem} missing on accounted endpoints") + cumulative = {} + for stem in _COUNTERS: + values = [series_stat(series[stem][u]) for u in accounted if u in series[stem]] + if values and len(values) == len(accounted) and None not in values: + cumulative[f"atom:{stem}"] = float(sum(values)) + validity["diagnostics"]["cumulative_stats_total"] = cumulative + + if profiling_window is None: + return refuse("WINDOW_UNRESOLVED", "no summary.phase_time_ranges.profiling window") + start, end = profiling_window + totals: dict[str, float | None] = {} + boundary: list[str] = [] + for stem in _COUNTERS: + present = [u for u in accounted if u in series[stem]] + if not present: + totals[stem] = None + continue + if len(present) != len(accounted): + totals[stem] = None + continue + inside = 0.0 + for url in present: + slices = _slices(series[stem][url]) + if slices is None: + return refuse("WINDOW_UNRESOLVED", f"{stem}@{url}: timeslices missing or invalid") + parts = _window_parts(slices, start, end) + if parts["gap"]: + return refuse("WINDOW_UNRESOLVED", f"{stem}@{url}: sampling gap in the window") + inside += parts["inside"] + if stem in (LOADED, REQUESTS): + continue + if parts["end_straddle"] > 0: + boundary.append(f"{stem}@{url}: activity in the slice straddling profiling end") + if parts["start_straddle"] > 0: + warm_series, _ = _series_by_endpoint(warmup_metrics or {}, stem) + warm = series_stat(warm_series[url]) if url in warm_series else None + if warm is None or parts["before"] + parts["start_straddle"] != warm: + boundary.append( + f"{stem}@{url}: activity in the slice straddling profiling start is not " + "proven warmup-only by warmup_metrics" + ) + totals[stem] = inside + validity["observed_scope"] = "profiling_window_interior_timeslices" + validity["window"] = {"start_ns": start, "end_ns": end, "boundary_issues": boundary} + + full, cached = totals[FULL], totals[CACHED] + offload, loaded, requests = totals[OFFLOAD], totals[LOADED], totals[REQUESTS] + if full is None or cached is None: + return refuse("MISSING_OBSERVATION", "cached/full counters missing on accounted endpoints") + if offload is not None: + validity["cpu_component"] = "MEASURED" + elif (kv_offloading or "").lower() == "none": + validity["cpu_component"] = "NO_CPU_TIER_CONFIRMED" + offload = 0.0 + else: + validity["cpu_component"] = "MISSING" + if not math.isfinite(full) or full <= 0: + return refuse("INVALID_COUNTER", "no prompt tokens in the accounting window") + if not math.isfinite(cached) or not 0 <= cached <= full: + return refuse( + "INVALID_COUNTER", f"cached tokens ({cached:.0f}) outside [0, full ({full:.0f})]" + ) + if offload is not None and ( + not math.isfinite(offload) or offload < 0 or not cached + offload <= full + ): + return refuse( + "INVALID_COUNTER", + f"cached+offload ({cached + offload:.0f}) exceeds full tokens ({full:.0f})", + ) + result.update(full=full, cached=cached, offload=offload, loaded=loaded, requests=requests) + result["gpu"] = cached / full + if offload is not None: + result.update(cpu=offload / full, overall=(cached + offload) / full) + + scope = dict(request_scope or {}) + client_requests = scope.get("profiling_requests") + client_tokens = scope.get("profiling_prompt_tokens") + conserved = ( + requests is not None + and client_tokens is not None + and requests == client_requests + and full == client_tokens + and not scope.get("error_records") + and not scope.get("cancelled_records") + ) + validity["request_scope"] = { + "server_window_admissions": requests, + "server_window_prompt_tokens": full, + **scope, + "conserved": conserved, + } + + if boundary: + reasons.extend(boundary) + validity["status"] = "WINDOW_UNRESOLVED" + elif validity["cpu_component"] == "MISSING": + reasons.append("CPU reuse counter missing and no confirmed absence of a CPU tier") + validity["status"] = "PARTIAL_COMPONENT" + elif not conserved: + reasons.append( + "server window admissions/prompt tokens do not match the successful profiling " + f"requests ({requests} vs {client_requests}; {full} vs {client_tokens})" + ) + validity["status"] = "REQUEST_SCOPE_UNRESOLVED" + else: + validity["status"] = "VALID_ZERO" if cached + offload == 0 else "VALID_COMPLETE" + validity["pricing_eligible"] = validity["status"] in PRICING_STATUSES + if not validity["pricing_eligible"]: + result["warnings"].append( + f"atom cache accounting: {validity['status']}; rates are telemetry, not pricing input" + ) + return result + + class AtomBackend(ServerMetricsBackend): name = "atom" + accepts_context = True def matches(self, metrics: dict[str, dict[str, Any]], framework: str) -> bool: metric_names = set(metrics) @@ -33,6 +307,8 @@ def populate( metrics: dict[str, dict[str, Any]], flat: dict[str, Any], nested: dict[str, Any], + *, + context: Mapping[str, Any] | None = None, ) -> None: prompt_total = sum_stat( metrics, @@ -47,29 +323,26 @@ def populate( flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) - cached_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_cached_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - full_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_full_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - cache_hit_rate = rate(cached_tokens, full_tokens) - external_tokens = sum_stat( + context = context or {} + accounting = account_cache( metrics, - _atom_counter_names("lmcache_loaded_tokens"), - preferred_keys=("total", "sum", "max", "avg"), + framework=str(context.get("framework") or ""), + endpoint_roles=context.get("endpoint_roles"), + profiling_window=context.get("profiling_window"), + warmup_metrics=context.get("warmup_metrics"), + request_scope=context.get("request_scope"), + disagg=context.get("disagg"), + kv_offloading=context.get("kv_offloading"), ) - external_hit_rate = rate(external_tokens, full_tokens) - overall_hit_rate = cache_hit_rate + sink = context.get("warnings") + if isinstance(sink, list): + sink.extend(accounting["warnings"]) - flat["server_gpu_cache_hit_rate"] = cache_hit_rate - flat["server_cpu_cache_hit_rate"] = external_hit_rate - flat["server_external_cache_hit_rate"] = external_hit_rate - flat["server_overall_cache_hit_rate"] = overall_hit_rate + flat["server_gpu_cache_hit_rate"] = accounting["gpu"] + flat["server_cpu_cache_hit_rate"] = accounting["cpu"] + # LocalCPUBackend is ATOM's only external tier, so external aliases the CPU reuse. + flat["server_external_cache_hit_rate"] = accounting["cpu"] + flat["server_overall_cache_hit_rate"] = accounting["overall"] flat["gpu_kv_cache_usage_pct"] = normalize_fraction( gauge_stat( metrics, @@ -81,16 +354,39 @@ def populate( nested["cache"].update( { - "gpu_cache_hit_rate": cache_hit_rate, - "cpu_cache_hit_rate": external_hit_rate, - "external_cache_hit_rate": external_hit_rate, - "overall_cache_hit_rate": overall_hit_rate, - "prefix_cache_hits": cached_tokens, - "prefix_cache_queries": full_tokens, - "external_prefix_cache_hits": external_tokens, - "external_prefix_cache_queries": full_tokens, + "gpu_cache_hit_rate": accounting["gpu"], + "cpu_cache_hit_rate": accounting["cpu"], + "external_cache_hit_rate": accounting["cpu"], + "overall_cache_hit_rate": accounting["overall"], + "prefix_cache_hits": accounting["cached"], + "prefix_cache_queries": accounting["full"], + "external_prefix_cache_hits": accounting["offload"], + "external_prefix_cache_queries": ( + accounting["full"] if accounting["offload"] is not None else None + ), } ) + nested["cache_validity"] = accounting["validity"] + by_source = nested["tokens"]["prompt_by_source"] + if accounting["full"] is not None and accounting["offload"] is not None: + by_source.update( + { + "gpu_cache_hit": accounting["cached"], + "cpu_or_external_cache_hit": accounting["offload"], + "computed": accounting["full"] - accounting["cached"] - accounting["offload"], + } + ) + by_source["raw"] = { + f"atom:{stem}": accounting[key] + for stem, key in ( + (FULL, "full"), + (CACHED, "cached"), + (OFFLOAD, "offload"), + (LOADED, "loaded"), + (REQUESTS, "requests"), + ) + if accounting[key] is not None + } nested["kv_cache"]["gpu_usage_pct"] = flat["gpu_kv_cache_usage_pct"] nested["tokens"].update( { diff --git a/inferencex-e2e/infx/results/agentic/backends/base.py b/inferencex-e2e/infx/results/agentic/backends/base.py index 427a4c1606..370897dd92 100644 --- a/inferencex-e2e/infx/results/agentic/backends/base.py +++ b/inferencex-e2e/infx/results/agentic/backends/base.py @@ -67,6 +67,7 @@ def empty_server_metrics(present: bool, metric_count: int) -> dict[str, Any]: "frontend_cached_tokens": None, "frontend_input_tokens": None, }, + "cache_validity": None, "kv_cache": { "gpu_usage_pct": None, "gpu_total_tokens": None, diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index d14962a61f..46cbc17684 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any from .backends import detect_backend @@ -12,6 +12,72 @@ empty_server_metrics, ) from .common import index_server_metrics +from .request_metrics import extract_per_record_ints + + +def request_scope_from_records( + records: list[dict[str, Any]], request_accounting: Mapping[str, Any] +) -> dict[str, Any]: + """Client-side profiling cohort in server token units (``usage_prompt_tokens``). + + ``profiling_prompt_tokens`` is None when any successful record lacks usage, so a partial + sum is never compared with server counters. ``error_records`` counts failed profiling + requests; accounting without that split falls back to the all-phase error total. + """ + prompt = extract_per_record_ints(records, "usage_prompt_tokens") + return { + "profiling_requests": len(records), + "profiling_prompt_tokens": sum(prompt) if len(prompt) == len(records) else None, + "error_records": request_accounting.get( + "records_profiling_error_dropped", request_accounting.get("records_error_dropped") + ), + "cancelled_records": sum( + 1 for record in records if (record.get("metadata") or {}).get("was_cancelled") + ), + } + + +def deployment_from_env(env: Mapping[str, str]) -> dict[str, Any]: + """Launcher-declared topology facts; absent or unrecognised values stay unknown (None).""" + disagg = {"true": True, "false": False}.get((env.get("DISAGG") or "").lower()) + return {"disagg": disagg, "kv_offloading": env.get("KV_OFFLOADING") or None} + + +def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: + """Map each scraped metrics URL to its launcher role, or None when roles are unknown. + + Mirrors ``infx.bench.agentic.replay._server_metrics_urls``: aggregated workers are + ``unified``; otherwise prefill endpoints precede decode endpoints. Caller-supplied + AIPERF_SERVER_METRICS_URLS and Dynamo frontends carry no roles, and an endpoint listed + under two roles is ambiguous. + """ + if env.get("AIPERF_SERVER_METRICS_URLS") or env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": + return None + if env.get("SRT_AGG_ENDPOINTS"): + groups = [("unified", env["SRT_AGG_ENDPOINTS"])] + else: + groups = [ + ("prefill", env.get("SRT_PREFILL_ENDPOINTS", "")), + ("decode", env.get("SRT_DECODE_ENDPOINTS", "")), + ] + roles: dict[str, str] = {} + for role, endpoints in groups: + for endpoint in endpoints.split(","): + if endpoint and roles.setdefault(f"http://{endpoint}/metrics", role) != role: + return None + return roles or None + + +def _profiling_window(server_metrics: dict[str, Any]) -> tuple[int, int] | None: + summary = server_metrics.get("summary") + ranges = summary.get("phase_time_ranges") if isinstance(summary, dict) else None + profiling = ranges.get("profiling") if isinstance(ranges, dict) else None + if not isinstance(profiling, dict): + return None + start, end = profiling.get("start_ns"), profiling.get("end_ns") + if isinstance(start, int) and isinstance(end, int) and start < end: + return start, end + return None def compute_server_metrics( @@ -20,6 +86,9 @@ def compute_server_metrics( framework: str, records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), + endpoint_roles: Mapping[str, str] | None = None, + request_scope: Mapping[str, Any] | None = None, + deployment: Mapping[str, Any] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() @@ -46,7 +115,20 @@ def compute_server_metrics( f"framework={framework.lower()!r}, metric_names=[{preview}]" ) nested["adapter"] = adapter.name - adapter.populate(metrics, flat, nested) + if getattr(adapter, "accepts_context", False): + warmup = server_metrics.get("warmup_metrics") + context = { + "framework": framework, + "endpoint_roles": endpoint_roles, + "profiling_window": _profiling_window(server_metrics), + "warmup_metrics": warmup if isinstance(warmup, dict) else None, + "request_scope": request_scope, + **dict(deployment or {}), + "warnings": warnings, + } + adapter.populate(metrics, flat, nested, context=context) + else: + adapter.populate(metrics, flat, nested) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": nested["kv_cache"]["gpu_total_tokens"] = None diff --git a/inferencex-e2e/infx/tests/bench/test_agentic_command.py b/inferencex-e2e/infx/tests/bench/test_agentic_command.py index 260809124b..b37558fb83 100644 --- a/inferencex-e2e/infx/tests/bench/test_agentic_command.py +++ b/inferencex-e2e/infx/tests/bench/test_agentic_command.py @@ -170,7 +170,9 @@ def test_standalone_srt_slurm_point_needs_no_kv_offload_declaration(tmp_path): WINDOW = { - "IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", "REQUIRE_POWER": "0", + "IS_MULTINODE": "true", + "ENABLE_AGENTX_POWER": "1", + "REQUIRE_POWER": "0", "SRT_MEASUREMENT_WINDOW_DIR": "/w", } MISSING = {"IS_MULTINODE": "true", "ENABLE_AGENTX_POWER": "1", "REQUIRE_POWER": "1"} @@ -439,3 +441,97 @@ def frontend(method: str, path: str) -> tuple[int, object]: assert "fake aiperf: profiled 1 request" in result.stdout aggregate = json.loads((tmp_path / "agentx.json").read_text()) assert (aggregate["conc"], aggregate["num_requests_total"]) == (8, 1) + + +FAKE_ATOM_AIPERF_SCRIPT = ( + FAKE_AIPERF_SCRIPT + + r""" +record["metrics"]["usage_prompt_tokens"] = {"value": 100, "unit": "tokens"} +(out / "profile_export.jsonl").write_text(json.dumps(record) + "\n") +urls = argv[argv.index("--server-metrics") + 1 :] +urls = urls[: next((i for i, url in enumerate(urls) if url.startswith("--")), len(urls))] +prefill = "http://" + os.environ["SRT_PREFILL_ENDPOINTS"] + "/metrics" +edges = [(0, 1000), (1000, 2500), (2500, 4000)] +counters = { + "prefix_cache_full_tokens": (0, 40, 60), + "prefix_cache_cached_tokens": (0, 20, 40), + "prefix_cache_offload_tokens": (0, 5, 5), + "prefix_cache_requests": (0, 1, 0), +} +def series(url, values): + slices = [{"start_ns": lo, "end_ns": hi, "total": v} for (lo, hi), v in zip(edges, values)] + return {"endpoint_url": url, "stats": {"total": sum(values)}, "timeslices": slices} +metrics = { + f"atom:{name}": { + "type": "counter", + "series": [series(url, values if url == prefill else (0, 50, 50)) for url in urls], + } + for name, values in counters.items() +} +summary = {"phase_time_ranges": {"profiling": {"start_ns": 1000, "end_ns": 4000}}} +(out / "server_metrics_export.json").write_text(json.dumps({"summary": summary, "metrics": metrics})) +""" +) + + +def test_srt_shim_hands_launcher_endpoint_roles_to_the_atom_producer(tmp_path, http_server): + tools = tmp_path / "tools" + tools.mkdir() + executable(tools / "uv", FAKE_UV) + (tools / "python3").symlink_to(sys.executable) + script = tmp_path / "fake_aiperf.py" + script.write_text(FAKE_ATOM_AIPERF_SCRIPT) + + def frontend(method: str, path: str) -> tuple[int, object]: + if path == "/v1/models": + return 200, {"data": [{"id": "served/model"}]} + return (405 if path == "/v1/chat/completions" else 200), {} + + workers = { + f"{side}_{key}": value + for side in ("PREFILL", "DECODE") + for key, value in { + "NUM_WORKERS": "1", + "TP": "8", + "PP_SIZE": "1", + "DCP_SIZE": "1", + "PCP_SIZE": "1", + "EP": "1", + "DP_ATTN": "false", + }.items() + } + env = _point( + tmp_path, + PATH=f"{tools}:/usr/bin:/bin", + HOME=str(tmp_path), + FAKE_AIPERF_SCRIPT=str(script), + REPLAY_ARGV=str(tmp_path / "replay-argv"), + HF_HUB_CACHE=str(tmp_path / "hf"), + AIPERF_RUNTIME_DIR=str(tmp_path / "runtime"), + EVAL_ONLY="true", + AIPERF_SERVER_URL=http_server(frontend), + SERVED_MODEL_NAME="served/model", + EVAL_ENDPOINT_READY_TIMEOUT_SECONDS="30", + EVAL_MODEL_STABILIZATION_SECONDS="0", + FRAMEWORK="atom-disagg", + DISAGG="true", + SRT_PREFILL_ENDPOINTS="10.0.0.1:6100", + SRT_DECODE_ENDPOINTS="10.0.0.2:6100", + **workers, + ) + + result = subprocess.run( + ["bash", str(SHIM)], env=env, capture_output=True, text=True, timeout=120, check=False + ) + + assert result.returncode == 0, result.stderr + argv = shlex.split((tmp_path / "replay-argv").read_text()) + scraped = argv[argv.index("--server-metrics") + 1 : argv.index("--server-metrics") + 3] + assert scraped == ["http://10.0.0.1:6100/metrics", "http://10.0.0.2:6100/metrics"] + server_metrics = json.loads((tmp_path / "agentx.json").read_text())["server_metrics"] + validity = server_metrics["cache_validity"] + assert validity["roles_source"] == "launcher_env" + assert validity["accounted_endpoints"] == ["http://10.0.0.1:6100/metrics"] + assert (validity["status"], validity["pricing_eligible"]) == ("VALID_COMPLETE", True) + cache = server_metrics["cache"] + assert (cache["gpu_cache_hit_rate"], cache["cpu_cache_hit_rate"]) == (0.6, 0.1) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py new file mode 100644 index 0000000000..b82c25f1d3 --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_accounting.py @@ -0,0 +1,493 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path + +import pytest + +from infx.bench.agentic.replay import _server_metrics_urls +from infx.results.agentic.server_metrics import compute_server_metrics, endpoint_roles_from_env +from infx.tests.results.agentic.test_process_agentic_result import ( + _assert_stable_server_metrics_schema, + _run_processor, + _write_fixture, +) + +PREFILL = "http://10.0.0.1:6100/metrics" +DECODE = "http://10.0.0.2:6100/metrics" +ROLES = {PREFILL: "prefill", DECODE: "decode"} +WINDOW = (1_000, 4_000) +# warmup | profiling | profiling | tail: no slice straddles the window edges. +EDGES = ((0, 1_000), (1_000, 2_500), (2_500, 4_000), (4_000, 5_000)) +ZERO = (0.0, 0.0, 0.0, 0.0) +SCOPE = {"profiling_requests": 5, "profiling_prompt_tokens": 1000, "error_records": 0} +PD = {"disagg": True, "kv_offloading": "dram"} +RATE_KEYS = ( + "gpu_cache_hit_rate", + "cpu_cache_hit_rate", + "external_cache_hit_rate", + "overall_cache_hit_rate", +) +NO_RATES = (None, None, None, None) + + +def _series(url: str, values: tuple, edges: tuple = EDGES) -> dict: + slices = [{"start_ns": lo, "end_ns": hi, "total": v} for (lo, hi), v in zip(edges, values)] + return {"endpoint_url": url, "stats": {"total": sum(values)}, "timeslices": slices} + + +def _export(counters: dict[str, dict[str, tuple]], edges: tuple = EDGES, window=WINDOW) -> dict: + metrics = { + f"atom:{stem}": {"type": "counter", "series": [_series(u, v, edges) for u, v in ep.items()]} + for stem, ep in counters.items() + } + export: dict = {"metrics": metrics} + if window is not None: + profiling = {"start_ns": window[0], "end_ns": window[1]} + export["summary"] = {"phase_time_ranges": {"profiling": profiling}} + return export + + +def _pd(offload: tuple | None = (0, 50, 50, 0), **override: tuple) -> dict: + """Interior prefill: full 1000, cached 600, offload 100, loaded 150, 5 admissions.""" + prefill = { + "prefix_cache_full_tokens": (0, 400, 600, 0), + "prefix_cache_cached_tokens": (0, 200, 400, 0), + "prefix_cache_offload_tokens": offload, + "lmcache_loaded_tokens": (0, 70, 80, 0), + "prefix_cache_requests": (0, 3, 2, 0), + } | override + decode = {"prefix_cache_full_tokens": (0, 500, 650, 0), "prefix_cache_requests": (0, 3, 2, 0)} + return _export( + { + stem: {PREFILL: values, DECODE: decode.get(stem, ZERO)} + for stem, values in prefill.items() + if values is not None + } + ) + + +def _run(export: dict, *, roles=ROLES, scope=SCOPE, deployment=PD, framework="atom-disagg"): + _, nested, warnings = compute_server_metrics( + export, + framework=framework, + records=[], + endpoint_roles=roles, + request_scope=scope, + deployment=deployment, + ) + return nested, warnings + + +def _rates(nested: dict) -> tuple: + return tuple(nested["cache"][key] for key in RATE_KEYS) + + +def test_complete_pd_observation_is_eligible_and_ignores_decode_and_load_volume() -> None: + nested, warnings = _run(_pd()) + validity = nested["cache_validity"] + + assert _rates(nested) == (0.6, 0.1, 0.1, pytest.approx(0.7)) + assert nested["cache"]["prefix_cache_queries"] == 1000 + assert nested["tokens"]["prompt_by_source"]["raw"]["atom:lmcache_loaded_tokens"] == 150 + assert validity["status"] == "VALID_COMPLETE" + assert validity["pricing_eligible"] is True + assert validity["observed_scope"] == "profiling_window_interior_timeslices" + assert validity["accounted_endpoints"] == [PREFILL] + assert validity["request_scope"]["conserved"] is True + assert warnings == [] + + +@pytest.mark.parametrize( + ("kv_offloading", "status", "cpu", "eligible"), + [ + ("dram", "PARTIAL_COMPONENT", None, False), + (None, "PARTIAL_COMPONENT", None, False), + ("none", "VALID_COMPLETE", 0.0, True), + ], +) +def test_missing_cpu_counter_is_partial_unless_no_cpu_tier_is_configured( + kv_offloading: str | None, status: str, cpu: float | None, eligible: bool +) -> None: + nested, _ = _run(_pd(offload=None), deployment={"disagg": True, "kv_offloading": kv_offloading}) + + assert nested["cache"]["gpu_cache_hit_rate"] == 0.6 + assert nested["cache"]["cpu_cache_hit_rate"] == cpu + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is eligible + + +@pytest.mark.parametrize( + ("offload", "kv_offloading"), + [(None, "none"), ((0, 0, 0, 0), "dram")], +) +def test_cached_above_full_is_refused_on_every_cpu_branch(offload, kv_offloading) -> None: + deployment = {"disagg": True, "kv_offloading": kv_offloading} + export = _pd(offload=offload, prefix_cache_cached_tokens=(0, 500, 700, 0)) + + nested, warnings = _run(export, deployment=deployment) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "INVALID_COUNTER" + assert nested["cache_validity"]["pricing_eligible"] is False + assert any("cached" in w for w in warnings) + + +def test_missing_counter_contract_zero_matches_an_explicit_zero() -> None: + no_cpu_tier = {"disagg": True, "kv_offloading": "none"} + + explicit, _ = _run(_pd(offload=(0, 0, 0, 0)), deployment=no_cpu_tier) + missing, _ = _run(_pd(offload=None), deployment=no_cpu_tier) + + assert _rates(explicit) == _rates(missing) == (0.6, 0.0, 0.0, 0.6) + assert explicit["cache_validity"]["cpu_component"] == "MEASURED" + assert missing["cache_validity"]["cpu_component"] == "NO_CPU_TIER_CONFIRMED" + assert explicit["cache_validity"]["pricing_eligible"] is True + assert missing["cache_validity"]["pricing_eligible"] is True + + +@pytest.mark.parametrize( + ("cached", "offload", "status", "rates"), + [ + ((0, 0, 0, 0), (0, 0, 0, 0), "VALID_ZERO", (0.0, 0.0, 0.0, 0.0)), + ((0, 0, 0, 0), (0, 100, 150, 0), "VALID_COMPLETE", (0.0, 0.25, 0.25, 0.25)), + ], +) +def test_zero_and_cpu_only_reuse_are_eligible_measurements(cached, offload, status, rates) -> None: + nested, _ = _run(_pd(offload=offload, prefix_cache_cached_tokens=cached)) + + assert _rates(nested) == rates + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is True + + +@pytest.mark.parametrize( + ("roles", "deployment", "reason"), + [ + (None, PD, "no launcher endpoint roles"), + (None, {"disagg": None}, "no launcher endpoint roles"), + ({DECODE: "decode"}, PD, f"unmapped=['{PREFILL}']"), + ({**ROLES, "http://10.0.0.3:6100/metrics": "prefill"}, PD, "missing=['http://10.0.0.3"), + ({PREFILL: "decode", DECODE: "decode"}, PD, "unmapped=[], missing=[]"), + ], +) +def test_unresolved_roles_publish_no_rates(roles, deployment, reason) -> None: + nested, warnings = _run(_pd(), roles=roles, deployment=deployment) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "ROLE_OR_COVERAGE_UNRESOLVED" + assert nested["cache_validity"]["pricing_eligible"] is False + assert any(reason in w for w in warnings) + + +def test_unified_fallback_needs_a_declared_non_disagg_single_endpoint() -> None: + single = _export({stem: {PREFILL: v} for stem, v in _pd_values().items()}) + two = _export({stem: {PREFILL: v, DECODE: v} for stem, v in _pd_values().items()}) + unified = {"disagg": False, "kv_offloading": "dram"} + + one, _ = _run(single, roles=None, deployment=unified, framework="atom") + many, _ = _run(two, roles=None, deployment=unified, framework="atom") + + assert one["cache_validity"]["roles_source"] == "single_endpoint_non_disagg" + assert one["cache_validity"]["pricing_eligible"] is True + assert many["cache_validity"]["status"] == "ROLE_OR_COVERAGE_UNRESOLVED" + + +def _pd_values() -> dict[str, tuple]: + return { + "prefix_cache_full_tokens": (0, 400, 600, 0), + "prefix_cache_cached_tokens": (0, 200, 400, 0), + "prefix_cache_offload_tokens": (0, 50, 50, 0), + "prefix_cache_requests": (0, 3, 2, 0), + } + + +@pytest.mark.parametrize( + ("override", "status", "reason"), + [ + ({"prefix_cache_cached_tokens": (0, -1, 400, 0)}, "WINDOW_UNRESOLVED", "invalid"), + ({"prefix_cache_full_tokens": (0, math.nan, 600, 0)}, "WINDOW_UNRESOLVED", "invalid"), + ( + {"prefix_cache_full_tokens": ZERO, "prefix_cache_cached_tokens": ZERO}, + "INVALID_COUNTER", + "no prompt tokens", + ), + ({"prefix_cache_cached_tokens": (0, 400, 550, 0)}, "INVALID_COUNTER", "exceeds full"), + ], +) +def test_invalid_counters_are_refused_not_clamped(override, status, reason) -> None: + nested, warnings = _run(_pd(**override)) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == status + assert any(reason in w for w in warnings) + + +def test_missing_timeslices_never_fall_back_to_stats_total() -> None: + export = _pd() + for entry in export["metrics"].values(): + for series in entry["series"]: + del series["timeslices"] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + cumulative = nested["cache_validity"]["diagnostics"]["cumulative_stats_total"] + assert cumulative["atom:prefix_cache_full_tokens"] == 1000 + + +def test_missing_profiling_window_is_unresolved() -> None: + export = _pd() + del export["summary"] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + + +# Window (1_200, 4_000): (1_000, 1_400) straddles the start, (3_600, 4_200) the end. +STRADDLE = ((0, 1_000), (1_000, 1_400), (1_400, 2_600), (2_600, 3_600), (3_600, 4_200)) + + +def _unified(full: tuple, cached: tuple, warmup: dict | None = None) -> dict: + export = _export( + { + "prefix_cache_full_tokens": {PREFILL: full}, + "prefix_cache_cached_tokens": {PREFILL: cached}, + "prefix_cache_offload_tokens": {PREFILL: (0, 0, 0, 0, 0)}, + "prefix_cache_requests": {PREFILL: (0, 0, 3, 2, 0)}, + }, + edges=STRADDLE, + window=(1_200, 4_000), + ) + if warmup is not None: + export["warmup_metrics"] = { + f"atom:{stem}": {"series": [{"endpoint_url": PREFILL, "stats": {"total": total}}]} + for stem, total in warmup.items() + } + return export + + +@pytest.mark.parametrize( + ("full", "cached", "warmup", "status"), + [ + ((0, 0, 400, 600, 0), (0, 0, 200, 400, 0), None, "VALID_COMPLETE"), + ((0, 0, 400, 600, 9), (0, 0, 200, 400, 0), None, "WINDOW_UNRESOLVED"), + ((300, 90, 400, 600, 0), (100, 30, 200, 400, 0), None, "WINDOW_UNRESOLVED"), + ( + (300, 90, 400, 600, 0), + (100, 30, 200, 400, 0), + {"prefix_cache_full_tokens": 390, "prefix_cache_cached_tokens": 130}, + "VALID_COMPLETE", + ), + ( + (300, 90, 400, 600, 0), + (100, 30, 200, 400, 0), + {"prefix_cache_full_tokens": 300, "prefix_cache_cached_tokens": 100}, + "WINDOW_UNRESOLVED", + ), + ], +) +def test_boundary_slice_activity_must_be_proven_warmup_only(full, cached, warmup, status) -> None: + unified = {"disagg": False, "kv_offloading": "dram"} + nested, _ = _run(_unified(full, cached, warmup), roles=None, deployment=unified) + + assert nested["cache_validity"]["status"] == status + assert nested["cache_validity"]["pricing_eligible"] is (status == "VALID_COMPLETE") + assert nested["cache"]["gpu_cache_hit_rate"] == 0.6 + + +def test_sampling_gap_inside_the_window_is_unresolved() -> None: + gappy = ((0, 1_000), (1_000, 2_000), (2_200, 4_000), (4_000, 5_000)) + export = _export( + {stem: {PREFILL: values} for stem, values in _pd_values().items()}, edges=gappy + ) + + nested, warnings = _run(export, roles=None, deployment={"disagg": False}) + + assert nested["cache_validity"]["status"] == "WINDOW_UNRESOLVED" + assert any("sampling gap" in w for w in warnings) + + +def test_duplicate_alias_series_for_one_endpoint_is_invalid() -> None: + export = _pd() + export["metrics"]["atom_prefix_cache_cached_tokens_total"] = export["metrics"][ + "atom:prefix_cache_cached_tokens" + ] + + nested, _ = _run(export) + + assert _rates(nested) == NO_RATES + assert nested["cache_validity"]["status"] == "INVALID_COUNTER" + + +@pytest.mark.parametrize( + "scope", + [ + {**SCOPE, "profiling_requests": 4, "profiling_prompt_tokens": 900}, + {**SCOPE, "error_records": 1}, + {**SCOPE, "profiling_prompt_tokens": None}, + None, + ], +) +def test_request_scope_mismatch_keeps_window_telemetry_but_blocks_pricing(scope) -> None: + nested, _ = _run(_pd(), scope=scope) + + assert _rates(nested) == (0.6, 0.1, 0.1, pytest.approx(0.7)) + assert nested["cache_validity"]["status"] == "REQUEST_SCOPE_UNRESOLVED" + assert nested["cache_validity"]["pricing_eligible"] is False + + +def test_unrelated_router_endpoint_is_not_a_missing_prefill() -> None: + export = _pd() + router = "http://10.0.0.9:8000/metrics" + export["metrics"]["atomesh:http_requests_total"] = { + "type": "counter", + "series": [_series(router, (0, 5, 5, 0))], + } + + nested, _ = _run(export, roles={**ROLES, router: "router"}) + + assert nested["cache_validity"]["status"] == "VALID_COMPLETE" + + +@pytest.mark.parametrize( + ("env", "expected"), + [ + ( + {"SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100,"}, + ROLES, + ), + ( + {"SRT_AGG_ENDPOINTS": "a:1,b:1", "SRT_PREFILL_ENDPOINTS": "c:1"}, + {"http://a:1/metrics": "unified", "http://b:1/metrics": "unified"}, + ), + ], +) +def test_endpoint_roles_cover_launcher_scrape_urls( + env: dict[str, str], expected: dict[str, str] +) -> None: + roles = endpoint_roles_from_env(env) + + assert roles == expected + assert set(roles) == set(_server_metrics_urls(env)) + + +@pytest.mark.parametrize( + "env", + [ + {"AIPERF_SERVER_METRICS_URLS": "http://x/metrics", "SRT_PREFILL_ENDPOINTS": "a:1"}, + {"SRTCTL_FRONTEND_TYPE": "dynamo", "SRT_PREFILL_ENDPOINTS": "a:1"}, + {"SRT_PREFILL_ENDPOINTS": "a:1", "SRT_DECODE_ENDPOINTS": "a:1"}, + {}, + ], +) +def test_endpoint_roles_unknown_without_launcher_metadata(env: dict[str, str]) -> None: + assert endpoint_roles_from_env(env) is None + + +DISAGG_ENV = { + "FRAMEWORK": "atom-disagg", + "IS_MULTINODE": "true", + "DISAGG": "true", + **{ + f"{side}_{key}": value + for side in ("PREFILL", "DECODE") + for key, value in { + "NUM_WORKERS": "1", + "TP": "8", + "PP_SIZE": "1", + "DCP_SIZE": "1", + "PCP_SIZE": "1", + "EP": "1", + "DP_ATTN": "false", + "HARDWARE": "mi355x", + }.items() + }, +} +ROLE_ENV = {"SRT_PREFILL_ENDPOINTS": "10.0.0.1:6100", "SRT_DECODE_ENDPOINTS": "10.0.0.2:6100"} + + +def _atom_result_dir(tmp_path: Path) -> tuple[Path, list[dict]]: + result_dir = _write_fixture(tmp_path) + artifacts = result_dir / "aiperf_artifacts" + (artifacts / "server_metrics_export.json").write_text(json.dumps(_pd())) + jsonl = artifacts / "profile_export.jsonl" + records = [json.loads(line) for line in jsonl.read_text().splitlines() if line.strip()] + for record in records: + record["metrics"]["usage_prompt_tokens"] = {"value": 1000 // len(records), "unit": "tokens"} + return result_dir, records + + +def _write_records(result_dir: Path, records: list[dict]) -> None: + jsonl = result_dir / "aiperf_artifacts" / "profile_export.jsonl" + jsonl.write_text("".join(json.dumps(record) + "\n" for record in records)) + + +def _failed(phase: str | None) -> dict: + metadata = {} if phase is None else {"benchmark_phase": phase} + return {"metadata": metadata, "error": {"type": "Timeout", "message": "request timed out"}} + + +def test_processor_entry_point_carries_validity_through_the_serializer(tmp_path: Path) -> None: + result_dir, records = _atom_result_dir(tmp_path) + _write_records(result_dir, records) + + agg = _run_processor(result_dir, tmp_path / "out", env_overrides={**DISAGG_ENV, **ROLE_ENV}) + unresolved = _run_processor(result_dir, tmp_path / "unresolved", env_overrides=DISAGG_ENV) + + _assert_stable_server_metrics_schema(agg) + assert len(records) == 5 + assert _rates(agg["server_metrics"]) == (0.6, 0.1, 0.1, 0.7) + assert agg["server_metrics"]["cache_validity"]["status"] == "VALID_COMPLETE" + assert agg["server_metrics"]["cache_validity"]["pricing_eligible"] is True + assert _rates(unresolved["server_metrics"]) == NO_RATES + assert unresolved["server_metrics"]["cache_validity"]["pricing_eligible"] is False + assert ( + agg["server_metrics"]["tokens"]["prompt_total"] + == (unresolved["server_metrics"]["tokens"]["prompt_total"]) + ) + + +@pytest.mark.parametrize( + ("failed_phases", "legacy", "status", "profiling_errors"), + [ + ((), False, "VALID_COMPLETE", 0), + (("warmup",), False, "VALID_COMPLETE", 0), + (("profiling",), False, "REQUEST_SCOPE_UNRESOLVED", 1), + (("warmup", "profiling"), False, "REQUEST_SCOPE_UNRESOLVED", 1), + ((), True, "VALID_COMPLETE", 0), + ((None,), True, "REQUEST_SCOPE_UNRESOLVED", 1), + ], + ids=["clean", "warmup-error", "profiling-error", "both-phases", "legacy-clean", "legacy-error"], +) +def test_only_profiling_errors_from_the_loader_block_pricing( + tmp_path: Path, + failed_phases: tuple[str | None, ...], + legacy: bool, + status: str, + profiling_errors: int, +) -> None: + result_dir, records = _atom_result_dir(tmp_path) + if legacy: + for record in records: + del record["metadata"]["benchmark_phase"] + _write_records(result_dir, [_failed(phase) for phase in failed_phases] + records) + + agg = _run_processor(result_dir, tmp_path / "out", env_overrides={**DISAGG_ENV, **ROLE_ENV}) + validity = agg["server_metrics"]["cache_validity"] + accounting = agg["request_accounting"] + + assert _rates(agg["server_metrics"]) == (0.6, 0.1, 0.1, 0.7) + assert validity["status"] == status + assert validity["pricing_eligible"] is (status == "VALID_COMPLETE") + assert validity["request_scope"]["error_records"] == profiling_errors + assert accounting["records_profiled"] == 5 + assert accounting["records_error_dropped"] == len(failed_phases) + assert accounting["records_profiling_error_dropped"] == profiling_errors + assert accounting["records_warmup_dropped"] == failed_phases.count("warmup") + expected_categories = {"Timeout": len(failed_phases)} if failed_phases else {} + assert accounting["error_categories"] == expected_categories diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index b76964b2ff..e420a19f5e 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -67,6 +67,7 @@ "records_dropped_total", "records_warmup_dropped", "records_error_dropped", + "records_profiling_error_dropped", "error_categories", } @@ -79,6 +80,7 @@ "kv_offload", "tokens", "sources", + "cache_validity", } SERVER_CACHE_KEYS = { "gpu_cache_hit_rate", @@ -336,8 +338,7 @@ def _run_processor( timeout=30, ) assert proc.returncode == 0, ( - f"processor exited {proc.returncode}\n" - f"stdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + f"processor exited {proc.returncode}\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" ) out = output_dir / "agg_test.json" assert out.exists(), f"missing output {out}; stdout:\n{proc.stdout}" @@ -393,27 +394,54 @@ def test_processor_omits_component_metadata_when_absent(tmp_path: Path): assert "kv_p2p_transfer" not in agg -@pytest.mark.parametrize("parser", [ - _optional_component_metadata, _optional_kv_offload_backend_metadata, -]) +@pytest.mark.parametrize( + "parser", + [ + _optional_component_metadata, + _optional_kv_offload_backend_metadata, + ], +) @pytest.mark.parametrize("raw", [None, "", "null"]) def test_optional_metadata_accepts_unset_values(parser, raw): env = {} if raw is None else {"TEST_METADATA": raw} assert parser(env, "TEST_METADATA") is None -@pytest.mark.parametrize(("parser", "raw", "message"), [ - (_optional_component_metadata, "{", "must contain valid JSON"), - (_optional_component_metadata, "[]", "must contain exactly 'name' and 'version'"), - (_optional_component_metadata, '{"name":"router"}', "must contain exactly 'name' and 'version'"), - (_optional_component_metadata, '{"name":"router","version":0}', "name and version must be non-empty strings"), - (_optional_kv_offload_backend_metadata, "{", "must contain valid JSON"), - (_optional_kv_offload_backend_metadata, "[]", "may contain only 'name' and 'version'"), - (_optional_kv_offload_backend_metadata, '{"name":"cache","extra":1}', "may contain only 'name' and 'version'"), - (_optional_kv_offload_backend_metadata, "{}", "must contain 'name' and optional 'version'"), - (_optional_kv_offload_backend_metadata, '{"version":"1"}', "must contain 'name' and optional 'version'"), - (_optional_kv_offload_backend_metadata, '{"name":"cache","version":""}', "values must be non-empty strings"), -]) +@pytest.mark.parametrize( + ("parser", "raw", "message"), + [ + (_optional_component_metadata, "{", "must contain valid JSON"), + (_optional_component_metadata, "[]", "must contain exactly 'name' and 'version'"), + ( + _optional_component_metadata, + '{"name":"router"}', + "must contain exactly 'name' and 'version'", + ), + ( + _optional_component_metadata, + '{"name":"router","version":0}', + "name and version must be non-empty strings", + ), + (_optional_kv_offload_backend_metadata, "{", "must contain valid JSON"), + (_optional_kv_offload_backend_metadata, "[]", "may contain only 'name' and 'version'"), + ( + _optional_kv_offload_backend_metadata, + '{"name":"cache","extra":1}', + "may contain only 'name' and 'version'", + ), + (_optional_kv_offload_backend_metadata, "{}", "must contain 'name' and optional 'version'"), + ( + _optional_kv_offload_backend_metadata, + '{"version":"1"}', + "must contain 'name' and optional 'version'", + ), + ( + _optional_kv_offload_backend_metadata, + '{"name":"cache","version":""}', + "values must be non-empty strings", + ), + ], +) def test_optional_metadata_preserves_cli_errors(parser, raw, message): with pytest.raises(SystemExit) as error: parser({"TEST_METADATA": raw}, "TEST_METADATA") @@ -795,49 +823,82 @@ def test_gpu_shape_defaults_empty_parallelism_to_one(env): assert _gpu_shape(env) == ({"pp": 1, "dcp_size": 1, "pcp_size": 1}, 1, 1, 1, "false") -@pytest.mark.parametrize("decode_workers,expected_gpus,expected_decode", [ - ("3", 78, (2, 11, 1, 9, 5)), - ("0", 48, (0, 0, 1, 1, 1)), -]) +@pytest.mark.parametrize( + "decode_workers,expected_gpus,expected_decode", + [ + ("3", 78, (2, 11, 1, 9, 5)), + ("0", 48, (0, 0, 1, 1, 1)), + ], +) def test_gpu_shape_counts_workers_and_normalizes_absent_decode( - decode_workers, expected_gpus, expected_decode, + decode_workers, + expected_gpus, + expected_decode, ): env = { - "IS_MULTINODE": "true", "PREFILL_NUM_WORKERS": "2", "PREFILL_TP": "3", - "PREFILL_PP_SIZE": "2", "PREFILL_PCP_SIZE": "4", "PREFILL_DCP_SIZE": "7", - "PREFILL_EP": "8", "PREFILL_DP_ATTN": "YES", "DECODE_NUM_WORKERS": decode_workers, - "DECODE_TP": "2", "DECODE_EP": "11", "DECODE_PP_SIZE": "1", - "DECODE_DCP_SIZE": "9", "DECODE_PCP_SIZE": "5", "DECODE_DP_ATTN": "false", - "PREFILL_GPUS": "999", "DECODE_GPUS": "999", + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "2", + "PREFILL_TP": "3", + "PREFILL_PP_SIZE": "2", + "PREFILL_PCP_SIZE": "4", + "PREFILL_DCP_SIZE": "7", + "PREFILL_EP": "8", + "PREFILL_DP_ATTN": "YES", + "DECODE_NUM_WORKERS": decode_workers, + "DECODE_TP": "2", + "DECODE_EP": "11", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "9", + "DECODE_PCP_SIZE": "5", + "DECODE_DP_ATTN": "false", + "PREFILL_GPUS": "999", + "DECODE_GPUS": "999", } fields, num_gpus, tp, ep, attention = _gpu_shape(env) assert num_gpus == expected_gpus assert fields["num_prefill_gpu"] == 48 assert fields["num_decode_gpu"] == (30 if decode_workers == "3" else 0) - assert tuple(fields[key] for key in ("decode_tp", "decode_ep", "decode_pp", - "decode_dcp_size", "decode_pcp_size")) == expected_decode + assert ( + tuple( + fields[key] + for key in ("decode_tp", "decode_ep", "decode_pp", "decode_dcp_size", "decode_pcp_size") + ) + == expected_decode + ) assert (tp, ep, attention) == ((5, 11, "true") if decode_workers == "3" else (3, 8, "true")) assert fields["prefill_dp_attention"] == "YES" assert fields["decode_dp_attention"] == "false" -@pytest.mark.parametrize("env,error_type,message", [ - ({"IS_MULTINODE": "true", "TP": "bad"}, ValueError, "invalid literal for int"), - ({"IS_MULTINODE": "true", "PREFILL_PP_SIZE": "0", "PREFILL_HARDWARE": "gpu"}, - SystemExit, "Multinode PP, DCP, and PCP sizes must be positive integers."), - ({"IS_MULTINODE": "true", "DECODE_NUM_WORKERS": "0", "DECODE_PCP_SIZE": "0"}, - SystemExit, "Multinode PP, DCP, and PCP sizes must be positive integers."), -]) +@pytest.mark.parametrize( + "env,error_type,message", + [ + ({"IS_MULTINODE": "true", "TP": "bad"}, ValueError, "invalid literal for int"), + ( + {"IS_MULTINODE": "true", "PREFILL_PP_SIZE": "0", "PREFILL_HARDWARE": "gpu"}, + SystemExit, + "Multinode PP, DCP, and PCP sizes must be positive integers.", + ), + ( + {"IS_MULTINODE": "true", "DECODE_NUM_WORKERS": "0", "DECODE_PCP_SIZE": "0"}, + SystemExit, + "Multinode PP, DCP, and PCP sizes must be positive integers.", + ), + ], +) def test_gpu_shape_preserves_parsing_and_validation_order(env, error_type, message): with pytest.raises(error_type, match=message): _gpu_shape(env) -@pytest.mark.parametrize("error,category", [ - ({"type": "HTTPStatusError", "message": "500 server error"}, "HTTPStatusError"), - (" \t\r\n", "unknown"), - ({"message": " \t\r\n"}, "unknown"), -]) +@pytest.mark.parametrize( + "error,category", + [ + ({"type": "HTTPStatusError", "message": "500 server error"}, "HTTPStatusError"), + (" \t\r\n", "unknown"), + ({"message": " \t\r\n"}, "unknown"), + ], +) def test_processor_surfaces_request_accounting(tmp_path: Path, error, category): result_dir = tmp_path / "results" artifact = result_dir / "aiperf_artifacts" @@ -895,6 +956,7 @@ def test_processor_surfaces_request_accounting(tmp_path: Path, error, category): "records_dropped_total": 2, "records_warmup_dropped": 1, "records_error_dropped": 1, + "records_profiling_error_dropped": 1, "error_categories": {category: 1}, } assert agg["server_metrics"]["tokens"]["requests_completed"] == 1 @@ -1336,9 +1398,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path, framework: s }, "sglang:realtime_tokens": { "type": "counter", - "series": [ - {"labels": {"mode": "prefill_compute"}, "stats": {"total": 500.0}} - ], + "series": [{"labels": {"mode": "prefill_compute"}, "stats": {"total": 500.0}}], }, } } @@ -1383,9 +1443,7 @@ def test_processor_normalizes_sglang_server_metrics(tmp_path: Path, framework: s if framework == "dynamo-sglang": assert agg["server_metrics"]["kv_cache"]["gpu_total_tokens"] is None assert agg["kv_cache_pool_tokens"] is None - assert any( - "rank replicas are not normalized" in warning for warning in agg["warnings"] - ) + assert any("rank replicas are not normalized" in warning for warning in agg["warnings"]) def test_processor_normalizes_trtllm_server_metrics(tmp_path: Path): @@ -1669,9 +1727,7 @@ def test_processor_uses_aiperf_theoretical_cache_metric(tmp_path: Path): assert proc.returncode == 0, proc.stderr agg = json.loads((tmp_path / "out" / "agg_test.json").read_text()) assert agg["request_metrics"]["cache"]["theoretical_cache_hit_rate"] == pytest.approx(0.25) - assert agg["request_metrics"]["tokens"]["output_expected"]["mean"] == pytest.approx( - 55.0 - ) + assert agg["request_metrics"]["tokens"]["output_expected"]["mean"] == pytest.approx(55.0) @pytest.mark.parametrize("cache_state", ["matching", "missing", "ambiguous", "no_identity"]) @@ -1740,29 +1796,61 @@ def test_processor_supports_per_run_subdir_layout(tmp_path: Path): def test_builder_uses_explicit_inputs_without_environment_or_file_access(monkeypatch): records = [ - _make_record(conv_id="root", turn_index=0, isl=100, osl=50, - ttft_ms=30, e2e_ms=1000, itl_ms=10, - start_ns=1_000_000_000, end_ns=2_000_000_000), - _make_record(conv_id="root::sa:child", turn_index=1, isl=200, osl=100, - ttft_ms=50, e2e_ms=2000, itl_ms=20, - start_ns=2_000_000_000, end_ns=4_000_000_000), + _make_record( + conv_id="root", + turn_index=0, + isl=100, + osl=50, + ttft_ms=30, + e2e_ms=1000, + itl_ms=10, + start_ns=1_000_000_000, + end_ns=2_000_000_000, + ), + _make_record( + conv_id="root::sa:child", + turn_index=1, + isl=200, + osl=100, + ttft_ms=50, + e2e_ms=2000, + itl_ms=20, + start_ns=2_000_000_000, + end_ns=4_000_000_000, + ), ] aggregate = {"metadata": {"dataset": {"hf_dataset_name": "example/traces"}}} - server_metrics = {"metrics": {"vllm:prefix_cache_hits": { - "series": [{"stats": {"total": 10}}], - }, "vllm:prefix_cache_queries": {"series": [{"stats": {"total": 40}}]}}} - traces = [{"id": "root", "requests": [ - {"type": "n", "out": 11}, {"type": "tool", "out": 999}, - {"type": "s", "output_length": 29}, - ]}] + server_metrics = { + "metrics": { + "vllm:prefix_cache_hits": { + "series": [{"stats": {"total": 10}}], + }, + "vllm:prefix_cache_queries": {"series": [{"stats": {"total": 40}}]}, + } + } + traces = [ + { + "id": "root", + "requests": [ + {"type": "n", "out": 11}, + {"type": "tool", "out": 999}, + {"type": "s", "output_length": 29}, + ], + } + ] logs = [ "INFO (EngineCore_DP0 pid=10) GPU KV cache size: 800 tokens\n" "INFO (EngineCore_DP0 pid=10) GPU KV cache size: 800 tokens", "INFO (EngineCore_DP0 pid=11) GPU KV cache size: 1,200 tokens", ] - accounting = {"records_total": 4, "records_profiled": 2, "records_dropped_total": 2, - "records_warmup_dropped": 1, "records_error_dropped": 1, - "error_categories": {"Timeout": 1}} + accounting = { + "records_total": 4, + "records_profiled": 2, + "records_dropped_total": 2, + "records_warmup_dropped": 1, + "records_error_dropped": 1, + "error_categories": {"Timeout": 1}, + } env = {"KV_OFFLOADING": "none", "TP": "4", "FRAMEWORK": "vllm"} before = deepcopy((records, aggregate, server_metrics, traces, logs, accounting, env)) @@ -1777,13 +1865,20 @@ class NoEnvironment(dict): isolated.setattr("builtins.open", forbidden) isolated.setattr(Path, "open", forbidden) result = build_result( - records, aggregate, server_metrics, MappingProxyType(env), - request_accounting=accounting, traces=iter(traces), server_logs=iter(logs), + records, + aggregate, + server_metrics, + MappingProxyType(env), + request_accounting=accounting, + traces=iter(traces), + server_logs=iter(logs), ) second = build_result(records, aggregate, {}, {**env, "TP": "2"}) assert result["request_metrics"]["throughput"]["per_gpu"] == { - "total_tput_tps": 37.5, "output_tput_tps": 12.5, "input_tput_tps": 25.0, + "total_tput_tps": 37.5, + "output_tput_tps": 12.5, + "input_tput_tps": 25.0, } assert second["request_metrics"]["throughput"]["per_gpu"]["total_tput_tps"] == 75.0 assert result["request_metrics"]["tokens"]["output_expected"]["mean"] == 20 @@ -1796,18 +1891,23 @@ class NoEnvironment(dict): assert (records, aggregate, server_metrics, traces, logs, accounting, env) == before -@pytest.mark.parametrize("framework,metric_names", [ - ("atom", ["atom:prompt_tokens"]), - ("trtllm", ["trtllm_kv_cache_max_blocks"]), - ("dynamo-sglang", ["sglang:num_requests_total"]), -]) +@pytest.mark.parametrize( + "framework,metric_names", + [ + ("atom", ["atom:prompt_tokens"]), + ("trtllm", ["trtllm_kv_cache_max_blocks"]), + ("dynamo-sglang", ["sglang:num_requests_total"]), + ], +) def test_builder_does_not_consume_logs_when_backend_does_not_use_them(framework, metric_names): def unavailable_logs(): raise OSError("unreadable optional log") yield result = build_result( - [], {}, {"metrics": {name: {"series": []} for name in metric_names}}, + [], + {}, + {"metrics": {name: {"series": []} for name in metric_names}}, {"KV_OFFLOADING": "none", "FRAMEWORK": framework}, server_logs=unavailable_logs(), ) @@ -1819,30 +1919,47 @@ def unavailable_logs(): ] -@pytest.mark.parametrize("override,message", [ - ({"KV_OFFLOADING": ""}, "Missing required environment variable: KV_OFFLOADING"), - ({"PP_SIZE": "0"}, "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers."), - ({"ROUTER_METADATA": "{"}, "ROUTER_METADATA must contain valid JSON"), - ({}, "invalid literal for int() with base 10: 'bad-output'"), -]) +@pytest.mark.parametrize( + "override,message", + [ + ({"KV_OFFLOADING": ""}, "Missing required environment variable: KV_OFFLOADING"), + ({"PP_SIZE": "0"}, "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers."), + ({"ROUTER_METADATA": "{"}, "ROUTER_METADATA must contain valid JSON"), + ({}, "invalid literal for int() with base 10: 'bad-output'"), + ], +) def test_processor_preserves_validation_before_optional_trace_reads(tmp_path, override, message): result_dir = _write_fixture(tmp_path) cache = tmp_path / "hf" snapshot = cache / "datasets--semianalysisai--cc-traces-weka-062126" / "snapshots" / "one" snapshot.mkdir(parents=True) - (snapshot / "traces.json").write_text(json.dumps({ - "id": "trace-A", "requests": [{"type": "n", "out": "bad-output"}], - })) + (snapshot / "traces.json").write_text( + json.dumps( + { + "id": "trace-A", + "requests": [{"type": "n", "out": "bad-output"}], + } + ) + ) out = tmp_path / "out" env = { - "PATH": os.environ.get("PATH", ""), "PYTHONPATH": str(REPO_ROOT), - "RESULT_DIR": str(result_dir), "AGENTIC_OUTPUT_DIR": str(out), - "RESULT_FILENAME": "agg_test", "KV_OFFLOADING": "none", - "FRAMEWORK": "vllm", "HF_HUB_CACHE": str(cache), **override, + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(REPO_ROOT), + "RESULT_DIR": str(result_dir), + "AGENTIC_OUTPUT_DIR": str(out), + "RESULT_FILENAME": "agg_test", + "KV_OFFLOADING": "none", + "FRAMEWORK": "vllm", + "HF_HUB_CACHE": str(cache), + **override, } proc = subprocess.run( [sys.executable, "-m", "infx.results.agentic.process_agentic_result"], - cwd=REPO_ROOT, env=env, text=True, capture_output=True, timeout=30, + cwd=REPO_ROOT, + env=env, + text=True, + capture_output=True, + timeout=30, ) assert proc.returncode == 1 assert proc.stderr.rstrip().endswith(message) @@ -1858,9 +1975,14 @@ def test_processor_trace_cache_precedence_and_last_duplicate(tmp_path, cache_env snapshot.mkdir(parents=True) trace = {"id": "trace-A", "requests": [{"type": "n", "out": 17}] * 3} (snapshot / "a.jsonl").write_text("{bad json\n" + json.dumps(trace) + "\n") - (snapshot / "z.json").write_text(json.dumps({ - **trace, "requests": [{"type": "s", "output_length": 23}] * 3, - })) + (snapshot / "z.json").write_text( + json.dumps( + { + **trace, + "requests": [{"type": "s", "output_length": 23}] * 3, + } + ) + ) (snapshot / "zz.json").write_text(json.dumps({"id": "trace-A", "requests": []})) env = {"HF_HUB_CACHE": "", "HUGGINGFACE_HUB_CACHE": "", "HF_HOME": ""} env[cache_env] = str(cache) @@ -1880,13 +2002,20 @@ def test_processor_trace_cache_precedence_and_last_duplicate(tmp_path, cache_env assert expected["std"] == 0 -@pytest.mark.parametrize("offsets,expected_mean,expected_p95,windows", [ - ([0, 500_000_000, 1_000_000_000, 1_500_000_000, 2_000_000_000], 2, 2, 2), - ([0, 1_000_000_000, 1_000_000_000, 2_000_000_000], 1.5, 1.95, 2), - ([0, 250_000_000, 500_000_000], 6, None, 0), -]) -def test_qps_windows_exclude_right_boundary_and_preserve_duplicates(offsets, expected_mean, expected_p95, windows): - records = [{"metadata": {"request_end_ns": 1_000_000_000 + offset}} for offset in reversed(offsets)] +@pytest.mark.parametrize( + "offsets,expected_mean,expected_p95,windows", + [ + ([0, 500_000_000, 1_000_000_000, 1_500_000_000, 2_000_000_000], 2, 2, 2), + ([0, 1_000_000_000, 1_000_000_000, 2_000_000_000], 1.5, 1.95, 2), + ([0, 250_000_000, 500_000_000], 6, None, 0), + ], +) +def test_qps_windows_exclude_right_boundary_and_preserve_duplicates( + offsets, expected_mean, expected_p95, windows +): + records = [ + {"metadata": {"request_end_ns": 1_000_000_000 + offset}} for offset in reversed(offsets) + ] flat, nested = compute_qps_stats(records) assert nested["samples"] == windows assert flat["mean_qps"] == pytest.approx(expected_mean) @@ -1898,10 +2027,11 @@ def test_qps_windows_exclude_right_boundary_and_preserve_duplicates(offsets, exp @pytest.mark.parametrize("invalid_turn", [-1, -3, 2]) def test_expected_output_tokens_ignore_out_of_range_trace_turns(invalid_turn): - records = [{"metadata": {"conversation_id": "root", "turn_index": index}} - for index in (0, 1, invalid_turn)] - traces = [{"id": "root", "requests": [{"type": "n", "out": 11}, - {"type": "s", "out": 29}]}] + records = [ + {"metadata": {"conversation_id": "root", "turn_index": index}} + for index in (0, 1, invalid_turn) + ] + traces = [{"id": "root", "requests": [{"type": "n", "out": 11}, {"type": "s", "out": 29}]}] _, metrics = compute_request_metrics(records, traces=traces) assert metrics["tokens"]["output_expected"]["mean"] == 20 assert metrics["tokens"]["output_expected"]["std"] == 9 From 4b2c5ea714d3fbe08a8f7e7e1e7948a8c52cb8b3 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 14 Oct 2026 11:31:11 +0000 Subject: [PATCH 07/16] fix(agentx): mount the node-local hub cache at $HF_HOME/hub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit srtctl's model pre-download runs without its bash wrapper, so it sees only the HF_HOME it exports itself, not the role's HF_HUB_CACHE. On the c72 Store point it therefore pulled all 438 GB into the container's /tmp/hf_home/hub while the workers loaded the copy already in /var/lib/hf-hub-cache. Mount that cache at $HF_HOME/hub so the pre-download checks, and if needed fills, the same directory the workers read. srtctl 的模型预下载不经过 bash 包装，只能看到它自己 export 的 HF_HOME， 看不到角色环境里的 HF_HUB_CACHE。因此在 Store c72 测试点上，它把整整 438 GB 下载到了容器内的 /tmp/hf_home/hub，而 worker 实际加载的是 /var/lib/hf-hub-cache 中已有的副本。现在把该缓存挂载到 $HF_HOME/hub， 使预下载检查（必要时填充）的正是 worker 读取的同一目录。 Co-Authored-By: Claude Opus 5.5 --- .../glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 52470a7ac9..809c06be2b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -27,9 +27,9 @@ base: <<: &environment PYTHONUNBUFFERED: "1" PYTHONHASHSEED: "0" - # srtctl fills the first node's hub cache before workers start only when HF_HOME is set. - HF_HOME: /tmp/hf_home - HF_HUB_CACHE: /mnt/hf_hub_cache + # srtctl's pre-download sees only HF_HOME, so the node-local cache is mounted at $HF_HOME/hub. + HF_HOME: /mnt/hf_home + HF_HUB_CACHE: /mnt/hf_home/hub AITER_LOG_LEVEL: WARNING AITER_QUICK_REDUCE_QUANTIZATION: INT4 AITER_USE_FLYDSL_MOE_SORTING: "1" @@ -81,7 +81,7 @@ base: container-remap-root: "" extra_mount: # The node-local hub cache the single-node GLM-5.2 points already fill. - - /var/lib/hf-hub-cache:/mnt/hf_hub_cache + - /var/lib/hf-hub-cache:/mnt/hf_home/hub health_check: max_attempts: 1080 interval_seconds: 5 From e3f27cd4a205ab6a9bcbdf32fd3a44d407e24bac Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Sat, 17 Oct 2026 08:24:03 +0000 Subject: [PATCH 08/16] feat(agentx): run every GLM-5.2 ATOM PD point on the Mooncake Store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LMCache prefills sometimes hung for good in the startup barrier after registering their 1 TiB pinned pool (3 of 8 starts on mi355x-amds); Store prefills never have. On the test runs the Store matched or beat LMCache: 1P2D c56 24,090 vs 22,379 tok/s/GPU (TTFT p50 1.33 vs 1.62 s), 1P3D c24 10,173 vs 10,051. Replace the LMCache variants with Store variants that share the 1P1D Store masters, prefill env and connector; reserve the owners' hugepages only on the head node, where srtctl starts the prefill. #4211 budgets the colocated 1P1D decode's DRAM too, so the 1P1D owners grow from 349 to 448 GiB (1924 GB, 962 GB per NUMA node); c112 went from 36,779 to 40,499 tok/s/GPU (TTFT p50 19.3 -> 14.0 s). 1P2D and 1P3D owners stay at 349 GiB, the prefill node's 1499 GB. Add 1P1D c128. LMCache prefill 注册 1 TiB pinned 池后有时在启动 barrier 永久卡住 （mi355x-amds 上 8 次启动卡 3 次），Store prefill 从未出现。测试中 Store 不逊于 LMCache：1P2D c56 24,090 对 22,379 tok/s/GPU（TTFT p50 1.33 对 1.62 s），1P3D c24 10,173 对 10,051。用共享 1P1D Store master、prefill 环境和 connector 的 Store variant 取代 LMCache variant；只在 srtctl 启动 prefill 的 head 节点上预留 owner 的 hugepage。 #4211 也为同节点的 1P1D decode 分配内存预算，因此 1P1D owner 从 349 GiB 扩到 448 GiB（共 1924 GB，每个 NUMA 节点 962 GB）；c112 从 36,779 升到 40,499 tok/s/GPU（TTFT p50 19.3 -> 14.0 s）。1P2D 和 1P3D owner 保持 349 GiB，即 prefill 节点的 1499 GB。新增 1P1D c128。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 119 ++++++++++++------ inferencex-e2e/configs/amd-master.yaml | 23 ++-- inferencex-e2e/perf-changelog.yaml | 4 +- 3 files changed, 92 insertions(+), 54 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 809c06be2b..103e6ba08d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -99,16 +99,18 @@ override_1p1d_mcstore: commands: # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - # Owners take 714752 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Owners take 917504 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. - >- - if [[ "$(hostname -s)" == "$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)" ]]; then + if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then + echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); - HUGEPAGES_TARGET=$(( held + 714752 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 + HUGEPAGES_TARGET=$(( held + 917504 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; for node in 0 1; do free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); - echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 357376"; - [[ $free -ge 357376 ]] || exit 1; + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 458752"; + [[ $free -ge 458752 ]] || exit 1; done; fi teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" @@ -133,19 +135,22 @@ override_1p1d_mcstore: readiness: port: 26051 timeout_seconds: 300 - - <<: *master + - &master_rdma1 + <<: *master name: mc-master-rdma1 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] readiness: port: 26151 timeout_seconds: 300 - - <<: *master + - &master_rdma2 + <<: *master name: mc-master-rdma2 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] readiness: port: 26251 timeout_seconds: 300 - - <<: *master + - &master_rdma3 + <<: *master name: mc-master-rdma3 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] readiness: @@ -154,7 +159,7 @@ override_1p1d_mcstore: - &owner <<: *master name: mc-owner-rdma0 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] env: <<: *store_env # Back the segment with the reserved 2 MiB pages; the NIC cannot register 64 GiB regions of 4 KiB pages. @@ -170,16 +175,16 @@ override_1p1d_mcstore: interval_seconds: 5 - <<: *owner name: mc-owner-rdma1 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - <<: *owner name: mc-owner-rdma2 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - <<: *owner name: mc-owner-rdma3 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] roles: prefill: - env: + env: &store_prefill_env # More than one load worker hung the PP prefill pipeline at C96. OFFLOAD_LOAD_WORKERS: "1" MC_NUM_QP_PER_EP: "1" @@ -188,7 +193,7 @@ override_1p1d_mcstore: args: num-speculative-tokens: 3 # Each stage's Store client uses its GPU's NIC and that NIC's pool. - extra-kv-connectors: + extra-kv-connectors: &store_connector - kv_connector: mooncake_store kv_role: offload kv_connector_extra_config: @@ -213,7 +218,7 @@ override_1p1d_mcstore: cudagraph-capture-sizes: '[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' num-speculative-tokens: 3 -override_1p2d_lmcache_c56: +override_1p2d_mcstore_c56: resources: &spread # A node per decode; srtctl otherwise packs two TP4 decodes onto one node. spread_workers: true @@ -226,44 +231,78 @@ override_1p2d_lmcache_c56: balance-abs-threshold: 2 balance-rel-threshold: 2.0 eviction-interval: 300 + host_setup: &prefill_budget_host_setup + nodes: all + timeout_seconds: 1800 + commands: + # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + # Owners take 714752 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. + - >- + if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then + echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; + held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); + HUGEPAGES_TARGET=$(( held + 714752 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 + bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; + for node in 0 1; do + free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 357376"; + [[ $free -ge 357376 ]] || exit 1; + done; fi + teardown: + - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" + services: &prefill_budget_services + - *master + - *master_rdma1 + - *master_rdma2 + - *master_rdma3 + - <<: *owner + name: mc-owner-rdma0 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] + - <<: *owner + name: mc-owner-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] roles: prefill: - env: &lmcache_env - LMCACHE_LOCAL_CPU: "True" - LMCACHE_NUMA_MODE: auto - LMCACHE_MAX_LOCAL_CPU_SIZE: "256" - LMCACHE_CHUNK_SIZE: "256" - # Every PP stage runs a lookup server, so each stage refreshes its own LRU. - LMCACHE_LOOKUP_SERVER_WORKER_IDS: "0,1,2,3" + env: *store_prefill_env args: num-speculative-tokens: 4 - extra-kv-connectors: &lmcache_connector - - kv_connector: lmcache_offload - kv_role: offload + extra-kv-connectors: *store_connector decode: nodes: 2 workers: 2 env: AITER_REUSE_IDENTICAL_COMM_GROUPS: "1" + MC_NUM_QP_PER_EP: "1" args: <<: *dcp_decode num-speculative-tokens: 4 -override_1p3d_lmcache_c12: +override_1p3d_mcstore_c12: resources: *spread frontend: args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services roles: prefill: - env: *lmcache_env + env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *lmcache_connector + extra-kv-connectors: *store_connector decode: nodes: 3 workers: 3 - env: + env: &tp4_decode_env PYTHONNOUSERSITE: "1" + MC_NUM_QP_PER_EP: "1" args: num-speculative-tokens: 5 gpu-memory-utilization: 0.95 @@ -272,21 +311,22 @@ override_1p3d_lmcache_c12: cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24]' disable-uvicorn-access-log: true -override_1p3d_lmcache_c16: +override_1p3d_mcstore_c16: resources: *spread frontend: args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services roles: prefill: - env: *lmcache_env + env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *lmcache_connector + extra-kv-connectors: *store_connector decode: nodes: 3 workers: 3 - env: - PYTHONNOUSERSITE: "1" + env: *tp4_decode_env args: num-speculative-tokens: 5 gpu-memory-utilization: 0.95 @@ -295,21 +335,22 @@ override_1p3d_lmcache_c16: cudagraph-capture-sizes: '[1,2,4,8,12,16,20,24,28,32]' disable-uvicorn-access-log: true -override_1p3d_lmcache_c24: +override_1p3d_mcstore_c24: resources: *spread frontend: args: *cache_aware + host_setup: *prefill_budget_host_setup + services: *prefill_budget_services roles: prefill: - env: *lmcache_env + env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *lmcache_connector + extra-kv-connectors: *store_connector decode: nodes: 3 workers: 3 - env: - PYTHONNOUSERSITE: "1" + env: *tp4_decode_env args: num-speculative-tokens: 5 gpu-memory-utilization: 0.95 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 7ddfc5e96c..33ba5ae0f0 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1500,45 +1500,42 @@ glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx scenarios: agentic-coding: - # The Store's 4 x 349 GiB owners: all 1499 GB of the 4-GPU prefill's share of the 3 TB cap. + # 1.0 gives each 4-GPU server 1499 GB: 1P1D's 4 x 448 GiB owners hold 962 GB per NUMA node of its prefill's and colocated decode's shares; 1P2D and 1P3D owners hold the prefill's 1499 GB. - dram-utilization: 1.0 search-space: - spec-decoding: "mtp" - conc-list: [ 56, 72, 96, 112 ] + conc-list: [ 56, 72, 96, 112, 128 ] kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.13" } srt-recipe: disagg-variants.yaml:override_1p1d_mcstore prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 1, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - # LMCache's 4 x 256 GiB stages: 1099 GB of that share. - - dram-utilization: 0.733 - search-space: - spec-decoding: "mtp" conc-list: [ 56 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p2d_lmcache_c56 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p2d_mcstore_c56 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 2, tp: 4, dcp-size: 4, ep: 1, dp-attn: false } - spec-decoding: "mtp" conc-list: [ 12 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c12 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c12 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: "mtp" conc-list: [ 16 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c16 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c16 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } - spec-decoding: "mtp" conc-list: [ 24 ] kv-offloading: dram - kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" } - srt-recipe: disagg-variants.yaml:override_1p3d_lmcache_c24 + kv-offload-backend: { name: mooncake, version: "0.3.13" } + srt-recipe: disagg-variants.yaml:override_1p3d_mcstore_c24 prefill: { num-worker: 1, tp: 1, pp: 4, ep: 1, dp-attn: false } decode: { num-worker: 3, tp: 4, ep: 1, dp-attn: false } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 98cfe55680..62bac6da66 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10079,8 +10079,8 @@ - agentic-coding description: - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - - "1P1D on one node at concurrency 56, 72, 96 and 112: a DCP4 decode on GPUs 4-7 with three draft tokens, and the prefill's prefix tier in a Mooncake Store (0.3.13) of four per-NIC pools, each one master and a 349 GiB owner on 2 MiB HugeTLB pages. After the cluster host setup drops the page cache, host_setup reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." - - "LMCache in the prefill (256 GiB per stage): 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens), and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). Each decode has a node of its own." + - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node: four per-NIC pools, each one master and an owner on 2 MiB HugeTLB pages. 1P1D owners hold 448 GiB each (1924 GB, 962 GB per NUMA node, within the prefill's and the colocated decode's 1499 GB DRAM shares); 1P2D and 1P3D owners hold 349 GiB each, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." + - "1P1D on one node at concurrency 56, 72, 96, 112 and 128 with a DCP4 decode on GPUs 4-7 and three draft tokens; 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens); and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). In 1P2D and 1P3D each decode has a node of its own." - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 From 1a153fd161a274cd676423505f766dd14ddfe978 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Sat, 17 Oct 2026 11:20:13 +0000 Subject: [PATCH 09/16] feat(agentx): give the GLM-5.2 1P1D points one shared Store pool, as in ATOM's CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ATOM's CI runs these 1P1D points at 46.1K tok/s/GPU (c112) with one master and two 768 GiB owners that all four PP stages share. Per-NIC pools here reached 36.8K at 4 x 349 GiB and 40.5K at 4 x 448 GiB: each stage only reads its own pool, and a prefix hit needs every stage to keep its chunk. Give 1P1D one master and two 768 GiB owners, one per NUMA node, each registered on rdma0-3, and set MC_ENABLE_DEST_DEVICE_AFFINITY on the prefill so each stage reaches an owner through the NIC of its own name on this rail-isolated fabric. 1P2D and 1P3D keep their tested per-NIC pools. ATOM CI 在这些 1P1D 点上（c112）跑到 46.1K tok/s/GPU，用的是一个 master 和两个 768 GiB owner，四个 PP stage 共享。这里按网卡分池时，4 x 349 GiB 为 36.8K，4 x 448 GiB 为 40.5K：每个 stage 只能读自己的池，而前缀命中 要求每个 stage 都保留各自的块。1P1D 改为一个 master 和两个 768 GiB owner （每个 NUMA 节点一个），各自注册在 rdma0-3 上，并在 prefill 上设置 MC_ENABLE_DEST_DEVICE_AFFINITY，使每个 stage 在这套按 rail 隔离的网络中 通过同名网卡访问 owner。1P2D 和 1P3D 保留已验证的按网卡分池。 Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 114 +++++++++--------- inferencex-e2e/configs/amd-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 103e6ba08d..21c171b2b7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -99,24 +99,24 @@ override_1p1d_mcstore: commands: # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - # Owners take 917504 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. + # Owners take 786432 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. - >- if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); - HUGEPAGES_TARGET=$(( held + 917504 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 + HUGEPAGES_TARGET=$(( held + 786432 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; for node in 0 1; do free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); - echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 458752"; - [[ $free -ge 458752 ]] || exit 1; + echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 393216"; + [[ $free -ge 393216 ]] || exit 1; done; fi teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" services: - &master - name: mc-master-rdma0 + name: mc-master type: generic placement: node: prefill @@ -135,31 +135,11 @@ override_1p1d_mcstore: readiness: port: 26051 timeout_seconds: 300 - - &master_rdma1 - <<: *master - name: mc-master-rdma1 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] - readiness: - port: 26151 - timeout_seconds: 300 - - &master_rdma2 - <<: *master - name: mc-master-rdma2 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] - readiness: - port: 26251 - timeout_seconds: 300 - - &master_rdma3 - <<: *master - name: mc-master-rdma3 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] - readiness: - port: 26351 - timeout_seconds: 300 - &owner <<: *master - name: mc-owner-rdma0 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] + name: mc-owner-numa0 + # Each owner registers on rdma0-3, so every stage reaches it on its own rail. + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, "--device_names=rdma0,rdma1,rdma2,rdma3", --global_segment_size=824633720832, --local_buffer_size=0, --threads=4] env: <<: *store_env # Back the segment with the reserved 2 MiB pages; the NIC cannot register 64 GiB regions of 4 KiB pages. @@ -174,35 +154,29 @@ override_1p1d_mcstore: timeout_seconds: 1800 interval_seconds: 5 - <<: *owner - name: mc-owner-rdma1 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26151, --metadata_server=http://127.0.0.1:26180/metadata, --protocol=rdma, --device_names=rdma1, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - - <<: *owner - name: mc-owner-rdma2 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26054, --master_server_address=127.0.0.1:26251, --metadata_server=http://127.0.0.1:26280/metadata, --protocol=rdma, --device_names=rdma2, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] - - <<: *owner - name: mc-owner-rdma3 - command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26055, --master_server_address=127.0.0.1:26351, --metadata_server=http://127.0.0.1:26380/metadata, --protocol=rdma, --device_names=rdma3, --global_segment_size=481036337152, --local_buffer_size=0, --threads=4] + name: mc-owner-numa1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_client, "--host={node_ip}", --port=26053, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, "--device_names=rdma0,rdma1,rdma2,rdma3", --global_segment_size=824633720832, --local_buffer_size=0, --threads=4] roles: prefill: - env: &store_prefill_env - # More than one load worker hung the PP prefill pipeline at C96. - OFFLOAD_LOAD_WORKERS: "1" - MC_NUM_QP_PER_EP: "1" - # Registers the worker's HBM transfer pool in 1 GiB regions. - MC_MAX_MR_SIZE: "1073741824" + env: + <<: &store_prefill_env + # More than one load worker hung the PP prefill pipeline at C96. + OFFLOAD_LOAD_WORKERS: "1" + MC_NUM_QP_PER_EP: "1" + # Registers the worker's HBM transfer pool in 1 GiB regions. + MC_MAX_MR_SIZE: "1073741824" + # Every owner sits on rdma0-3 and this fabric is rail-isolated, so each stage must reach an owner through the NIC of its own name. + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" args: num-speculative-tokens: 3 - # Each stage's Store client uses its GPU's NIC and that NIC's pool. - extra-kv-connectors: &store_connector + # One pool shared by all four stages, as in ATOM's CI; each client still uses its GPU's NIC. + extra-kv-connectors: - kv_connector: mooncake_store kv_role: offload kv_connector_extra_config: mooncake_store.protocol: rdma - mooncake_store.pools: - rdma0: {master: "127.0.0.1:26051", metadata: "http://127.0.0.1:26080/metadata"} - rdma1: {master: "127.0.0.1:26151", metadata: "http://127.0.0.1:26180/metadata"} - rdma2: {master: "127.0.0.1:26251", metadata: "http://127.0.0.1:26280/metadata"} - rdma3: {master: "127.0.0.1:26351", metadata: "http://127.0.0.1:26380/metadata"} + mooncake_store.master: "127.0.0.1:26051" + mooncake_store.metadata: "http://127.0.0.1:26080/metadata" decode: nodes: colocate env: @@ -253,10 +227,26 @@ override_1p2d_mcstore_c56: teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" services: &prefill_budget_services - - *master - - *master_rdma1 - - *master_rdma2 - - *master_rdma3 + - <<: *master + name: mc-master-rdma0 + - <<: *master + name: mc-master-rdma1 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26151, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26180, --metrics_port=26190, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26151 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma2 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26251, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26280, --metrics_port=26290, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26251 + timeout_seconds: 300 + - <<: *master + name: mc-master-rdma3 + command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "1", mooncake_master, --rpc_port=26351, --rpc_thread_num=32, --enable_http_metadata_server=true, --http_metadata_server_host=0.0.0.0, --http_metadata_server_port=26380, --metrics_port=26390, --default_kv_lease_ttl=10000, --eviction_high_watermark_ratio=0.90, --eviction_ratio=0.05, --allocation_strategy=random, --memory_allocator=offset, --client_ttl=10] + readiness: + port: 26351 + timeout_seconds: 300 - <<: *owner name: mc-owner-rdma0 command: [python3, /app/ATOM/.github/scripts/atomesh/numa_exec.py, "0", mooncake_client, "--host={node_ip}", --port=26052, --master_server_address=127.0.0.1:26051, --metadata_server=http://127.0.0.1:26080/metadata, --protocol=rdma, --device_names=rdma0, --global_segment_size=374735896576, --local_buffer_size=0, --threads=4] @@ -274,7 +264,17 @@ override_1p2d_mcstore_c56: env: *store_prefill_env args: num-speculative-tokens: 4 - extra-kv-connectors: *store_connector + # Each stage's Store client uses its GPU's NIC and that NIC's pool. + extra-kv-connectors: &per_nic_store_connector + - kv_connector: mooncake_store + kv_role: offload + kv_connector_extra_config: + mooncake_store.protocol: rdma + mooncake_store.pools: + rdma0: {master: "127.0.0.1:26051", metadata: "http://127.0.0.1:26080/metadata"} + rdma1: {master: "127.0.0.1:26151", metadata: "http://127.0.0.1:26180/metadata"} + rdma2: {master: "127.0.0.1:26251", metadata: "http://127.0.0.1:26280/metadata"} + rdma3: {master: "127.0.0.1:26351", metadata: "http://127.0.0.1:26380/metadata"} decode: nodes: 2 workers: 2 @@ -296,7 +296,7 @@ override_1p3d_mcstore_c12: env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *store_connector + extra-kv-connectors: *per_nic_store_connector decode: nodes: 3 workers: 3 @@ -322,7 +322,7 @@ override_1p3d_mcstore_c16: env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *store_connector + extra-kv-connectors: *per_nic_store_connector decode: nodes: 3 workers: 3 @@ -346,7 +346,7 @@ override_1p3d_mcstore_c24: env: *store_prefill_env args: num-speculative-tokens: 5 - extra-kv-connectors: *store_connector + extra-kv-connectors: *per_nic_store_connector decode: nodes: 3 workers: 3 diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 33ba5ae0f0..7ad0141164 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1500,7 +1500,7 @@ glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: srt-recipe-dir: glm5.2/atom/mi355x-fp4/agentx scenarios: agentic-coding: - # 1.0 gives each 4-GPU server 1499 GB: 1P1D's 4 x 448 GiB owners hold 962 GB per NUMA node of its prefill's and colocated decode's shares; 1P2D and 1P3D owners hold the prefill's 1499 GB. + # 1.0 gives each 4-GPU server 1499 GB: 1P1D's 2 x 768 GiB owners hold 825 GB per NUMA node of its prefill's and colocated decode's shares; 1P2D and 1P3D owners hold the prefill's 1499 GB. - dram-utilization: 1.0 search-space: - spec-decoding: "mtp" diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 62bac6da66..89809ae5ab 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10079,7 +10079,7 @@ - agentic-coding description: - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node: four per-NIC pools, each one master and an owner on 2 MiB HugeTLB pages. 1P1D owners hold 448 GiB each (1924 GB, 962 GB per NUMA node, within the prefill's and the colocated decode's 1499 GB DRAM shares); 1P2D and 1P3D owners hold 349 GiB each, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its two owners' pages free; teardown releases them." + - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node, with owners on 2 MiB HugeTLB pages. 1P1D, as in ATOM's CI, has one pool shared by all four PP stages: one master and two 768 GiB owners, one per NUMA node (1649 GB, within the prefill's and the colocated decode's 1499 GB DRAM shares), each registered on rdma0-3, with MC_ENABLE_DEST_DEVICE_AFFINITY so each stage reaches them through the NIC of its own name on this rail-isolated fabric. 1P2D and 1P3D keep a pool per prefill NIC, each one master and a 349 GiB owner, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its owners' pages free; teardown releases them." - "1P1D on one node at concurrency 56, 72, 96, 112 and 128 with a DCP4 decode on GPUs 4-7 and three draft tokens; 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens); and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). In 1P2D and 1P3D each decode has a node of its own." - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." From 9740fde54d406f27ac2b9b38c369a710663c3c69 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 07:44:46 +0000 Subject: [PATCH 10/16] fix: account ATOMesh cache reuse on prefill workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅汇总 prefill/unified 缓存计数，CPU reuse 使用 offload counter， overall 合并 GPU 和 CPU reuse；缺少增量或角色时保留未知值。 --- .../infx/results/agentic/__init__.py | 1 + .../infx/results/agentic/backends/atom.py | 111 ++++++++-- .../infx/results/agentic/server_metrics.py | 37 +++- .../agentic/test_atom_cache_counters.py | 195 ++++++++++++++++++ .../agentic/test_process_agentic_result.py | 49 +++++ 5 files changed, 374 insertions(+), 19 deletions(-) create mode 100644 inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 5cf7469691..92a78f8584 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -218,6 +218,7 @@ def build_result( framework=framework, records=records, server_logs=server_logs, + env=env, ) if "total_tput_tps" in request_flat and num_gpus > 0: diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 947024b111..7c6071adff 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -2,12 +2,24 @@ from __future__ import annotations +import logging +import math +from collections.abc import Mapping from typing import Any -from infx.results.agentic.common import gauge_stat, normalize_fraction, rate, sum_stat +from infx.results.agentic.common import ( + gauge_stat, + metric_series, + normalize_fraction, + rate, + series_stat, + sum_stat, +) from .base import ServerMetricsBackend, counter_int +logger = logging.getLogger(__name__) + def _atom_names(suffix: str) -> list[str]: """Accept direct worker metrics and Atomesh's colon-normalized form.""" @@ -19,6 +31,73 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] +def _counter_total(metrics: dict[str, dict[str, Any]], stem: str, endpoint: str) -> float | None: + names = _atom_counter_names(f"prefix_cache_{stem}_tokens") + series = [s for s in metric_series(metrics, names) if s.get("endpoint_url") == endpoint] + if not series or any( + (value := series_stat(s, preferred_keys=("total",))) is None + or not math.isfinite(value) + or value < 0 + for s in series + ): + logger.warning("ATOM %s cache counter missing or invalid at %s", stem, endpoint) + return None + total = sum_stat( + metrics, + names, + preferred_keys=("total",), + series_filter=lambda s: s.get("endpoint_url") == endpoint, + ) + return total if total is not None and math.isfinite(total) else None + + +def _cache_counts( + metrics: dict[str, dict[str, Any]], + endpoint_roles: Mapping[str, str] | None, + disaggregated: bool | None, +) -> tuple[float | None, float | None, float | None]: + names = [ + name + for stem in ("full", "cached", "offload") + for name in _atom_counter_names(f"prefix_cache_{stem}_tokens") + ] + endpoints = [s.get("endpoint_url") for s in metric_series(metrics, names)] + if any(not isinstance(endpoint, str) or not endpoint for endpoint in endpoints): + return None, None, None + observed = set(endpoints) + if endpoint_roles is None: + accounted = observed if disaggregated is False and len(observed) == 1 else set() + else: + # Decode counts the prompt again without serving its prefill cache reuse. + accounted = {url for url, role in endpoint_roles.items() if role in {"prefill", "unified"}} + if observed - endpoint_roles.keys(): + accounted = set() + if not accounted: + logger.warning("ATOM cache endpoint roles missing or ambiguous") + return None, None, None + full_total = cached_total = offload_total = 0.0 + cpu_complete = True + for endpoint in accounted: + full, cached, offload = ( + _counter_total(metrics, stem, endpoint) for stem in ("full", "cached", "offload") + ) + if full is None or cached is None or cached > full: + return None, None, None + full_total += full + cached_total += cached + if offload is None or cached + offload > full: + cpu_complete = False + else: + offload_total += offload + if full_total <= 0 or not all(math.isfinite(v) for v in (full_total, cached_total)): + return None, None, None + return ( + full_total, + cached_total, + offload_total if cpu_complete and math.isfinite(offload_total) else None, + ) + + class AtomBackend(ServerMetricsBackend): name = "atom" @@ -33,6 +112,9 @@ def populate( metrics: dict[str, dict[str, Any]], flat: dict[str, Any], nested: dict[str, Any], + *, + endpoint_roles: Mapping[str, str] | None = None, + disaggregated: bool | None = None, ) -> None: prompt_total = sum_stat( metrics, @@ -47,24 +129,17 @@ def populate( flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) - cached_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_cached_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) - full_tokens = sum_stat( - metrics, - _atom_counter_names("prefix_cache_full_tokens"), - preferred_keys=("total", "sum", "max", "avg"), + full_tokens, cached_tokens, external_tokens = _cache_counts( + metrics, endpoint_roles, disaggregated ) cache_hit_rate = rate(cached_tokens, full_tokens) - external_tokens = sum_stat( - metrics, - _atom_counter_names("lmcache_loaded_tokens"), - preferred_keys=("total", "sum", "max", "avg"), - ) + # Offload is CPU reuse beyond the GPU prefix; loaded tokens measure transfer volume. external_hit_rate = rate(external_tokens, full_tokens) - overall_hit_rate = cache_hit_rate + overall_hit_rate = ( + rate(cached_tokens + external_tokens, full_tokens) + if cached_tokens is not None and external_tokens is not None + else None + ) flat["server_gpu_cache_hit_rate"] = cache_hit_rate flat["server_cpu_cache_hit_rate"] = external_hit_rate @@ -88,7 +163,9 @@ def populate( "prefix_cache_hits": cached_tokens, "prefix_cache_queries": full_tokens, "external_prefix_cache_hits": external_tokens, - "external_prefix_cache_queries": full_tokens, + "external_prefix_cache_queries": ( + full_tokens if external_tokens is not None else None + ), } ) nested["kv_cache"]["gpu_usage_pct"] = flat["gpu_kv_cache_usage_pct"] diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index d14962a61f..e100011642 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -2,10 +2,11 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any from .backends import detect_backend +from .backends.atom import AtomBackend from .backends.base import ( SERVER_CACHE_FLAT_FIELDS, apply_profile_totals, @@ -14,12 +15,34 @@ from .common import index_server_metrics +def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: + if env.get("AIPERF_SERVER_METRICS_URLS") or env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": + return {} + groups = ( + [("unified", env["SRT_AGG_ENDPOINTS"])] + if env.get("SRT_AGG_ENDPOINTS") + else [ + ("prefill", env.get("SRT_PREFILL_ENDPOINTS", "")), + ("decode", env.get("SRT_DECODE_ENDPOINTS", "")), + ] + ) + roles: dict[str, str] = {} + for role, endpoints in groups: + for endpoint in endpoints.split(","): + if endpoint: + url = f"http://{endpoint}/metrics" + if roles.setdefault(url, role) != role: + return {} + return roles or None + + def compute_server_metrics( server_metrics: dict[str, Any], *, framework: str, records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), + env: Mapping[str, str] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: metrics = index_server_metrics(server_metrics) flat = SERVER_CACHE_FLAT_FIELDS.copy() @@ -46,7 +69,17 @@ def compute_server_metrics( f"framework={framework.lower()!r}, metric_names=[{preview}]" ) nested["adapter"] = adapter.name - adapter.populate(metrics, flat, nested) + if isinstance(adapter, AtomBackend): + source_env = env or {} + adapter.populate( + metrics, + flat, + nested, + endpoint_roles=endpoint_roles_from_env(source_env), + disaggregated={"true": True, "false": False}.get(source_env.get("DISAGG", "").lower()), + ) + else: + adapter.populate(metrics, flat, nested) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": nested["kv_cache"]["gpu_total_tokens"] = None diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py new file mode 100644 index 0000000000..2d0e6f433d --- /dev/null +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import math + +import pytest + +from infx.results.agentic.server_metrics import compute_server_metrics + + +def _export(full: dict, cached: dict, offload: dict) -> dict: + return { + "metrics": { + f"atom:prefix_cache_{stem}_tokens": { + "series": [ + {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": value}} + for endpoint, value in values.items() + ] + } + for stem, values in (("full", full), ("cached", cached), ("offload", offload)) + } + } + + +def _compute(export: dict, env: dict) -> tuple[dict, dict]: + flat, nested, _ = compute_server_metrics(export, framework="mooncake-atom", records=[], env=env) + return flat, nested + + +def _assert_rates( + flat: dict, nested: dict, gpu: float | None, cpu: float | None, overall: float | None +): + for name, expected in (("gpu", gpu), ("cpu", cpu), ("external", cpu), ("overall", overall)): + actual = flat[f"server_{name}_cache_hit_rate"] + assert actual == (pytest.approx(expected) if expected is not None else None) + assert nested["cache"][f"{name}_cache_hit_rate"] == actual + + +@pytest.mark.parametrize("decode_cached", [800, math.nan]) +def test_prefill_counters_exclude_decode_and_transfer_volume(decode_cached: float) -> None: + export = _export({"p": 1000, "d": 1000}, {"p": 600, "d": decode_cached}, {"p": 100, "d": -1}) + export["metrics"]["atom:lmcache_loaded_tokens"] = { + "series": [{"endpoint_url": "http://p/metrics", "stats": {"total": 900}}] + } + flat, nested = _compute( + export, {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "d", "DISAGG": "true"} + ) + _assert_rates(flat, nested, 0.6, 0.1, 0.7) + assert nested["cache"]["prefix_cache_queries"] == 1000 + assert nested["cache"]["external_prefix_cache_hits"] == 100 + assert nested["cache"]["external_prefix_cache_queries"] == 1000 + + +@pytest.mark.parametrize( + ("env", "offload", "expected"), + [ + ({"SRT_AGG_ENDPOINTS": "worker", "SRT_PREFILL_ENDPOINTS": "unused"}, 50, 0.5), + ({"DISAGG": "false"}, 0, 0), + ], +) +def test_unified_zero_gpu_and_offload_counter_alias( + env: dict, offload: int, expected: float +) -> None: + export = _export({"worker": 100}, {"worker": 0}, {"worker": offload}) + export["metrics"] = { + name.replace("atom:", "atom_") + "_total": value + for name, value in export["metrics"].items() + } + flat, nested = _compute(export, env) + _assert_rates(flat, nested, 0, expected, expected) + + +def test_multiple_prefills_and_labelled_series_are_token_weighted() -> None: + export = _export( + {"p": 100, "q": 900, "idle": 0}, + {"p": 80, "q": 20, "idle": 0}, + {"p": 10, "q": 180, "idle": 0}, + ) + for stem, values in (("full", (400, 500)), ("cached", (5, 15)), ("offload", (80, 100))): + series = export["metrics"][f"atom:prefix_cache_{stem}_tokens"]["series"] + series[:] = [s for s in series if s["endpoint_url"] != "http://q/metrics"] + series.extend( + { + "endpoint_url": "http://q/metrics", + "labels": {"engine": str(i)}, + "stats": {"total": v}, + } + for i, v in enumerate(values) + ) + flat, nested = _compute(export, {"SRT_PREFILL_ENDPOINTS": "p,q,idle"}) + _assert_rates(flat, nested, 0.1, 0.19, 0.29) + + +@pytest.mark.parametrize( + "env", + [ + {}, + {"DISAGG": "true"}, + {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "p,d"}, + {"SRT_PREFILL_ENDPOINTS": "p,missing", "SRT_DECODE_ENDPOINTS": "d"}, + {"SRT_PREFILL_ENDPOINTS": "p"}, + { + "SRT_PREFILL_ENDPOINTS": "p", + "SRT_DECODE_ENDPOINTS": "d", + "AIPERF_SERVER_METRICS_URLS": "custom", + }, + { + "SRT_PREFILL_ENDPOINTS": "p", + "SRT_DECODE_ENDPOINTS": "d", + "SRTCTL_FRONTEND_TYPE": "dynamo", + }, + ], +) +def test_unknown_conflicting_or_incomplete_roles_do_not_mix_endpoints(env: dict) -> None: + flat, nested = _compute( + _export({"p": 100, "d": 100}, {"p": 60, "d": 0}, {"p": 10, "d": 0}), env + ) + _assert_rates(flat, nested, None, None, None) + + +def test_single_endpoint_without_explicit_unified_topology_stays_unknown() -> None: + flat, nested = _compute(_export({"worker": 100}, {"worker": 60}, {"worker": 10}), {}) + _assert_rates(flat, nested, None, None, None) + + +@pytest.mark.parametrize( + ("offload", "cpu", "overall"), + [({"p": 0, "q": 0}, 0, 0.2), ({}, None, None), ({"p": 10}, None, None)], +) +def test_missing_or_partial_offload_is_distinct_from_zero( + offload: dict, cpu: float | None, overall: float | None +) -> None: + flat, nested = _compute( + _export({"p": 100, "q": 100}, {"p": 20, "q": 20}, offload), + {"SRT_PREFILL_ENDPOINTS": "p,q"}, + ) + _assert_rates(flat, nested, 0.2, cpu, overall) + assert nested["cache"]["external_prefix_cache_queries"] == (200 if cpu is not None else None) + + +@pytest.mark.parametrize( + ("stem", "value"), + [ + ("full", 0), + ("cached", -1), + ("cached", 101), + ("cached", math.nan), + ("cached", math.inf), + ("cached", None), + ], +) +def test_invalid_counted_counter_does_not_produce_rates(stem: str, value: object) -> None: + export = _export({"worker": 100}, {"worker": 20}, {"worker": 10}) + export["metrics"][f"atom:prefix_cache_{stem}_tokens"]["series"][0]["stats"]["total"] = value + flat, nested = _compute(export, {"DISAGG": "false"}) + _assert_rates(flat, nested, None, None, None) + + +def test_unreadable_series_does_not_get_silently_skipped() -> None: + export = _export({"worker": 100}, {"worker": 20}, {"worker": 10}) + export["metrics"]["atom:prefix_cache_cached_tokens"]["series"].append( + { + "endpoint_url": "http://worker/metrics", + "labels": {"engine": "other"}, + "stats": {"max": 5}, + } + ) + flat, nested = _compute(export, {"DISAGG": "false"}) + _assert_rates(flat, nested, None, None, None) + + +@pytest.mark.parametrize("offload", [30, math.nan]) +def test_invalid_worker_cpu_count_cannot_hide_in_another_workers_denominator( + offload: float, +) -> None: + flat, nested = _compute( + _export({"p": 100, "q": 900}, {"p": 80, "q": 20}, {"p": offload, "q": 0}), + {"SRT_PREFILL_ENDPOINTS": "p,q"}, + ) + _assert_rates(flat, nested, 0.1, None, None) + + +def test_unknown_cache_roles_preserve_other_atom_telemetry() -> None: + export = _export({"p": 100, "d": 100}, {"p": 60, "d": 0}, {"p": 10, "d": 0}) + export["metrics"].update( + { + "atom:prompt_tokens": {"series": [{"stats": {"total": 200}}]}, + "atom:generation_tokens": {"series": [{"stats": {"total": 30}}]}, + "atom:kv_cache_usage_ratio": {"series": [{"stats": {"max": 0.75}}]}, + } + ) + flat, nested = _compute(export, {}) + _assert_rates(flat, nested, None, None, None) + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 200 + assert flat["total_generation_tokens"] == nested["tokens"]["generation_total"] == 30 + assert flat["gpu_kv_cache_usage_pct"] == nested["kv_cache"]["gpu_usage_pct"] == 0.75 diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index e162f49685..29aeeeef94 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -365,6 +365,55 @@ def test_processor_records_run_attempt(tmp_path: Path): assert "run_attempt" not in local +def test_processor_uses_launcher_roles_for_atom_cache_counts(tmp_path: Path) -> None: + result_dir = _write_fixture(tmp_path) + metrics = { + f"atom:prefix_cache_{stem}_tokens": { + "series": [ + {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": value}} + for endpoint, value in values.items() + ] + } + for stem, values in ( + ("full", {"prefill:8000": 1000, "decode:8000": 1000}), + ("cached", {"prefill:8000": 600, "decode:8000": 0}), + ("offload", {"prefill:8000": 100, "decode:8000": 0}), + ) + } + (result_dir / "aiperf_artifacts" / "server_metrics_export.json").write_text( + json.dumps({"metrics": metrics}) + ) + agg = _run_processor( + result_dir, + tmp_path / "out", + { + "FRAMEWORK": "mooncake-atom", + "DISAGG": "true", + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "4", + "PREFILL_EP": "1", + "DECODE_NUM_WORKERS": "1", + "DECODE_TP": "4", + "DECODE_EP": "1", + "SRT_PREFILL_ENDPOINTS": "prefill:8000", + "SRT_DECODE_ENDPOINTS": "decode:8000", + "SRT_AGG_ENDPOINTS": "", + "AIPERF_SERVER_METRICS_URLS": "", + "SRTCTL_FRONTEND_TYPE": "", + "KV_OFFLOADING": "dram", + "KV_OFFLOAD_BACKEND": "lmcache", + "KV_OFFLOAD_BACKEND_METADATA": '{"name":"lmcache","version":"0.4.5"}', + }, + ) + cache = agg["server_metrics"]["cache"] + assert cache["gpu_cache_hit_rate"] == 0.6 + assert cache["cpu_cache_hit_rate"] == cache["external_cache_hit_rate"] == 0.1 + assert cache["overall_cache_hit_rate"] == 0.7 + assert cache["prefix_cache_queries"] == cache["external_prefix_cache_queries"] == 1000 + assert agg["num_gpus"] == 8 + + def test_processor_preserves_dataset_provenance(tmp_path: Path): result_dir = _write_fixture(tmp_path) output_dir = tmp_path / "out" From 984280caef3ac62f0a498e8f539da9904ea0a939 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 20:28:52 +0000 Subject: [PATCH 11/16] fix(results): preserve ATOM unified counters and prefill totals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅在确认关闭 CPU offload 且无计数器时补零，保留单端点自定义 metrics 的角色回退，并从输入总量中排除 decode 重复计数。补充相应行为回归。 --- .../infx/results/agentic/backends/atom.py | 30 ++++++++++++++-- .../infx/results/agentic/server_metrics.py | 9 ++++- .../agentic/test_atom_cache_counters.py | 34 +++++++++++++++++-- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 7c6071adff..17929bb0c8 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -55,6 +55,7 @@ def _cache_counts( metrics: dict[str, dict[str, Any]], endpoint_roles: Mapping[str, str] | None, disaggregated: bool | None, + cpu_offload_disabled: bool = False, ) -> tuple[float | None, float | None, float | None]: names = [ name @@ -77,10 +78,16 @@ def _cache_counts( return None, None, None full_total = cached_total = offload_total = 0.0 cpu_complete = True + offload_absent = not any( + s.get("endpoint_url") in accounted + for s in metric_series(metrics, _atom_counter_names("prefix_cache_offload_tokens")) + ) for endpoint in accounted: full, cached, offload = ( _counter_total(metrics, stem, endpoint) for stem in ("full", "cached", "offload") ) + if cpu_offload_disabled and offload_absent: + offload = 0.0 if full is None or cached is None or cached > full: return None, None, None full_total += full @@ -115,11 +122,28 @@ def populate( *, endpoint_roles: Mapping[str, str] | None = None, disaggregated: bool | None = None, + cpu_offload_disabled: bool = False, ) -> None: + prompt_names = _atom_counter_names("prompt_tokens") + prompt_endpoints = {s.get("endpoint_url") for s in metric_series(metrics, prompt_names)} + if endpoint_roles is None: + accounted = ( + prompt_endpoints if disaggregated is False and len(prompt_endpoints) == 1 else set() + ) + else: + accounted = { + url for url, role in endpoint_roles.items() if role in {"prefill", "unified"} + } + if prompt_endpoints - endpoint_roles.keys(): + accounted = set() + accounted = {url for url in accounted if isinstance(url, str) and url} + if accounted - prompt_endpoints: + accounted = set() prompt_total = sum_stat( metrics, - _atom_counter_names("prompt_tokens"), - preferred_keys=("total", "sum", "max", "avg"), + prompt_names, + preferred_keys=("total",), + series_filter=lambda s: s.get("endpoint_url") in accounted, ) generation_total = sum_stat( metrics, @@ -130,7 +154,7 @@ def populate( flat["total_generation_tokens"] = counter_int(generation_total) full_tokens, cached_tokens, external_tokens = _cache_counts( - metrics, endpoint_roles, disaggregated + metrics, endpoint_roles, disaggregated, cpu_offload_disabled ) cache_hit_rate = rate(cached_tokens, full_tokens) # Offload is CPU reuse beyond the GPU prefix; loaded tokens measure transfer volume. diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index e100011642..2ef47da115 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -16,8 +16,10 @@ def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: - if env.get("AIPERF_SERVER_METRICS_URLS") or env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": + if env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": return {} + if env.get("AIPERF_SERVER_METRICS_URLS"): + return None groups = ( [("unified", env["SRT_AGG_ENDPOINTS"])] if env.get("SRT_AGG_ENDPOINTS") @@ -77,6 +79,11 @@ def compute_server_metrics( nested, endpoint_roles=endpoint_roles_from_env(source_env), disaggregated={"true": True, "false": False}.get(source_env.get("DISAGG", "").lower()), + cpu_offload_disabled=(source_env.get("KV_OFFLOADING", "").strip().lower() == "none") + and not ( + source_env.get("KV_OFFLOAD_BACKEND") + or source_env.get("KV_OFFLOAD_BACKEND_METADATA") + ), ) else: adapter.populate(metrics, flat, nested) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py index 2d0e6f433d..40ea3d9b89 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py @@ -41,9 +41,16 @@ def test_prefill_counters_exclude_decode_and_transfer_volume(decode_cached: floa export["metrics"]["atom:lmcache_loaded_tokens"] = { "series": [{"endpoint_url": "http://p/metrics", "stats": {"total": 900}}] } + export["metrics"]["atom:prompt_tokens"] = { + "series": [ + {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": 1000}} + for endpoint in ("p", "d") + ] + } flat, nested = _compute( export, {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "d", "DISAGG": "true"} ) + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1000 _assert_rates(flat, nested, 0.6, 0.1, 0.7) assert nested["cache"]["prefix_cache_queries"] == 1000 assert nested["cache"]["external_prefix_cache_hits"] == 100 @@ -55,6 +62,7 @@ def test_prefill_counters_exclude_decode_and_transfer_volume(decode_cached: floa [ ({"SRT_AGG_ENDPOINTS": "worker", "SRT_PREFILL_ENDPOINTS": "unused"}, 50, 0.5), ({"DISAGG": "false"}, 0, 0), + ({"DISAGG": "false", "AIPERF_SERVER_METRICS_URLS": "http://worker/metrics"}, 50, 0.5), ], ) def test_unified_zero_gpu_and_offload_counter_alias( @@ -188,8 +196,30 @@ def test_unknown_cache_roles_preserve_other_atom_telemetry() -> None: "atom:kv_cache_usage_ratio": {"series": [{"stats": {"max": 0.75}}]}, } ) - flat, nested = _compute(export, {}) + flat, nested, _ = compute_server_metrics( + export, + framework="mooncake-atom", + records=[{"metrics": {"input_sequence_length": {"value": 123}}}], + env={}, + ) _assert_rates(flat, nested, None, None, None) - assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 200 + assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 123 assert flat["total_generation_tokens"] == nested["tokens"]["generation_total"] == 30 assert flat["gpu_kv_cache_usage_pct"] == nested["kv_cache"]["gpu_usage_pct"] == 0.75 + + +@pytest.mark.parametrize( + ("env", "cpu", "overall"), + [ + ({"KV_OFFLOADING": " NONE "}, 0, 0.6), + ({"KV_OFFLOADING": "dram"}, None, None), + ({"KV_OFFLOADING": "none", "KV_OFFLOAD_BACKEND": "mooncake"}, None, None), + ], +) +def test_missing_offload_is_zero_only_with_confirmed_disabled_tier( + env: dict, cpu: float | None, overall: float | None +) -> None: + flat, nested = _compute( + _export({"worker": 100}, {"worker": 60}, {}), {"DISAGG": "false", **env} + ) + _assert_rates(flat, nested, 0.6, cpu, overall) From c14c0fde87eae59e1bf6194098255bf7078c9953 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Sun, 25 Oct 2026 17:25:34 +0000 Subject: [PATCH 12/16] fix(results): validate ATOM cache counters per label domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：按端点与完整标签域校验缓存分层计数；仅在 CPU offload 明确关闭且现有计数均为零时补齐缺失零值，拒绝无效或矛盾的 offload 数据。 --- .../infx/results/agentic/backends/atom.py | 76 ++++++++++++------- .../agentic/test_atom_cache_counters.py | 38 ++++++++-- 2 files changed, 78 insertions(+), 36 deletions(-) diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 183ca99832..32543a005b 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -31,24 +31,31 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] -def _counter_total(metrics: dict[str, dict[str, Any]], stem: str, endpoint: str) -> float | None: +def _counter_values( + metrics: dict[str, dict[str, Any]], stem: str, endpoint: str +) -> dict[tuple, float] | None: names = _atom_counter_names(f"prefix_cache_{stem}_tokens") - series = [s for s in metric_series(metrics, names) if s.get("endpoint_url") == endpoint] - if not series or any( - (value := series_stat(s, preferred_keys=("total",))) is None - or not math.isfinite(value) - or value < 0 - for s in series - ): - logger.warning("ATOM %s cache counter missing or invalid at %s", stem, endpoint) - return None - total = sum_stat( - metrics, - names, - preferred_keys=("total",), - series_filter=lambda s: s.get("endpoint_url") == endpoint, - ) - return total if total is not None and math.isfinite(total) else None + values = {} + for series in metric_series(metrics, names): + if series.get("endpoint_url") != endpoint: + continue + labels = series.get("labels") + labels = {} if labels is None else labels + value = series_stat(series, preferred_keys=("total",)) + if ( + not isinstance(labels, dict) + or any(not isinstance(k, str) or not isinstance(v, str) for k, v in labels.items()) + or value is None + or not math.isfinite(value) + or value < 0 + ): + return None + key = tuple(sorted(labels.items())) + if key in values: + logger.warning("ATOM %s cache counter duplicated at %s", stem, endpoint) + return None + values[key] = value + return values def _cache_counts( @@ -78,24 +85,35 @@ def _cache_counts( return None, None, None full_total = cached_total = offload_total = 0.0 cpu_complete = True - offload_absent = not any( - s.get("endpoint_url") in accounted - for s in metric_series(metrics, _atom_counter_names("prefix_cache_offload_tokens")) - ) for endpoint in accounted: full, cached, offload = ( - _counter_total(metrics, stem, endpoint) for stem in ("full", "cached", "offload") + _counter_values(metrics, stem, endpoint) for stem in ("full", "cached", "offload") ) - if cpu_offload_disabled and offload_absent: - offload = 0.0 - if full is None or cached is None or cached > full: + if ( + not full + or cached is None + or full.keys() != cached.keys() + or any(cached[key] > full[key] for key in full) + ): return None, None, None - full_total += full - cached_total += cached - if offload is None or cached + offload > full: + if cpu_offload_disabled: + offload = ( + dict.fromkeys(full, 0.0) + if offload is not None + and offload.keys() <= full.keys() + and not any(offload.values()) + else None + ) + full_total += sum(full.values()) + cached_total += sum(cached.values()) + if ( + offload is None + or offload.keys() != full.keys() + or any(cached[key] + offload[key] > full[key] for key in full) + ): cpu_complete = False else: - offload_total += offload + offload_total += sum(offload.values()) if full_total <= 0 or not all(math.isfinite(v) for v in (full_total, cached_total)): return None, None, None return ( diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py index 40ea3d9b89..8356e5bd39 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py @@ -177,13 +177,17 @@ def test_unreadable_series_does_not_get_silently_skipped() -> None: @pytest.mark.parametrize("offload", [30, math.nan]) +@pytest.mark.parametrize("labelled", [False, True]) def test_invalid_worker_cpu_count_cannot_hide_in_another_workers_denominator( - offload: float, + offload: float, labelled: bool ) -> None: - flat, nested = _compute( - _export({"p": 100, "q": 900}, {"p": 80, "q": 20}, {"p": offload, "q": 0}), - {"SRT_PREFILL_ENDPOINTS": "p,q"}, - ) + export = _export({"p": 100, "q": 900}, {"p": 80, "q": 20}, {"p": offload, "q": 0}) + if labelled: + for metric in export["metrics"].values(): + for series in metric["series"]: + series["labels"] = {"engine": series["endpoint_url"]} + series["endpoint_url"] = "http://worker/metrics" + flat, nested = _compute(export, {"SRT_PREFILL_ENDPOINTS": "worker" if labelled else "p,q"}) _assert_rates(flat, nested, 0.1, None, None) @@ -216,10 +220,30 @@ def test_unknown_cache_roles_preserve_other_atom_telemetry() -> None: ({"KV_OFFLOADING": "none", "KV_OFFLOAD_BACKEND": "mooncake"}, None, None), ], ) +@pytest.mark.parametrize("offload", [{}, {"p": 0}]) def test_missing_offload_is_zero_only_with_confirmed_disabled_tier( - env: dict, cpu: float | None, overall: float | None + env: dict, cpu: float | None, overall: float | None, offload: dict ) -> None: flat, nested = _compute( - _export({"worker": 100}, {"worker": 60}, {}), {"DISAGG": "false", **env} + _export({"p": 100, "q": 100}, {"p": 60, "q": 60}, offload), + {"SRT_PREFILL_ENDPOINTS": "p,q", **env}, ) _assert_rates(flat, nested, 0.6, cpu, overall) + + +@pytest.mark.parametrize("offload", [10, math.nan, None]) +def test_disabled_cpu_tier_does_not_hide_invalid_present_offload(offload: object) -> None: + flat, nested = _compute( + _export({"p": 100, "q": 100}, {"p": 60, "q": 60}, {"p": offload}), + {"SRT_PREFILL_ENDPOINTS": "p,q", "KV_OFFLOADING": "none"}, + ) + _assert_rates(flat, nested, 0.6, None, None) + + +def test_disabled_cpu_tier_does_not_fill_an_unmatched_label_domain() -> None: + export = _export({"worker": 100}, {"worker": 60}, {"worker": 0}) + export["metrics"]["atom:prefix_cache_offload_tokens"]["series"][0]["labels"] = { + "engine": "other" + } + flat, nested = _compute(export, {"DISAGG": "false", "KV_OFFLOADING": "none"}) + _assert_rates(flat, nested, 0.6, None, None) From 45743f9f4d922afc97c1f647ce87f26816380616 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Tue, 27 Oct 2026 20:18:06 +0000 Subject: [PATCH 13/16] chore(changelog): record GLM-5.2 ATOM PD target quantization and KV dtype The entry omitted the recipe's online ptpc_fp8 target quantization (MTP layer 78 excluded), kv_cache_dtype fp8 and INT4 quick all-reduce, and the draft's resulting precision. Co-authored-by: Cursor --- inferencex-e2e/perf-changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index e9abf9e807..384632df23 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11446,6 +11446,7 @@ - agentic-coding description: - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." + - "Both roles quantize the target's remaining BF16 linear layers to ptpc_fp8 at load with online_quant_config, which excludes lm_head, the embedding, the router gates, every expert and the MTP layer (model.layers.78.*), so the routed experts keep their MXFP4 weights. Both roles also set kv_cache_dtype fp8 and AITER_QUICK_REDUCE_QUANTIZATION=INT4. The MTP draft (model.layers.78 of amd/GLM-5.2-MXFP4, which ships all-BF16 including its routed experts) stays BF16 and uses the FP8 KV cache and FP4 indexer cache like the target." - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node, with owners on 2 MiB HugeTLB pages. 1P1D, as in ATOM's CI, has one pool shared by all four PP stages: one master and two 768 GiB owners, one per NUMA node (1649 GB, within the prefill's and the colocated decode's 1499 GB DRAM shares), each registered on rdma0-3, with MC_ENABLE_DEST_DEVICE_AFFINITY so each stage reaches them through the NIC of its own name on this rail-isolated fabric. 1P2D and 1P3D keep a pool per prefill NIC, each one master and a 349 GiB owner, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its owners' pages free; teardown releases them." - "1P1D on one node at concurrency 56, 72, 96, 112 and 128 with a DCP4 decode on GPUs 4-7 and three draft tokens; 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens); and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). In 1P2D and 1P3D each decode has a node of its own." - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." From cdbec8e227d68130308efd3f096f0c5cd4d4af07 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Wed, 28 Oct 2026 13:33:45 +0000 Subject: [PATCH 14/16] chore(agentx): drop the ATOM cache-counter tooling from #4397 #4397 was closed in favor of SemiAnalysisAI/InferenceX-app#1533, which reads the ATOM P/D counters from server_metrics_export.json at ingest. This PR now changes only the recipe, its master config entry, the changelog and the srt-slurm patch. Co-Authored-By: Claude Opus 5.5 --- .../infx/results/agentic/__init__.py | 1 - .../infx/results/agentic/backends/atom.py | 157 ++--------- .../infx/results/agentic/server_metrics.py | 44 +--- .../agentic/test_atom_cache_counters.py | 249 ------------------ .../agentic/test_process_agentic_result.py | 49 ---- 5 files changed, 21 insertions(+), 479 deletions(-) delete mode 100644 inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py diff --git a/inferencex-e2e/infx/results/agentic/__init__.py b/inferencex-e2e/infx/results/agentic/__init__.py index 638ab24a84..67ada94c91 100644 --- a/inferencex-e2e/infx/results/agentic/__init__.py +++ b/inferencex-e2e/infx/results/agentic/__init__.py @@ -231,7 +231,6 @@ def build_result( framework=framework, records=records, server_logs=server_logs, - env=env, ) if "total_tput_tps" in request_flat and num_gpus > 0: diff --git a/inferencex-e2e/infx/results/agentic/backends/atom.py b/inferencex-e2e/infx/results/agentic/backends/atom.py index 32543a005b..9e5c661de2 100644 --- a/inferencex-e2e/infx/results/agentic/backends/atom.py +++ b/inferencex-e2e/infx/results/agentic/backends/atom.py @@ -2,24 +2,12 @@ from __future__ import annotations -import logging -import math -from collections.abc import Mapping from typing import Any -from infx.results.agentic.common import ( - gauge_stat, - metric_series, - normalize_fraction, - rate, - series_stat, - sum_stat, -) +from infx.results.agentic.common import gauge_stat, normalize_fraction, rate, sum_stat from .base import ServerMetricsBackend, counter_int -logger = logging.getLogger(__name__) - def _atom_names(suffix: str) -> list[str]: """Accept direct worker metrics and Atomesh's colon-normalized form.""" @@ -31,98 +19,6 @@ def _atom_counter_names(stem: str) -> list[str]: return [*_atom_names(stem), *_atom_names(f"{stem}_total")] -def _counter_values( - metrics: dict[str, dict[str, Any]], stem: str, endpoint: str -) -> dict[tuple, float] | None: - names = _atom_counter_names(f"prefix_cache_{stem}_tokens") - values = {} - for series in metric_series(metrics, names): - if series.get("endpoint_url") != endpoint: - continue - labels = series.get("labels") - labels = {} if labels is None else labels - value = series_stat(series, preferred_keys=("total",)) - if ( - not isinstance(labels, dict) - or any(not isinstance(k, str) or not isinstance(v, str) for k, v in labels.items()) - or value is None - or not math.isfinite(value) - or value < 0 - ): - return None - key = tuple(sorted(labels.items())) - if key in values: - logger.warning("ATOM %s cache counter duplicated at %s", stem, endpoint) - return None - values[key] = value - return values - - -def _cache_counts( - metrics: dict[str, dict[str, Any]], - endpoint_roles: Mapping[str, str] | None, - disaggregated: bool | None, - cpu_offload_disabled: bool = False, -) -> tuple[float | None, float | None, float | None]: - names = [ - name - for stem in ("full", "cached", "offload") - for name in _atom_counter_names(f"prefix_cache_{stem}_tokens") - ] - endpoints = [s.get("endpoint_url") for s in metric_series(metrics, names)] - if any(not isinstance(endpoint, str) or not endpoint for endpoint in endpoints): - return None, None, None - observed = set(endpoints) - if endpoint_roles is None: - accounted = observed if disaggregated is False and len(observed) == 1 else set() - else: - # Decode counts the prompt again without serving its prefill cache reuse. - accounted = {url for url, role in endpoint_roles.items() if role in {"prefill", "unified"}} - if observed - endpoint_roles.keys(): - accounted = set() - if not accounted: - logger.warning("ATOM cache endpoint roles missing or ambiguous") - return None, None, None - full_total = cached_total = offload_total = 0.0 - cpu_complete = True - for endpoint in accounted: - full, cached, offload = ( - _counter_values(metrics, stem, endpoint) for stem in ("full", "cached", "offload") - ) - if ( - not full - or cached is None - or full.keys() != cached.keys() - or any(cached[key] > full[key] for key in full) - ): - return None, None, None - if cpu_offload_disabled: - offload = ( - dict.fromkeys(full, 0.0) - if offload is not None - and offload.keys() <= full.keys() - and not any(offload.values()) - else None - ) - full_total += sum(full.values()) - cached_total += sum(cached.values()) - if ( - offload is None - or offload.keys() != full.keys() - or any(cached[key] + offload[key] > full[key] for key in full) - ): - cpu_complete = False - else: - offload_total += sum(offload.values()) - if full_total <= 0 or not all(math.isfinite(v) for v in (full_total, cached_total)): - return None, None, None - return ( - full_total, - cached_total, - offload_total if cpu_complete and math.isfinite(offload_total) else None, - ) - - class AtomBackend(ServerMetricsBackend): name = "atom" @@ -138,32 +34,12 @@ def populate( metrics: dict[str, dict[str, Any]], flat: dict[str, Any], nested: dict[str, Any], - *, - endpoint_roles: Mapping[str, str] | None = None, - disaggregated: bool | None = None, - cpu_offload_disabled: bool = False, ) -> None: """Fill token totals, prefix and LMCache hit rates and KV usage from ATOM ``metrics``.""" - prompt_names = _atom_counter_names("prompt_tokens") - prompt_endpoints = {s.get("endpoint_url") for s in metric_series(metrics, prompt_names)} - if endpoint_roles is None: - accounted = ( - prompt_endpoints if disaggregated is False and len(prompt_endpoints) == 1 else set() - ) - else: - accounted = { - url for url, role in endpoint_roles.items() if role in {"prefill", "unified"} - } - if prompt_endpoints - endpoint_roles.keys(): - accounted = set() - accounted = {url for url in accounted if isinstance(url, str) and url} - if accounted - prompt_endpoints: - accounted = set() prompt_total = sum_stat( metrics, - prompt_names, - preferred_keys=("total",), - series_filter=lambda s: s.get("endpoint_url") in accounted, + _atom_counter_names("prompt_tokens"), + preferred_keys=("total", "sum", "max", "avg"), ) generation_total = sum_stat( metrics, @@ -173,17 +49,24 @@ def populate( flat["total_prompt_tokens"] = counter_int(prompt_total) flat["total_generation_tokens"] = counter_int(generation_total) - full_tokens, cached_tokens, external_tokens = _cache_counts( - metrics, endpoint_roles, disaggregated, cpu_offload_disabled + cached_tokens = sum_stat( + metrics, + _atom_counter_names("prefix_cache_cached_tokens"), + preferred_keys=("total", "sum", "max", "avg"), + ) + full_tokens = sum_stat( + metrics, + _atom_counter_names("prefix_cache_full_tokens"), + preferred_keys=("total", "sum", "max", "avg"), ) cache_hit_rate = rate(cached_tokens, full_tokens) - # Offload is CPU reuse beyond the GPU prefix; loaded tokens measure transfer volume. - external_hit_rate = rate(external_tokens, full_tokens) - overall_hit_rate = ( - rate(cached_tokens + external_tokens, full_tokens) - if cached_tokens is not None and external_tokens is not None - else None + external_tokens = sum_stat( + metrics, + _atom_counter_names("lmcache_loaded_tokens"), + preferred_keys=("total", "sum", "max", "avg"), ) + external_hit_rate = rate(external_tokens, full_tokens) + overall_hit_rate = cache_hit_rate flat["server_gpu_cache_hit_rate"] = cache_hit_rate flat["server_cpu_cache_hit_rate"] = external_hit_rate @@ -207,9 +90,7 @@ def populate( "prefix_cache_hits": cached_tokens, "prefix_cache_queries": full_tokens, "external_prefix_cache_hits": external_tokens, - "external_prefix_cache_queries": ( - full_tokens if external_tokens is not None else None - ), + "external_prefix_cache_queries": full_tokens, } ) nested["kv_cache"]["gpu_usage_pct"] = flat["gpu_kv_cache_usage_pct"] diff --git a/inferencex-e2e/infx/results/agentic/server_metrics.py b/inferencex-e2e/infx/results/agentic/server_metrics.py index 5d176339fe..0ba5ffdb55 100644 --- a/inferencex-e2e/infx/results/agentic/server_metrics.py +++ b/inferencex-e2e/infx/results/agentic/server_metrics.py @@ -2,11 +2,10 @@ from __future__ import annotations -from collections.abc import Iterable, Mapping +from collections.abc import Iterable from typing import Any from .backends import detect_backend -from .backends.atom import AtomBackend from .backends.base import ( SERVER_CACHE_FLAT_FIELDS, apply_profile_totals, @@ -15,36 +14,12 @@ from .common import index_server_metrics -def endpoint_roles_from_env(env: Mapping[str, str]) -> dict[str, str] | None: - if env.get("SRTCTL_FRONTEND_TYPE") == "dynamo": - return {} - if env.get("AIPERF_SERVER_METRICS_URLS"): - return None - groups = ( - [("unified", env["SRT_AGG_ENDPOINTS"])] - if env.get("SRT_AGG_ENDPOINTS") - else [ - ("prefill", env.get("SRT_PREFILL_ENDPOINTS", "")), - ("decode", env.get("SRT_DECODE_ENDPOINTS", "")), - ] - ) - roles: dict[str, str] = {} - for role, endpoints in groups: - for endpoint in endpoints.split(","): - if endpoint: - url = f"http://{endpoint}/metrics" - if roles.setdefault(url, role) != role: - return {} - return roles or None - - def compute_server_metrics( server_metrics: dict[str, Any], *, framework: str, records: list[dict[str, Any]], server_logs: Iterable[str | None] = (), - env: Mapping[str, str] | None = None, ) -> tuple[dict[str, Any], dict[str, Any], list[str]]: """Return flat and nested server metrics plus warnings from aiperf's ``server_metrics`` export. @@ -75,22 +50,7 @@ def compute_server_metrics( f"framework={framework.lower()!r}, metric_names=[{preview}]" ) nested["adapter"] = adapter.name - if isinstance(adapter, AtomBackend): - source_env = env or {} - adapter.populate( - metrics, - flat, - nested, - endpoint_roles=endpoint_roles_from_env(source_env), - disaggregated={"true": True, "false": False}.get(source_env.get("DISAGG", "").lower()), - cpu_offload_disabled=(source_env.get("KV_OFFLOADING", "").strip().lower() == "none") - and not ( - source_env.get("KV_OFFLOAD_BACKEND") - or source_env.get("KV_OFFLOAD_BACKEND_METADATA") - ), - ) - else: - adapter.populate(metrics, flat, nested) + adapter.populate(metrics, flat, nested) if framework.lower() == "dynamo-sglang" and adapter.name == "sglang": nested["kv_cache"]["gpu_total_tokens"] = None diff --git a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py b/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py deleted file mode 100644 index 8356e5bd39..0000000000 --- a/inferencex-e2e/infx/tests/results/agentic/test_atom_cache_counters.py +++ /dev/null @@ -1,249 +0,0 @@ -from __future__ import annotations - -import math - -import pytest - -from infx.results.agentic.server_metrics import compute_server_metrics - - -def _export(full: dict, cached: dict, offload: dict) -> dict: - return { - "metrics": { - f"atom:prefix_cache_{stem}_tokens": { - "series": [ - {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": value}} - for endpoint, value in values.items() - ] - } - for stem, values in (("full", full), ("cached", cached), ("offload", offload)) - } - } - - -def _compute(export: dict, env: dict) -> tuple[dict, dict]: - flat, nested, _ = compute_server_metrics(export, framework="mooncake-atom", records=[], env=env) - return flat, nested - - -def _assert_rates( - flat: dict, nested: dict, gpu: float | None, cpu: float | None, overall: float | None -): - for name, expected in (("gpu", gpu), ("cpu", cpu), ("external", cpu), ("overall", overall)): - actual = flat[f"server_{name}_cache_hit_rate"] - assert actual == (pytest.approx(expected) if expected is not None else None) - assert nested["cache"][f"{name}_cache_hit_rate"] == actual - - -@pytest.mark.parametrize("decode_cached", [800, math.nan]) -def test_prefill_counters_exclude_decode_and_transfer_volume(decode_cached: float) -> None: - export = _export({"p": 1000, "d": 1000}, {"p": 600, "d": decode_cached}, {"p": 100, "d": -1}) - export["metrics"]["atom:lmcache_loaded_tokens"] = { - "series": [{"endpoint_url": "http://p/metrics", "stats": {"total": 900}}] - } - export["metrics"]["atom:prompt_tokens"] = { - "series": [ - {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": 1000}} - for endpoint in ("p", "d") - ] - } - flat, nested = _compute( - export, {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "d", "DISAGG": "true"} - ) - assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 1000 - _assert_rates(flat, nested, 0.6, 0.1, 0.7) - assert nested["cache"]["prefix_cache_queries"] == 1000 - assert nested["cache"]["external_prefix_cache_hits"] == 100 - assert nested["cache"]["external_prefix_cache_queries"] == 1000 - - -@pytest.mark.parametrize( - ("env", "offload", "expected"), - [ - ({"SRT_AGG_ENDPOINTS": "worker", "SRT_PREFILL_ENDPOINTS": "unused"}, 50, 0.5), - ({"DISAGG": "false"}, 0, 0), - ({"DISAGG": "false", "AIPERF_SERVER_METRICS_URLS": "http://worker/metrics"}, 50, 0.5), - ], -) -def test_unified_zero_gpu_and_offload_counter_alias( - env: dict, offload: int, expected: float -) -> None: - export = _export({"worker": 100}, {"worker": 0}, {"worker": offload}) - export["metrics"] = { - name.replace("atom:", "atom_") + "_total": value - for name, value in export["metrics"].items() - } - flat, nested = _compute(export, env) - _assert_rates(flat, nested, 0, expected, expected) - - -def test_multiple_prefills_and_labelled_series_are_token_weighted() -> None: - export = _export( - {"p": 100, "q": 900, "idle": 0}, - {"p": 80, "q": 20, "idle": 0}, - {"p": 10, "q": 180, "idle": 0}, - ) - for stem, values in (("full", (400, 500)), ("cached", (5, 15)), ("offload", (80, 100))): - series = export["metrics"][f"atom:prefix_cache_{stem}_tokens"]["series"] - series[:] = [s for s in series if s["endpoint_url"] != "http://q/metrics"] - series.extend( - { - "endpoint_url": "http://q/metrics", - "labels": {"engine": str(i)}, - "stats": {"total": v}, - } - for i, v in enumerate(values) - ) - flat, nested = _compute(export, {"SRT_PREFILL_ENDPOINTS": "p,q,idle"}) - _assert_rates(flat, nested, 0.1, 0.19, 0.29) - - -@pytest.mark.parametrize( - "env", - [ - {}, - {"DISAGG": "true"}, - {"SRT_PREFILL_ENDPOINTS": "p", "SRT_DECODE_ENDPOINTS": "p,d"}, - {"SRT_PREFILL_ENDPOINTS": "p,missing", "SRT_DECODE_ENDPOINTS": "d"}, - {"SRT_PREFILL_ENDPOINTS": "p"}, - { - "SRT_PREFILL_ENDPOINTS": "p", - "SRT_DECODE_ENDPOINTS": "d", - "AIPERF_SERVER_METRICS_URLS": "custom", - }, - { - "SRT_PREFILL_ENDPOINTS": "p", - "SRT_DECODE_ENDPOINTS": "d", - "SRTCTL_FRONTEND_TYPE": "dynamo", - }, - ], -) -def test_unknown_conflicting_or_incomplete_roles_do_not_mix_endpoints(env: dict) -> None: - flat, nested = _compute( - _export({"p": 100, "d": 100}, {"p": 60, "d": 0}, {"p": 10, "d": 0}), env - ) - _assert_rates(flat, nested, None, None, None) - - -def test_single_endpoint_without_explicit_unified_topology_stays_unknown() -> None: - flat, nested = _compute(_export({"worker": 100}, {"worker": 60}, {"worker": 10}), {}) - _assert_rates(flat, nested, None, None, None) - - -@pytest.mark.parametrize( - ("offload", "cpu", "overall"), - [({"p": 0, "q": 0}, 0, 0.2), ({}, None, None), ({"p": 10}, None, None)], -) -def test_missing_or_partial_offload_is_distinct_from_zero( - offload: dict, cpu: float | None, overall: float | None -) -> None: - flat, nested = _compute( - _export({"p": 100, "q": 100}, {"p": 20, "q": 20}, offload), - {"SRT_PREFILL_ENDPOINTS": "p,q"}, - ) - _assert_rates(flat, nested, 0.2, cpu, overall) - assert nested["cache"]["external_prefix_cache_queries"] == (200 if cpu is not None else None) - - -@pytest.mark.parametrize( - ("stem", "value"), - [ - ("full", 0), - ("cached", -1), - ("cached", 101), - ("cached", math.nan), - ("cached", math.inf), - ("cached", None), - ], -) -def test_invalid_counted_counter_does_not_produce_rates(stem: str, value: object) -> None: - export = _export({"worker": 100}, {"worker": 20}, {"worker": 10}) - export["metrics"][f"atom:prefix_cache_{stem}_tokens"]["series"][0]["stats"]["total"] = value - flat, nested = _compute(export, {"DISAGG": "false"}) - _assert_rates(flat, nested, None, None, None) - - -def test_unreadable_series_does_not_get_silently_skipped() -> None: - export = _export({"worker": 100}, {"worker": 20}, {"worker": 10}) - export["metrics"]["atom:prefix_cache_cached_tokens"]["series"].append( - { - "endpoint_url": "http://worker/metrics", - "labels": {"engine": "other"}, - "stats": {"max": 5}, - } - ) - flat, nested = _compute(export, {"DISAGG": "false"}) - _assert_rates(flat, nested, None, None, None) - - -@pytest.mark.parametrize("offload", [30, math.nan]) -@pytest.mark.parametrize("labelled", [False, True]) -def test_invalid_worker_cpu_count_cannot_hide_in_another_workers_denominator( - offload: float, labelled: bool -) -> None: - export = _export({"p": 100, "q": 900}, {"p": 80, "q": 20}, {"p": offload, "q": 0}) - if labelled: - for metric in export["metrics"].values(): - for series in metric["series"]: - series["labels"] = {"engine": series["endpoint_url"]} - series["endpoint_url"] = "http://worker/metrics" - flat, nested = _compute(export, {"SRT_PREFILL_ENDPOINTS": "worker" if labelled else "p,q"}) - _assert_rates(flat, nested, 0.1, None, None) - - -def test_unknown_cache_roles_preserve_other_atom_telemetry() -> None: - export = _export({"p": 100, "d": 100}, {"p": 60, "d": 0}, {"p": 10, "d": 0}) - export["metrics"].update( - { - "atom:prompt_tokens": {"series": [{"stats": {"total": 200}}]}, - "atom:generation_tokens": {"series": [{"stats": {"total": 30}}]}, - "atom:kv_cache_usage_ratio": {"series": [{"stats": {"max": 0.75}}]}, - } - ) - flat, nested, _ = compute_server_metrics( - export, - framework="mooncake-atom", - records=[{"metrics": {"input_sequence_length": {"value": 123}}}], - env={}, - ) - _assert_rates(flat, nested, None, None, None) - assert flat["total_prompt_tokens"] == nested["tokens"]["prompt_total"] == 123 - assert flat["total_generation_tokens"] == nested["tokens"]["generation_total"] == 30 - assert flat["gpu_kv_cache_usage_pct"] == nested["kv_cache"]["gpu_usage_pct"] == 0.75 - - -@pytest.mark.parametrize( - ("env", "cpu", "overall"), - [ - ({"KV_OFFLOADING": " NONE "}, 0, 0.6), - ({"KV_OFFLOADING": "dram"}, None, None), - ({"KV_OFFLOADING": "none", "KV_OFFLOAD_BACKEND": "mooncake"}, None, None), - ], -) -@pytest.mark.parametrize("offload", [{}, {"p": 0}]) -def test_missing_offload_is_zero_only_with_confirmed_disabled_tier( - env: dict, cpu: float | None, overall: float | None, offload: dict -) -> None: - flat, nested = _compute( - _export({"p": 100, "q": 100}, {"p": 60, "q": 60}, offload), - {"SRT_PREFILL_ENDPOINTS": "p,q", **env}, - ) - _assert_rates(flat, nested, 0.6, cpu, overall) - - -@pytest.mark.parametrize("offload", [10, math.nan, None]) -def test_disabled_cpu_tier_does_not_hide_invalid_present_offload(offload: object) -> None: - flat, nested = _compute( - _export({"p": 100, "q": 100}, {"p": 60, "q": 60}, {"p": offload}), - {"SRT_PREFILL_ENDPOINTS": "p,q", "KV_OFFLOADING": "none"}, - ) - _assert_rates(flat, nested, 0.6, None, None) - - -def test_disabled_cpu_tier_does_not_fill_an_unmatched_label_domain() -> None: - export = _export({"worker": 100}, {"worker": 60}, {"worker": 0}) - export["metrics"]["atom:prefix_cache_offload_tokens"]["series"][0]["labels"] = { - "engine": "other" - } - flat, nested = _compute(export, {"DISAGG": "false", "KV_OFFLOADING": "none"}) - _assert_rates(flat, nested, 0.6, None, None) diff --git a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py index c736a237db..363ce7ba1a 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_process_agentic_result.py @@ -365,55 +365,6 @@ def test_processor_records_run_attempt(tmp_path: Path): assert "run_attempt" not in local -def test_processor_uses_launcher_roles_for_atom_cache_counts(tmp_path: Path) -> None: - result_dir = _write_fixture(tmp_path) - metrics = { - f"atom:prefix_cache_{stem}_tokens": { - "series": [ - {"endpoint_url": f"http://{endpoint}/metrics", "stats": {"total": value}} - for endpoint, value in values.items() - ] - } - for stem, values in ( - ("full", {"prefill:8000": 1000, "decode:8000": 1000}), - ("cached", {"prefill:8000": 600, "decode:8000": 0}), - ("offload", {"prefill:8000": 100, "decode:8000": 0}), - ) - } - (result_dir / "aiperf_artifacts" / "server_metrics_export.json").write_text( - json.dumps({"metrics": metrics}) - ) - agg = _run_processor( - result_dir, - tmp_path / "out", - { - "FRAMEWORK": "mooncake-atom", - "DISAGG": "true", - "IS_MULTINODE": "true", - "PREFILL_NUM_WORKERS": "1", - "PREFILL_TP": "4", - "PREFILL_EP": "1", - "DECODE_NUM_WORKERS": "1", - "DECODE_TP": "4", - "DECODE_EP": "1", - "SRT_PREFILL_ENDPOINTS": "prefill:8000", - "SRT_DECODE_ENDPOINTS": "decode:8000", - "SRT_AGG_ENDPOINTS": "", - "AIPERF_SERVER_METRICS_URLS": "", - "SRTCTL_FRONTEND_TYPE": "", - "KV_OFFLOADING": "dram", - "KV_OFFLOAD_BACKEND": "lmcache", - "KV_OFFLOAD_BACKEND_METADATA": '{"name":"lmcache","version":"0.4.5"}', - }, - ) - cache = agg["server_metrics"]["cache"] - assert cache["gpu_cache_hit_rate"] == 0.6 - assert cache["cpu_cache_hit_rate"] == cache["external_cache_hit_rate"] == 0.1 - assert cache["overall_cache_hit_rate"] == 0.7 - assert cache["prefix_cache_queries"] == cache["external_prefix_cache_queries"] == 1000 - assert agg["num_gpus"] == 8 - - def test_processor_preserves_dataset_provenance(tmp_path: Path): result_dir = _write_fixture(tmp_path) output_dir = tmp_path / "out" From da9e0abb542d27f0b6238b4fc8dafce952720e7d Mon Sep 17 00:00:00 2001 From: seungrokj Date: Wed, 28 Oct 2026 09:37:00 -0700 Subject: [PATCH 15/16] docs(changelog): sync GLM-5.2 MI355X ATOM disagg block with the diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Name the config key, the image digest and its ATOM commit (6fd1bb1be, ROCm 7.2.4). - List the nine points with their prefill and decode layouts and dram-utilization 1.0. - Give per-role server settings: prefill eager at gpu-memory-utilization 0.85, decode gpu-memory-utilization, admission and CUDA graph sizes, draft tokens 3/4/5, and AToMesh round-robin at 1P1D vs cache-aware at 1P2D and 1P3D. - Correct the draft line: the MTP draft runs as shipped (all BF16) and shares the target's FP8 KV cache and FP8 indexer cache, not an FP4 indexer cache; at ATOM 6fd1bb1be a GLM model's index_cache_dtype follows kv_cache_dtype unless set. - 写明配置键、镜像 digest 及其 ATOM commit（6fd1bb1be，ROCm 7.2.4）。 - 列出九个测试点的预填充与解码布局，以及 dram-utilization 1.0。 - 写明各角色的服务参数：预填充以 eager 模式运行、gpu-memory-utilization 0.85；解码的 gpu-memory-utilization、准入数与 CUDA graph 大小；草稿 token 数 3/4/5；AToMesh 在 1P1D 使用 round-robin，在 1P2D 和 1P3D 使用 cache-aware。 - 更正草稿模型一行：MTP 草稿模型按出厂精度（全部 BF16）运行，与目标模型共用 FP8 KV 缓存和 FP8 indexer 缓存，而非 FP4 indexer 缓存；在 ATOM 6fd1bb1be 上，GLM 模型未设置 index_cache_dtype 时沿用 kv_cache_dtype。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/perf-changelog.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 9c73367c3f..c88d962143 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -11495,10 +11495,12 @@ scenario-type: - agentic-coding description: - - "Add GLM-5.2 MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 (digest-pinned). The prefill is one CPP4 worker (PP4 x TP1 on four GPUs)." - - "Both roles quantize the target's remaining BF16 linear layers to ptpc_fp8 at load with online_quant_config, which excludes lm_head, the embedding, the router gates, every expert and the MTP layer (model.layers.78.*), so the routed experts keep their MXFP4 weights. Both roles also set kv_cache_dtype fp8 and AITER_QUICK_REDUCE_QUANTIZATION=INT4. The MTP draft (model.layers.78 of amd/GLM-5.2-MXFP4, which ships all-BF16 including its routed experts) stays BF16 and uses the FP8 KV cache and FP4 indexer cache like the target." + - "Add glm5.2-fp4-mi355x-atom-disagg-agentic-mtp: amd/GLM-5.2-MXFP4 on MI355X with ATOM prefill/decode disaggregation behind AToMesh, Mooncake RDMA from prefill to decode and MTP, on rocm/atom-dev:nightly_202610131249 pinned by digest sha256:d5b3fcd99bd44e3489740d1b026037f49f2e3f6160dc70b70c3fd23a125a4905 (ROCm/ATOM 6fd1bb1be, ROCm 7.2.4; AToMesh at the same commit)." + - "Nine points with dram-utilization 1.0, each with one CPP4 prefill worker (PP4 x TP1 on GPUs 0-3, layers split 20/20/20/18): 1P1D at concurrency 56, 72, 96, 112 and 128 with a TP4 + DCP4 decode on GPUs 4-7 of the prefill node; 1P2D at concurrency 56 with two TP4 + DCP4 decodes; and 1P3D at concurrency 12, 16 and 24 with three TP4 decodes. In 1P2D and 1P3D each decode has a node of its own (spread_workers)." + - "Both roles set kv_cache_dtype fp8, block size 16, AITER_QUICK_REDUCE_QUANTIZATION=INT4 and online_quant_config ptpc_fp8, which quantizes the target's remaining BF16 linear layers to FP8 at load and excludes lm_head, the embedding, the router gates, every routed expert (which keep their MXFP4 weights) and the MTP layer (model.layers.78.*)." + - "The prefill runs eager with gpu-memory-utilization 0.85, max-num-seqs 512, max-num-batched-tokens 8192 and one offload load worker. The DCP4 decodes use gpu-memory-utilization 0.94, max-num-seqs 512, max-num-batched-tokens 2048 and FULL CUDA graphs up to batch 256; the 1P3D TP4 decodes use gpu-memory-utilization 0.95, max-num-batched-tokens 16384 and max-num-seqs and CUDA graphs up to 2x concurrency (24, 32, 48). Draft tokens are 3 at 1P1D, 4 at 1P2D and 5 at 1P3D. AToMesh routes round-robin at 1P1D and cache-aware (cache threshold 0.8) at 1P2D and 1P3D." - "Every point keeps its prefill's prefix tier in a Mooncake Store (0.3.13) on the prefill node, with owners on 2 MiB HugeTLB pages. 1P1D, as in ATOM's CI, has one pool shared by all four PP stages: one master and two 768 GiB owners, one per NUMA node (1649 GB, within the prefill's and the colocated decode's 1499 GB DRAM shares), each registered on rdma0-3, with MC_ENABLE_DEST_DEVICE_AFFINITY so each stage reaches them through the NIC of its own name on this rail-isolated fabric. 1P2D and 1P3D keep a pool per prefill NIC, each one master and a 349 GiB owner, the prefill's 1499 GB. After the cluster host setup drops the page cache, host_setup on the head node, where srtctl starts the prefill, reserves the owners' pages plus 2% above the hugepages host services hold and fails unless each NUMA node has its owners' pages free; teardown releases them." - - "1P1D on one node at concurrency 56, 72, 96, 112 and 128 with a DCP4 decode on GPUs 4-7 and three draft tokens; 1P2D with two DCP4 decodes at concurrency 56 (four draft tokens); and 1P3D with three TP4 decodes at concurrency 12, 16 and 24 (five draft tokens). In 1P2D and 1P3D each decode has a node of its own." - "Workers read the checkpoint from each node's local Hugging Face hub cache, as the single-node GLM-5.2 points do; srtctl fills the first node's cache before they start." - "Patch srt-slurm so its ATOM backend passes -tp as the worker's GPUs over its pipeline and prefill-context stages (NVIDIA/srt-slurm#604); unpatched, the PP4 prefill launches with -tp 4." + - "The MTP draft runs as shipped: model.layers.78 of amd/GLM-5.2-MXFP4 is all BF16, including its 256 routed experts (only the router bias is F32), and at ATOM 6fd1bb1be the model.layers.78.* online exclusion keeps every draft layer out of ptpc_fp8. The draft shares the target's FP8 KV cache and FP8 indexer cache (the recipe sets no index_cache_dtype, so it follows kv_cache_dtype)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4045 From 07f244be0102e46ab6c9cd3facdd387247c310f4 Mon Sep 17 00:00:00 2001 From: Jasen2201 Date: Thu, 29 Oct 2026 00:30:12 +0000 Subject: [PATCH 16/16] refactor(agentx): reserve the GLM-5.2 Store owners' hugepages in the mi355x-amds host hook The GLM-5.2 ATOM PD recipe grew the 2 MiB HugeTLB pool for its Mooncake Store owners with an inline shell block in each host_setup. setup.sh now does it when a recipe sets HEAD_NODE_HUGEPAGES_PER_NUMA: on the head node it grows the pool 2% above the pages host services hold and fails unless every NUMA node ends with its share free. Unset, setup.sh behaves as before, so no other recipe on the cluster changes. The recipe passes the same page counts (393216 per NUMA at 1P1D, 357376 at 1P2D and 1P3D), so the pool, its checks and the teardown are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../mi355x-fp4/agentx/disagg-variants.yaml | 34 +++---------------- .../srt-slurm/hooks/mi355x-amds/setup.sh | 24 +++++++++++++ 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml index 21c171b2b7..3bb574da8a 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/atom/mi355x-fp4/agentx/disagg-variants.yaml @@ -97,21 +97,8 @@ override_1p1d_mcstore: nodes: all timeout_seconds: 1800 commands: - # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. - - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - # Owners take 786432 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. - # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. - - >- - if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then - echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; - held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); - HUGEPAGES_TARGET=$(( held + 786432 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 - bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; - for node in 0 1; do - free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); - echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 393216"; - [[ $free -ge 393216 ]] || exit 1; - done; fi + # Reserves the Store owners' 2 MiB pages on the head node, where srtctl starts the prefill. + - HEAD_NODE_HUGEPAGES_PER_NUMA=393216 IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" services: @@ -209,21 +196,8 @@ override_1p2d_mcstore_c56: nodes: all timeout_seconds: 1800 commands: - # A recipe host_setup replaces the cluster default, which also drops the page cache under DRAM KV offloading. - - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" - # Owners take 714752 2 MiB pages, half per NUMA node; reserve 2% more above the pages host services hold. - # Only the prefill node holds owners; srtctl starts the prefill on the head node, which launches this srun. - - >- - if hostname -I | tr ' ' '\n' | grep -qxF "${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}"; then - echo "[hugepage] $(hostname -s) is the head node; reserving the owners' pages"; - held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' /proc/meminfo); - HUGEPAGES_TARGET=$(( held + 714752 * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 - bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" || exit 1; - for node in 0 1; do - free=$(cat /sys/devices/system/node/node$node/hugepages/hugepages-2048kB/free_hugepages); - echo "[hugepage] NUMA$node: $free free 2 MiB pages, owners need 357376"; - [[ $free -ge 357376 ]] || exit 1; - done; fi + # Reserves the Store owners' 2 MiB pages on the head node, where srtctl starts the prefill. + - HEAD_NODE_HUGEPAGES_PER_NUMA=357376 IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" teardown: - HUGEPAGES_TARGET=0 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/hugepage_sanity.sh" services: &prefill_budget_services diff --git a/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh b/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh index 2c281bf90b..e167f3d718 100755 --- a/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh +++ b/inferencex-e2e/runners/srt-slurm/hooks/mi355x-amds/setup.sh @@ -59,3 +59,27 @@ if (( after_free > reserved )); then echo "Host preparation could not reclaim all unused hugepages" >&2 exit 1 fi + +# A recipe whose head-node service registers NUMA-bound 2 MiB HugeTLB memory with +# the NIC (a Mooncake Store owner) sets HEAD_NODE_HUGEPAGES_PER_NUMA; unset, no +# pages are reserved. The container cannot grow the pool, so it is grown here, 2% +# above the pages host services hold, and each NUMA node must end with its share +# free. The recipe's teardown releases them (HUGEPAGES_TARGET=0 hugepage_sanity.sh). +pages_per_numa="${HEAD_NODE_HUGEPAGES_PER_NUMA:-}" +if [[ -n "$pages_per_numa" ]]; then + launch_ip="${SLURM_LAUNCH_NODE_IPADDR:?srun did not set SLURM_LAUNCH_NODE_IPADDR}" + if [[ " $(hostname -I) " == *" $launch_ip "* ]]; then + numa_nodes=(/sys/devices/system/node/node[0-9]*) + echo "[hugepage] $(hostname -s) is the head node; reserving $pages_per_numa 2 MiB pages on each of ${#numa_nodes[@]} NUMA nodes" + held=$(awk '/^HugePages_Total:/ {total = $2} /^HugePages_Free:/ {free = $2} /^HugePages_Rsvd:/ {rsvd = $2} END {print total - free + rsvd}' "$meminfo") + HUGEPAGES_TARGET=$(( held + pages_per_numa * ${#numa_nodes[@]} * 102 / 100 )) HUGEPAGE_GROW_TOLERANCE_PCT=1 \ + bash "$(dirname "${BASH_SOURCE[0]}")/hugepage_sanity.sh" + for numa_node in "${numa_nodes[@]}"; do + numa_free=$(cat "$numa_node/hugepages/hugepages-2048kB/free_hugepages") + echo "[hugepage] ${numa_node##*/}: $numa_free free 2 MiB pages, $pages_per_numa needed" + if (( numa_free < pages_per_numa )); then + exit 1 + fi + done + fi +fi