diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..56b6a87647 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml @@ -0,0 +1,205 @@ +# DeepSeek-V4.1-Flash AgentX on MI355X with ATOM DSpark (five draft tokens), +# following ROCm/ATOM recipes/DeepSeek-V4.1-Flash-Agentic.md at the image's +# revision (7fc0926c698e3f19df9dca23091e399851d3d0ec). TP2 and TP4, no expert +# parallelism; the KV cache is GPU-resident. +base: + schema: 2 + name: dsv41flash-fp4-mi355x-atom-agentic + model: + path: hf:deepseek-ai/DeepSeek-V4.1-Flash + container: rocm/atom-dev:nightly_202610070333 + precision: fp4 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: atomesh + container_image: rocm/atom-dev@sha256:f00a9dc588a380038fdad54cd57a956043cb316ddda54ebb1b62a24f8caa343b + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: atom + roles: + agg: + nodes: 1 + workers: 1 + args: + served-model-name: deepseek-ai/DeepSeek-V4.1-Flash + trust-remote-code: true + kv_cache_dtype: bf16 + index-cache-dtype: fp4 + gpu-memory-utilization: 0.9 + max-num-seqs: 128 + max-num-batched-tokens: 16384 + attn-prefill-chunk-size: 16384 + enable_prefix_caching: true + block-size: 16 + state-checkpoint-interval-tokens: 8192 + level: 3 + cudagraph-mode: FULL + cudagraph-capture-sizes: '[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' + # The checkpoint's bundled DSpark head. Throughput runs get the golden + # AL (dsv41flash_dspark thinking_on, K5) from infx/srt_slurm/synthetic_acceptance.py; + # evals keep real acceptance. + method: dspark + num-speculative-tokens: 5 + tool-call-parser: dsml_v41 + env: + OMP_NUM_THREADS: '4' + ATOM_NUMA_BIND: '0' + ATOM_DISABLE_MMAP: 'true' + AITER_LOG_LEVEL: WARNING + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + MODEL: deepseek-ai/DeepSeek-V4.1-Flash + # Full-context corpus for this 1M-context recipe. + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + # The upstream one-hour profile warms five requests per lane. + AIPERF_WARMUP_REQUESTS_PER_LANE: '5' + # Long AgentX stalls exceed AIPerf's 30 s TCP_USER_TIMEOUT default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT: '300' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'atom:' + +# One variant per point. Every point admits 128 sequences and captures every +# decode size through 32 plus 48, 64 and 128. + +override_tp2_c1: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '1' + +override_tp2_c2: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '2' + +override_tp2_c4: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '4' + +override_tp2_c5: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '5' + +override_tp2_c6: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '6' + +override_tp2_c8: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '8' + +override_tp2_c16: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '16' + +override_tp2_c32: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '32' + +override_tp2_c64: + roles: + agg: + gpus: 2 + # Run at least 4 decode passes after each prefill and batch the prefills + # that arrive meanwhile; at this load the batching also raises throughput. + env: + ATOM_PREFILL_DECODE_INTERVAL: '4' + benchmark: + env: + CONC: '64' + +override_tp2_c128: + roles: + agg: + gpus: 2 + benchmark: + env: + CONC: '128' + +override_tp4_c1: + roles: + agg: + gpus: 4 + benchmark: + env: + CONC: '1' + +override_tp4_c4: + roles: + agg: + gpus: 4 + benchmark: + env: + CONC: '4' + +override_tp4_c5: + roles: + agg: + gpus: 4 + benchmark: + env: + CONC: '5' + +override_tp4_c6: + roles: + agg: + gpus: 4 + benchmark: + env: + CONC: '6' + +override_tp4_c8: + roles: + agg: + gpus: 4 + benchmark: + env: + CONC: '8' + +override_tp4_c32: + roles: + agg: + gpus: 4 + # Run at least 16 decode passes after each prefill and batch the prefills + # that arrive meanwhile, so new turns stall decode less often. + env: + ATOM_PREFILL_DECODE_INTERVAL: '16' + benchmark: + env: + CONC: '32' diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 1658598f1a..10a264bc70 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -1515,6 +1515,22 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/vllm/mi355x-fp4-mtp/agentic.yaml } +# DeepSeek-V4.1-Flash on ATOM, following the upstream TP2 / TP4 AgentX recipe. +# Five-token DSpark uses golden AL 3.51 for throughput and real acceptance for eval. +dsv41flash-fp4-mi355x-atom-agentic-dspark: + image: rocm/atom-dev:nightly_202610070333 + model: deepseek-ai/DeepSeek-V4.1-Flash + model-prefix: dsv41flash + runner: cluster:mi355x-amds + precision: fp4 + framework: atom + multinode: false + scenarios: + agentic-coding: + - search-space: + - { tp: 2, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 5, 6, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 4, 5, 6, 8, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml } + # The golden acceptance curve remains the provisional GLM-5.2-derived curve # documented in infx/golden_al_distribution/glm5.3_mtp.yaml. glm5.3-fp8-mi355x-tilert-agentic: diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index e17c99ad38..dc1fafbd61 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -112,7 +112,7 @@ Other offloading tiers, including NVMe KV cache offloading, are outside the init | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| -| DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (vLLM: DSpark, Engram UVA offload; SGLang: DSpark arms added per SKU from 2026-09-17; ATOM: MI355X TP2/TP4 DSpark added 2026-09-23, removed in #3463; GPU validation pending) | | +| DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (vLLM: DSpark, Engram UVA offload; SGLang: DSpark arms added per SKU from 2026-09-17; ATOM: MI355X TP2/TP4 DSpark added 2026-09-23, removed in #3463, restored on srt-slurm 2026-09-30; GPU validation pending) | | | GLM-5.3 | `glm5.3` | 2026-09-22 ([#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)) | Agentic coding (MTP only, per the deprecation policy) | Single-turn 1k1k (deprecated for all models before this model was added; never run) | | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26 ([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | Agentic coding | | | Kimi-K3 | `kimik3` | 2026-07-27 ([#2391](https://github.com/SemiAnalysisAI/InferenceX/pull/2391)) | Agentic coding (DSpark may be disabled for better Pareto points) | Standalone non-DSpark A/B baseline (not required from day 0) | diff --git a/inferencex-e2e/docs/MODELS_zh.md b/inferencex-e2e/docs/MODELS_zh.md index 2033aa0a64..b053918b21 100644 --- a/inferencex-e2e/docs/MODELS_zh.md +++ b/inferencex-e2e/docs/MODELS_zh.md @@ -112,7 +112,7 @@ InferenceX 支持 SGLang 和 vLLM 双方的维护者,并响应 AI 实验室和 | 模型架构类别 | 前缀 | 加入日期 | 启用场景 | 已弃用场景 | |---|---|---|---|---| -| DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | 智能体编码(vLLM:DSpark、Engram UVA 卸载;SGLang:自 2026-09-17 起按 SKU 添加 DSpark 配置;ATOM:2026-09-23 添加 MI355X TP2/TP4 DSpark,后于 #3463 移除;GPU 待验证) | | +| DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | 智能体编码(vLLM:DSpark、Engram UVA 卸载;SGLang:自 2026-09-17 起按 SKU 添加 DSpark 配置;ATOM:2026-09-23 添加 MI355X TP2/TP4 DSpark,后于 #3463 移除,2026-09-30 以 srt-slurm 恢复;GPU 待验证) | | | GLM-5.3 | `glm5.3` | 2026-09-22([#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)) | 智能体编码(仅 MTP,见弃用政策) | 单轮 1k1k(在本模型加入前已对所有模型弃用,从未运行) | | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | 智能体编码 | | | Kimi-K3 | `kimik3` | 2026-07-27 ([#2391](https://github.com/SemiAnalysisAI/InferenceX/pull/2391)) | 智能体编码(可关闭 DSpark 以获得更优帕累托点) | 独立非 DSpark A/B 基线(自第 0 天起即不要求) | diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 5b4c52f578..a93c093d41 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9397,3 +9397,16 @@ description: - "Accept native SRT power samples v3 with optional GPU temperature while preserving v1/v2 replay and existing energy validation. Temperature remains in raw telemetry for ingestion and display." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3685 + +- config-keys: + - dsv41flash-fp4-mi355x-atom-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Restore DeepSeek-V4.1-Flash FP4 ATOM AgentX on MI355X (#3387) on the native srt-slurm recipe benchmarks/single_node/srt-slurm-recipes/dsv41flash/atom/mi355x-fp4-mtp/agentic.yaml; the legacy script and master-config entry were removed in #3461 / #3463. Points change from #3387: TP2 concurrency [1, 2, 4, 5, 6, 8, 16, 32, 64, 128] (adds 4, 5, 6 and 128), TP4 concurrency [1, 4, 5, 6, 8, 32] (adds 1, 4, 5 and 6; drops 2, 16 and 64); no expert parallelism or KV offload. Concurrency 1-8 covers the batch sizes that V4.1 mono decode serves (up to 8 requests a step)." + - "Update the image from rocm/atom-dev:nightly_202609250902 to nightly_202610070333 (digest sha256:2f605a3e8b459a0385f784edb0335c11892277fee650fe42044e4af39e36dd01; ATOM 0.1.7.dev60+g7fc0926c6, ROCm 7.2.4). ROCm/ATOM#2419 adds group32 FP8 weight preshuffle and BF16 RoPE tables for DeepSeek-V4.1. ROCm/ATOM#2479 adds V4.1 mono decode, which is on by default and serves decode steps of up to 8 requests for this TP2/TP4, BF16-KV, five-token DSpark deployment, and the FP4 index plane that the upstream recipes/DeepSeek-V4.1-Flash-Agentic.md now uses. The image's AITER includes ROCm/aiter#6174, which adds gfx950 tuned rows for the six DeepSeek-V4.1-Flash TP4 bf16 GEMM (N, K) pairs that previously fell back to torch, including the DSpark verify batches and the TP4 lm_head. ROCm/ATOM#2483 replaces the Triton mono step fence, which Triton 3.7 on gfx950 compiled into a loop that could spin for ever: on nightly_202610061452 and nightly_202610061600 the TP2 c1 canary hung in warmup on the DSpark draft's first mono step and never served (runs 37486079757 and 37493871230)." + - "Server settings follow the upstream recipe as before (BF16 KV, FP4 index cache (FP8 before; follows the upstream recipe at 7fc0926c), max-num-seqs 128, 16K batching/prefill chunks, prefix caching with block size 16, 8K state checkpoints, level 3, FULL graphs, dsml_v41 tool parsing), except that every point now captures the dense list [1..32, 48, 64, 128] instead of only concurrency 32. Five-token DSpark takes the thinking_on golden AL 3.51 from infx/srt_slurm/synthetic_acceptance.py for throughput and real acceptance for eval. Replay semianalysis_cc_traces_weka_062126 for 3600 seconds per point with five warmup requests per lane." + - "No data-type or precision change to the DSpark draft (layers 37-39, loaded inline from the checkpoint): no --online_quant_config is passed, so the draft keeps its checkpoint precision; ROCm/ATOM#2419 only lets its fused wkv GEMM consume the preshuffled FP8 layout and fuses inverse RoPE into the output projection. --kv_cache_dtype bf16 and --index-cache-dtype fp4 touch cache storage only." + - "TP4 concurrency 32 sets ATOM_PREFILL_DECODE_INTERVAL=16: after each prefill, ATOM runs at least 16 decode passes and batches the prefills that arrive in the meantime (ATOM's PrefillDelayer, off by default). Prefill no longer interrupts decode as often, so P90 interactivity rises while TTFT rises. Local 3600 s replay: 108.2 -> 150.9 tok/s/user, ITL p90 9.24 -> 6.63 ms, TTFT p50 0.51 -> 1.14 s, throughput unchanged." + - "TP2 concurrency 64 sets ATOM_PREFILL_DECODE_INTERVAL=4. At this load each batched prefill forward serves several requests, so throughput rises as well as interactivity. Local 3600 s replay: 30.3 -> 38.8 tok/s/user, 105.4k -> 115.2k tok/s/GPU, 6487 -> 6941 requests, TTFT p90 2.3 -> 6.4 s. All other points are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3603