From e3ee574d423b781585eac0eb2f7f034206977aad Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:34:35 -0500 Subject: [PATCH 01/12] chore: remove the standalone llm-d launcher Signed-off-by: Cam Quilici --- inferencex-e2e/benchmarks/llm-d/Dockerfile | 44 -- inferencex-e2e/benchmarks/llm-d/README.md | 16 - inferencex-e2e/benchmarks/llm-d/binaries.env | 32 - inferencex-e2e/benchmarks/llm-d/envoy.yaml | 129 ---- .../benchmarks/llm-d/epp-config.yaml | 61 -- .../benchmarks/llm-d/extract-binaries.sh | 40 -- .../dsv4-fp4-gb200-low-latency.yaml | 192 ------ .../dsv4-fp4-gb200-mid-curve-megamoe.yaml | 196 ------ .../benchmarks/multi_node/llm-d/README.md | 142 ----- .../benchmarks/multi_node/llm-d/job.slurm | 282 --------- .../benchmarks/multi_node/llm-d/server.sh | 573 ------------------ .../benchmarks/multi_node/llm-d/submit.sh | 123 ---- .../benchmarks/multi_node/runtime_settings.sh | 7 - .../infx/launch/drivers/__init__.py | 3 +- inferencex-e2e/infx/launch/drivers/llmd.py | 161 ----- .../infx/launch/drivers/srt/models.py | 5 - inferencex-e2e/infx/launch/policy.py | 3 - inferencex-e2e/infx/launch/request.py | 14 - inferencex-e2e/infx/matrix/validation.py | 7 - .../infx/tests/launch/test_llmd_driver.py | 137 ----- .../infx/tests/launch/test_srt_policy.py | 1 - .../infx/tests/matrix/test_validation.py | 12 - 22 files changed, 1 insertion(+), 2179 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/llm-d/Dockerfile delete mode 100644 inferencex-e2e/benchmarks/llm-d/README.md delete mode 100644 inferencex-e2e/benchmarks/llm-d/binaries.env delete mode 100644 inferencex-e2e/benchmarks/llm-d/envoy.yaml delete mode 100644 inferencex-e2e/benchmarks/llm-d/epp-config.yaml delete mode 100755 inferencex-e2e/benchmarks/llm-d/extract-binaries.sh delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d/README.md delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm delete mode 100755 inferencex-e2e/benchmarks/multi_node/llm-d/server.sh delete mode 100755 inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh delete mode 100644 inferencex-e2e/infx/launch/drivers/llmd.py delete mode 100644 inferencex-e2e/infx/tests/launch/test_llmd_driver.py diff --git a/inferencex-e2e/benchmarks/llm-d/Dockerfile b/inferencex-e2e/benchmarks/llm-d/Dockerfile deleted file mode 100644 index 2cd2bb6dff..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Combined image for the InferenceX llmd-vllm framework. -# -# Base = vllm/vllm-openai (vLLM with the OpenAI-compatible API server). -# We add the EPP, the routing-sidecar, and Envoy on top so every node in -# a SLURM allocation can play any role (prefill, decode, or coordinator) -# from a single image. -# -# Configs (epp-config.yaml, envoy.yaml, per-topology recipes) are NOT -# baked in. They are mounted at runtime by job.slurm so config-only -# iteration does not require an image rebuild. See -# benchmarks/multi_node/llm-d/job.slurm for the expected mount layout. - -# All FROM-referenced images are declared as global-scope ARGs. BuildKit -# does NOT expand a build-arg inside `COPY --from=${VAR}` directly ("variable -# expansion is not supported for --from"); the supported pattern is to bind -# each ARG to a named FROM stage in the global scope and COPY --from that -# stage alias. Defaults MUST match benchmarks/llm-d/binaries.env, which is the -# single source of truth (it also drives the mounted-binary path via -# extract-binaries.sh). Override at build time from that file, e.g.: -# source benchmarks/llm-d/binaries.env -# docker build \ -# --build-arg EPP_FROM_IMAGE="$EPP_FROM_IMAGE" \ -# --build-arg ROUTING_SIDECAR_IMAGE="$ROUTING_SIDECAR_IMAGE" \ -# --build-arg ENVOY_FROM_IMAGE="$ENVOY_FROM_IMAGE" ... -ARG VLLM_BASE=vllm/vllm-openai:v0.26.0 -ARG EPP_FROM_IMAGE=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0 -ARG ROUTING_SIDECAR_IMAGE=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0 -ARG ENVOY_FROM_IMAGE=envoyproxy/envoy:distroless-v1.33.2 - -# Stage aliases for the three add-on binary sources (see note above). -FROM ${EPP_FROM_IMAGE} AS epp_src -FROM ${ROUTING_SIDECAR_IMAGE} AS sidecar_src -FROM ${ENVOY_FROM_IMAGE} AS envoy_src - -FROM ${VLLM_BASE} - -COPY --from=epp_src \ - /app/epp /usr/local/bin/epp - -COPY --from=sidecar_src \ - /app/pd-sidecar /usr/local/bin/pd-sidecar - -COPY --from=envoy_src \ - /usr/local/bin/envoy /usr/local/bin/ diff --git a/inferencex-e2e/benchmarks/llm-d/README.md b/inferencex-e2e/benchmarks/llm-d/README.md deleted file mode 100644 index 027a18eee5..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# llmd-vllm framework artifacts - -This directory holds the static, baked-into-the-image pieces of the -`llmd-vllm` benchmark framework. - -| File | Purpose | -|---|---| -| `Dockerfile` | Combined image: vLLM (DeepEP-enabled), EPP, routing-sidecar, Envoy. One image, every node uses what its role requires. | -| `epp-config.yaml` | Fallback EPP scheduling config. Used when no recipe overrides it via `CONFIG_FILE`. `disagg-profile-handler` + `kv-cache-utilization-scorer` + `random-picker` over the file-discovery endpoint set. | -| `envoy.yaml` | Static Envoy: listener `:8080`, ext_proc to `127.0.0.1:9002`, ORIGINAL_DST cluster reading `x-gateway-destination-endpoint`. | - -The runtime pieces (per-node `server.sh`, the SLURM job script, recipe -files, and the endpoint discovery mechanism) live under -`benchmarks/multi_node/llm-d/` and `benchmarks/multi_node/llm-d-recipes/`. -See the README in `benchmarks/multi_node/llm-d/` for the endpoints-file -generation flow. diff --git a/inferencex-e2e/benchmarks/llm-d/binaries.env b/inferencex-e2e/benchmarks/llm-d/binaries.env deleted file mode 100644 index 4782f489b8..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/binaries.env +++ /dev/null @@ -1,32 +0,0 @@ -# Single source of truth for the non-vLLM executables the llmd-vllm -# framework needs (EPP, P/D sidecar, Envoy). -# -# This file is sourced by benchmarks/multi_node/llm-d/job.slurm to -# optionally bind-mount extracted binaries (via $LLMD_BIN_DIR) in place -# of the combined image's baked-in copies. extract-binaries.sh pulls -# each image once and copies the binary out to $LLMD_BIN_DIR; the -# Dockerfile reads the same URLs when building the combined image. -# -# By default this is a no-op: we build and use an image that already -# contains all binaries, so the job.slurm mount loop skips them (its -# -x check fails until extract-binaries.sh has populated $LLMD_BIN_DIR). - -# --- Endpoint Picker (EPP / inference scheduler) --- -EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" -EPP_BIN_PATH="/app/epp" - -# --- P/D routing sidecar (pd-sidecar) --- -ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0" -ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar" - -# --- Envoy front proxy --- -ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2" -ENVOY_BIN_PATH="/usr/local/bin/envoy" - -# Platform the binaries must target (GB200 = Grace = arm64). -LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/arm64}" - -# Shared-filesystem dir the extracted binaries live in, mounted into -# /usr/local/bin by job.slurm. Defaults to the same Lustre area the -# launcher already caches enroot squash images in (proven writable). -LLMD_BIN_DIR="${LLMD_BIN_DIR:-/mnt/lustre01/users-public/sa-shared/llm-d-bins}" diff --git a/inferencex-e2e/benchmarks/llm-d/envoy.yaml b/inferencex-e2e/benchmarks/llm-d/envoy.yaml deleted file mode 100644 index ccba51ba0b..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/envoy.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# Envoy front door for the llmd-vllm framework. -# -# Listener : 0.0.0.0:8080 (benchmark client target) -# ext_proc : EPP on 127.0.0.1:9002 -# Cluster : ORIGINAL_DST, picks the address from the -# x-gateway-destination-endpoint header that EPP sets. - -static_resources: - listeners: - - name: main - address: - socket_address: { address: 0.0.0.0, port_value: 8080 } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress_http - codec_type: AUTO - stream_idle_timeout: 0s - request_timeout: 0s - # Write per-request access logs to their own file so - # they survive an SIGTERM-on-failure: when envoy's - # stdout is captured via `> envoy.log`, glibc - # block-buffers it, and the buffer is lost when srun - # SIGTERMs envoy after a failed bench. FileAccessLog - # opens its own fd and flushes per record. - access_log: - - name: envoy.access_loggers.file - typed_config: - "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog - path: /benchmark_logs/envoy_access.log - log_format: - text_format_source: - inline_string: >- - "[ACCESS] %REQ(:METHOD)% %REQ(:PATH)% %PROTOCOL% - status=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% - upstream=%UPSTREAM_HOST% cluster=%UPSTREAM_CLUSTER% - dst=%REQ(x-gateway-destination-endpoint)% - duration=%DURATION%ms\n" - route_config: - name: route - virtual_hosts: - - name: vh - domains: ["*"] - routes: - - match: { prefix: "/" } - route: - cluster: original_dst - timeout: 0s - http_filters: - - name: envoy.filters.http.ext_proc - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor - grpc_service: - envoy_grpc: - cluster_name: epp - timeout: 10s - # message_timeout caps how long Envoy will wait for any - # one ext_proc message ack from EPP. Generation can take - # many seconds; 1000s mirrors the upstream llm-d guide. - message_timeout: 1000s - # FULL_DUPLEX_STREAMED for both directions: the dev EPP - # (ghcr.io/llm-d/llm-d-router-endpoint-picker-dev:main) - # does not ack BUFFERED body mode and Envoy times out - # with 504. Trailer modes also have to be SEND for the - # request lifecycle to terminate cleanly. - processing_mode: - request_header_mode: SEND - response_header_mode: SEND - request_body_mode: FULL_DUPLEX_STREAMED - response_body_mode: FULL_DUPLEX_STREAMED - request_trailer_mode: SEND - response_trailer_mode: SEND - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: epp - type: STATIC - connect_timeout: 1s - # Without an explicit circuit_breakers block Envoy applies its DEFAULT - # per-cluster thresholds: max_connections/max_pending_requests/max_requests - # all = 1024. The ext_proc filter holds one long-lived HTTP/2 stream to - # this EPP cluster for the ENTIRE lifetime of every request (header -> - # FULL_DUPLEX_STREAMED body -> response -> trailers, up to message_timeout - # 1000s). At conc4096 that means only 1024 active + 1024 pending = 2048 - # requests ever reach EPP; the rest get a fast 500 and never see the - # engine. This capped every high-tpt run at exactly 1024 completions - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 40000 - max_pending_requests: 40000 - max_requests: 40000 - max_retries: 1024 - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - load_assignment: - cluster_name: epp - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: { address: 127.0.0.1, port_value: 9002 } - - name: original_dst - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - connect_timeout: 5s - # Same default-1024 circuit-breaker trap as the epp cluster above: this is - # the cluster that actually carries request bodies to the picked - # prefill/decode endpoint, so its budget must also clear max-concurrency. - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 40000 - max_pending_requests: 40000 - max_requests: 40000 - max_retries: 1024 - original_dst_lb_config: - use_http_header: true - http_header_name: x-gateway-destination-endpoint - -admin: - address: - socket_address: { address: 0.0.0.0, port_value: 9901 } diff --git a/inferencex-e2e/benchmarks/llm-d/epp-config.yaml b/inferencex-e2e/benchmarks/llm-d/epp-config.yaml deleted file mode 100644 index 3ff0eea87f..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/epp-config.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Default EPP scheduling config (fallback when CONFIG_FILE is unset). -# -# Mirrors the upstream llm-d well-lit-path P/D guide: -# guides/pd-disaggregation/router/pd-disaggregation.values.yaml -# in github.com/llm-d/llm-d. Plugins, scheduling profiles, and scorer -# weights are unchanged from upstream. -# -# Single delta vs upstream: file-discovery. The upstream guide assumes -# a Kubernetes control plane drives endpoint discovery; in our SLURM -# setup the coordinator node writes /tmp/endpoints.yaml at job start -# (see benchmarks/multi_node/llm-d/README.md) and EPP loads it via the -# file-discovery plugin instead. - -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - # Endpoint discovery (replaces upstream's K8s discovery). - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - # P/D routing - identical to upstream pd-disaggregation guide. - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: active-request-scorer - - type: max-score-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: max-score-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: max-score-picker - -dataLayer: - discovery: - pluginRef: file-disc diff --git a/inferencex-e2e/benchmarks/llm-d/extract-binaries.sh b/inferencex-e2e/benchmarks/llm-d/extract-binaries.sh deleted file mode 100755 index bdb1968518..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/extract-binaries.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Extract the EPP, pd-sidecar, and Envoy binaries into $LLMD_BIN_DIR so a stock -# vllm/vllm-openai image can mount them at runtime (see job.slurm) instead of -# rebuilding a combined image on every vLLM bump. Needs docker (arm64, or x86 -# with --platform emulation); re-run only when a source image in binaries.env -# changes. Idempotent. -# Usage: [LLMD_BIN_DIR=dir] [LLMD_BIN_PLATFORM=linux/amd64] ./extract-binaries.sh - -set -eo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "$HERE/binaries.env" - -echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)" -mkdir -p "$LLMD_BIN_DIR" - -# (source image, path inside image, output filename) -extract() { - local image="$1" src="$2" out="$3" - echo " $out <- $image:$src" - local cid - cid="$(docker create --platform "$LLMD_BIN_PLATFORM" "$image")" - # shellcheck disable=SC2064 - trap "docker rm -f '$cid' >/dev/null 2>&1 || true" RETURN - docker cp "$cid:$src" "$LLMD_BIN_DIR/$out" - chmod +x "$LLMD_BIN_DIR/$out" -} - -extract "$EPP_FROM_IMAGE" "$EPP_BIN_PATH" epp -extract "$ROUTING_SIDECAR_IMAGE" "$ROUTING_SIDECAR_BIN_PATH" pd-sidecar -extract "$ENVOY_FROM_IMAGE" "$ENVOY_BIN_PATH" envoy - -echo "Done. Contents of $LLMD_BIN_DIR:" -ls -la "$LLMD_BIN_DIR" - -# epp/pd-sidecar are static Go binaries; envoy is dynamically linked C++. -echo -echo "NOTE: verify 'ldd $LLMD_BIN_DIR/envoy' resolves cleanly inside the" -echo " target vLLM image before relying on the mounted-binary path." diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml b/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml deleted file mode 100644 index e29106062d..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml +++ /dev/null @@ -1,192 +0,0 @@ -# DeepSeek-V4-Pro (FP4) on GB200, low-latency P/D disagg via llmd-vllm. -# -# Latency end of the curve. Engine config mirrors the upstream srt-slurm -# recipe disagg-gb200-low-latency.yaml (less the dynamo-only frontend keys), -# served via the InferenceX llmd-vllm path (no NATS/etcd infra node). It is -# the direct sibling of the dsv4-fp4-gb200-dynamo-vllm conc=1 low-latency -# point (same topology: 1 prefill DEP8 + 1 decode pure-TP8). -# -# Topology (set by the nvidia-master.yaml key via additional-settings): -# low-latency key (conc 1): 1P1D = 1 prefill + 1 decode engine, 4 GB200 -# nodes / 16 GPUs. PREFILL_NODES=2 DECODE_NODES=2. -# -# The ONLY difference from the mid-curve-megamoe recipe is the DECODE engine: -# 1. DECODE is PURE TP=8 (tensor-parallel-size 8, NO DP, NO EP) - ONE model -# replica across 8 GPUs / 2 GB200 nodes, launched headless multi-node TP -# (leader + follower). This is the latency-optimal decode: no all-to-all -# MoE routing, no DP-attention. `enable-expert-parallel: false` + -# decode ep:1/dp-attn:false in the master key is what selects the -# pure-TP path in server.sh. Decode knobs are latency-sized: max-num-seqs -# 256, max-num-batched-tokens 256, max-cudagraph-capture-size 256, -# FULL_DECODE_ONLY graphs, gpu-memory-utilization 0.9. -# 2. PREFILL is IDENTICAL to the mid-curve/high-tpt prefill (copied verbatim, -# DEP8 with enable-ep-weight-filter + moe-backend deep_gemm_mega_moe). An -# earlier srt-slurm-style low-latency prefill (weight-offload, no filter, -# moe-backend auto) warmed up ALL expert shapes (~4069-step DeepGEMM) and -# never became ready in the bring-up window; the mid-curve prefill's -# weight-filter collapses that warmup, and it is already proven on GB200. -# -# ---- EPP scheduling config (identical to the mid-curve recipe) ---- -# EPP routing is engine-config-agnostic, so the scheduling profiles match the -# upstream generic wide-ep-lws router config byte-for-byte: -# prefill = prefix-cache(3) + queue(2) + active-request(2) -# decode = active-request only -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: active-request-scorer - - type: weighted-random-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - -dataLayer: - discovery: - pluginRef: file-disc - -# ---- Per-role vLLM flags ---- -# Prefill: DEP=8 (TP=1, DP=8, EP). IDENTICAL to the mid-curve/high-tpt prefill -# block - copied verbatim (all args + env). enable-ep-weight-filter + -# moe-backend deep_gemm_mega_moe keep only the served experts resident and -# collapse the DeepGEMM warmup, so bring-up is fast. A pure srt-slurm-style -# low-latency prefill (weight-offload, no filter, moe-backend auto) instead -# warmed up ALL expert shapes (~4069-step DeepGEMM) and never became ready -# inside the bring-up window (e2e runs 28742781217 / 28744015942). Only the -# DECODE below differs from mid-curve: pure TP=8 instead of DEP=8. -prefill: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --enforce-eager - --gpu-memory-utilization 0.95 - --max-model-len 9280 - --max-num-seqs 16 - --max-num-batched-tokens 32768 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-async-scheduling - --block-size 256 - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - --no-enable-flashinfer-autotune - --numa-bind - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - # dynamo-vllm parity (2026-06-27): GPUDirect-RDMA over the GB200 - # Grace-Blackwell C2C link - lifts cross-node NCCL collective (DP8/EP8 - # MoE all-to-all) bandwidth. Dynamo sets it; we did not. - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # Added rc (IB verbs) so cross-node KV falls back to InfiniBand, not - # TCP, when MNNVL/cuda_ipc is unavailable. Prior value lacked rc, so - # the fallback was tcp (slow). Primary path is still MNNVL via - # cuda_ipc + cumem allocator. - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - # dynamo-vllm parity (2026-06-27): no NVSHMEM remote transport (we don't - # use DeepEP), plus explicit NIC->PE mapping. server.sh defaults - # NVSHMEM_REMOTE_TRANSPORT to ibgda but honors this override and clears - # the image's NVSHMEM_HCA_LIST so this PE mapping takes effect. - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" - # Harden distributed coordination against transient gloo/NCCL resets under - # load (matters for the low-middle 1P4D point; harmless at conc=1). Matches - # the DEP8 recipe + srt-slurm (TORCH_DISTRIBUTED_DEFAULT_TIMEOUT 1800). - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# Decode: PURE TP=8 (one replica across 8 GPUs / 2 nodes). NO EP, NO DP. -# Mirrors the srt-slurm low-latency decode vllm_config. tensor-parallel-size -# is supplied by server.sh from the master key (tp:8); the recipe sets EP off. -decode: - tp: 8 - enable-expert-parallel: false - extra-args: >- - --kv-cache-dtype fp8 - --max-model-len 16384 - --max-num-seqs 256 - --max-num-batched-tokens 256 - --max-cudagraph-capture-size 256 - --gpu-memory-utilization 0.9 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-enable-flashinfer-autotune - --block-size 256 - --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --tokenizer-mode deepseek_v4 - --no-disable-hybrid-kv-cache-manager - --enable-sleep-mode - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # rc added so KV falls back to InfiniBand, not TCP (see prefill note). - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - VLLM_USE_NCCL_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# ---- SLURM resource directives ---- -slurm: - time_limit: "08:00:00" diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml b/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml deleted file mode 100644 index 89292bc09c..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml +++ /dev/null @@ -1,196 +0,0 @@ -# DeepSeek-V4-Pro (FP4) on GB200, MegaMOE P/D disagg via llmd-vllm. -# -# This file defines only the COMMON per-engine shape + engine env. The -# multi-engine topology (how many prefill/decode engines, node counts, -# concurrency) is set by the nvidia-master.yaml key via additional-settings, -# NOT here - so the same recipe backs BOTH supported topologies: -# mid-curve key (conc 256/512/1024): 1P1D = 1 prefill + 1 decode engine, -# 4 GB200 nodes / 16 GPUs. PREFILL_NODES=2 DECODE_NODES=2. -# high-tpt key (conc 4096): 2P1D = 2 prefill + 1 decode engine, -# 6 GB200 nodes / 24 GPUs. PREFILL_NODES=4 PREFILL_WORKERS=2 DECODE_NODES=2. -# -# Per-engine shape (identical in every engine; GB200 = 4 GPUs/node): -# Each engine is DEP8 - TP=1 DP=8 EP=8, spanning 2 nodes (LWS_GROUP_SIZE=2), -# with DP-attention + deep_gemm_mega_moe + EP weight filter. Decode engines -# additionally use FULL_DECODE_ONLY graphs. -# -# Selected via additional-settings: CONFIG_FILE=, GPUS_PER_NODE=4, -# plus the PREFILL_NODES/PREFILL_WORKERS/DECODE_NODES for the chosen topology. -# -# -# ---- EPP scheduling config (matched to upstream wide-ep-lws) ---- -# Scorers match the upstream generic wide-ep-lws router config -# (llm-d guides/wide-ep-lws/router/wide-ep-lws.values.yaml): -# prefill = prefix-cache(3) + queue(2) + active-request(2) -# decode = active-request only -# EPP routing is engine-config-agnostic, so this recipe file also backs the -# high-tpt (2P1D conc4096) key. -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: active-request-scorer - - type: weighted-random-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - -dataLayer: - discovery: - pluginRef: file-disc - -# ---- Per-role vLLM flags ---- -# Engine config mirrors the upstream srt-slurm mid-curve-megamoe recipe -# byte-for-byte (less the dynamo-only frontend keys). The MegaMOE shape -# differs from the low-latency 1P+1D in three ways: -# 1. Decode is DEP=8 (TP=1, DP=8, EP) instead of pure TP=8. This is -# what the "MegaMOE" name refers to: every MoE rank runs experts -# locally with DP-attention to hide cross-rank latency, instead of -# sharding tensors. -# 2. moe-backend = deep_gemm_mega_moe on both sides (the higher-perf -# path; the simple recipe doesn't override moe-backend). -# 3. enable-ep-weight-filter on both sides keeps only the experts a -# rank actually serves resident on GPU; without it, DEP=8 OOMs -# because every rank tries to hold all experts. -prefill: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --enforce-eager - --gpu-memory-utilization 0.92 - --max-model-len 9280 - --max-num-seqs 16 - --max-num-batched-tokens 32768 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-async-scheduling - --block-size 256 - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - --no-enable-flashinfer-autotune - --numa-bind - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - # dynamo-vllm parity (2026-06-27): GPUDirect-RDMA over the GB200 - # Grace-Blackwell C2C link - lifts cross-node NCCL collective (DP8/EP8 - # MoE all-to-all) bandwidth. Dynamo sets it; we did not. - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # Added rc (IB verbs) so cross-node KV falls back to InfiniBand, not - # TCP, when MNNVL/cuda_ipc is unavailable. Prior value lacked rc, so - # the fallback was tcp (slow). Primary path is still MNNVL via - # cuda_ipc + cumem allocator. - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - # dynamo-vllm parity (2026-06-27): no NVSHMEM remote transport (we don't - # use DeepEP), plus explicit NIC->PE mapping. server.sh defaults - # NVSHMEM_REMOTE_TRANSPORT to ibgda but honors this override and clears - # the image's NVSHMEM_HCA_LIST so this PE mapping takes effect. - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" - # Harden distributed coordination against transient gloo/NCCL connection - # resets under the conc4096 warmup burst. max-tpt in-sweep died with - # "gloo tcp Connection reset by peer" -> EngineDeadError (no OOM/CUDA/HW - # fault); a longer PG timeout keeps a momentarily-stalled peer from being - # dropped. Value matches srt-slurm (TORCH_DISTRIBUTED_DEFAULT_TIMEOUT 1800). - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -decode: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --max-num-seqs 512 - --max-num-batched-tokens 512 - --max-cudagraph-capture-size 512 - --gpu-memory-utilization 0.9 - --max-model-len 9280 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-enable-flashinfer-autotune - --block-size 256 - --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # rc added so KV falls back to InfiniBand, not TCP (see prefill note). - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - # See prefill note: harden the decode's cross-node DP8/EP8 coordination - # against transient gloo resets under the conc4096 warmup burst. - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# ---- SLURM resource directives ---- -slurm: - time_limit: "08:00:00" diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/README.md b/inferencex-e2e/benchmarks/multi_node/llm-d/README.md deleted file mode 100644 index 55c0575fec..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/README.md +++ /dev/null @@ -1,142 +0,0 @@ -# llmd-vllm multi-node SLURM scaffolding - -This directory holds standalone SLURM-side orchestration for the `llmd-vllm` -benchmark framework. InferenceX itself owns the SLURM job, with no vendor -multi-node tool involved. This scaffolding is separate from the active -srt-slurm launch path. - -| File | Role | -|---|---| -| `submit.sh` | sbatch wrapper. Validates env, exports tuning vars, returns `JOB_ID`. May read `slurm.time_limit` from the recipe to override `TIME_LIMIT`. | -| `job.slurm` | sbatch entrypoint. Allocates `PREFILL_NODES + DECODE_NODES` nodes, derives per-node IPs, runs one Docker container per node via `srun`, threads role assignment env into each. | -| `server.sh` | Per-node entry. Reads `NODE_RANK = SLURM_PROCID`, picks role, starts vLLM (with the wide-EP / DeepEP / NIXL flag set from the llm-d wide-EP-lws guide), starts the pd-sidecar on each leader, and on the decode leader additionally writes `endpoints.yaml`, starts EPP + Envoy, runs `python3 -m infx.bench fixed-seq point` per concurrency (then `python3 -m infx.bench eval` when `RUN_EVAL=true`), and signals `job.slurm` to end the job. | - -## Topology - -For an `xP` prefill nodes / `yD` decode nodes run, total nodes = `xP + yD`. -There is **no dedicated coordinator node**. The decode leader doubles as -the coordinator (EPP + Envoy + bench). - -| Rank | Role | -|---|---| -| `0` | prefill leader (`LWS_WORKER_INDEX=0`, DP rank 0) + pd-sidecar | -| `1 .. xP-1` | prefill workers | -| `xP` | decode leader + pd-sidecar + EPP + Envoy + benchmark client | -| `xP+1 .. xP+yD-1` | decode workers | - -Each instance (prefill or decode) is one vLLM engine spanning multiple -nodes via `--data-parallel-hybrid-lb`. With `xP=2, yD=2, -GPUS_PER_NODE=8` you get DP=16 prefill + DP=16 decode (the wide-EP -reference). Per-rank split: `--data-parallel-size 16 ---data-parallel-size-local 8 --data-parallel-start-rank -$((LWS_WORKER_INDEX * 8))`. - -## How `endpoints.yaml` is generated (file-discovery contract) - -The EPP runs in **no-Kubernetes mode**, using the `file-discovery` plugin -from `llm-d-inference-scheduler` (branch `filediscovery-4`). At startup, -it reads `/tmp/endpoints.yaml`. The file lists every backend the EPP can -route to, with role labels. - -The file is generated at runtime by `server.sh` on the decode leader -(rank `PREFILL_NODES`). Because all node IPs are only known after -`sbatch` allocates the job, the file cannot be baked into the image and -is not part of the repo. - -Generation flow: - -1. `submit.sh` calls `sbatch -N (xP+yD)`. `sbatch` allocates nodes. -2. `job.slurm` resolves each node's IP via `srun ip route get 1.1.1.1`, - slices them into `PREFILL_LEADER_IP` (= IPS[0]) and `DECODE_LEADER_IP` - (= IPS[PREFILL_NODES]), and passes both into the container as env - vars. -3. On the decode leader, `server.sh` writes `/tmp/endpoints.yaml` - inside the container with one entry per node: - - ```yaml - endpoints: - - name: prefill-0 - address: - port: "8200" # vLLM port (EPP hits prefill vLLM directly) - labels: - llm-d.ai/role: prefill - - name: prefill-1 # one entry per prefill node - address: - port: "8200" - labels: - llm-d.ai/role: prefill - - name: decode-0 - address: - port: "8000" # pd-sidecar port - labels: - llm-d.ai/role: decode - - name: decode-1 # one entry per decode node - address: - port: "8000" - labels: - llm-d.ai/role: decode - ``` - -4. The EPP (started immediately after) loads the file via - `dataLayer.discovery.pluginRef: file-disc` (see - `benchmarks/llm-d/epp-config.yaml`). The plugin enumerates the - endpoints into the EPP datastore before the EPP starts serving - `ext_proc`, so Envoy never gets a request before discovery is ready. -5. The `disagg-profile-handler` in the EPP config uses `prefill-filter` - and `decode-filter` to pick the right backend per request phase, - matching on the `llm-d.ai/role` label. - -### Why one entry per node (not per DP rank) - -Each instance is a vLLM engine that spans multiple nodes via -`--data-parallel-hybrid-lb`. With hybrid-lb, every node runs its own -api-server (prefill) or pd-sidecar (decode) on a fixed port -(`VLLM_PORT`=8200 / `SIDECAR_PORT`=8000) and internally load-balances -its own local DP ranks. So `server.sh` `add_role()` emits one endpoint -per node (`prefill-0..N-1`, `decode-0..M-1`) and lets EPP fan out -across nodes, while each node's hybrid-lb spreads work across its local -ranks. - -Per-node (rather than per-DP-rank) emission is what the multi-engine -high-tpt topology needs: with `PREFILL_WORKERS>1` the prefill nodes form -several independent DEP engines, and one endpoint per node is what -exposes every engine to EPP. Emitting per-DP-rank instead would address -ranks directly and bypass each node's internal hybrid-lb. - -### Live reload - -`watchFile: false` in `epp-config.yaml`. Endpoints are static for the -job lifetime - no reason to pay for `fsnotify` here. Set `watchFile: -true` (and rewrite `/tmp/endpoints.yaml` from the coordinator) only if -you want to drain or add an instance mid-run. - -### Validation rules (enforced by the plugin) - -- `address` must be a literal IPv4 address (no IPv6, no hostnames). -- `port` is a string in `1..65535`. -- File capped at 1 MiB. -- Names must be unique within their namespace (we use the default - namespace, so they must be globally unique in the file). - -The IPs we collect from `ip route get 1.1.1.1` are always IPv4 on the -H200 / B200 cluster's primary fabric. If you point at a different -interface and it returns an IPv6 address, EPP will reject the file at -startup. - -## Recipe files - -`benchmarks/multi_node/llm-d-recipes/.yaml` is selected via -`CONFIG_FILE=.yaml` in the master config's `additional-settings`. -Each recipe carries: - -- top-level `plugins:` / `schedulingProfiles:` / `dataLayer:` - fed into - the EPP via `--config-file`. Lets you change routing strategy without - rebuilding the image. -- `prefill:` / `decode:` blocks with `extra-args` (appended to the vLLM - launch command on each node of that role) and `env` (exported before - vLLM starts). -- `slurm.time_limit` - overrides `TIME_LIMIT` for that recipe. - -When `CONFIG_FILE` is unset or the file is missing, the EPP falls back -to `/etc/epp/config.yaml` baked into the image, and vLLM runs with no -extra flags beyond the wide-EP common set in `server.sh`. diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm b/inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm deleted file mode 100644 index 65e9dde4e0..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=llm-d-bench -#SBATCH --ntasks-per-node=1 -# --output, --error, -N, -n, --time set by submit.sh -# -# Allocates PREFILL_NODES + DECODE_NODES nodes, derives per-node IPs, then -# srun-runs server.sh inside one Docker container per node. NODE_RANK -# (= SLURM_PROCID) drives role selection inside server.sh. - -set -eo pipefail - -echo "=== llm-d job start ===" -echo "UTC: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" - -# Repo root. $(pwd) = the sbatch submit dir, which the wrapper sets to -# benchmarks/multi_node/llm-d/ before invoking submit.sh, so 3 up = -# repo root. Using $(dirname "$0") would resolve to a SLURM staging -# copy under /var/spool/... and miss the checkout entirely. -DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) -export DI_REPO_DIR -echo "REPO DIR: ${DI_REPO_DIR}" - -ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) -echo "Allocated nodes ($TOTAL_NODES): $(echo "$ALL_NODES" | tr '\n' ' ')" - -if [[ "$TOTAL_NODES" -ne "$NUM_NODES" ]]; then - echo "Error: SLURM allocated $TOTAL_NODES nodes, expected $NUM_NODES" >&2 - exit 1 -fi - -# Per-node IPs in rank order. -IPS=() -for NODE in $ALL_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --nodelist="$NODE" \ - bash -c 'ip route get 1.1.1.1 | awk "/src/ {print \$7}"') - IPS+=("$IP") -done -echo "Node IPs: ${IPS[*]}" - -if ! srun --nodes="$NUM_NODES" --ntasks-per-node=1 test -d "$MODEL_DIR"; then - echo "FATAL: model not found on all $NUM_NODES nodes: $MODEL_DIR" >&2 - exit 1 -fi - -# Rank slicing: -# prefill leader = rank 0 -# prefill workers = ranks 1 .. PREFILL_NODES-1 -# decode leader = rank PREFILL_NODES (also coordinator: EPP + Envoy + bench) -# decode workers = ranks PREFILL_NODES+1 .. NUM_NODES-1 -PREFILL_LEADER_IP="${IPS[0]}" -DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" - -# DP leader addresses for vLLM --data-parallel-address (rank 0 of each instance). -PREFILL_DP_ADDR="$PREFILL_LEADER_IP" -DECODE_DP_ADDR="$DECODE_LEADER_IP" - -ALL_IP_LIST=$(IFS=,; echo "${IPS[*]}") - -SANITIZED_USER=$(echo "$USER" | tr -c 'a-zA-Z0-9_.-' '_') -DOCKER_CONT_NAME="llmd_bench_${SANITIZED_USER}_${SLURM_JOB_ID}" -export DOCKER_CONT_NAME -: "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" -DOCKER_MOUNT_PATH="/workspace" - -# Stop the step and exit 0 once the coordinator is done, so the job ends COMPLETED, not CANCELLED. -BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" -rm -f "$BENCH_DONE_MARKER" - -run_until_bench_done() { - "$@" & - local step=$! rc=0 - while kill -0 "$step" 2>/dev/null; do - if [[ -f "$BENCH_DONE_MARKER" ]]; then - echo "[${SLURM_JOB_ID}] coordinator finished; stopping the srun step" - kill -TERM "$step" 2>/dev/null || true - break - fi - sleep 5 - done - wait "$step" || rc=$? - [[ -f "$BENCH_DONE_MARKER" ]] && return 0 - echo "[${SLURM_JOB_ID}] srun step exited rc=$rc before the coordinator finished" >&2 - return "$rc" -} - -# Container engine: 'docker' (default) for clusters where the SLURM -# user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' -# for clusters that require enroot+pyxis srun (e.g. gb200-nv, where the -# SLURM user is not in the docker group). Same env contract on both -# paths; only the per-node container launch differs. -source "$DI_REPO_DIR/benchmarks/check_env.sh" -check_env_vars \ - LLMD_CONTAINER_ENGINE VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS \ - NVSHMEM_REMOTE_TRANSPORT NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE LLMD_API_SERVER_COUNT -echo "LLMD_CONTAINER_ENGINE=$LLMD_CONTAINER_ENGINE" - -check_env_vars INFERENCEX_RUNTIME_ENV_VARS -RUNTIME_DOCKER_ENV=() -for setting in $INFERENCEX_RUNTIME_ENV_VARS; do - RUNTIME_DOCKER_ENV+=(-e "$setting") -done - -if [[ "$LLMD_CONTAINER_ENGINE" == "docker" ]]; then - # One docker run per node, one task per node. server.sh dispatches by NODE_RANK. - run_until_bench_done srun \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - bash -lc " -set -eo pipefail -echo \"Rank \$SLURM_PROCID on \$(hostname)\" - -docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f || true - -exec docker run --rm \ - --init \ - --entrypoint bash \ - --stop-timeout 10 \ - --user 0:0 \ - --network host \ - --ipc host \ - --gpus all \ - --ulimit memlock=-1 --ulimit stack=67108864 \ - --shm-size 32G \ - --cap-add SYS_PTRACE --cap-add IPC_LOCK --cap-add SYS_RAWIO \ - --device /dev/infiniband \ - --security-opt seccomp=unconfined \ - --privileged \ - -v ${MODEL_DIR}:/models:ro \ - -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ - -v ${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro \ - -v ${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro \ - -v ${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro \ - -e SLURM_JOB_ID=\$SLURM_JOB_ID \ - -e NODE_RANK=\$SLURM_PROCID \ - -e NUM_NODES=$NUM_NODES \ - -e PREFILL_NODES=$PREFILL_NODES \ - -e DECODE_NODES=$DECODE_NODES \ - -e PREFILL_WORKERS=$PREFILL_WORKERS \ - -e DECODE_WORKERS=$DECODE_WORKERS \ - -e ALL_IPS=$ALL_IP_LIST \ - -e PREFILL_LEADER_IP=$PREFILL_LEADER_IP \ - -e DECODE_LEADER_IP=$DECODE_LEADER_IP \ - -e PREFILL_DP_ADDR=$PREFILL_DP_ADDR \ - -e DECODE_DP_ADDR=$DECODE_DP_ADDR \ - -e MODEL_DIR=/models \ - -e MODEL_NAME=$MODEL_NAME \ - -e GPUS_PER_NODE=$GPUS_PER_NODE \ - -e PREFILL_DP_SIZE=$PREFILL_DP_SIZE \ - -e DECODE_DP_SIZE=$DECODE_DP_SIZE \ - -e BENCH_INPUT_LEN=$BENCH_INPUT_LEN \ - -e BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN \ - -e BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY \ - -e BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE \ - -e BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO \ - -e BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER \ - -e BENCHMARK_LOGS_DIR=/benchmark_logs \ - -e RUN_EVAL=$RUN_EVAL \ - -e EVAL_ONLY=$EVAL_ONLY \ - -e EVAL_CONC=$EVAL_CONC \ - -e EVAL_FRAMEWORK=$EVAL_FRAMEWORK \ - -e EVAL_LIMIT=$EVAL_LIMIT \ - -e EVAL_SUITE=$EVAL_SUITE \ - -e IS_AGENTIC=$IS_AGENTIC \ - -e SCENARIO_TYPE=$SCENARIO_TYPE \ - -e FRAMEWORK=$FRAMEWORK \ - -e PRECISION=$PRECISION \ - -e MODEL_PREFIX=$MODEL_PREFIX \ - -e RUNNER_TYPE=$RUNNER_TYPE \ - -e RESULT_FILENAME=$RESULT_FILENAME \ - -e SPEC_DECODING=$SPEC_DECODING \ - -e IS_MULTINODE=$IS_MULTINODE \ - -e CONFIG_FILE=$CONFIG_FILE \ - ${RUNTIME_DOCKER_ENV[*]} \ - -e VLLM_RANDOMIZE_DP_DUMMY_INPUTS=$VLLM_RANDOMIZE_DP_DUMMY_INPUTS \ - -e VLLM_LOGGING_LEVEL=$VLLM_LOGGING_LEVEL \ - -e UCX_TLS=$UCX_TLS \ - -e NVSHMEM_REMOTE_TRANSPORT=$NVSHMEM_REMOTE_TRANSPORT \ - -e NVSHMEM_IB_ENABLE_IBGDA=$NVSHMEM_IB_ENABLE_IBGDA \ - -e NVSHMEM_SYMMETRIC_SIZE=$NVSHMEM_SYMMETRIC_SIZE \ - -e LLMD_API_SERVER_COUNT=$LLMD_API_SERVER_COUNT \ - --name \"${DOCKER_CONT_NAME}_\$SLURM_PROCID\" \ - \"\$DOCKER_IMAGE_NAME\" -lc ' - set -o pipefail - ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ - 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log - ' -" - - srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true - -elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then - : "${LLMD_SQUASH_FILE:?LLMD_SQUASH_FILE must be set when LLMD_CONTAINER_ENGINE=pyxis}" - if [[ ! -s "$LLMD_SQUASH_FILE" ]]; then - echo "Error: LLMD_SQUASH_FILE does not exist or is empty: $LLMD_SQUASH_FILE" >&2 - exit 1 - fi - - # Pre-export non-SLURM static vars so pyxis --container-env= - # whitelisting can pick them up. Vars that already come from submit.sh - # via export are no-ops here; we re-state them for clarity. ALL_IPS - # is computed locally above as $ALL_IP_LIST. - export ALL_IPS="$ALL_IP_LIST" - export NUM_NODES PREFILL_NODES DECODE_NODES PREFILL_LEADER_IP DECODE_LEADER_IP - export PREFILL_WORKERS DECODE_WORKERS - export PREFILL_DP_ADDR DECODE_DP_ADDR MODEL_NAME GPUS_PER_NODE - export PREFILL_DP_SIZE DECODE_DP_SIZE - export BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY - export BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER - export RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE - export IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX - export RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE - - export VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS NVSHMEM_REMOTE_TRANSPORT NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE LLMD_API_SERVER_COUNT - PYXIS_ENV_LIST="NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP" - PYXIS_ENV_LIST+=",PREFILL_WORKERS,DECODE_WORKERS" - PYXIS_ENV_LIST+=",PREFILL_DP_ADDR,DECODE_DP_ADDR,MODEL_NAME,GPUS_PER_NODE" - PYXIS_ENV_LIST+=",PREFILL_DP_SIZE,DECODE_DP_SIZE" - PYXIS_ENV_LIST+=",BENCH_INPUT_LEN,BENCH_OUTPUT_LEN,BENCH_MAX_CONCURRENCY" - PYXIS_ENV_LIST+=",BENCH_REQUEST_RATE,BENCH_RANDOM_RANGE_RATIO,BENCH_NUM_PROMPTS_MULTIPLIER" - PYXIS_ENV_LIST+=",RUN_EVAL,EVAL_ONLY,EVAL_CONC,EVAL_FRAMEWORK,EVAL_LIMIT,EVAL_SUITE" - PYXIS_ENV_LIST+=",IS_AGENTIC,SCENARIO_TYPE,FRAMEWORK,PRECISION,MODEL_PREFIX" - PYXIS_ENV_LIST+=",RUNNER_TYPE,RESULT_FILENAME,SPEC_DECODING,IS_MULTINODE,CONFIG_FILE" - - PYXIS_ENV_LIST+=",VLLM_RANDOMIZE_DP_DUMMY_INPUTS,VLLM_ENGINE_READY_TIMEOUT_S,VLLM_LOGGING_LEVEL,UCX_TLS,NVSHMEM_REMOTE_TRANSPORT,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_SYMMETRIC_SIZE,LLMD_API_SERVER_COUNT" - - PYXIS_ENV_LIST+=",${INFERENCEX_RUNTIME_ENV_VARS// /,}" - - PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" - PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" - PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro" - - # Optional: mount the epp / pd-sidecar / envoy binaries from a shared - # filesystem instead of relying on them being baked into the image. - # This lets a STOCK vllm/vllm-openai image be used directly (no - # combined-image rebuild per vLLM version bump) - see - # benchmarks/llm-d/binaries.env + extract-binaries.sh. Each mount is - # gated on the file existing, so this is a no-op when the binaries - # have not been extracted (the baked-image path keeps working), and - # harmless when they have (mounting a binary over the identical one). - # shellcheck source=/dev/null - [[ -f "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" ]] && \ - source "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" - for _bin in epp pd-sidecar envoy; do - if [[ -n "${LLMD_BIN_DIR:-}" && -x "${LLMD_BIN_DIR}/${_bin}" ]]; then - PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" - echo "Mounting ${LLMD_BIN_DIR}/${_bin} -> /usr/local/bin/${_bin}" - fi - done - - # MODEL_DIR / BENCHMARK_LOGS_DIR / NODE_RANK are translated to their - # in-container values inside bash -lc (host MODEL_DIR is the source - # path of the bind mount, but server.sh expects /models inside). - run_until_bench_done srun \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - --container-image="$LLMD_SQUASH_FILE" \ - --container-name="${DOCKER_CONT_NAME}" \ - --container-mounts="$PYXIS_MOUNTS" \ - --container-remap-root \ - --container-writable \ - --container-env="$PYXIS_ENV_LIST" \ - bash -lc ' -set -o pipefail -echo "Rank $SLURM_PROCID on $(hostname)" -export NODE_RANK="$SLURM_PROCID" -export MODEL_DIR=/models -export BENCHMARK_LOGS_DIR=/benchmark_logs -'"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ - 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' - -else - echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 - exit 1 -fi diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/server.sh b/inferencex-e2e/benchmarks/multi_node/llm-d/server.sh deleted file mode 100755 index 08c70f2140..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/server.sh +++ /dev/null @@ -1,573 +0,0 @@ -#!/usr/bin/env bash -# -# Per-node entrypoint for the llmd-vllm wide-EP P/D disagg benchmark. -# NODE_RANK is set by srun (= $SLURM_PROCID) in job.slurm. -# -# Roles: -# Rank 0 -> prefill leader (DP rank 0) -# Ranks 1 .. PREFILL_NODES-1 -> prefill workers -# Rank PREFILL_NODES -> decode leader (DP rank 0) + pd-sidecar -# + EPP + Envoy + benchmark client (coordinator) -# Ranks PREFILL_NODES+1 .. -> decode workers -# -# Each instance (prefill or decode) is one vLLM engine spanning its role's nodes -# via --data-parallel-hybrid-lb; the leader accepts traffic, workers serve their -# local DP ranks. - -set -eo pipefail - -source /workspace/benchmarks/check_env.sh -# Root in this container must not leave __pycache__ in the /workspace mount. -export PYTHONDONTWRITEBYTECODE=1 - -check_env_vars \ - NODE_RANK PREFILL_NODES DECODE_NODES GPUS_PER_NODE PREFILL_WORKERS \ - DECODE_WORKERS EVAL_ONLY RUN_EVAL MODEL_DIR MODEL_NAME -VLLM_PORT=8200 -SIDECAR_PORT=8000 -ENVOY_PORT=8080 -EPP_GRPC_PORT=9002 -EPP_HEALTH_PORT=9003 -EPP_METRICS_PORT=9090 - -# Weights live at MODEL_DIR (/models, bind-mounted by job.slurm); the eval reads them as MODEL. -# MODEL_NAME is the served-model-name, not a filesystem path. -export MODEL="${MODEL_DIR}" - -# Container-side benchmark commands run from the /workspace checkout. -infx_bench() { - PYTHONSAFEPATH=1 PYTHONPATH="/workspace${PYTHONPATH:+:$PYTHONPATH}" python3 -m infx.bench "$@" -} - -# Resolved without iproute2 (`ip` is absent on the arm64 vLLM base); python3's -# socket layer exposes the kernel's source-IP / iface choice. -_HOST_INFO=$(python3 -c ' -import socket -s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -try: - s.connect(("1.1.1.1", 80)) - ip = s.getsockname()[0] -finally: - s.close() -iface = "" -try: - with open("/proc/net/route") as f: - f.readline() # header - for line in f: - parts = line.split() - if parts[1] == "00000000": # default route dest - iface = parts[0]; break -except OSError: - pass -print(ip, iface) -' 2>/dev/null) || true -HOST_IP=$(echo "$_HOST_INFO" | awk '{print $1}') -DEFAULT_IFACE=$(echo "$_HOST_INFO" | awk '{print $2}') -DEFAULT_IFACE="${DEFAULT_IFACE:-eth0}" - -VLLM_LOG="/benchmark_logs/vllm_rank${NODE_RANK}.log" -SIDECAR_LOG="/benchmark_logs/sidecar_rank${NODE_RANK}.log" -EPP_LOG="/benchmark_logs/epp.log" -ENVOY_LOG="/benchmark_logs/envoy.log" - -echo "=== rank=$NODE_RANK host=$HOST_IP model=$MODEL ===" - -# A role's nodes split into PREFILL_WORKERS / DECODE_WORKERS independent DP/EP -# engines, each spanning (role_nodes / role_workers) nodes with its own DP -# coordinator (leader IP) and rank range. workers=1 => one engine over all role -# nodes (1P+1D / mid-curve); >1 => high-tpt (e.g. 2 prefill : 1 decode, DEP8 each). -IFS=',' read -r -a _ALL_IPS <<< "${ALL_IPS:-}" - -if [[ "$NODE_RANK" -lt "$PREFILL_NODES" ]]; then - ROLE="prefill" - DP_SIZE="$PREFILL_DP_SIZE" - _local_rank="$NODE_RANK" - _nodes_per_worker=$(( PREFILL_NODES / PREFILL_WORKERS )) - LWS_WORKER_INDEX=$(( _local_rank % _nodes_per_worker )) - LWS_GROUP_SIZE="$_nodes_per_worker" - _group_leader_rank=$(( (_local_rank / _nodes_per_worker) * _nodes_per_worker )) -elif [[ "$NODE_RANK" -lt $((PREFILL_NODES + DECODE_NODES)) ]]; then - ROLE="decode" - DP_SIZE="$DECODE_DP_SIZE" - _local_rank=$(( NODE_RANK - PREFILL_NODES )) - _nodes_per_worker=$(( DECODE_NODES / DECODE_WORKERS )) - LWS_WORKER_INDEX=$(( _local_rank % _nodes_per_worker )) - LWS_GROUP_SIZE="$_nodes_per_worker" - _group_leader_rank=$(( PREFILL_NODES + (_local_rank / _nodes_per_worker) * _nodes_per_worker )) -else - echo "ERROR: NODE_RANK=$NODE_RANK out of range" >&2 - exit 1 -fi - -# Each engine's DP coordinator = its leader node's IP (ALL_IPS[leader rank]); -# fall back to the role leader env when ALL_IPS is unset. -if [[ -n "${_ALL_IPS[${_group_leader_rank}]:-}" ]]; then - DP_ADDR="${_ALL_IPS[${_group_leader_rank}]}" -elif [[ "$ROLE" == "prefill" ]]; then - DP_ADDR="$PREFILL_DP_ADDR" -else - DP_ADDR="$DECODE_DP_ADDR" -fi - -DP_SIZE_LOCAL="$GPUS_PER_NODE" -START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) - -# Defaults: TP=1, DP=role_total, EP on (the H200 1P+1D shape). Recipe overrides below. -TP_SIZE=1 -ROLE_ENABLE_EP=true - -echo "ROLE=$ROLE DP_SIZE=$DP_SIZE DP_ADDR=$DP_ADDR LWS_WORKER_INDEX=$LWS_WORKER_INDEX START_RANK=$START_RANK" - -# Per-role keys: tp (int -> --tensor-parallel-size), enable-expert-parallel -# (bool -> --enable-expert-parallel + DP/wide-EP knobs), extra-args (appended -# verbatim), env (map, exported before vllm serve). Absent keys keep the -# defaults above, so a recipe with neither tp nor EP is a plain TP=1 DP+EP run. -ROLE_EXTRA_ARGS="" -if [[ -n "${CONFIG_FILE:-}" ]]; then - RECIPE_PATH="/etc/llmd-recipes/${CONFIG_FILE}" - if [[ -f "$RECIPE_PATH" ]]; then - echo "Loading $ROLE recipe from $RECIPE_PATH" - eval "$(python3 - <&2 - fi -fi -echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" - -export GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-$DEFAULT_IFACE} -export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-$DEFAULT_IFACE} -check_env_vars \ - VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS NVSHMEM_REMOTE_TRANSPORT \ - NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE LLMD_API_SERVER_COUNT -export VLLM_SKIP_P2P_CHECK=1 -# Randomized DP dummy inputs make idle DP ranks fan their lockstep dummy passes -# across all experts (full MoE all-to-all), wasting prefill bandwidth; a recipe -# may set this to 0. -export VLLM_RANDOMIZE_DP_DUMMY_INPUTS -export VLLM_USE_DEEP_GEMM=1 -# Cold-start budget for engine-core readiness. DSV4-Pro on GB200 cold-starts in -# ~9-11 min (weight load + DeepGEMM JIT warmup + cudagraph capture + NIXL/UCX -# handshake); the 600s vLLM default is too tight, so allow 30 min. -export VLLM_ENGINE_READY_TIMEOUT_S -# DeepGEMM JIT links -l:libcuda.so.1 at warmup; the compat dir is on -# LD_LIBRARY_PATH (runtime) but not LIBRARY_PATH (link time). Prepend it, plus -# the arch-specific toolkit lib dir resolved from `uname -m`. -case "$(uname -m)" in - aarch64|arm64) _NCT_LIB=/usr/lib/aarch64-linux-gnu ;; - *) _NCT_LIB=/usr/lib/x86_64-linux-gnu ;; -esac -export LIBRARY_PATH=/usr/local/cuda/compat:${_NCT_LIB}:${LIBRARY_PATH:-} -export VLLM_NIXL_SIDE_CHANNEL_HOST="$HOST_IP" -export VLLM_LOGGING_LEVEL - -# Pin NIXL/UCX to IB verbs (rc) so cross-node KV rides the IB HCAs (job.slurm -# exposes /dev/infiniband + IPC_LOCK); cuda_copy/cuda_ipc cover intra-node. -export UCX_TLS - -# Single-node-per-role recipes avoid DeepEP / NVSHMEM ibgda, so leave these off -# there to avoid triggering ibgda code paths that are not needed. -if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - export NVIDIA_GDRCOPY=enabled - # ibgda default kept for future DeepEP/wide-EP recipes; a recipe may override - # NVSHMEM_REMOTE_TRANSPORT to none. - export NVSHMEM_REMOTE_TRANSPORT - export NVSHMEM_IB_ENABLE_IBGDA - export NVSHMEM_SYMMETRIC_SIZE - export NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME=${NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME:-$DEFAULT_IFACE} - # NVSHMEM ignores NVSHMEM_HCA_PE_MAPPING when NVSHMEM_HCA_LIST is set, so - # clear the latter when the recipe provides an explicit PE mapping. - if [[ -n "${NVSHMEM_HCA_PE_MAPPING:-}" ]]; then - unset NVSHMEM_HCA_LIST 2>/dev/null || true - fi -fi - -if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then - KV_ROLE="$KV_ROLE_OVERRIDE" -elif [[ "$ROLE" == "prefill" ]]; then - KV_ROLE="kv_producer" -else - KV_ROLE="kv_consumer" -fi -KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" - -COMMON_ARGS=( - --port "$VLLM_PORT" - --served-model-name "$MODEL_NAME" - --trust-remote-code - --disable-access-log-for-endpoints=/health,/metrics - --tensor-parallel-size "$TP_SIZE" - --kv_transfer_config "$KV_TRANSFER_CONFIG" -) -# One frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps throughput, -# so run several (LLMD_API_SERVER_COUNT). Incompatible with --headless, so the -# headless-worker branch below drops it. With --data-parallel-hybrid-lb each node's -# api-server balances its local DP ranks, so VLLM_PORT is also the health port. -HEALTH_PORT="$VLLM_PORT" -API_SERVER_COUNT="${LLMD_API_SERVER_COUNT}" -# Multiple frontends only help the DP (wide-EP) path. A pure-TP engine has a single -# core with one frontend, so it keeps the default (and avoids --api-server-count -# interacting with the --headless multi-node TP launch below). -if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=(--api-server-count "$API_SERVER_COUNT") -fi -# Set to 1 by the pure-TP multi-node branch below on --headless followers, which -# run no local api-server; gates the post-launch health wait. -IS_HEADLESS_FOLLOWER=0 -# --moe-backend is model-specific (DSR1-FP8 wants deep_gemm, gpt-oss-MXFP4 -# rejects it), so each recipe sets its own via extra-args. - -# EP/DP knobs only when the recipe enables EP. Pure tensor-parallel roles skip -# them (vLLM rejects --data-parallel-size combined with TP>1). -if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=( - --enable-expert-parallel - --data-parallel-size "$DP_SIZE" - ) - if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - COMMON_ARGS+=( - --data-parallel-hybrid-lb - --data-parallel-size-local "$DP_SIZE_LOCAL" - --data-parallel-address "$DP_ADDR" - --data-parallel-rpc-port 5555 - --data-parallel-start-rank "$START_RANK" - ) - fi -elif [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - # Pure TP spanning >1 node (e.g. DSV4-Pro decode TP=8 on GB200's 4-GPU - # nodes): use vLLM's headless multi-node API - leader binds --master-addr, - # followers join --headless with matching --nnodes/--node-rank. - COMMON_ARGS+=( - --master-addr "$DP_ADDR" - --nnodes "$LWS_GROUP_SIZE" - --node-rank "$LWS_WORKER_INDEX" - ) - if [[ "$LWS_WORKER_INDEX" -gt 0 ]]; then - COMMON_ARGS+=(--headless) - IS_HEADLESS_FOLLOWER=1 - fi -fi - -echo "Starting vLLM ($ROLE) DP=$DP_SIZE local=$DP_SIZE_LOCAL start_rank=$START_RANK group_size=$LWS_GROUP_SIZE" -# shellcheck disable=SC2086 -vllm serve "$MODEL" "${COMMON_ARGS[@]}" $ROLE_EXTRA_ARGS \ - > "$VLLM_LOG" 2>&1 & -VLLM_PID=$! - -# Each rank waits for its own engine /health before continuing (for wide-EP this -# blocks the bench until worker DP shards are up; a no-op for single-node). A -# pure-TP --headless follower runs no local api-server (only the TP-group leader -# binds a health port, and its /health only reports ready once every TP worker -# has joined), so it skips the wait and stays alive via the final `wait`. -if [[ "$IS_HEADLESS_FOLLOWER" -eq 1 ]]; then - echo "vLLM headless TP follower on rank $NODE_RANK (worker_index=$LWS_WORKER_INDEX): no local api-server, skipping health wait" -else - infx_bench wait --url "http://0.0.0.0:$HEALTH_PORT/health" --pid "$VLLM_PID" --log "$VLLM_LOG" - echo "vLLM ready on rank $NODE_RANK ($ROLE worker_index=$LWS_WORKER_INDEX, health port $HEALTH_PORT)" -fi - -# The sidecar forwards a prefill request, reads kv_transfer_params from vLLM's -# response, then hits its local decode vLLM, whose NIXLv2 connector pulls KV -# directly from prefill vLLM. -# -# DEP8 (EP on, hybrid-LB): every decode node runs an api-server for its local DP -# ranks, so every decode node runs a sidecar and endpoints.yaml lists one decode -# endpoint per node. Pure-TP: only the TP-group leader has an api-server -# (followers are --headless), so only the leader runs a sidecar and only leaders -# are listed as endpoints. -if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX" -eq 0 ) ]]; then - SIDECAR_CONNECTOR="nixlv2" - SIDECAR_FLAGS=(--port="$SIDECAR_PORT" --vllm-port="$VLLM_PORT" - --kv-connector="$SIDECAR_CONNECTOR" --secure-proxy=false - --enable-prefiller-sampling) - SIDECAR_HEALTH_PORT="$SIDECAR_PORT" - echo "Starting pd-sidecar (decode node_rank=$NODE_RANK worker_index=$LWS_WORKER_INDEX): ${SIDECAR_FLAGS[*]}" - pd-sidecar "${SIDECAR_FLAGS[@]}" > "$SIDECAR_LOG" 2>&1 & - SIDECAR_PID=$! - infx_bench wait --url "http://0.0.0.0:$SIDECAR_HEALTH_PORT/health" --pid "$SIDECAR_PID" \ - --log "$SIDECAR_LOG" - echo "pd-sidecar ready on $HOST_IP:$SIDECAR_HEALTH_PORT" -fi - -# Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT - - # namespace must match EPP's --pool-namespace (file-discovery filters by it; - # the schema default 'default' would drop every entry). See README.md. - python3 - < DEP8 hybrid-LB (an api-server per node); -# EP off => pure-TP (only each TP-group leader has an api-server). -decode_ep = ('$ROLE_ENABLE_EP' == 'true') -VLLM_PORT = int('$VLLM_PORT') -SIDECAR_PORT = int('$SIDECAR_PORT') -# ALL_IPS is rank-ordered: ranks [0:pn] are prefill nodes, [pn:pn+dn] decode. -prefill_ips = all_ips[:pn] or [os.environ['PREFILL_LEADER_IP']] -decode_ips = all_ips[pn:pn + dn] or [os.environ['DECODE_LEADER_IP']] -endpoints = [] - -def add_role(role, ips, base_port, group_size=1): - # group_size == 1: one endpoint per node (DEP8 hybrid-LB: each node's - # api-server / sidecar load-balances its local DP ranks). - # group_size > 1: one endpoint per TP-group leader (pure-TP: followers are - # --headless with no api-server), i.e. every group_size-th node IP. - serving_ips = ips[::group_size] if group_size > 1 else ips - for i, ip in enumerate(serving_ips): - endpoints.append({'name': f'{role}-{i}', 'namespace': NS, 'address': ip, - 'port': str(base_port), 'labels': {'llm-d.ai/role': role}}) - -# Prefill (DEP8 in every current recipe): one endpoint per node, EPP hits vLLM -# directly (VLLM_PORT). Decode: EPP hits the pd-sidecar (SIDECAR_PORT); one -# endpoint per node for DEP8, or one per TP-group leader for pure-TP. -add_role('prefill', prefill_ips, VLLM_PORT) -decode_group = 1 if decode_ep else max(1, dn // decode_workers) -add_role('decode', decode_ips, SIDECAR_PORT, group_size=decode_group) -yaml.safe_dump({'endpoints': endpoints}, open('/tmp/endpoints.yaml', 'w')) -print(f'endpoints.yaml ({len(endpoints)} endpoints):') -print(open('/tmp/endpoints.yaml').read()) -PY - - # EPP - # Config: when a recipe is set, project it down to the keys EPP's strict - # decoder accepts (it rejects the per-role vLLM / slurm keys); else use the - # default mounted at /etc/epp/config.yaml. - if [[ -n "$CONFIG_FILE" && -f "/etc/llmd-recipes/$CONFIG_FILE" ]]; then - EPP_CONFIG="/tmp/epp-config-from-recipe.yaml" - python3 - < ext_proc trips, Envoy 500s). - epp \ - --pool-name=epp \ - --pool-namespace=inferencex \ - --config-file="$EPP_CONFIG" \ - --grpc-port="$EPP_GRPC_PORT" \ - --grpc-health-port="$EPP_HEALTH_PORT" \ - --metrics-port="$EPP_METRICS_PORT" \ - --secure-serving=false \ - > "$EPP_LOG" 2>&1 & - EPP_PID=$! - - # Wait for EPP's gRPC listener before starting Envoy (Envoy's ext_proc dials - # it). gRPC has no plain HTTP /health, so probe the TCP listener directly. - echo "Waiting for EPP on 127.0.0.1:$EPP_GRPC_PORT" - EPP_WAIT_DEADLINE=$(( $(date +%s) + 60 )) - until (echo > "/dev/tcp/127.0.0.1/$EPP_GRPC_PORT") 2>/dev/null; do - if ! kill -0 "$EPP_PID" 2>/dev/null; then - echo "ERROR: EPP died before binding $EPP_GRPC_PORT" >&2 - exit 1 - fi - if [[ "$(date +%s)" -ge "$EPP_WAIT_DEADLINE" ]]; then - echo "ERROR: EPP did not bind $EPP_GRPC_PORT within 60s" >&2 - exit 1 - fi - sleep 1 - done - echo "EPP listening on $EPP_GRPC_PORT" - - # Envoy - envoy -c /etc/envoy/envoy.yaml > "$ENVOY_LOG" 2>&1 & - ENVOY_PID=$! - - # Probe admin /ready (9901); /health on :8080 routes through ext_proc -> EPP - # and needs request routing metadata, so it would 503 until traffic flows. - echo "Waiting for envoy admin on 127.0.0.1:9901/ready" - ENVOY_WAIT_DEADLINE=$(( $(date +%s) + 120 )) - until [[ "$(curl --output /dev/null --silent --write-out '%{http_code}' \ - "http://127.0.0.1:9901/ready" 2>/dev/null)" == "200" ]]; do - if ! kill -0 "$ENVOY_PID" 2>/dev/null; then - echo "ERROR: envoy died before admin /ready returned 200" >&2 - tail -n 80 "$ENVOY_LOG" >&2 || true - exit 1 - fi - if [[ "$(date +%s)" -ge "$ENVOY_WAIT_DEADLINE" ]]; then - echo "ERROR: envoy admin /ready did not return 200 within 120s" >&2 - tail -n 80 "$ENVOY_LOG" >&2 || true - exit 1 - fi - sleep 2 - done - echo "Envoy admin ready; listener should be on $ENVOY_PORT" - - # Gate on ALL prefill vLLM /health endpoints. Prefill ranks only wait on their own - # local /health, and with PREFILL_WORKERS>1 every prefill node must be probed, not - # just IPS[0]. curl gets explicit connect/max timeouts so a blackholed endpoint - # trips the deadline instead of hanging the run (a timeout-less curl once wedged - # a 2P run for 7h). - _prefill_ips=( "${_ALL_IPS[@]:0:${PREFILL_NODES}}" ) - [[ ${#_prefill_ips[@]} -gt 0 ]] || _prefill_ips=( "$PREFILL_LEADER_IP" ) - - # On failure, dump enough to tell a server-not-ready problem (TCP connects but - # /health is slow) apart from a network/subnet problem (TCP connect refused or - # times out, host unreachable). Uses only bash builtins + coreutils: the arm64 - # serving image has no iproute2 / nc. - _diag_prefill_endpoint() { - local ip="$1" port="$2" - { - echo "=== NET DIAG: decode -> prefill ${ip}:${port} ===" - echo "[diag] decode node: $(hostname -f 2>/dev/null || hostname) local-ips: $(hostname -I 2>/dev/null)" - echo "[diag] ifaces: DEFAULT_IFACE=${DEFAULT_IFACE:-} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-}" - # Local source address the kernel would pick to reach ip: reveals which - # subnet/interface the route uses, without needing iproute2. - python3 - "$ip" <<'PY' 2>&1 || true -import socket, struct, sys -ip = sys.argv[1] -# Source address the kernel picks to reach ip (which local iface/subnet). -try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect((ip, 80)) - print(f"[diag] local source IP toward {ip}: {s.getsockname()[0]}") - s.close() -except Exception as e: - print(f"[diag] no route to {ip}: {e}") -# Subnet check via /proc/net/route (no iproute2 needed). Fields are little-endian -# hex. On-link route (gateway 0.0.0.0) == same subnet; a non-zero gateway means the -# target is reached across a router == DIFFERENT subnet. -def _ntoa(v): # little-endian int -> dotted quad - return socket.inet_ntoa(struct.pack(' best[0]: - best = (plen, iface, gw, dest) - if best is None: - print(f"[diag] {ip}: NO matching route -> unreachable") - else: - plen, iface, gw, dest = best - if gw == 0: - print(f"[diag] {ip}: ON-LINK via {iface}, subnet {_ntoa(dest)}/{plen} -> SAME subnet (no router hop)") - else: - print(f"[diag] {ip}: via GATEWAY {_ntoa(gw)} on {iface}, subnet {_ntoa(dest)}/{plen} -> DIFFERENT subnet (crosses a router)") -except Exception as e: - print(f"[diag] subnet classification failed: {e}") -PY - # L4: raw TCP connect (bash /dev/tcp, 5s cap) - port reachable-but-slow - # vs unreachable/filtered. - if timeout 5 bash -c "exec 3<>/dev/tcp/${ip}/${port}" 2>/dev/null; then - echo "[diag] TCP connect ${ip}:${port} OK -> port open; server up but /health slow/not-ready (NOT a network issue)" - else - echo "[diag] TCP connect ${ip}:${port} FAILED/timed out -> closed, filtered, or unreachable (LIKELY network/subnet/firewall issue)" - fi - if command -v ping >/dev/null 2>&1; then - ping -c 2 -W 2 "$ip" 2>&1 || echo "[diag] ping ${ip} failed (ICMP blocked or host down)" - fi - curl -v --connect-timeout 5 --max-time 8 "http://${ip}:${port}/health" 2>&1 || true - echo "=== END NET DIAG ${ip}:${port} ===" - } >&2 - } - - # Log the decode->prefill target layout up front so a subnet/interface - # mismatch is visible even on a run that eventually succeeds. Every prefill - # node serves on VLLM_PORT (hybrid LB). - echo "[diag] decode-leader $(hostname 2>/dev/null) local-ips: $(hostname -I 2>/dev/null); prefill targets: ${_prefill_ips[*]}" - echo "Waiting for prefill vLLM /health on ${#_prefill_ips[@]} node(s): ${_prefill_ips[*]}" - PREFILL_WAIT_DEADLINE=$(( $(date +%s) + 300 )) - for _pidx in "${!_prefill_ips[@]}"; do - _pip="${_prefill_ips[$_pidx]}" - _pport="$VLLM_PORT" - until curl --output /dev/null --silent --fail \ - --connect-timeout 5 --max-time 10 \ - "http://$_pip:$_pport/health"; do - if [[ "$(date +%s)" -ge "$PREFILL_WAIT_DEADLINE" ]]; then - echo "ERROR: prefill vLLM at $_pip:$_pport not ready within 5 min" >&2 - _diag_prefill_endpoint "$_pip" "$_pport" - exit 1 - fi - sleep 5 - done - echo "Prefill vLLM at $_pip:$_pport is ready" - done - echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" - - # Benchmark sweep. BENCH_MAX_CONCURRENCY is 'x'-delimited from submit.sh (e.g. "1024x512"). - IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" - # GPU counts are embedded in the result filename as _gpus_/_ctx_/_gen_ so the CI - # "Process result" step can parse them. - # ctx = prefill GPUs, gen = decode GPUs. - _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) - _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) - _bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus )) - if [[ "${EVAL_ONLY}" != "true" ]]; then - for max_concurrency in "${CONCURRENCIES[@]}"; do - num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER )) - [[ "$num_prompts" -lt 16 ]] && num_prompts=16 - # Bench against Envoy (EPP routes to decode; the sidecar pulls from prefill via - # NIXL). The tokenizer is the checkpoint: served-model-name is not an HF repo id. - # Non-fatal: a failed or timed-out conc point must not abort the sweep or (under - # set -e) skip the allocation release below. - infx_bench fixed-seq point \ - --base-url "http://0.0.0.0:$ENVOY_PORT" \ - --model "$MODEL_NAME" \ - --backend openai \ - --tokenizer "$MODEL" \ - --isl "$BENCH_INPUT_LEN" \ - --osl "$BENCH_OUTPUT_LEN" \ - --random-range-ratio "$BENCH_RANDOM_RANGE_RATIO" \ - --conc "$max_concurrency" \ - --num-prompts "$num_prompts" \ - --result "$BENCHMARK_LOGS_DIR/${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}.json" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" - done - fi - - # Eval (optional) - if [[ "${RUN_EVAL}" == "true" ]]; then - if [[ -n "${EVAL_CONC:-}" ]]; then - EVAL_CONCURRENT_REQUESTS="$EVAL_CONC" - else - EVAL_CONCURRENT_REQUESTS=$(printf '%s' "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - # /workspace is the runner checkout, where the workflow looks for results*.json. - infx_bench eval --endpoint "http://127.0.0.1:$ENVOY_PORT" \ - --concurrency "$EVAL_CONCURRENT_REQUESTS" --stage-to /workspace - fi - - # job.slurm stops the srun step once this marker exists. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" -else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait -fi diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh b/inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh deleted file mode 100755 index ee3b986981..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# -# Submit a multi-node llmd-vllm wide-EP P/D disagg benchmark job to SLURM. -# Submits the standalone llm-d Slurm job and prints JOB_ID on -# stdout so the runner can poll for completion. -# -# Topology (matches the llm-d wide-EP guide reference): -# 1 prefill instance with DP=PREFILL_NODES * GPUS_PER_NODE -# 1 decode instance with DP=DECODE_NODES * GPUS_PER_NODE -# each instance spans PREFILL_NODES / DECODE_NODES nodes via vLLM -# --data-parallel-hybrid-lb. Total nodes = PREFILL_NODES + DECODE_NODES. - -set -eo pipefail - -# Repo root resolved from this script's location, so paths below are -# independent of the caller's $PWD (the wrapper cd's into llm-d/ before -# invoking this script). -REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -source "$REPO_ROOT/benchmarks/check_env.sh" -check_env_vars \ - SLURM_ACCOUNT SLURM_PARTITION TIME_LIMIT MODEL_PATH MODEL_NAME \ - CONTAINER_IMAGE RUNNER_NAME BENCHMARK_LOGS_DIR GPUS_PER_NODE PREFILL_WORKERS \ - DECODE_WORKERS BENCH_NUM_PROMPTS_MULTIPLIER RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK \ - IS_AGENTIC FRAMEWORK SPEC_DECODING IS_MULTINODE -if [[ $# -ne 7 ]]; then - echo "Usage: submit.sh prefill_nodes decode_nodes isl osl concurrencies request_rate random_range_ratio" >&2 - exit 1 -fi - -PREFILL_NODES=$1 -DECODE_NODES=$2 -ISL=$3 -OSL=$4 -CONCURRENCIES=$5 -REQUEST_RATE=${6} -RANDOM_RANGE_RATIO=${7} - -NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) - -export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE -export MODEL_DIR=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export NUM_NODES=$NUM_NODES -export PREFILL_NODES=$PREFILL_NODES -export DECODE_NODES=$DECODE_NODES -export GPUS_PER_NODE=$GPUS_PER_NODE -# Each role's nodes split into this many INDEPENDENT DP/EP engines (default 1 = one -# engine over all role nodes), so DP_SIZE is PER-ENGINE. Matches how dynamo/AMD and -# upstream oci-high-tpt run 2P high-tpt (2 prefill : 1 decode). -export PREFILL_WORKERS -export DECODE_WORKERS -if (( PREFILL_NODES % PREFILL_WORKERS != 0 )); then - echo "Error: PREFILL_NODES ($PREFILL_NODES) not divisible by PREFILL_WORKERS ($PREFILL_WORKERS)" >&2 - exit 1 -fi -if (( DECODE_NODES % DECODE_WORKERS != 0 )); then - echo "Error: DECODE_NODES ($DECODE_NODES) not divisible by DECODE_WORKERS ($DECODE_WORKERS)" >&2 - exit 1 -fi -export PREFILL_DP_SIZE=$(( PREFILL_NODES / PREFILL_WORKERS * GPUS_PER_NODE )) -export DECODE_DP_SIZE=$(( DECODE_NODES / DECODE_WORKERS * GPUS_PER_NODE )) -export BENCH_INPUT_LEN=$ISL -export BENCH_OUTPUT_LEN=$OSL -export BENCH_MAX_CONCURRENCY=$CONCURRENCIES -export BENCH_REQUEST_RATE=$REQUEST_RATE -export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO -export BENCH_NUM_PROMPTS_MULTIPLIER - -export RUN_EVAL -export EVAL_ONLY -export EVAL_CONC="${EVAL_CONC:-}" -export EVAL_FRAMEWORK -export EVAL_LIMIT="${EVAL_LIMIT:-}" -export EVAL_SUITE="${EVAL_SUITE:-}" -export IS_AGENTIC -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export FRAMEWORK -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING -export IS_MULTINODE -export CONFIG_FILE="${CONFIG_FILE:-}" - -# Recipe may override SLURM time limit (longer topologies need more wall time). -if [[ -n "$CONFIG_FILE" ]]; then - RECIPE_PATH="${REPO_ROOT}/benchmarks/multi_node/llm-d-recipes/${CONFIG_FILE}" - if [[ -f "$RECIPE_PATH" ]]; then - RECIPE_TIME=$(python3 -c " -import yaml, sys -r = yaml.safe_load(open('$RECIPE_PATH')) -t = r.get('slurm', {}).get('time_limit', '') -print(t) -" 2>/dev/null || true) - [[ -n "$RECIPE_TIME" ]] && TIME_LIMIT="$RECIPE_TIME" - fi -fi - -mkdir -p "$BENCHMARK_LOGS_DIR" - -JOB_ID=$(sbatch \ - --parsable \ - --exclusive \ - -N "$NUM_NODES" \ - -n "$NUM_NODES" \ - --ntasks-per-node=1 \ - --gres=gpu:"$GPUS_PER_NODE" \ - --time "$TIME_LIMIT" \ - --partition "$SLURM_PARTITION" \ - --account "$SLURM_ACCOUNT" \ - --job-name "$RUNNER_NAME" \ - --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" \ - --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" \ - "$(dirname "$0")/job.slurm") - -if [[ -z "$JOB_ID" ]]; then - echo "Error: sbatch failed" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh b/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh index 36366b2696..87ea9cb7fc 100644 --- a/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh +++ b/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh @@ -19,11 +19,4 @@ case "$FRAMEWORK" in fi fi ;; - llmd-vllm) - export BENCH_NUM_PROMPTS_MULTIPLIER=10 - export LLMD_CONTAINER_ENGINE=docker VLLM_RANDOMIZE_DP_DUMMY_INPUTS=1 - export VLLM_ENGINE_READY_TIMEOUT_S=1800 VLLM_LOGGING_LEVEL=INFO UCX_TLS=cuda_copy,cuda_ipc,rc - export NVSHMEM_REMOTE_TRANSPORT=ibgda NVSHMEM_IB_ENABLE_IBGDA=true NVSHMEM_SYMMETRIC_SIZE=16G - export LLMD_API_SERVER_COUNT=4 - ;; esac diff --git a/inferencex-e2e/infx/launch/drivers/__init__.py b/inferencex-e2e/infx/launch/drivers/__init__.py index e377318a3c..bb49084a0d 100644 --- a/inferencex-e2e/infx/launch/drivers/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/__init__.py @@ -13,7 +13,7 @@ from infx.launch import policy from infx.launch.backends import backend_class from infx.launch.context import Launch, LaunchError -from infx.launch.drivers import llmd, script, srt +from infx.launch.drivers import script, srt from infx.launch.policy import LaunchPath, launch_path if TYPE_CHECKING: @@ -36,7 +36,6 @@ class Route: LaunchPath.SRT_MULTI: Route("slurm", srt.run_multinode), LaunchPath.SRT_NATIVE: Route("slurm", srt.run_multinode), LaunchPath.SCRIPT: Route(None, script.run), - LaunchPath.LLMD: Route("slurm", llmd.run), } diff --git a/inferencex-e2e/infx/launch/drivers/llmd.py b/inferencex-e2e/infx/launch/drivers/llmd.py deleted file mode 100644 index 95b6f636be..0000000000 --- a/inferencex-e2e/infx/launch/drivers/llmd.py +++ /dev/null @@ -1,161 +0,0 @@ -"""llm-d vLLM multinode jobs submitted through benchmarks/multi_node/llm-d/submit.sh.""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import sys -from pathlib import Path - -from infx.bench.proc import echo -from infx.launch import artifacts, policy -from infx.launch.backends.base import BackendError -from infx.launch.backends.slurm import cli -from infx.launch.context import Launch, LaunchError -from infx.launch.drivers.srt import models -from infx.launch.drivers.srt.run import slurm_backend -from infx.launch.request import LlmdRequest, RequestError - -CANCEL_TIMEOUT_S = 600.0 -DEFAULT_TIME_LIMIT = "08:00:00" -LLMD_DIR = "benchmarks/multi_node/llm-d" - - -def _find_eval_dir(logs_dir: Path) -> Path | None: - for root, dirs, _files in os.walk(logs_dir): - if "eval_results" in dirs: - return Path(root) / "eval_results" - return None - - -def _stage_agentic(logs_dir: Path, workspace: Path) -> None: - agentic = logs_dir / "agentic" - if not agentic.is_dir(): - return - staged = workspace / "LOGS" / "agentic" - staged.mkdir(parents=True, exist_ok=True) - for entry in agentic.iterdir(): - destination = staged / entry.name - if entry.is_dir(): - shutil.copytree(entry, destination, dirs_exist_ok=True) - elif entry.is_file(): - shutil.copy2(entry, destination) - - -def run(launch: Launch) -> int: - """Submit the llm-d Slurm job, follow its log, and stage benchmark artifacts.""" - event = launch.life.event - backend = slurm_backend(launch) - request = LlmdRequest.from_env(launch.request.env) - if backend.settings.squash is None: - raise LaunchError(f"llmd-vllm: cluster {launch.cluster.id!r} has no slurm.squash") - if not request.disagg: - raise LaunchError("llmd-vllm supports only P/D disaggregated points") - - checkpoint = models.checkpoint(launch.cluster, request) - if checkpoint is None: - raise LaunchError( - f"cluster {launch.cluster.id!r} stages no checkpoint for MODEL={request.model}" - ) - model_path = models.host_path(launch.cluster, checkpoint) - if not checkpoint.node_local and not (model_path / "config.json").is_file(): - raise LaunchError(f"model checkpoint is unavailable: {model_path / 'config.json'}") - - with event.stage("prepare"): - squash = backend.prepare_image(request.image) - logs_dir = request.workspace / "benchmark_logs" - logs_dir.mkdir(parents=True, exist_ok=True) - - account = backend.settings.account or cli.default_account() - if not account: - raise RequestError.missing("SLURM_ACCOUNT") - - env = policy.runtime_env( - launch.cluster, - request, - models.job_env(launch.cluster, request, str(model_path)), - { - "SLURM_PARTITION": backend.settings.partition, - "SLURM_ACCOUNT": account, - "MODEL_PATH": str(model_path), - "MODEL_NAME": request.model, - "CONTAINER_IMAGE": request.image, - "GPUS_PER_NODE": str(launch.cluster.gpus_per_node), - "TIME_LIMIT": request.env.get("TIME_LIMIT") or DEFAULT_TIME_LIMIT, - "PREFILL_WORKERS": str(request.prefill_num_workers), - "DECODE_WORKERS": str(request.decode_num_workers), - "LLMD_CONTAINER_ENGINE": "pyxis", - "LLMD_SQUASH_FILE": squash.reference, - "BENCHMARK_LOGS_DIR": str(logs_dir), - }, - ) - - argv = [ - "bash", - "submit.sh", - str(request.prefill_nodes), - str(request.decode_nodes), - str(request.isl), - str(request.osl), - "x".join(map(str, request.conc_list)), - "inf", - request.random_range_ratio, - ] - with event.stage("submit"): - echo(argv, env) - submitted = subprocess.run( - argv, - stdout=subprocess.PIPE, - stderr=sys.stderr, - text=True, - env=env, - cwd=request.workspace / LLMD_DIR, - check=False, - ) - job_id = submitted.stdout.strip() - if submitted.returncode != 0 or not job_id: - event.fail("SubmitFailed", "llm-d submit.sh failed before returning a Slurm job id") - return 1 - if not (job_id.isascii() and job_id.isdigit()): - message = f"llm-d submit.sh printed {job_id!r} instead of a Slurm job id" - event.fail("SubmitFailed", message) - return 1 - - log_file = logs_dir / f"slurm_job-{job_id}.out" - job = backend.attach(job_id, log=log_file, outputs=logs_dir) - print(f"Submitted llm-d job: {job_id}", flush=True) - - launch.life.callback( - artifacts.bundle_server_logs, logs_dir, request.workspace / "multinode_server_logs.tar.gz" - ) - launch.life.callback(backend.cancel, job, wait_s=CANCEL_TIMEOUT_S) - - try: - backend.stream_logs(job) - except BackendError: - return 1 - - status = backend.state(job) - rc = 0 if status.succeeded else 1 - - with event.stage("collect"): - for result_file in sorted(logs_dir.glob(f"{request.result_filename}*.json")): - try: - artifacts.copy_to_workspace(result_file, request.workspace / result_file.name) - except artifacts.ArtifactError as error: - event.error(error) - rc = 1 - - if request.is_agentic and not request.eval_only: - _stage_agentic(logs_dir, request.workspace) - - if request.run_eval: - eval_dir = _find_eval_dir(logs_dir) or logs_dir / "eval_results" - try: - artifacts.copy_eval_artifacts(eval_dir, request.workspace) - except artifacts.ArtifactError as error: - event.error(error) - rc = 1 - - return rc diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index da8abdb6cb..5f0e9b33cb 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -44,11 +44,6 @@ class Override: Override(Match(model_glob="*/DeepSeek-V4-Pro-0813"), entry="DeepSeek-V4-Pro-0813"), ), "gb200-nv": ( - Override( - Match(any_of("dsv4"), any_of("fp4"), any_of("llmd-vllm")), - entry="DeepSeek-V4-Pro@numa1", - served_name="deepseek-ai/DeepSeek-V4-Pro", - ), Override( Match(any_of("dsr1"), any_of("fp4"), any_of("dynamo-sglang")), entry="deepseek-r1-0528-fp4-v2", diff --git a/inferencex-e2e/infx/launch/policy.py b/inferencex-e2e/infx/launch/policy.py index 3a6ed913e8..d6bf0edba4 100644 --- a/inferencex-e2e/infx/launch/policy.py +++ b/inferencex-e2e/infx/launch/policy.py @@ -57,7 +57,6 @@ class LaunchPath(StrEnum): SRT_NATIVE = "srt-native" SRT_BATCH = "srt-batch" SCRIPT = "script" - LLMD = "llmd" NATIVE_SRT_LANES: dict[str, tuple[Match, ...]] = { @@ -77,8 +76,6 @@ class LaunchPath(StrEnum): def launch_path(cluster_id: str, request: LaunchRequest) -> LaunchPath: if request.is_multinode: - if request.framework == "llmd-vllm": - return LaunchPath.LLMD if any(lane(request) for lane in NATIVE_SRT_LANES.get(cluster_id, ())): return LaunchPath.SRT_NATIVE return LaunchPath.SRT_MULTI diff --git a/inferencex-e2e/infx/launch/request.py b/inferencex-e2e/infx/launch/request.py index 88150eaa61..2c463a062d 100644 --- a/inferencex-e2e/infx/launch/request.py +++ b/inferencex-e2e/infx/launch/request.py @@ -181,17 +181,3 @@ class ScriptRequest(LaunchRequest): model: str = Field(alias="MODEL") gpu_count: int = Field(alias="GPU_COUNT") salloc_time_limit: int = Field(alias="SALLOC_TIME_LIMIT") - - -class LlmdRequest(SrtRequest): - """An llm-d vLLM multinode job submitted through benchmarks/multi_node/llm-d.""" - - model: str = Field(alias="MODEL") - disagg: TrueFlag = Field(alias="DISAGG") - prefill_nodes: int = Field(alias="PREFILL_NODES") - decode_nodes: int = Field(alias="DECODE_NODES") - prefill_num_workers: int = Field(1, alias="PREFILL_NUM_WORKERS") - decode_num_workers: int = Field(1, alias="DECODE_NUM_WORKERS") - isl: int = Field(alias="ISL") - osl: int = Field(alias="OSL") - random_range_ratio: str = Field(alias="RANDOM_RANGE_RATIO") diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index a8dbbd7bc2..ea040e79c1 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -20,7 +20,6 @@ DEFAULT_AGENTIC_DURATION_SECONDS = 3600 SINGLE_NODE_RECIPES = "benchmarks/single_node/srt-slurm-recipes" MULTI_NODE_RECIPES = "benchmarks/multi_node/srt-slurm-recipes" -LLMD_FRAMEWORK = "llmd-vllm" RECIPE_SELECTOR = re.compile(r"base|override_[\w-]+|zip_override_[\w-]+\[\d+\]") @@ -872,12 +871,6 @@ def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: def _validate_srt_recipes(self: Any) -> Any: entries = _master_search_space_entries(self) - if self.multinode and self.framework == LLMD_FRAMEWORK: - if self.srt_recipe_dir is not None or any( - entry.srt_recipe or entry.eval_srt_recipe for entry in entries - ): - raise ValueError(f"{LLMD_FRAMEWORK} selects recipes with CONFIG_FILE, not srt-recipe") - return self if self.srt_recipe_dir is None: raise ValueError(f"srt-slurm configs require {Fields.SRT_RECIPE_DIR.value}") for entry in entries: diff --git a/inferencex-e2e/infx/tests/launch/test_llmd_driver.py b/inferencex-e2e/infx/tests/launch/test_llmd_driver.py deleted file mode 100644 index 1123286573..0000000000 --- a/inferencex-e2e/infx/tests/launch/test_llmd_driver.py +++ /dev/null @@ -1,137 +0,0 @@ -"""The llm-d driver: run submit.sh, attach, stage artifacts.""" - -import json -import subprocess -import tarfile -from pathlib import Path - -import pytest - -from infx.tests.launch.fake_slurm import ( - base_env, - install_fakes, - launch, - make_workspace, - runner_for, - sandbox_runner_config, -) - -LLMD_SUBMIT = """#!/usr/bin/env bash -set -e -env > "$GITHUB_WORKSPACE/submitted.env" -echo "$PWD $*" > "$GITHUB_WORKSPACE/submitted.args" -logs="$BENCHMARK_LOGS_DIR" -job="$logs/slurm_job-4299" -mkdir -p "$logs/agentic/conc_128" "$job/eval_results" -echo '{"conc": 128}' > "$logs/point-identity_conc128.json" -echo trace > "$logs/agentic/conc_128/profile.json" -echo '{"score": 1}' > "$job/eval_results/results_gsm8k.json" -echo 'server log' > "$logs/server.log" -echo 'benchmark done' > "$logs/slurm_job-4299.out" -echo 'worker warning' > "$logs/slurm_job-4299.err" -echo 'submitting' >&2 -[[ "${NO_JOB_ID:-}" == 1 ]] && exit 1 -echo 4299 -""" - - -@pytest.fixture -def harness(tmp_path): - """Sandboxed gb200-nv cluster, fake Slurm binaries, and a workspace with a fake submit.sh.""" - sandbox = tmp_path / "sandbox" - sandbox.mkdir() - config = sandbox_runner_config(sandbox) - workspace = make_workspace(tmp_path / "workspace") - wrapper = workspace / "benchmarks/multi_node/llm-d/submit.sh" - wrapper.parent.mkdir(parents=True, exist_ok=True) - wrapper.write_text(LLMD_SUBMIT) - wrapper.chmod(0o755) - - model_root = sandbox / "mnt/numa1/models/DeepSeek-V4-Pro" - model_root.mkdir(parents=True) - (model_root / "config.json").write_text("{}\n") - - logs = tmp_path / "logs" - env = base_env( - fakes=install_fakes(tmp_path / "bin"), logs=logs, workspace=workspace, sandbox=sandbox - ) - env.update( - RUNNER_NAME=runner_for("gb200-nv"), - IS_MULTINODE="true", - IS_AGENTIC="1", - RUN_EVAL="true", - EVAL_ONLY="false", - FRAMEWORK="llmd-vllm", - MODEL="deepseek-ai/DeepSeek-V4-Pro", - MODEL_PREFIX="dsv4", - PRECISION="fp4", - SPEC_DECODING="mtp", - THINKING_MODE="thinking_on", - PREFILL_NODES="2", - DECODE_NODES="2", - ISL="8192", - OSL="1024", - RANDOM_RANGE_RATIO="0.8", - CONC="256", - CONC_LIST="256", - DISAGG="true", - IMAGE="vllm/vllm-openai:v0.21.0", - RESULT_FILENAME="point-identity", - ENROOT_IMPORT_TIME_LIMIT="10", - ) - return config, workspace, env - - -def run_launch(harness) -> subprocess.CompletedProcess[str]: - config, workspace, env = harness - return launch(env, config, workspace) - - -def test_llmd_driver_submits_the_wrapper_and_stages_artifacts(harness): - result = run_launch(harness) - config, workspace, env = harness - - assert result.returncode == 0, result.stdout + result.stderr - submitted = dict( - line.split("=", 1) for line in (workspace / "submitted.env").read_text().splitlines() if "=" in line - ) - model_path = f"{config.parent}/mnt/numa1/models/DeepSeek-V4-Pro" - assert submitted["MODEL_PATH"] == model_path - assert submitted["MODEL_NAME"] == env["MODEL"] - assert submitted["LLMD_CONTAINER_ENGINE"] == "pyxis" - assert submitted["LLMD_SQUASH_FILE"] - assert submitted["BENCHMARK_LOGS_DIR"] == f"{workspace}/benchmark_logs" - assert submitted["SLURM_PARTITION"] == "batch" - assert submitted["SLURM_ACCOUNT"] == "benchmark" - assert submitted["GPUS_PER_NODE"] == "4" - assert submitted["CONTAINER_IMAGE"] == env["IMAGE"] - assert (workspace / "submitted.args").read_text().split() == [ - str(workspace / "benchmarks/multi_node/llm-d"), - *"2 2 8192 1024 256 inf 0.8".split(), - ] - - assert json.loads((workspace / "point-identity_conc128.json").read_text()) == {"conc": 128} - assert (workspace / "LOGS/agentic/conc_128/profile.json").read_text() == "trace\n" - assert json.loads((workspace / "results_gsm8k.json").read_text()) == {"score": 1} - with tarfile.open(workspace / "multinode_server_logs.tar.gz") as bundle: - assert "./server.log" in bundle.getnames() - assert "submitting" in result.stderr - - -def test_llmd_driver_fails_when_the_wrapper_prints_no_job_id(harness): - _, workspace, env = harness - env["NO_JOB_ID"] = "1" - - result = launch(env, harness[0], workspace) - - assert result.returncode == 1 - assert "submit.sh failed before returning a Slurm job id" in result.stderr - - -def test_llmd_driver_leaves_node_local_checkpoints_to_the_job(harness): - config, workspace, env = harness - (config.parent / "mnt/numa1/models/DeepSeek-V4-Pro/config.json").unlink() - - result = launch(env, config, workspace) - - assert result.returncode == 0, result.stdout + result.stderr diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index a6e52eee69..6f8e47c0e3 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -55,7 +55,6 @@ def cluster(tmp_path, single_node_models: str = "staged") -> Cluster: ("b200-nscale", dict(MULTI, MODEL_PREFIX="glm5.3", PRECISION="fp8", FRAMEWORK="tilert", SPEC_DECODING="mtp"), LaunchPath.SRT_MULTI), ("mi355x-amds", dict(IS_MULTINODE="true", FRAMEWORK="atom-disagg"), LaunchPath.SRT_MULTI), ("mi355x-amds", dict(MULTI, FRAMEWORK="sglang-disagg"), LaunchPath.SRT_MULTI), - ("gb200-nv", dict(MULTI, FRAMEWORK="llmd-vllm", MODEL_PREFIX="dsv4", PRECISION="fp4", SPEC_DECODING="mtp"), LaunchPath.LLMD), ("gb200-nv", dict(MULTI, FRAMEWORK="tilert"), LaunchPath.SRT_MULTI), ("b300-dsxe", dict(SINGLE, MODEL_PREFIX="dsv41flash", FRAMEWORK="sglang", IS_AGENTIC="1"), LaunchPath.SRT_BATCH), ("b300-dsxe", dict(SINGLE, MODEL_PREFIX="dsv41flash", FRAMEWORK="sglang", IS_AGENTIC="1", INFX_BATCH_REENTRY="1"), LaunchPath.SRT_SINGLE), diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index a46620c758..54413d0a28 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -1060,18 +1060,6 @@ def test_recipe_settings_are_not_additional_settings( with pytest.raises(ValidationError, match=replacement): MultiNodeMasterConfigEntry(**valid_multinode_master_config) - def test_llmd_configs_keep_config_file(self, valid_multinode_master_config): - config = valid_multinode_master_config - config["framework"] = "llmd-vllm" - del config["srt-recipe-dir"] - row = config["scenarios"]["fixed-seq-len"][0]["search-space"][0] - del row["srt-recipe"] - row["prefill"]["additional-settings"].append("CONFIG_FILE=llmd.yaml") - assert validate_master_config({"llmd": config}, Path()) == {"llmd": config} - row["srt-recipe"] = "recipe.yaml" - with pytest.raises(ValidationError, match="llmd-vllm selects recipes with CONFIG_FILE"): - MultiNodeMasterConfigEntry(**config) - def test_eval_srt_recipe_is_multinode_only(self, valid_single_node_master_config): config = valid_single_node_master_config config["runner"] = "cluster:b200-nscale" From 625116a5a71a521164b2071874ca753d2a8038ed Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:56:26 -0500 Subject: [PATCH 02/12] feat: add DeepSeek-V4-Pro-0813 llm-d AgentX on GB200 through srt-slurm Port #2719's llm-d submission to srt-slurm's llm-d frontend: disagg 1P1D DEP8/DEP8 and 2P1D DEP8/DEP16 with Mooncake on prefill, and agg TP8, DEP8, and DEP8 with Mooncake. Drop the MiniMax-M3 smoke keys. Co-authored-by: Ilya Markov Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 239 +++++++++++++++ .../vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 274 ++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 102 +++++++ .../infx/launch/drivers/srt/lanes.py | 2 +- inferencex-e2e/perf-changelog.yaml | 10 + 5 files changed, 626 insertions(+), 1 deletion(-) create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml create mode 100644 inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml new file mode 100644 index 0000000000..a998dc8771 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -0,0 +1,239 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router +# (srt-slurm frontend.type: llm-d): one aggregated worker over two trays, as TP8, +# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. Image, vLLM flags, +# environment, and EPP schedulers follow #2719's llm-d recipes. srt-slurm launches a +# DEP8 worker as one `vllm serve` that balances its eight DP ranks itself, so the EPP +# routes to the worker rather than to one rank, and VLLM_USE_RUST_FRONTEND is left +# unset (the Python API server). +# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro-0813 + container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} + container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + dynamo: + install: false + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + observability: + enabled: false + tachometer: + enabled: false + frontend: + type: llm-d + enable_multiple_frontends: false + # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + epp_config: + plugins: + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 5858 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: max-score-picker + engine: + type: vllm + connector: + roles: + agg: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.88 + max-num-batched-tokens: 8192 + block-size: 256 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + enable-cumem-allocator: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +# TP8 with five DSpark tokens, no EP, and a load-only scheduler. +override_tp8: + name: dsv4-gb200-vllm-agentic-dspark-agg-tp8-llm-d + frontend: + epp_config: + plugins: + - type: active-request-scorer + - type: queue-scorer + - type: weighted-random-picker + schedulingProfiles: + - name: default + plugins: + - pluginRef: active-request-scorer + weight: 2 + - pluginRef: queue-scorer + weight: 2 + - pluginRef: weighted-random-picker + roles: + agg: + env: + PYTHONHASHSEED: + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: + VLLM_USE_DEEP_GEMM: + NVSHMEM_REMOTE_TRANSPORT: + NVSHMEM_IB_ENABLE_IBGDA: + NVSHMEM_CUMEM_HANDLE_TYPE: + NVSHMEM_DISABLE_CUDA_VMM: + VLLM_USE_NCCL_SYMM_MEM: "0" + VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: auto + UCX_TLS: cuda_copy,cuda_ipc,tcp + UCX_MEMTYPE_REG_WHOLE: "n" + args: + tensor-parallel-size: 8 + data-parallel-size: + enable-expert-parallel: + moe-backend: + enable-ep-weight-filter: + numa-bind: + gpu-memory-utilization: 0.85 + disable-custom-all-reduce: true + speculative-config: '{"method":"dspark","num_speculative_tokens":5,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + +override_dep8: + name: dsv4-gb200-vllm-agentic-dspark-agg-dep8-llm-d + +# DEP8 with an embedded Mooncake store at 45% of each tray's DRAM (~900 GB over four +# GPUs); NIXL, SimpleCPUOffloadConnector, and MooncakeStoreConnector in a MultiConnector. +override_dep8_mooncake: + name: dsv4-gb200-vllm-agentic-dspark-agg-dep8-mooncake-llm-d + services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "101GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + frontend: + epp_config: + plugins: + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 40810 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 20000 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 1499703 + - type: active-request-scorer + - type: queue-scorer + - type: max-score-picker + roles: + agg: + env: + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + gpu-memory-utilization: 0.87 + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":42949672960}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml new file mode 100644 index 0000000000..ba5558e892 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -0,0 +1,274 @@ +# DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router +# (srt-slurm frontend.type: llm-d): DEP8 prefill workers over two trays each and one +# DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix +# cache through an embedded Mooncake store. Image, vLLM flags, environment, and EPP +# scheduler follow #2719's disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml, with these +# differences: +# - srt-slurm launches each worker as one `vllm serve` over its trays that balances its +# DP ranks itself, so the EPP routes to the worker rather than to one rank. +# - Prefill drafts three DSpark tokens like decode (#2719: one): the srt launch planner +# gives both roles the decode role's golden acceptance length. +# - VLLM_USE_RUST_FRONTEND is left unset (the Python API server). +# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. + +schema: 2 + +base: + model: + path: deepseek-v4-pro-0813 + container: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + precision: fp4 + identity: + model: {repo: deepseek-ai/DeepSeek-V4-Pro-0813} + container: {image: "quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95"} + dynamo: + install: false + slurm: {time_limit: "8:00:00"} + health_check: {max_attempts: 2160, interval_seconds: 10} + resources: + gpu_type: gb200 + gpus_per_node: 4 + observability: + enabled: false + tachometer: + enabled: false + services: + - name: mooncake-master + type: mooncake-master + options: + store_config: + metadata_server: "P2PHANDSHAKE" + # Each prefill rank's embedded store: 2/3 of a tray's ~900 GB DRAM over 4 GPUs. + global_segment_size: "150GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_3,mlx5_4" + mode: "embedded" + enable_offload: false + frontend: + type: llm-d + enable_multiple_frontends: false + # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + epp_config: + plugins: + - type: prefill-filter + - type: decode-filter + - type: inflight-load-producer + - type: approx-prefix-cache-producer + parameters: + autoTune: false + blockSizeTokens: 256 + maxPrefixTokensToMatch: 1048576 + maxPrefixBlocksToMatch: 4096 + lruCapacityPerServer: 43949 + - type: prefix-cache-affinity-filter + parameters: + peakPrefillThroughput: 4783 + maxTTFTPenaltyMs: 30000 + - type: prefix-cache-scorer + - type: token-load-scorer + parameters: + queueThresholdTokens: 3000000 + - type: active-request-scorer + - type: queue-scorer + - type: always-disagg-pd-decider + - type: disagg-profile-handler + parameters: + deciders: + prefill: always-disagg-pd-decider + - type: max-score-picker + name: prefill-picker + - type: max-score-picker + name: decode-picker + schedulingProfiles: + - name: prefill + plugins: + - pluginRef: prefill-filter + - pluginRef: prefix-cache-affinity-filter + - pluginRef: prefix-cache-scorer + weight: 6 + - pluginRef: token-load-scorer + weight: 3 + - pluginRef: queue-scorer + weight: 3 + - pluginRef: prefill-picker + - name: decode + plugins: + - pluginRef: decode-filter + - pluginRef: active-request-scorer + - pluginRef: decode-picker + engine: + type: vllm + connector: + roles: + prefill: + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_CONNECTOR_PREFETCH_KV_CAP: "0.65" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED: "1" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_RPC_CLIENT_IO_THREADS: "32" + MC_TE_RPC_CLIENT_IO_THREADS: "32" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + # NIXL hands KV to decode; the Mooncake store shares prefix cache across prefill ranks. + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + max-num-seqs: 16 + max-model-len: 1048576 + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + enable-cumem-allocator: true + block-size: 256 + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + decode: + nodes: 2 + workers: 1 + gpus: 8 + env: + PYTHONHASHSEED: "0" + HF_HUB_CACHE: /hf_hub_cache + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_NCCL_SYMM_MEM: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + VLLM_USE_DEEP_GEMM: "1" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_NO_USAGE_STATS: "1" + VLLM_LOGGING_LEVEL: INFO + TQDM_DISABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + TORCH_SYMMMEM: NVSHMEM + DG_JIT_CACHE_DIR: /tmp/dg-cache-dsv4-agentx + NVSHMEM_REMOTE_TRANSPORT: "none" + NVSHMEM_IB_ENABLE_IBGDA: "false" + NVSHMEM_CUMEM_HANDLE_TYPE: FABRIC + NVSHMEM_DISABLE_CUDA_VMM: "0" + UCX_TLS: cuda_copy,cuda_ipc,rc,tcp + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MEMTYPE_CACHE: "n" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + NCCL_P2P_LEVEL: NVL + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" + args: + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + trust-remote-code: true + tensor-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}}' + kv-cache-dtype: fp8 + gpu-memory-utilization: 0.95 + max-num-batched-tokens: 256 + max-num-seqs: 32 + max-cudagraph-capture-size: 256 + max-model-len: 1048576 + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + enable-cumem-allocator: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + tokenizer-mode: deepseek_v4 + tool-call-parser: deepseek_v4 + enable-auto-tool-choice: true + reasoning-parser: deepseek_v4 + moe-backend: deep_gemm_mega_moe + enable-ep-weight-filter: true + no-disable-hybrid-kv-cache-manager: true + no-enable-flashinfer-autotune: true + numa-bind: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + sbatch_directives: {cpus-per-task: "144", mem: "0"} + srun_options: {container-remap-root: ""} + benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + MODEL: deepseek-ai/DeepSeek-V4-Pro-0813 + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" + +override_1p1d: + name: dsv4-gb200-vllm-agentic-dspark-disagg-1p1d-dep8-dep8-llm-d + roles: + prefill: + nodes: 2 + workers: 1 + +override_2p1d: + name: dsv4-gb200-vllm-agentic-dspark-disagg-2p1d-dep8-dep16-llm-d + roles: + prefill: + nodes: 4 + workers: 2 + decode: + nodes: 4 + gpus: 16 + args: + data-parallel-size: 16 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 5d7f3ffa3f..f408a46b36 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5743,6 +5743,108 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: ep: 1 dp-attn: false +# DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200 behind the llm-d router (srt-slurm +# frontend.type: llm-d). The image bundles epp and pd-sidecar from llm-d-router v0.10.0. +dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: vllm + router: { name: llm-d-router, version: "0.10.0" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.666667 + search-space: + # 1P DEP8 + 1D DEP8 (4 trays), Mooncake prefix cache on prefill. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [64, 128, 160, 256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_1p1d" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 2P DEP8 + 1D DEP16 (8 trays), Mooncake prefix cache on prefill. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [256, 384, 576] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_2p1d" + decode: + num-worker: 1 + tp: 8 + ep: 16 + dp-attn: true + +# Aggregated TP8 and DEP8 over two trays; only the Mooncake arm offloads KV to DRAM. +dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg: + image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: vllm + router: { name: llm-d-router, version: "0.10.0" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.45 + search-space: + - spec-decoding: mtp + kv-offloading: none + conc-list: [1, 2, 4, 8, 12] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_tp8" + - spec-decoding: mtp + kv-offloading: none + conc-list: [12, 16, 32] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: mooncake } + conc-list: [52, 72] + num-nodes: 2 + worker: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8_mooncake" + dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index cb7a33e639..c12d82c3b1 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -59,7 +59,7 @@ class SrtLane: ), ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane(frameworks=_DYNAMO), ("gb200-nv", LaunchPath.SRT_MULTI): SrtLane( - frameworks=_DYNAMO, + frameworks=any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm", "vllm"), setup_scripts={"dynamo-sglang": "install-torchao.sh"}, mounts=( LaneMount( diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 254ed843f4..af1dee0332 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10778,3 +10778,13 @@ - "Enable --monokernel-backend amd on concurrency <= 16." - "Enable SGLANG_HIP_FP4_INDEXER_ROWGROUP=1 on all points. Enable SGLANG_HIP_DSV41_INDEX_K_PAGE8=1 only on the non-monokernel points (TP2 c32, DP2 c32 and c64); the monokernel refuses that layout." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4367 + +- config-keys: + - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg + - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend, porting #2719's llm-d submission: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." + - "Same image, vLLM flags, environment, and EPP schedulers as #2719. Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 From 95bd86f3471b0dde8566459601c0e6591e9371a6 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:52:52 -0500 Subject: [PATCH 03/12] fix: declare the 2P1D decode worker as TP16 so results count 32 GPUs Signed-off-by: Cam Quilici --- inferencex-e2e/configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index f408a46b36..d2d0455dd1 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5791,7 +5791,7 @@ dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_2p1d" decode: num-worker: 1 - tp: 8 + tp: 16 ep: 16 dp-attn: true From 14a694d60542b07b8d8293c14e5e2018b3b5cab7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:55:45 -0500 Subject: [PATCH 04/12] fix: stage gb200-nv dsv4 vllm runs on the shared run root Signed-off-by: Cam Quilici --- inferencex-e2e/infx/launch/drivers/srt/lanes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index c12d82c3b1..4e53d31cdc 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -71,7 +71,7 @@ class SrtLane: ), shared_run_root=( Match(any_of("minimaxm3", "kimik3", "qwen3.5", "glm5.2")), - Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm")), + Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm", "vllm")), Match(any_of("dsv4"), frameworks=any_of("dynamo-sglang"), agentic=True), ), ), From 11f834a571fcb8376a29ea14691cf028e13d06be Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:56:10 -0500 Subject: [PATCH 05/12] chore: drop PR references from the llm-d recipes Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 5 ++--- .../dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 15 ++++++--------- inferencex-e2e/perf-changelog.yaml | 4 ++-- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index a998dc8771..da2fb35ff4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -1,7 +1,6 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router # (srt-slurm frontend.type: llm-d): one aggregated worker over two trays, as TP8, -# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. Image, vLLM flags, -# environment, and EPP schedulers follow #2719's llm-d recipes. srt-slurm launches a +# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. srt-slurm launches a # DEP8 worker as one `vllm serve` that balances its eight DP ranks itself, so the EPP # routes to the worker rather than to one rank, and VLLM_USE_RUST_FRONTEND is left # unset (the Python API server). @@ -31,7 +30,7 @@ base: frontend: type: llm-d enable_multiple_frontends: false - # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + # srtctl adds the file-discovery plugin. epp_config: plugins: - type: inflight-load-producer diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index ba5558e892..f4c419cc81 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -1,14 +1,11 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router # (srt-slurm frontend.type: llm-d): DEP8 prefill workers over two trays each and one # DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix -# cache through an embedded Mooncake store. Image, vLLM flags, environment, and EPP -# scheduler follow #2719's disagg-gb200-1p1d-dep8-dep8-dspark-agentic.yaml, with these -# differences: -# - srt-slurm launches each worker as one `vllm serve` over its trays that balances its -# DP ranks itself, so the EPP routes to the worker rather than to one rank. -# - Prefill drafts three DSpark tokens like decode (#2719: one): the srt launch planner -# gives both roles the decode role's golden acceptance length. -# - VLLM_USE_RUST_FRONTEND is left unset (the Python API server). +# cache through an embedded Mooncake store. srt-slurm launches each worker as one +# `vllm serve` over its trays that balances its DP ranks itself, so the EPP routes to the +# worker rather than to one rank. Prefill drafts three DSpark tokens like decode: the srt +# launch planner gives both roles the decode role's golden acceptance length. +# VLLM_USE_RUST_FRONTEND is left unset (the Python API server). # Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. schema: 2 @@ -48,7 +45,7 @@ base: frontend: type: llm-d enable_multiple_frontends: false - # #2719's scheduler, without its file-discovery plugin (srtctl adds its own). + # srtctl adds the file-discovery plugin. epp_config: plugins: - type: prefill-filter diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index af1dee0332..7f79bd6372 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10785,6 +10785,6 @@ scenario-type: - agentic-coding description: - - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend, porting #2719's llm-d submission: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." - - "Same image, vLLM flags, environment, and EPP schedulers as #2719. Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." + - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." + - "Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 From 74e90f4e1a14b475a2accc2176a1c306f9ec2319 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 16:00:36 -0500 Subject: [PATCH 06/12] fix: label the llm-d keys llmd-vllm Signed-off-by: Cam Quilici --- inferencex-e2e/configs/nvidia-master.yaml | 8 ++++---- inferencex-e2e/infx/launch/drivers/srt/lanes.py | 4 ++-- inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py | 1 + inferencex-e2e/perf-changelog.yaml | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index d2d0455dd1..143d5285fa 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5745,13 +5745,13 @@ minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp: # DeepSeek-V4-Pro-0813 (DSpark) FP4 GB200 behind the llm-d router (srt-slurm # frontend.type: llm-d). The image bundles epp and pd-sidecar from llm-d-router v0.10.0. -dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: +dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb200-nv precision: fp4 - framework: vllm + framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } kv-p2p-transfer: nixl multinode: true @@ -5796,13 +5796,13 @@ dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg: dp-attn: true # Aggregated TP8 and DEP8 over two trays; only the Mooncake arm offloads KV to DRAM. -dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg: +dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg: image: quay.io/rh-ee-imarkov/llm-d-nokube-vllm:dspark-0814-nightly@sha256:00742fdd10e572172d49559e44c357a68c388933c74db2c4e819bf61377a6a95 model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:gb200-nv precision: fp4 - framework: vllm + framework: llmd-vllm router: { name: llm-d-router, version: "0.10.0" } multinode: true disagg: false diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index 4e53d31cdc..4ffbeca23b 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -59,7 +59,7 @@ class SrtLane: ), ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane(frameworks=_DYNAMO), ("gb200-nv", LaunchPath.SRT_MULTI): SrtLane( - frameworks=any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm", "vllm"), + frameworks=any_of("dynamo-sglang", "dynamo-trt", "dynamo-vllm", "llmd-vllm"), setup_scripts={"dynamo-sglang": "install-torchao.sh"}, mounts=( LaneMount( @@ -71,7 +71,7 @@ class SrtLane: ), shared_run_root=( Match(any_of("minimaxm3", "kimik3", "qwen3.5", "glm5.2")), - Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm", "vllm")), + Match(any_of("dsv4"), frameworks=any_of("dynamo-vllm", "llmd-vllm")), Match(any_of("dsv4"), frameworks=any_of("dynamo-sglang"), agentic=True), ), ), diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 77f28799bc..b7b47655af 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -24,6 +24,7 @@ "vllm-disagg": "vllm", "vllm": "vllm", "dynamo-vllm": "vllm", + "llmd-vllm": "vllm", "dynamo-sglang": "sglang", "trt": "trtllm", "dynamo-trt": "trtllm", diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 7f79bd6372..08b667be48 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10780,8 +10780,8 @@ pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4367 - config-keys: - - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-disagg - - dsv4-fp4-gb200-vllm-llm-d-agentic-mtp-agg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg scenario-type: - agentic-coding description: From 28e57e3a5d7f144af1694d8f53f241f94b42146a Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 16:34:17 -0500 Subject: [PATCH 07/12] fix: pin huggingface-hub below 2 when installing lm-eval Signed-off-by: Cam Quilici --- inferencex-e2e/infx/bench/eval/lm_eval.py | 4 +++- inferencex-e2e/infx/tests/bench/test_eval_command.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/infx/bench/eval/lm_eval.py b/inferencex-e2e/infx/bench/eval/lm_eval.py index cc44aea3c5..98eafba81e 100644 --- a/inferencex-e2e/infx/bench/eval/lm_eval.py +++ b/inferencex-e2e/infx/bench/eval/lm_eval.py @@ -32,7 +32,9 @@ def install(environ: Mapping[str, str]) -> None: return if "atom" in environ.get("IMAGE", ""): _pip(environ, "uninstall", "torchvision") - _pip(environ, "install", "lm-eval[api]") + # lm-eval does not depend on transformers, so without this range its resolution + # upgrades huggingface-hub past what transformers 5.x accepts. + _pip(environ, "install", "lm-eval[api]", "huggingface-hub>=1.5,<2") pinned = ("install", "--no-deps", "--reinstall") git = shutil.which("git", path=environ.get("PATH")) if git and _pip(environ, *pinned, f"git+{REPOSITORY}.git@{REF}"): diff --git a/inferencex-e2e/infx/tests/bench/test_eval_command.py b/inferencex-e2e/infx/tests/bench/test_eval_command.py index de2a82fb36..e7a7490093 100644 --- a/inferencex-e2e/infx/tests/bench/test_eval_command.py +++ b/inferencex-e2e/infx/tests/bench/test_eval_command.py @@ -400,7 +400,7 @@ def uv_step(argv: list[str]) -> str: """Name one recorded ``uv`` call of the lm-eval install into this interpreter.""" assert argv[0] == "pip" and argv[-2:] == ["--python", sys.executable] assert "--break-system-packages" in argv - spec = argv[-3] + spec = next(arg for arg in argv[2:] if not arg.startswith("-")) if argv[1] == "uninstall": return f"uninstall {spec}" if {"--no-deps", "--reinstall"} <= set(argv): From 907f8f24303e82c77ab0c3bd5792b2731ff30410 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 10 Oct 2026 00:10:52 -0500 Subject: [PATCH 08/12] fix: keep decode HTTP connections alive past the llm-d sidecar's idle timeout Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index f4c419cc81..f007fcc066 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -187,6 +187,9 @@ base: VLLM_LOG_STATS_INTERVAL: "1" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # Outlast the decode sidecar's 90 s idle-connection timeout, so it never reuses a + # connection the server has already closed (connection reset, 502). + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_USE_DEEP_GEMM: "1" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" From 73bfacca6c45882b04d03f76f0522e0c832a0af3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Sat, 10 Oct 2026 00:56:46 -0500 Subject: [PATCH 09/12] fix: keep agg HTTP connections alive past Envoy's upstream idle timeout Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index da2fb35ff4..150cb3f00e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -82,6 +82,9 @@ base: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + # Outlast Envoy's default 1 h upstream idle timeout, so it never reuses a connection + # the server has already closed (503 UC). + VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "3900" VLLM_DSV4_MEGA_FP8_COMBINE: "1" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" From 4f5357a1af63445d7a161e1d4cbfe7baaa17abdd Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Mon, 12 Oct 2026 17:43:11 +0200 Subject: [PATCH 10/12] fix(srt-slurm): restore adaptive DSpark verification for llm-d AgentX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match Sep 28 speculative policy on GB200 llm-d recipes (adaptive disagg/DEP8, golden AL only on agg TP8) and skip golden injection when adaptive is enabled. 在 GB200 llm-d AgentX 配方中恢复 9 月 28 日的投机策略（disagg/DEP8 自适应验证、 仅 agg TP8 使用黄金 AL），并在启用自适应时跳过黄金接受长度注入。 --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 2 +- .../vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 8 +-- .../infx/srt_slurm/synthetic_acceptance.py | 11 +++- .../srt_slurm/test_synthetic_acceptance.py | 55 +++++++++++++++++++ inferencex-e2e/perf-changelog.yaml | 9 +++ 5 files changed, 79 insertions(+), 6 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index 150cb3f00e..c74530894e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -124,7 +124,7 @@ base: no-enable-flashinfer-autotune: true numa-bind: true compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - speculative-config: '{"method":"dspark","num_speculative_tokens":3,"enable_adaptive_verification":false,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"enable_adaptive_verification":true,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4"}' attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' sbatch_directives: {cpus-per-task: "144", mem: "0"} srun_options: {container-remap-root: ""} diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index f007fcc066..43fed8d374 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -3,8 +3,8 @@ # DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix # cache through an embedded Mooncake store. srt-slurm launches each worker as one # `vllm serve` over its trays that balances its DP ranks itself, so the EPP routes to the -# worker rather than to one rank. Prefill drafts three DSpark tokens like decode: the srt -# launch planner gives both roles the decode role's golden acceptance length. +# worker rather than to one rank. Prefill drafts one DSpark token and decode three, both with +# adaptive verification on throughput (golden synthetic acceptance applies only to agg TP8). # VLLM_USE_RUST_FRONTEND is left unset (the Python API server). # Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. @@ -158,7 +158,7 @@ base: long-prefill-token-threshold: 1024 max-num-seqs: 16 max-model-len: 1048576 - speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + speculative-config: '{"method":"dspark","num_speculative_tokens":1,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}' enable-cumem-allocator: true block-size: 256 tokenizer-mode: deepseek_v4 @@ -224,7 +224,7 @@ base: max-num-seqs: 32 max-cudagraph-capture-size: 256 max-model-len: 1048576 - speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":false}' + speculative-config: '{"method":"dspark","num_speculative_tokens":3,"draft_sample_method":"probabilistic","attention_backend":"FLASHINFER_MLA_SPARSE_DSV4","enable_adaptive_verification":true}' enable-cumem-allocator: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index b7b47655af..6ab1a17a40 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -186,7 +186,16 @@ def build_overrides( if worker_engine == "vllm": if not worker_spec: continue - if worker_al is not None: + if worker_spec.get("enable_adaptive_verification") is True: + if ( + worker_spec.get("rejection_sample_method") == "synthetic" + or "synthetic_acceptance_length" in worker_spec + ): + worker_spec["rejection_sample_method"] = "block" + worker_spec.pop("synthetic_acceptance_length", None) + else: + continue + elif worker_al is not None: worker_spec.update( rejection_sample_method="synthetic", synthetic_acceptance_length=worker_al ) diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index f68e0a8e75..0e5c429cee 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -147,6 +147,61 @@ def test_thinking_mode_and_decode_priority(golden_dir: Path) -> None: assert result["roles"]["prefill"] == {"args": {"tensor-parallel-size": 8}} +def test_adaptive_dspark_skips_golden_acceptance_on_throughput(golden_dir: Path) -> None: + recipe = { + "roles": { + "prefill": { + "args": { + "speculative-config": json.dumps( + { + "method": "dspark", + "num_speculative_tokens": 1, + "enable_adaptive_verification": True, + } + ) + } + }, + "decode": { + "args": { + "speculative-config": json.dumps( + { + "method": "dspark", + "num_speculative_tokens": 3, + "enable_adaptive_verification": True, + } + ) + } + }, + } + } + original = copy.deepcopy(recipe) + result = apply_native( + recipe, + build_overrides( + recipe, + "llmd-vllm", + {**ENV, "SPEC_DECODING": "dspark"}, + golden_dir=golden_dir, + ), + ) + assert result == original + tp8_recipe = vllm_recipe( + "dspark", num_speculative_tokens=3, enable_adaptive_verification=False + ) + tp8 = apply_native( + tp8_recipe, + build_overrides( + tp8_recipe, + "vllm", + {**ENV, "SPEC_DECODING": "dspark"}, + golden_dir=golden_dir, + ), + ) + spec = json.loads(tp8["roles"]["agg"]["args"]["speculative-config"]) + assert spec["synthetic_acceptance_length"] == 2.7 + assert spec["rejection_sample_method"] == "synthetic" + + @pytest.mark.parametrize("sampling", [None, "unknown"]) def test_kimi_curve_requires_an_explicit_supported_sampler( golden_dir: Path, sampling: str | None diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 08b667be48..b1d4980396 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10788,3 +10788,12 @@ - "Add DeepSeek-V4-Pro-0813 DSpark FP4/FP8-KV AgentX on GB200 behind the llm-d router (EPP and pd-sidecar from llm-d-router v0.10.0, Envoy) through srt-slurm's llm-d frontend: disagg 1P1D DEP8/DEP8 (conc 64/128/160/256) and 2P1D DEP8/DEP16 (conc 256/384/576) with NIXL P/D KV and a Mooncake prefix-cache store on prefill at 66.7% node DRAM; agg TP8 (conc 1-12), DEP8 (conc 12-32), and DEP8 with Mooncake at 45% node DRAM (conc 52/72)." - "Each DEP worker is one vllm serve that balances its DP ranks itself, so the EPP routes to workers. Throughput uses committed golden synthetic acceptance (TP8 K=5, DEP K=3 on both prefill and decode); evals use real verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Match the Sep 28 standalone llm-d AgentX speculative policy: disagg and agg DEP8 (including Mooncake) use adaptive verification with prefill K=1 and decode/agg K=3; only agg TP8 keeps golden synthetic acceptance at K=5 (3.61). The srt-slurm synthetic-acceptance connector skips golden AL injection when enable_adaptive_verification is true." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 From 51617ba7b504698eac6acae03490a1af3c71237c Mon Sep 17 00:00:00 2001 From: Markov Ilya Date: Tue, 13 Oct 2026 22:23:20 +0200 Subject: [PATCH 11/12] feat(benchmark): llm-d DEP8 external LB, Rust FE, drop c576 disagg point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align AgentX llm-d recipes with Sep 28 rank-level EPP routing and stop sweeping 2P×1D concurrency 576 after warmup stalls. 为 GB200 AgentX llm-d 启用每 DP rank 的外部负载均衡与 Rust 前端，并从主配置搜索空间中移除反复 warmup 失败的 2P×1D 并发 576。 --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 10 ++++++---- .../dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 13 ++++++++----- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 9 +++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index c74530894e..3d094b0497 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -1,9 +1,8 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router # (srt-slurm frontend.type: llm-d): one aggregated worker over two trays, as TP8, -# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. srt-slurm launches a -# DEP8 worker as one `vllm serve` that balances its eight DP ranks itself, so the EPP -# routes to the worker rather than to one rank, and VLLM_USE_RUST_FRONTEND is left -# unset (the Python API server). +# DEP8, or DEP8 with an embedded Mooncake prefix-cache store. DEP8 uses +# data-parallel-external-lb so srtctl launches one Rust-fronted `vllm serve` per DP rank +# and the EPP routes to each rank (agg TP8 stays a single TP8 worker). # Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_. schema: 2 @@ -77,6 +76,7 @@ base: PYTHONHASHSEED: "0" HF_HUB_CACHE: /hf_hub_cache VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_USE_NCCL_SYMM_MEM: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" @@ -108,6 +108,7 @@ base: trust-remote-code: true tensor-parallel-size: 1 data-parallel-size: 8 + data-parallel-external-lb: true enable-expert-parallel: true kv-cache-dtype: fp8 gpu-memory-utilization: 0.88 @@ -179,6 +180,7 @@ override_tp8: args: tensor-parallel-size: 8 data-parallel-size: + data-parallel-external-lb: enable-expert-parallel: moe-backend: enable-ep-weight-filter: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index 43fed8d374..7285b9c0f0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -1,11 +1,10 @@ # DeepSeek-V4-Pro-0813 (DSpark) FP4 AgentX on GB200 behind the llm-d router # (srt-slurm frontend.type: llm-d): DEP8 prefill workers over two trays each and one # DEP8 or DEP16 decode worker. KV moves over NIXL and the prefill workers share prefix -# cache through an embedded Mooncake store. srt-slurm launches each worker as one -# `vllm serve` over its trays that balances its DP ranks itself, so the EPP routes to the -# worker rather than to one rank. Prefill drafts one DSpark token and decode three, both with -# adaptive verification on throughput (golden synthetic acceptance applies only to agg TP8). -# VLLM_USE_RUST_FRONTEND is left unset (the Python API server). +# cache through an embedded Mooncake store. DEP8 roles use data-parallel-external-lb so +# srtctl launches one `vllm serve` per DP rank (Rust frontend) and the EPP routes to each +# rank. Prefill drafts one DSpark token and decode three, both with adaptive verification on +# throughput (golden synthetic acceptance applies only to agg TP8). # Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. schema: 2 @@ -104,6 +103,7 @@ base: PYTHONHASHSEED: "0" HF_HUB_CACHE: /hf_hub_cache VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_USE_NCCL_SYMM_MEM: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" @@ -149,6 +149,7 @@ base: trust-remote-code: true tensor-parallel-size: 1 data-parallel-size: 8 + data-parallel-external-lb: true enable-expert-parallel: true # NIXL hands KV to decode; the Mooncake store shares prefix cache across prefill ranks. kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' @@ -179,6 +180,7 @@ base: PYTHONHASHSEED: "0" HF_HUB_CACHE: /hf_hub_cache VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" VLLM_SERVER_DEV_MODE: "1" VLLM_USE_NCCL_SYMM_MEM: "1" VLLM_ENGINE_READY_TIMEOUT_S: "3600" @@ -216,6 +218,7 @@ base: trust-remote-code: true tensor-parallel-size: 1 data-parallel-size: 8 + data-parallel-external-lb: true enable-expert-parallel: true kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false,"kv_lease_duration":1800}}' kv-cache-dtype: fp8 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 143d5285fa..70e7a43038 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5781,7 +5781,7 @@ dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg: - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake } - conc-list: [256, 384, 576] + conc-list: [256, 384] prefill: num-worker: 2 tp: 8 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index b1d4980396..8c82ba91cd 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10797,3 +10797,12 @@ description: - "Match the Sep 28 standalone llm-d AgentX speculative policy: disagg and agg DEP8 (including Mooncake) use adaptive verification with prefill K=1 and decode/agg K=3; only agg TP8 keeps golden synthetic acceptance at K=5 (3.61). The srt-slurm synthetic-acceptance connector skips golden AL injection when enable_adaptive_verification is true." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg + - dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg + scenario-type: + - agentic-coding + description: + - "Route llm-d DEP8 disagg and agg workers with data-parallel-external-lb (one vLLM API per DP rank for the EPP) and VLLM_USE_RUST_FRONTEND=1, matching the Sep 28 standalone llm-d wide-EP launch; agg TP8 keeps a single TP8 worker without external LB. Drop 2P×1D disagg agentic concurrency 576 from the master search space after repeated warmup stalls." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3741 From 5a436aafe2da24145885b1b9d0d054485b6e4309 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 22 Oct 2026 09:45:24 -0500 Subject: [PATCH 12/12] fix: select the llm-d recipes with srt-recipe-dir and srt-recipe llmd-vllm now launches through srt-slurm, which reads SRT_RECIPE rather than a CONFIG_FILE additional-setting. Signed-off-by: Cam Quilici --- .../dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml | 2 +- .../vllm/gb200-fp4/agentx/disagg-llm-d.yaml | 2 +- inferencex-e2e/configs/nvidia-master.yaml | 17 +++++++---------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml index 3d094b0497..e87b83bb98 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml @@ -3,7 +3,7 @@ # DEP8, or DEP8 with an embedded Mooncake prefix-cache store. DEP8 uses # data-parallel-external-lb so srtctl launches one Rust-fronted `vllm serve` per DP rank # and the EPP routes to each rank (agg TP8 stays a single TP8 worker). -# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_. +# Select one with srt-recipe: agg-llm-d.yaml:override_. schema: 2 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml index 7285b9c0f0..fd6c847285 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml @@ -5,7 +5,7 @@ # srtctl launches one `vllm serve` per DP rank (Rust frontend) and the EPP routes to each # rank. Prefill drafts one DSpark token and decode three, both with adaptive verification on # throughput (golden synthetic acceptance applies only to agg TP8). -# Select one with CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_. +# Select one with srt-recipe: disagg-llm-d.yaml:override_. schema: 2 diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 70e7a43038..464fa85db8 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -5756,6 +5756,7 @@ dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg: kv-p2p-transfer: nixl multinode: true disagg: true + srt-recipe-dir: dsv4/vllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.666667 @@ -5765,13 +5766,12 @@ dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake } conc-list: [64, 128, 160, 256] + srt-recipe: disagg-llm-d.yaml:override_1p1d prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_1p1d" decode: num-worker: 1 tp: 8 @@ -5782,13 +5782,12 @@ dsv4-fp4-gb200-llmd-vllm-agentic-mtp-disagg: kv-offloading: dram kv-offload-backend: { name: mooncake } conc-list: [256, 384] + srt-recipe: disagg-llm-d.yaml:override_2p1d prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/disagg-llm-d.yaml:override_2p1d" decode: num-worker: 1 tp: 16 @@ -5806,6 +5805,7 @@ dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg: router: { name: llm-d-router, version: "0.10.0" } multinode: true disagg: false + srt-recipe-dir: dsv4/vllm/gb200-fp4/agentx scenarios: agentic-coding: - dram-utilization: 0.45 @@ -5814,36 +5814,33 @@ dsv4-fp4-gb200-llmd-vllm-agentic-mtp-agg: kv-offloading: none conc-list: [1, 2, 4, 8, 12] num-nodes: 2 + srt-recipe: agg-llm-d.yaml:override_tp8 worker: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_tp8" - spec-decoding: mtp kv-offloading: none conc-list: [12, 16, 32] num-nodes: 2 + srt-recipe: agg-llm-d.yaml:override_dep8 worker: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8" - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: mooncake } conc-list: [52, 72] num-nodes: 2 + srt-recipe: agg-llm-d.yaml:override_dep8_mooncake worker: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/dsv4/vllm/gb200-fp4/agentx/agg-llm-d.yaml:override_dep8_mooncake" dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106