From e3ee574d423b781585eac0eb2f7f034206977aad Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 15:34:35 -0500 Subject: [PATCH] chore: remove the standalone llm-d launcher Signed-off-by: Cam Quilici --- inferencex-e2e/benchmarks/llm-d/Dockerfile | 44 -- inferencex-e2e/benchmarks/llm-d/README.md | 16 - inferencex-e2e/benchmarks/llm-d/binaries.env | 32 - inferencex-e2e/benchmarks/llm-d/envoy.yaml | 129 ---- .../benchmarks/llm-d/epp-config.yaml | 61 -- .../benchmarks/llm-d/extract-binaries.sh | 40 -- .../dsv4-fp4-gb200-low-latency.yaml | 192 ------ .../dsv4-fp4-gb200-mid-curve-megamoe.yaml | 196 ------ .../benchmarks/multi_node/llm-d/README.md | 142 ----- .../benchmarks/multi_node/llm-d/job.slurm | 282 --------- .../benchmarks/multi_node/llm-d/server.sh | 573 ------------------ .../benchmarks/multi_node/llm-d/submit.sh | 123 ---- .../benchmarks/multi_node/runtime_settings.sh | 7 - .../infx/launch/drivers/__init__.py | 3 +- inferencex-e2e/infx/launch/drivers/llmd.py | 161 ----- .../infx/launch/drivers/srt/models.py | 5 - inferencex-e2e/infx/launch/policy.py | 3 - inferencex-e2e/infx/launch/request.py | 14 - inferencex-e2e/infx/matrix/validation.py | 7 - .../infx/tests/launch/test_llmd_driver.py | 137 ----- .../infx/tests/launch/test_srt_policy.py | 1 - .../infx/tests/matrix/test_validation.py | 12 - 22 files changed, 1 insertion(+), 2179 deletions(-) delete mode 100644 inferencex-e2e/benchmarks/llm-d/Dockerfile delete mode 100644 inferencex-e2e/benchmarks/llm-d/README.md delete mode 100644 inferencex-e2e/benchmarks/llm-d/binaries.env delete mode 100644 inferencex-e2e/benchmarks/llm-d/envoy.yaml delete mode 100644 inferencex-e2e/benchmarks/llm-d/epp-config.yaml delete mode 100755 inferencex-e2e/benchmarks/llm-d/extract-binaries.sh delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d/README.md delete mode 100644 inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm delete mode 100755 inferencex-e2e/benchmarks/multi_node/llm-d/server.sh delete mode 100755 inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh delete mode 100644 inferencex-e2e/infx/launch/drivers/llmd.py delete mode 100644 inferencex-e2e/infx/tests/launch/test_llmd_driver.py diff --git a/inferencex-e2e/benchmarks/llm-d/Dockerfile b/inferencex-e2e/benchmarks/llm-d/Dockerfile deleted file mode 100644 index 2cd2bb6dff..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -# Combined image for the InferenceX llmd-vllm framework. -# -# Base = vllm/vllm-openai (vLLM with the OpenAI-compatible API server). -# We add the EPP, the routing-sidecar, and Envoy on top so every node in -# a SLURM allocation can play any role (prefill, decode, or coordinator) -# from a single image. -# -# Configs (epp-config.yaml, envoy.yaml, per-topology recipes) are NOT -# baked in. They are mounted at runtime by job.slurm so config-only -# iteration does not require an image rebuild. See -# benchmarks/multi_node/llm-d/job.slurm for the expected mount layout. - -# All FROM-referenced images are declared as global-scope ARGs. BuildKit -# does NOT expand a build-arg inside `COPY --from=${VAR}` directly ("variable -# expansion is not supported for --from"); the supported pattern is to bind -# each ARG to a named FROM stage in the global scope and COPY --from that -# stage alias. Defaults MUST match benchmarks/llm-d/binaries.env, which is the -# single source of truth (it also drives the mounted-binary path via -# extract-binaries.sh). Override at build time from that file, e.g.: -# source benchmarks/llm-d/binaries.env -# docker build \ -# --build-arg EPP_FROM_IMAGE="$EPP_FROM_IMAGE" \ -# --build-arg ROUTING_SIDECAR_IMAGE="$ROUTING_SIDECAR_IMAGE" \ -# --build-arg ENVOY_FROM_IMAGE="$ENVOY_FROM_IMAGE" ... -ARG VLLM_BASE=vllm/vllm-openai:v0.26.0 -ARG EPP_FROM_IMAGE=ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0 -ARG ROUTING_SIDECAR_IMAGE=ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0 -ARG ENVOY_FROM_IMAGE=envoyproxy/envoy:distroless-v1.33.2 - -# Stage aliases for the three add-on binary sources (see note above). -FROM ${EPP_FROM_IMAGE} AS epp_src -FROM ${ROUTING_SIDECAR_IMAGE} AS sidecar_src -FROM ${ENVOY_FROM_IMAGE} AS envoy_src - -FROM ${VLLM_BASE} - -COPY --from=epp_src \ - /app/epp /usr/local/bin/epp - -COPY --from=sidecar_src \ - /app/pd-sidecar /usr/local/bin/pd-sidecar - -COPY --from=envoy_src \ - /usr/local/bin/envoy /usr/local/bin/ diff --git a/inferencex-e2e/benchmarks/llm-d/README.md b/inferencex-e2e/benchmarks/llm-d/README.md deleted file mode 100644 index 027a18eee5..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# llmd-vllm framework artifacts - -This directory holds the static, baked-into-the-image pieces of the -`llmd-vllm` benchmark framework. - -| File | Purpose | -|---|---| -| `Dockerfile` | Combined image: vLLM (DeepEP-enabled), EPP, routing-sidecar, Envoy. One image, every node uses what its role requires. | -| `epp-config.yaml` | Fallback EPP scheduling config. Used when no recipe overrides it via `CONFIG_FILE`. `disagg-profile-handler` + `kv-cache-utilization-scorer` + `random-picker` over the file-discovery endpoint set. | -| `envoy.yaml` | Static Envoy: listener `:8080`, ext_proc to `127.0.0.1:9002`, ORIGINAL_DST cluster reading `x-gateway-destination-endpoint`. | - -The runtime pieces (per-node `server.sh`, the SLURM job script, recipe -files, and the endpoint discovery mechanism) live under -`benchmarks/multi_node/llm-d/` and `benchmarks/multi_node/llm-d-recipes/`. -See the README in `benchmarks/multi_node/llm-d/` for the endpoints-file -generation flow. diff --git a/inferencex-e2e/benchmarks/llm-d/binaries.env b/inferencex-e2e/benchmarks/llm-d/binaries.env deleted file mode 100644 index 4782f489b8..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/binaries.env +++ /dev/null @@ -1,32 +0,0 @@ -# Single source of truth for the non-vLLM executables the llmd-vllm -# framework needs (EPP, P/D sidecar, Envoy). -# -# This file is sourced by benchmarks/multi_node/llm-d/job.slurm to -# optionally bind-mount extracted binaries (via $LLMD_BIN_DIR) in place -# of the combined image's baked-in copies. extract-binaries.sh pulls -# each image once and copies the binary out to $LLMD_BIN_DIR; the -# Dockerfile reads the same URLs when building the combined image. -# -# By default this is a no-op: we build and use an image that already -# contains all binaries, so the job.slurm mount loop skips them (its -# -x check fails until extract-binaries.sh has populated $LLMD_BIN_DIR). - -# --- Endpoint Picker (EPP / inference scheduler) --- -EPP_FROM_IMAGE="ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0" -EPP_BIN_PATH="/app/epp" - -# --- P/D routing sidecar (pd-sidecar) --- -ROUTING_SIDECAR_IMAGE="ghcr.io/llm-d/llm-d-router-disagg-sidecar:v0.9.0" -ROUTING_SIDECAR_BIN_PATH="/app/pd-sidecar" - -# --- Envoy front proxy --- -ENVOY_FROM_IMAGE="envoyproxy/envoy:distroless-v1.33.2" -ENVOY_BIN_PATH="/usr/local/bin/envoy" - -# Platform the binaries must target (GB200 = Grace = arm64). -LLMD_BIN_PLATFORM="${LLMD_BIN_PLATFORM:-linux/arm64}" - -# Shared-filesystem dir the extracted binaries live in, mounted into -# /usr/local/bin by job.slurm. Defaults to the same Lustre area the -# launcher already caches enroot squash images in (proven writable). -LLMD_BIN_DIR="${LLMD_BIN_DIR:-/mnt/lustre01/users-public/sa-shared/llm-d-bins}" diff --git a/inferencex-e2e/benchmarks/llm-d/envoy.yaml b/inferencex-e2e/benchmarks/llm-d/envoy.yaml deleted file mode 100644 index ccba51ba0b..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/envoy.yaml +++ /dev/null @@ -1,129 +0,0 @@ -# Envoy front door for the llmd-vllm framework. -# -# Listener : 0.0.0.0:8080 (benchmark client target) -# ext_proc : EPP on 127.0.0.1:9002 -# Cluster : ORIGINAL_DST, picks the address from the -# x-gateway-destination-endpoint header that EPP sets. - -static_resources: - listeners: - - name: main - address: - socket_address: { address: 0.0.0.0, port_value: 8080 } - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - stat_prefix: ingress_http - codec_type: AUTO - stream_idle_timeout: 0s - request_timeout: 0s - # Write per-request access logs to their own file so - # they survive an SIGTERM-on-failure: when envoy's - # stdout is captured via `> envoy.log`, glibc - # block-buffers it, and the buffer is lost when srun - # SIGTERMs envoy after a failed bench. FileAccessLog - # opens its own fd and flushes per record. - access_log: - - name: envoy.access_loggers.file - typed_config: - "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog - path: /benchmark_logs/envoy_access.log - log_format: - text_format_source: - inline_string: >- - "[ACCESS] %REQ(:METHOD)% %REQ(:PATH)% %PROTOCOL% - status=%RESPONSE_CODE% flags=%RESPONSE_FLAGS% - upstream=%UPSTREAM_HOST% cluster=%UPSTREAM_CLUSTER% - dst=%REQ(x-gateway-destination-endpoint)% - duration=%DURATION%ms\n" - route_config: - name: route - virtual_hosts: - - name: vh - domains: ["*"] - routes: - - match: { prefix: "/" } - route: - cluster: original_dst - timeout: 0s - http_filters: - - name: envoy.filters.http.ext_proc - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor - grpc_service: - envoy_grpc: - cluster_name: epp - timeout: 10s - # message_timeout caps how long Envoy will wait for any - # one ext_proc message ack from EPP. Generation can take - # many seconds; 1000s mirrors the upstream llm-d guide. - message_timeout: 1000s - # FULL_DUPLEX_STREAMED for both directions: the dev EPP - # (ghcr.io/llm-d/llm-d-router-endpoint-picker-dev:main) - # does not ack BUFFERED body mode and Envoy times out - # with 504. Trailer modes also have to be SEND for the - # request lifecycle to terminate cleanly. - processing_mode: - request_header_mode: SEND - response_header_mode: SEND - request_body_mode: FULL_DUPLEX_STREAMED - response_body_mode: FULL_DUPLEX_STREAMED - request_trailer_mode: SEND - response_trailer_mode: SEND - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - clusters: - - name: epp - type: STATIC - connect_timeout: 1s - # Without an explicit circuit_breakers block Envoy applies its DEFAULT - # per-cluster thresholds: max_connections/max_pending_requests/max_requests - # all = 1024. The ext_proc filter holds one long-lived HTTP/2 stream to - # this EPP cluster for the ENTIRE lifetime of every request (header -> - # FULL_DUPLEX_STREAMED body -> response -> trailers, up to message_timeout - # 1000s). At conc4096 that means only 1024 active + 1024 pending = 2048 - # requests ever reach EPP; the rest get a fast 500 and never see the - # engine. This capped every high-tpt run at exactly 1024 completions - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 40000 - max_pending_requests: 40000 - max_requests: 40000 - max_retries: 1024 - typed_extension_protocol_options: - envoy.extensions.upstreams.http.v3.HttpProtocolOptions: - "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions - explicit_http_config: - http2_protocol_options: {} - load_assignment: - cluster_name: epp - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: { address: 127.0.0.1, port_value: 9002 } - - name: original_dst - type: ORIGINAL_DST - lb_policy: CLUSTER_PROVIDED - connect_timeout: 5s - # Same default-1024 circuit-breaker trap as the epp cluster above: this is - # the cluster that actually carries request bodies to the picked - # prefill/decode endpoint, so its budget must also clear max-concurrency. - circuit_breakers: - thresholds: - - priority: DEFAULT - max_connections: 40000 - max_pending_requests: 40000 - max_requests: 40000 - max_retries: 1024 - original_dst_lb_config: - use_http_header: true - http_header_name: x-gateway-destination-endpoint - -admin: - address: - socket_address: { address: 0.0.0.0, port_value: 9901 } diff --git a/inferencex-e2e/benchmarks/llm-d/epp-config.yaml b/inferencex-e2e/benchmarks/llm-d/epp-config.yaml deleted file mode 100644 index 3ff0eea87f..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/epp-config.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Default EPP scheduling config (fallback when CONFIG_FILE is unset). -# -# Mirrors the upstream llm-d well-lit-path P/D guide: -# guides/pd-disaggregation/router/pd-disaggregation.values.yaml -# in github.com/llm-d/llm-d. Plugins, scheduling profiles, and scorer -# weights are unchanged from upstream. -# -# Single delta vs upstream: file-discovery. The upstream guide assumes -# a Kubernetes control plane drives endpoint discovery; in our SLURM -# setup the coordinator node writes /tmp/endpoints.yaml at job start -# (see benchmarks/multi_node/llm-d/README.md) and EPP loads it via the -# file-discovery plugin instead. - -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - # Endpoint discovery (replaces upstream's K8s discovery). - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - # P/D routing - identical to upstream pd-disaggregation guide. - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: kv-cache-utilization-scorer - - type: active-request-scorer - - type: max-score-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: kv-cache-utilization-scorer - weight: 2 - - pluginRef: max-score-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: max-score-picker - -dataLayer: - discovery: - pluginRef: file-disc diff --git a/inferencex-e2e/benchmarks/llm-d/extract-binaries.sh b/inferencex-e2e/benchmarks/llm-d/extract-binaries.sh deleted file mode 100755 index bdb1968518..0000000000 --- a/inferencex-e2e/benchmarks/llm-d/extract-binaries.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Extract the EPP, pd-sidecar, and Envoy binaries into $LLMD_BIN_DIR so a stock -# vllm/vllm-openai image can mount them at runtime (see job.slurm) instead of -# rebuilding a combined image on every vLLM bump. Needs docker (arm64, or x86 -# with --platform emulation); re-run only when a source image in binaries.env -# changes. Idempotent. -# Usage: [LLMD_BIN_DIR=dir] [LLMD_BIN_PLATFORM=linux/amd64] ./extract-binaries.sh - -set -eo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=/dev/null -source "$HERE/binaries.env" - -echo "Extracting llm-d binaries -> $LLMD_BIN_DIR (platform $LLMD_BIN_PLATFORM)" -mkdir -p "$LLMD_BIN_DIR" - -# (source image, path inside image, output filename) -extract() { - local image="$1" src="$2" out="$3" - echo " $out <- $image:$src" - local cid - cid="$(docker create --platform "$LLMD_BIN_PLATFORM" "$image")" - # shellcheck disable=SC2064 - trap "docker rm -f '$cid' >/dev/null 2>&1 || true" RETURN - docker cp "$cid:$src" "$LLMD_BIN_DIR/$out" - chmod +x "$LLMD_BIN_DIR/$out" -} - -extract "$EPP_FROM_IMAGE" "$EPP_BIN_PATH" epp -extract "$ROUTING_SIDECAR_IMAGE" "$ROUTING_SIDECAR_BIN_PATH" pd-sidecar -extract "$ENVOY_FROM_IMAGE" "$ENVOY_BIN_PATH" envoy - -echo "Done. Contents of $LLMD_BIN_DIR:" -ls -la "$LLMD_BIN_DIR" - -# epp/pd-sidecar are static Go binaries; envoy is dynamically linked C++. -echo -echo "NOTE: verify 'ldd $LLMD_BIN_DIR/envoy' resolves cleanly inside the" -echo " target vLLM image before relying on the mounted-binary path." diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml b/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml deleted file mode 100644 index e29106062d..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-low-latency.yaml +++ /dev/null @@ -1,192 +0,0 @@ -# DeepSeek-V4-Pro (FP4) on GB200, low-latency P/D disagg via llmd-vllm. -# -# Latency end of the curve. Engine config mirrors the upstream srt-slurm -# recipe disagg-gb200-low-latency.yaml (less the dynamo-only frontend keys), -# served via the InferenceX llmd-vllm path (no NATS/etcd infra node). It is -# the direct sibling of the dsv4-fp4-gb200-dynamo-vllm conc=1 low-latency -# point (same topology: 1 prefill DEP8 + 1 decode pure-TP8). -# -# Topology (set by the nvidia-master.yaml key via additional-settings): -# low-latency key (conc 1): 1P1D = 1 prefill + 1 decode engine, 4 GB200 -# nodes / 16 GPUs. PREFILL_NODES=2 DECODE_NODES=2. -# -# The ONLY difference from the mid-curve-megamoe recipe is the DECODE engine: -# 1. DECODE is PURE TP=8 (tensor-parallel-size 8, NO DP, NO EP) - ONE model -# replica across 8 GPUs / 2 GB200 nodes, launched headless multi-node TP -# (leader + follower). This is the latency-optimal decode: no all-to-all -# MoE routing, no DP-attention. `enable-expert-parallel: false` + -# decode ep:1/dp-attn:false in the master key is what selects the -# pure-TP path in server.sh. Decode knobs are latency-sized: max-num-seqs -# 256, max-num-batched-tokens 256, max-cudagraph-capture-size 256, -# FULL_DECODE_ONLY graphs, gpu-memory-utilization 0.9. -# 2. PREFILL is IDENTICAL to the mid-curve/high-tpt prefill (copied verbatim, -# DEP8 with enable-ep-weight-filter + moe-backend deep_gemm_mega_moe). An -# earlier srt-slurm-style low-latency prefill (weight-offload, no filter, -# moe-backend auto) warmed up ALL expert shapes (~4069-step DeepGEMM) and -# never became ready in the bring-up window; the mid-curve prefill's -# weight-filter collapses that warmup, and it is already proven on GB200. -# -# ---- EPP scheduling config (identical to the mid-curve recipe) ---- -# EPP routing is engine-config-agnostic, so the scheduling profiles match the -# upstream generic wide-ep-lws router config byte-for-byte: -# prefill = prefix-cache(3) + queue(2) + active-request(2) -# decode = active-request only -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: active-request-scorer - - type: weighted-random-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - -dataLayer: - discovery: - pluginRef: file-disc - -# ---- Per-role vLLM flags ---- -# Prefill: DEP=8 (TP=1, DP=8, EP). IDENTICAL to the mid-curve/high-tpt prefill -# block - copied verbatim (all args + env). enable-ep-weight-filter + -# moe-backend deep_gemm_mega_moe keep only the served experts resident and -# collapse the DeepGEMM warmup, so bring-up is fast. A pure srt-slurm-style -# low-latency prefill (weight-offload, no filter, moe-backend auto) instead -# warmed up ALL expert shapes (~4069-step DeepGEMM) and never became ready -# inside the bring-up window (e2e runs 28742781217 / 28744015942). Only the -# DECODE below differs from mid-curve: pure TP=8 instead of DEP=8. -prefill: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --enforce-eager - --gpu-memory-utilization 0.95 - --max-model-len 9280 - --max-num-seqs 16 - --max-num-batched-tokens 32768 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-async-scheduling - --block-size 256 - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - --no-enable-flashinfer-autotune - --numa-bind - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - # dynamo-vllm parity (2026-06-27): GPUDirect-RDMA over the GB200 - # Grace-Blackwell C2C link - lifts cross-node NCCL collective (DP8/EP8 - # MoE all-to-all) bandwidth. Dynamo sets it; we did not. - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # Added rc (IB verbs) so cross-node KV falls back to InfiniBand, not - # TCP, when MNNVL/cuda_ipc is unavailable. Prior value lacked rc, so - # the fallback was tcp (slow). Primary path is still MNNVL via - # cuda_ipc + cumem allocator. - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - # dynamo-vllm parity (2026-06-27): no NVSHMEM remote transport (we don't - # use DeepEP), plus explicit NIC->PE mapping. server.sh defaults - # NVSHMEM_REMOTE_TRANSPORT to ibgda but honors this override and clears - # the image's NVSHMEM_HCA_LIST so this PE mapping takes effect. - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" - # Harden distributed coordination against transient gloo/NCCL resets under - # load (matters for the low-middle 1P4D point; harmless at conc=1). Matches - # the DEP8 recipe + srt-slurm (TORCH_DISTRIBUTED_DEFAULT_TIMEOUT 1800). - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# Decode: PURE TP=8 (one replica across 8 GPUs / 2 nodes). NO EP, NO DP. -# Mirrors the srt-slurm low-latency decode vllm_config. tensor-parallel-size -# is supplied by server.sh from the master key (tp:8); the recipe sets EP off. -decode: - tp: 8 - enable-expert-parallel: false - extra-args: >- - --kv-cache-dtype fp8 - --max-model-len 16384 - --max-num-seqs 256 - --max-num-batched-tokens 256 - --max-cudagraph-capture-size 256 - --gpu-memory-utilization 0.9 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-enable-flashinfer-autotune - --block-size 256 - --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --tokenizer-mode deepseek_v4 - --no-disable-hybrid-kv-cache-manager - --enable-sleep-mode - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # rc added so KV falls back to InfiniBand, not TCP (see prefill note). - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - VLLM_USE_NCCL_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# ---- SLURM resource directives ---- -slurm: - time_limit: "08:00:00" diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml b/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml deleted file mode 100644 index 89292bc09c..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d-recipes/dsv4-fp4-gb200-mid-curve-megamoe.yaml +++ /dev/null @@ -1,196 +0,0 @@ -# DeepSeek-V4-Pro (FP4) on GB200, MegaMOE P/D disagg via llmd-vllm. -# -# This file defines only the COMMON per-engine shape + engine env. The -# multi-engine topology (how many prefill/decode engines, node counts, -# concurrency) is set by the nvidia-master.yaml key via additional-settings, -# NOT here - so the same recipe backs BOTH supported topologies: -# mid-curve key (conc 256/512/1024): 1P1D = 1 prefill + 1 decode engine, -# 4 GB200 nodes / 16 GPUs. PREFILL_NODES=2 DECODE_NODES=2. -# high-tpt key (conc 4096): 2P1D = 2 prefill + 1 decode engine, -# 6 GB200 nodes / 24 GPUs. PREFILL_NODES=4 PREFILL_WORKERS=2 DECODE_NODES=2. -# -# Per-engine shape (identical in every engine; GB200 = 4 GPUs/node): -# Each engine is DEP8 - TP=1 DP=8 EP=8, spanning 2 nodes (LWS_GROUP_SIZE=2), -# with DP-attention + deep_gemm_mega_moe + EP weight filter. Decode engines -# additionally use FULL_DECODE_ONLY graphs. -# -# Selected via additional-settings: CONFIG_FILE=, GPUS_PER_NODE=4, -# plus the PREFILL_NODES/PREFILL_WORKERS/DECODE_NODES for the chosen topology. -# -# -# ---- EPP scheduling config (matched to upstream wide-ep-lws) ---- -# Scorers match the upstream generic wide-ep-lws router config -# (llm-d guides/wide-ep-lws/router/wide-ep-lws.values.yaml): -# prefill = prefix-cache(3) + queue(2) + active-request(2) -# decode = active-request only -# EPP routing is engine-config-agnostic, so this recipe file also backs the -# high-tpt (2P1D conc4096) key. -apiVersion: llm-d.ai/v1alpha1 -kind: EndpointPickerConfig - -plugins: - - name: file-disc - type: file-discovery - parameters: - path: /tmp/endpoints.yaml - watchFile: false - - - type: disagg-headers-handler - - type: always-disagg-pd-decider - - type: disagg-profile-handler - parameters: - deciderPluginName: always-disagg-pd-decider - - type: prefill-filter - - type: decode-filter - - type: prefix-cache-scorer - - type: queue-scorer - - type: active-request-scorer - - type: weighted-random-picker - -schedulingProfiles: - - name: prefill - plugins: - - pluginRef: prefill-filter - - pluginRef: prefix-cache-scorer - weight: 3 - - pluginRef: queue-scorer - weight: 2 - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - - name: decode - plugins: - - pluginRef: decode-filter - - pluginRef: active-request-scorer - weight: 2 - - pluginRef: weighted-random-picker - -dataLayer: - discovery: - pluginRef: file-disc - -# ---- Per-role vLLM flags ---- -# Engine config mirrors the upstream srt-slurm mid-curve-megamoe recipe -# byte-for-byte (less the dynamo-only frontend keys). The MegaMOE shape -# differs from the low-latency 1P+1D in three ways: -# 1. Decode is DEP=8 (TP=1, DP=8, EP) instead of pure TP=8. This is -# what the "MegaMOE" name refers to: every MoE rank runs experts -# locally with DP-attention to hide cross-rank latency, instead of -# sharding tensors. -# 2. moe-backend = deep_gemm_mega_moe on both sides (the higher-perf -# path; the simple recipe doesn't override moe-backend). -# 3. enable-ep-weight-filter on both sides keeps only the experts a -# rank actually serves resident on GPU; without it, DEP=8 OOMs -# because every rank tries to hold all experts. -prefill: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --enforce-eager - --gpu-memory-utilization 0.92 - --max-model-len 9280 - --max-num-seqs 16 - --max-num-batched-tokens 32768 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-async-scheduling - --block-size 256 - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - --no-enable-flashinfer-autotune - --numa-bind - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - # dynamo-vllm parity (2026-06-27): GPUDirect-RDMA over the GB200 - # Grace-Blackwell C2C link - lifts cross-node NCCL collective (DP8/EP8 - # MoE all-to-all) bandwidth. Dynamo sets it; we did not. - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # Added rc (IB verbs) so cross-node KV falls back to InfiniBand, not - # TCP, when MNNVL/cuda_ipc is unavailable. Prior value lacked rc, so - # the fallback was tcp (slow). Primary path is still MNNVL via - # cuda_ipc + cumem allocator. - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - # dynamo-vllm parity (2026-06-27): no NVSHMEM remote transport (we don't - # use DeepEP), plus explicit NIC->PE mapping. server.sh defaults - # NVSHMEM_REMOTE_TRANSPORT to ibgda but honors this override and clears - # the image's NVSHMEM_HCA_LIST so this PE mapping takes effect. - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - VLLM_HTTP_TIMEOUT_KEEP_ALIVE: "120" - # Harden distributed coordination against transient gloo/NCCL connection - # resets under the conc4096 warmup burst. max-tpt in-sweep died with - # "gloo tcp Connection reset by peer" -> EngineDeadError (no OOM/CUDA/HW - # fault); a longer PG timeout keeps a momentarily-stalled peer from being - # dropped. Value matches srt-slurm (TORCH_DISTRIBUTED_DEFAULT_TIMEOUT 1800). - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -decode: - tp: 1 - enable-expert-parallel: true - extra-args: >- - --kv-cache-dtype fp8 - --max-num-seqs 512 - --max-num-batched-tokens 512 - --max-cudagraph-capture-size 512 - --gpu-memory-utilization 0.9 - --max-model-len 9280 - --enable-cumem-allocator - --no-enable-prefix-caching - --no-enable-flashinfer-autotune - --block-size 256 - --compilation-config {"cudagraph_mode":"FULL_DECODE_ONLY","mode":0} - --tokenizer-mode deepseek_v4 - --moe-backend deep_gemm_mega_moe - --enable-ep-weight-filter - --no-disable-hybrid-kv-cache-manager - env: - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_P2P_LEVEL: "NVL" - NCCL_NET_GDR_C2C: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - # rc added so KV falls back to InfiniBand, not TCP (see prefill note). - UCX_TLS: "cuda_copy,cuda_ipc,rc,tcp" - UCX_CUDA_IPC_ENABLE_MNNVL: "y" - NVSHMEM_REMOTE_TRANSPORT: "none" - NVSHMEM_ENABLE_NIC_PE_MAPPING: "1" - NVSHMEM_HCA_PE_MAPPING: "mlx5_0:1:1,mlx5_1:1:1,mlx5_3:1:1,mlx5_4:1:1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TILELANG_CLEANUP_TEMP_FILES: "1" - NVSHMEM_CUMEM_HANDLE_TYPE: "FABRIC" - NVSHMEM_DISABLE_CUDA_VMM: "0" - VLLM_SKIP_P2P_CHECK: "1" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_USE_DEEP_GEMM: "1" - VLLM_USE_RUST_FRONTEND: "1" - NVIDIA_GDRCOPY: "enabled" - # See prefill note: harden the decode's cross-node DP8/EP8 coordination - # against transient gloo resets under the conc4096 warmup burst. - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: "1800" - -# ---- SLURM resource directives ---- -slurm: - time_limit: "08:00:00" diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/README.md b/inferencex-e2e/benchmarks/multi_node/llm-d/README.md deleted file mode 100644 index 55c0575fec..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/README.md +++ /dev/null @@ -1,142 +0,0 @@ -# llmd-vllm multi-node SLURM scaffolding - -This directory holds standalone SLURM-side orchestration for the `llmd-vllm` -benchmark framework. InferenceX itself owns the SLURM job, with no vendor -multi-node tool involved. This scaffolding is separate from the active -srt-slurm launch path. - -| File | Role | -|---|---| -| `submit.sh` | sbatch wrapper. Validates env, exports tuning vars, returns `JOB_ID`. May read `slurm.time_limit` from the recipe to override `TIME_LIMIT`. | -| `job.slurm` | sbatch entrypoint. Allocates `PREFILL_NODES + DECODE_NODES` nodes, derives per-node IPs, runs one Docker container per node via `srun`, threads role assignment env into each. | -| `server.sh` | Per-node entry. Reads `NODE_RANK = SLURM_PROCID`, picks role, starts vLLM (with the wide-EP / DeepEP / NIXL flag set from the llm-d wide-EP-lws guide), starts the pd-sidecar on each leader, and on the decode leader additionally writes `endpoints.yaml`, starts EPP + Envoy, runs `python3 -m infx.bench fixed-seq point` per concurrency (then `python3 -m infx.bench eval` when `RUN_EVAL=true`), and signals `job.slurm` to end the job. | - -## Topology - -For an `xP` prefill nodes / `yD` decode nodes run, total nodes = `xP + yD`. -There is **no dedicated coordinator node**. The decode leader doubles as -the coordinator (EPP + Envoy + bench). - -| Rank | Role | -|---|---| -| `0` | prefill leader (`LWS_WORKER_INDEX=0`, DP rank 0) + pd-sidecar | -| `1 .. xP-1` | prefill workers | -| `xP` | decode leader + pd-sidecar + EPP + Envoy + benchmark client | -| `xP+1 .. xP+yD-1` | decode workers | - -Each instance (prefill or decode) is one vLLM engine spanning multiple -nodes via `--data-parallel-hybrid-lb`. With `xP=2, yD=2, -GPUS_PER_NODE=8` you get DP=16 prefill + DP=16 decode (the wide-EP -reference). Per-rank split: `--data-parallel-size 16 ---data-parallel-size-local 8 --data-parallel-start-rank -$((LWS_WORKER_INDEX * 8))`. - -## How `endpoints.yaml` is generated (file-discovery contract) - -The EPP runs in **no-Kubernetes mode**, using the `file-discovery` plugin -from `llm-d-inference-scheduler` (branch `filediscovery-4`). At startup, -it reads `/tmp/endpoints.yaml`. The file lists every backend the EPP can -route to, with role labels. - -The file is generated at runtime by `server.sh` on the decode leader -(rank `PREFILL_NODES`). Because all node IPs are only known after -`sbatch` allocates the job, the file cannot be baked into the image and -is not part of the repo. - -Generation flow: - -1. `submit.sh` calls `sbatch -N (xP+yD)`. `sbatch` allocates nodes. -2. `job.slurm` resolves each node's IP via `srun ip route get 1.1.1.1`, - slices them into `PREFILL_LEADER_IP` (= IPS[0]) and `DECODE_LEADER_IP` - (= IPS[PREFILL_NODES]), and passes both into the container as env - vars. -3. On the decode leader, `server.sh` writes `/tmp/endpoints.yaml` - inside the container with one entry per node: - - ```yaml - endpoints: - - name: prefill-0 - address: - port: "8200" # vLLM port (EPP hits prefill vLLM directly) - labels: - llm-d.ai/role: prefill - - name: prefill-1 # one entry per prefill node - address: - port: "8200" - labels: - llm-d.ai/role: prefill - - name: decode-0 - address: - port: "8000" # pd-sidecar port - labels: - llm-d.ai/role: decode - - name: decode-1 # one entry per decode node - address: - port: "8000" - labels: - llm-d.ai/role: decode - ``` - -4. The EPP (started immediately after) loads the file via - `dataLayer.discovery.pluginRef: file-disc` (see - `benchmarks/llm-d/epp-config.yaml`). The plugin enumerates the - endpoints into the EPP datastore before the EPP starts serving - `ext_proc`, so Envoy never gets a request before discovery is ready. -5. The `disagg-profile-handler` in the EPP config uses `prefill-filter` - and `decode-filter` to pick the right backend per request phase, - matching on the `llm-d.ai/role` label. - -### Why one entry per node (not per DP rank) - -Each instance is a vLLM engine that spans multiple nodes via -`--data-parallel-hybrid-lb`. With hybrid-lb, every node runs its own -api-server (prefill) or pd-sidecar (decode) on a fixed port -(`VLLM_PORT`=8200 / `SIDECAR_PORT`=8000) and internally load-balances -its own local DP ranks. So `server.sh` `add_role()` emits one endpoint -per node (`prefill-0..N-1`, `decode-0..M-1`) and lets EPP fan out -across nodes, while each node's hybrid-lb spreads work across its local -ranks. - -Per-node (rather than per-DP-rank) emission is what the multi-engine -high-tpt topology needs: with `PREFILL_WORKERS>1` the prefill nodes form -several independent DEP engines, and one endpoint per node is what -exposes every engine to EPP. Emitting per-DP-rank instead would address -ranks directly and bypass each node's internal hybrid-lb. - -### Live reload - -`watchFile: false` in `epp-config.yaml`. Endpoints are static for the -job lifetime - no reason to pay for `fsnotify` here. Set `watchFile: -true` (and rewrite `/tmp/endpoints.yaml` from the coordinator) only if -you want to drain or add an instance mid-run. - -### Validation rules (enforced by the plugin) - -- `address` must be a literal IPv4 address (no IPv6, no hostnames). -- `port` is a string in `1..65535`. -- File capped at 1 MiB. -- Names must be unique within their namespace (we use the default - namespace, so they must be globally unique in the file). - -The IPs we collect from `ip route get 1.1.1.1` are always IPv4 on the -H200 / B200 cluster's primary fabric. If you point at a different -interface and it returns an IPv6 address, EPP will reject the file at -startup. - -## Recipe files - -`benchmarks/multi_node/llm-d-recipes/.yaml` is selected via -`CONFIG_FILE=.yaml` in the master config's `additional-settings`. -Each recipe carries: - -- top-level `plugins:` / `schedulingProfiles:` / `dataLayer:` - fed into - the EPP via `--config-file`. Lets you change routing strategy without - rebuilding the image. -- `prefill:` / `decode:` blocks with `extra-args` (appended to the vLLM - launch command on each node of that role) and `env` (exported before - vLLM starts). -- `slurm.time_limit` - overrides `TIME_LIMIT` for that recipe. - -When `CONFIG_FILE` is unset or the file is missing, the EPP falls back -to `/etc/epp/config.yaml` baked into the image, and vLLM runs with no -extra flags beyond the wide-EP common set in `server.sh`. diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm b/inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm deleted file mode 100644 index 65e9dde4e0..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/job.slurm +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=llm-d-bench -#SBATCH --ntasks-per-node=1 -# --output, --error, -N, -n, --time set by submit.sh -# -# Allocates PREFILL_NODES + DECODE_NODES nodes, derives per-node IPs, then -# srun-runs server.sh inside one Docker container per node. NODE_RANK -# (= SLURM_PROCID) drives role selection inside server.sh. - -set -eo pipefail - -echo "=== llm-d job start ===" -echo "UTC: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" - -# Repo root. $(pwd) = the sbatch submit dir, which the wrapper sets to -# benchmarks/multi_node/llm-d/ before invoking submit.sh, so 3 up = -# repo root. Using $(dirname "$0") would resolve to a SLURM staging -# copy under /var/spool/... and miss the checkout entirely. -DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) -export DI_REPO_DIR -echo "REPO DIR: ${DI_REPO_DIR}" - -ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) -echo "Allocated nodes ($TOTAL_NODES): $(echo "$ALL_NODES" | tr '\n' ' ')" - -if [[ "$TOTAL_NODES" -ne "$NUM_NODES" ]]; then - echo "Error: SLURM allocated $TOTAL_NODES nodes, expected $NUM_NODES" >&2 - exit 1 -fi - -# Per-node IPs in rank order. -IPS=() -for NODE in $ALL_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --nodelist="$NODE" \ - bash -c 'ip route get 1.1.1.1 | awk "/src/ {print \$7}"') - IPS+=("$IP") -done -echo "Node IPs: ${IPS[*]}" - -if ! srun --nodes="$NUM_NODES" --ntasks-per-node=1 test -d "$MODEL_DIR"; then - echo "FATAL: model not found on all $NUM_NODES nodes: $MODEL_DIR" >&2 - exit 1 -fi - -# Rank slicing: -# prefill leader = rank 0 -# prefill workers = ranks 1 .. PREFILL_NODES-1 -# decode leader = rank PREFILL_NODES (also coordinator: EPP + Envoy + bench) -# decode workers = ranks PREFILL_NODES+1 .. NUM_NODES-1 -PREFILL_LEADER_IP="${IPS[0]}" -DECODE_LEADER_IP="${IPS[$PREFILL_NODES]}" - -# DP leader addresses for vLLM --data-parallel-address (rank 0 of each instance). -PREFILL_DP_ADDR="$PREFILL_LEADER_IP" -DECODE_DP_ADDR="$DECODE_LEADER_IP" - -ALL_IP_LIST=$(IFS=,; echo "${IPS[*]}") - -SANITIZED_USER=$(echo "$USER" | tr -c 'a-zA-Z0-9_.-' '_') -DOCKER_CONT_NAME="llmd_bench_${SANITIZED_USER}_${SLURM_JOB_ID}" -export DOCKER_CONT_NAME -: "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" -DOCKER_MOUNT_PATH="/workspace" - -# Stop the step and exit 0 once the coordinator is done, so the job ends COMPLETED, not CANCELLED. -BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" -rm -f "$BENCH_DONE_MARKER" - -run_until_bench_done() { - "$@" & - local step=$! rc=0 - while kill -0 "$step" 2>/dev/null; do - if [[ -f "$BENCH_DONE_MARKER" ]]; then - echo "[${SLURM_JOB_ID}] coordinator finished; stopping the srun step" - kill -TERM "$step" 2>/dev/null || true - break - fi - sleep 5 - done - wait "$step" || rc=$? - [[ -f "$BENCH_DONE_MARKER" ]] && return 0 - echo "[${SLURM_JOB_ID}] srun step exited rc=$rc before the coordinator finished" >&2 - return "$rc" -} - -# Container engine: 'docker' (default) for clusters where the SLURM -# user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' -# for clusters that require enroot+pyxis srun (e.g. gb200-nv, where the -# SLURM user is not in the docker group). Same env contract on both -# paths; only the per-node container launch differs. -source "$DI_REPO_DIR/benchmarks/check_env.sh" -check_env_vars \ - LLMD_CONTAINER_ENGINE VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS \ - NVSHMEM_REMOTE_TRANSPORT NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE LLMD_API_SERVER_COUNT -echo "LLMD_CONTAINER_ENGINE=$LLMD_CONTAINER_ENGINE" - -check_env_vars INFERENCEX_RUNTIME_ENV_VARS -RUNTIME_DOCKER_ENV=() -for setting in $INFERENCEX_RUNTIME_ENV_VARS; do - RUNTIME_DOCKER_ENV+=(-e "$setting") -done - -if [[ "$LLMD_CONTAINER_ENGINE" == "docker" ]]; then - # One docker run per node, one task per node. server.sh dispatches by NODE_RANK. - run_until_bench_done srun \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - bash -lc " -set -eo pipefail -echo \"Rank \$SLURM_PROCID on \$(hostname)\" - -docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f || true - -exec docker run --rm \ - --init \ - --entrypoint bash \ - --stop-timeout 10 \ - --user 0:0 \ - --network host \ - --ipc host \ - --gpus all \ - --ulimit memlock=-1 --ulimit stack=67108864 \ - --shm-size 32G \ - --cap-add SYS_PTRACE --cap-add IPC_LOCK --cap-add SYS_RAWIO \ - --device /dev/infiniband \ - --security-opt seccomp=unconfined \ - --privileged \ - -v ${MODEL_DIR}:/models:ro \ - -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ - -v ${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro \ - -v ${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro \ - -v ${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro \ - -e SLURM_JOB_ID=\$SLURM_JOB_ID \ - -e NODE_RANK=\$SLURM_PROCID \ - -e NUM_NODES=$NUM_NODES \ - -e PREFILL_NODES=$PREFILL_NODES \ - -e DECODE_NODES=$DECODE_NODES \ - -e PREFILL_WORKERS=$PREFILL_WORKERS \ - -e DECODE_WORKERS=$DECODE_WORKERS \ - -e ALL_IPS=$ALL_IP_LIST \ - -e PREFILL_LEADER_IP=$PREFILL_LEADER_IP \ - -e DECODE_LEADER_IP=$DECODE_LEADER_IP \ - -e PREFILL_DP_ADDR=$PREFILL_DP_ADDR \ - -e DECODE_DP_ADDR=$DECODE_DP_ADDR \ - -e MODEL_DIR=/models \ - -e MODEL_NAME=$MODEL_NAME \ - -e GPUS_PER_NODE=$GPUS_PER_NODE \ - -e PREFILL_DP_SIZE=$PREFILL_DP_SIZE \ - -e DECODE_DP_SIZE=$DECODE_DP_SIZE \ - -e BENCH_INPUT_LEN=$BENCH_INPUT_LEN \ - -e BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN \ - -e BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY \ - -e BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE \ - -e BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO \ - -e BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER \ - -e BENCHMARK_LOGS_DIR=/benchmark_logs \ - -e RUN_EVAL=$RUN_EVAL \ - -e EVAL_ONLY=$EVAL_ONLY \ - -e EVAL_CONC=$EVAL_CONC \ - -e EVAL_FRAMEWORK=$EVAL_FRAMEWORK \ - -e EVAL_LIMIT=$EVAL_LIMIT \ - -e EVAL_SUITE=$EVAL_SUITE \ - -e IS_AGENTIC=$IS_AGENTIC \ - -e SCENARIO_TYPE=$SCENARIO_TYPE \ - -e FRAMEWORK=$FRAMEWORK \ - -e PRECISION=$PRECISION \ - -e MODEL_PREFIX=$MODEL_PREFIX \ - -e RUNNER_TYPE=$RUNNER_TYPE \ - -e RESULT_FILENAME=$RESULT_FILENAME \ - -e SPEC_DECODING=$SPEC_DECODING \ - -e IS_MULTINODE=$IS_MULTINODE \ - -e CONFIG_FILE=$CONFIG_FILE \ - ${RUNTIME_DOCKER_ENV[*]} \ - -e VLLM_RANDOMIZE_DP_DUMMY_INPUTS=$VLLM_RANDOMIZE_DP_DUMMY_INPUTS \ - -e VLLM_LOGGING_LEVEL=$VLLM_LOGGING_LEVEL \ - -e UCX_TLS=$UCX_TLS \ - -e NVSHMEM_REMOTE_TRANSPORT=$NVSHMEM_REMOTE_TRANSPORT \ - -e NVSHMEM_IB_ENABLE_IBGDA=$NVSHMEM_IB_ENABLE_IBGDA \ - -e NVSHMEM_SYMMETRIC_SIZE=$NVSHMEM_SYMMETRIC_SIZE \ - -e LLMD_API_SERVER_COUNT=$LLMD_API_SERVER_COUNT \ - --name \"${DOCKER_CONT_NAME}_\$SLURM_PROCID\" \ - \"\$DOCKER_IMAGE_NAME\" -lc ' - set -o pipefail - ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ - 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log - ' -" - - srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true - -elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then - : "${LLMD_SQUASH_FILE:?LLMD_SQUASH_FILE must be set when LLMD_CONTAINER_ENGINE=pyxis}" - if [[ ! -s "$LLMD_SQUASH_FILE" ]]; then - echo "Error: LLMD_SQUASH_FILE does not exist or is empty: $LLMD_SQUASH_FILE" >&2 - exit 1 - fi - - # Pre-export non-SLURM static vars so pyxis --container-env= - # whitelisting can pick them up. Vars that already come from submit.sh - # via export are no-ops here; we re-state them for clarity. ALL_IPS - # is computed locally above as $ALL_IP_LIST. - export ALL_IPS="$ALL_IP_LIST" - export NUM_NODES PREFILL_NODES DECODE_NODES PREFILL_LEADER_IP DECODE_LEADER_IP - export PREFILL_WORKERS DECODE_WORKERS - export PREFILL_DP_ADDR DECODE_DP_ADDR MODEL_NAME GPUS_PER_NODE - export PREFILL_DP_SIZE DECODE_DP_SIZE - export BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY - export BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER - export RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE - export IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX - export RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE - - export VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS NVSHMEM_REMOTE_TRANSPORT NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE LLMD_API_SERVER_COUNT - PYXIS_ENV_LIST="NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP" - PYXIS_ENV_LIST+=",PREFILL_WORKERS,DECODE_WORKERS" - PYXIS_ENV_LIST+=",PREFILL_DP_ADDR,DECODE_DP_ADDR,MODEL_NAME,GPUS_PER_NODE" - PYXIS_ENV_LIST+=",PREFILL_DP_SIZE,DECODE_DP_SIZE" - PYXIS_ENV_LIST+=",BENCH_INPUT_LEN,BENCH_OUTPUT_LEN,BENCH_MAX_CONCURRENCY" - PYXIS_ENV_LIST+=",BENCH_REQUEST_RATE,BENCH_RANDOM_RANGE_RATIO,BENCH_NUM_PROMPTS_MULTIPLIER" - PYXIS_ENV_LIST+=",RUN_EVAL,EVAL_ONLY,EVAL_CONC,EVAL_FRAMEWORK,EVAL_LIMIT,EVAL_SUITE" - PYXIS_ENV_LIST+=",IS_AGENTIC,SCENARIO_TYPE,FRAMEWORK,PRECISION,MODEL_PREFIX" - PYXIS_ENV_LIST+=",RUNNER_TYPE,RESULT_FILENAME,SPEC_DECODING,IS_MULTINODE,CONFIG_FILE" - - PYXIS_ENV_LIST+=",VLLM_RANDOMIZE_DP_DUMMY_INPUTS,VLLM_ENGINE_READY_TIMEOUT_S,VLLM_LOGGING_LEVEL,UCX_TLS,NVSHMEM_REMOTE_TRANSPORT,NVSHMEM_IB_ENABLE_IBGDA,NVSHMEM_SYMMETRIC_SIZE,LLMD_API_SERVER_COUNT" - - PYXIS_ENV_LIST+=",${INFERENCEX_RUNTIME_ENV_VARS// /,}" - - PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" - PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" - PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" - PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro" - - # Optional: mount the epp / pd-sidecar / envoy binaries from a shared - # filesystem instead of relying on them being baked into the image. - # This lets a STOCK vllm/vllm-openai image be used directly (no - # combined-image rebuild per vLLM version bump) - see - # benchmarks/llm-d/binaries.env + extract-binaries.sh. Each mount is - # gated on the file existing, so this is a no-op when the binaries - # have not been extracted (the baked-image path keeps working), and - # harmless when they have (mounting a binary over the identical one). - # shellcheck source=/dev/null - [[ -f "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" ]] && \ - source "${DI_REPO_DIR}/benchmarks/llm-d/binaries.env" - for _bin in epp pd-sidecar envoy; do - if [[ -n "${LLMD_BIN_DIR:-}" && -x "${LLMD_BIN_DIR}/${_bin}" ]]; then - PYXIS_MOUNTS+=",${LLMD_BIN_DIR}/${_bin}:/usr/local/bin/${_bin}:ro" - echo "Mounting ${LLMD_BIN_DIR}/${_bin} -> /usr/local/bin/${_bin}" - fi - done - - # MODEL_DIR / BENCHMARK_LOGS_DIR / NODE_RANK are translated to their - # in-container values inside bash -lc (host MODEL_DIR is the source - # path of the bind mount, but server.sh expects /models inside). - run_until_bench_done srun \ - --kill-on-bad-exit=1 \ - --signal=TERM@30 \ - --unbuffered \ - --container-image="$LLMD_SQUASH_FILE" \ - --container-name="${DOCKER_CONT_NAME}" \ - --container-mounts="$PYXIS_MOUNTS" \ - --container-remap-root \ - --container-writable \ - --container-env="$PYXIS_ENV_LIST" \ - bash -lc ' -set -o pipefail -echo "Rank $SLURM_PROCID on $(hostname)" -export NODE_RANK="$SLURM_PROCID" -export MODEL_DIR=/models -export BENCHMARK_LOGS_DIR=/benchmark_logs -'"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ - 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' - -else - echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 - exit 1 -fi diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/server.sh b/inferencex-e2e/benchmarks/multi_node/llm-d/server.sh deleted file mode 100755 index 08c70f2140..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/server.sh +++ /dev/null @@ -1,573 +0,0 @@ -#!/usr/bin/env bash -# -# Per-node entrypoint for the llmd-vllm wide-EP P/D disagg benchmark. -# NODE_RANK is set by srun (= $SLURM_PROCID) in job.slurm. -# -# Roles: -# Rank 0 -> prefill leader (DP rank 0) -# Ranks 1 .. PREFILL_NODES-1 -> prefill workers -# Rank PREFILL_NODES -> decode leader (DP rank 0) + pd-sidecar -# + EPP + Envoy + benchmark client (coordinator) -# Ranks PREFILL_NODES+1 .. -> decode workers -# -# Each instance (prefill or decode) is one vLLM engine spanning its role's nodes -# via --data-parallel-hybrid-lb; the leader accepts traffic, workers serve their -# local DP ranks. - -set -eo pipefail - -source /workspace/benchmarks/check_env.sh -# Root in this container must not leave __pycache__ in the /workspace mount. -export PYTHONDONTWRITEBYTECODE=1 - -check_env_vars \ - NODE_RANK PREFILL_NODES DECODE_NODES GPUS_PER_NODE PREFILL_WORKERS \ - DECODE_WORKERS EVAL_ONLY RUN_EVAL MODEL_DIR MODEL_NAME -VLLM_PORT=8200 -SIDECAR_PORT=8000 -ENVOY_PORT=8080 -EPP_GRPC_PORT=9002 -EPP_HEALTH_PORT=9003 -EPP_METRICS_PORT=9090 - -# Weights live at MODEL_DIR (/models, bind-mounted by job.slurm); the eval reads them as MODEL. -# MODEL_NAME is the served-model-name, not a filesystem path. -export MODEL="${MODEL_DIR}" - -# Container-side benchmark commands run from the /workspace checkout. -infx_bench() { - PYTHONSAFEPATH=1 PYTHONPATH="/workspace${PYTHONPATH:+:$PYTHONPATH}" python3 -m infx.bench "$@" -} - -# Resolved without iproute2 (`ip` is absent on the arm64 vLLM base); python3's -# socket layer exposes the kernel's source-IP / iface choice. -_HOST_INFO=$(python3 -c ' -import socket -s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -try: - s.connect(("1.1.1.1", 80)) - ip = s.getsockname()[0] -finally: - s.close() -iface = "" -try: - with open("/proc/net/route") as f: - f.readline() # header - for line in f: - parts = line.split() - if parts[1] == "00000000": # default route dest - iface = parts[0]; break -except OSError: - pass -print(ip, iface) -' 2>/dev/null) || true -HOST_IP=$(echo "$_HOST_INFO" | awk '{print $1}') -DEFAULT_IFACE=$(echo "$_HOST_INFO" | awk '{print $2}') -DEFAULT_IFACE="${DEFAULT_IFACE:-eth0}" - -VLLM_LOG="/benchmark_logs/vllm_rank${NODE_RANK}.log" -SIDECAR_LOG="/benchmark_logs/sidecar_rank${NODE_RANK}.log" -EPP_LOG="/benchmark_logs/epp.log" -ENVOY_LOG="/benchmark_logs/envoy.log" - -echo "=== rank=$NODE_RANK host=$HOST_IP model=$MODEL ===" - -# A role's nodes split into PREFILL_WORKERS / DECODE_WORKERS independent DP/EP -# engines, each spanning (role_nodes / role_workers) nodes with its own DP -# coordinator (leader IP) and rank range. workers=1 => one engine over all role -# nodes (1P+1D / mid-curve); >1 => high-tpt (e.g. 2 prefill : 1 decode, DEP8 each). -IFS=',' read -r -a _ALL_IPS <<< "${ALL_IPS:-}" - -if [[ "$NODE_RANK" -lt "$PREFILL_NODES" ]]; then - ROLE="prefill" - DP_SIZE="$PREFILL_DP_SIZE" - _local_rank="$NODE_RANK" - _nodes_per_worker=$(( PREFILL_NODES / PREFILL_WORKERS )) - LWS_WORKER_INDEX=$(( _local_rank % _nodes_per_worker )) - LWS_GROUP_SIZE="$_nodes_per_worker" - _group_leader_rank=$(( (_local_rank / _nodes_per_worker) * _nodes_per_worker )) -elif [[ "$NODE_RANK" -lt $((PREFILL_NODES + DECODE_NODES)) ]]; then - ROLE="decode" - DP_SIZE="$DECODE_DP_SIZE" - _local_rank=$(( NODE_RANK - PREFILL_NODES )) - _nodes_per_worker=$(( DECODE_NODES / DECODE_WORKERS )) - LWS_WORKER_INDEX=$(( _local_rank % _nodes_per_worker )) - LWS_GROUP_SIZE="$_nodes_per_worker" - _group_leader_rank=$(( PREFILL_NODES + (_local_rank / _nodes_per_worker) * _nodes_per_worker )) -else - echo "ERROR: NODE_RANK=$NODE_RANK out of range" >&2 - exit 1 -fi - -# Each engine's DP coordinator = its leader node's IP (ALL_IPS[leader rank]); -# fall back to the role leader env when ALL_IPS is unset. -if [[ -n "${_ALL_IPS[${_group_leader_rank}]:-}" ]]; then - DP_ADDR="${_ALL_IPS[${_group_leader_rank}]}" -elif [[ "$ROLE" == "prefill" ]]; then - DP_ADDR="$PREFILL_DP_ADDR" -else - DP_ADDR="$DECODE_DP_ADDR" -fi - -DP_SIZE_LOCAL="$GPUS_PER_NODE" -START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) - -# Defaults: TP=1, DP=role_total, EP on (the H200 1P+1D shape). Recipe overrides below. -TP_SIZE=1 -ROLE_ENABLE_EP=true - -echo "ROLE=$ROLE DP_SIZE=$DP_SIZE DP_ADDR=$DP_ADDR LWS_WORKER_INDEX=$LWS_WORKER_INDEX START_RANK=$START_RANK" - -# Per-role keys: tp (int -> --tensor-parallel-size), enable-expert-parallel -# (bool -> --enable-expert-parallel + DP/wide-EP knobs), extra-args (appended -# verbatim), env (map, exported before vllm serve). Absent keys keep the -# defaults above, so a recipe with neither tp nor EP is a plain TP=1 DP+EP run. -ROLE_EXTRA_ARGS="" -if [[ -n "${CONFIG_FILE:-}" ]]; then - RECIPE_PATH="/etc/llmd-recipes/${CONFIG_FILE}" - if [[ -f "$RECIPE_PATH" ]]; then - echo "Loading $ROLE recipe from $RECIPE_PATH" - eval "$(python3 - <&2 - fi -fi -echo "Resolved $ROLE TP_SIZE=$TP_SIZE ROLE_ENABLE_EP=$ROLE_ENABLE_EP" - -export GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-$DEFAULT_IFACE} -export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-$DEFAULT_IFACE} -check_env_vars \ - VLLM_RANDOMIZE_DP_DUMMY_INPUTS VLLM_ENGINE_READY_TIMEOUT_S VLLM_LOGGING_LEVEL UCX_TLS NVSHMEM_REMOTE_TRANSPORT \ - NVSHMEM_IB_ENABLE_IBGDA NVSHMEM_SYMMETRIC_SIZE LLMD_API_SERVER_COUNT -export VLLM_SKIP_P2P_CHECK=1 -# Randomized DP dummy inputs make idle DP ranks fan their lockstep dummy passes -# across all experts (full MoE all-to-all), wasting prefill bandwidth; a recipe -# may set this to 0. -export VLLM_RANDOMIZE_DP_DUMMY_INPUTS -export VLLM_USE_DEEP_GEMM=1 -# Cold-start budget for engine-core readiness. DSV4-Pro on GB200 cold-starts in -# ~9-11 min (weight load + DeepGEMM JIT warmup + cudagraph capture + NIXL/UCX -# handshake); the 600s vLLM default is too tight, so allow 30 min. -export VLLM_ENGINE_READY_TIMEOUT_S -# DeepGEMM JIT links -l:libcuda.so.1 at warmup; the compat dir is on -# LD_LIBRARY_PATH (runtime) but not LIBRARY_PATH (link time). Prepend it, plus -# the arch-specific toolkit lib dir resolved from `uname -m`. -case "$(uname -m)" in - aarch64|arm64) _NCT_LIB=/usr/lib/aarch64-linux-gnu ;; - *) _NCT_LIB=/usr/lib/x86_64-linux-gnu ;; -esac -export LIBRARY_PATH=/usr/local/cuda/compat:${_NCT_LIB}:${LIBRARY_PATH:-} -export VLLM_NIXL_SIDE_CHANNEL_HOST="$HOST_IP" -export VLLM_LOGGING_LEVEL - -# Pin NIXL/UCX to IB verbs (rc) so cross-node KV rides the IB HCAs (job.slurm -# exposes /dev/infiniband + IPC_LOCK); cuda_copy/cuda_ipc cover intra-node. -export UCX_TLS - -# Single-node-per-role recipes avoid DeepEP / NVSHMEM ibgda, so leave these off -# there to avoid triggering ibgda code paths that are not needed. -if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - export NVIDIA_GDRCOPY=enabled - # ibgda default kept for future DeepEP/wide-EP recipes; a recipe may override - # NVSHMEM_REMOTE_TRANSPORT to none. - export NVSHMEM_REMOTE_TRANSPORT - export NVSHMEM_IB_ENABLE_IBGDA - export NVSHMEM_SYMMETRIC_SIZE - export NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME=${NVSHMEM_BOOTSTRAP_UID_SOCK_IFNAME:-$DEFAULT_IFACE} - # NVSHMEM ignores NVSHMEM_HCA_PE_MAPPING when NVSHMEM_HCA_LIST is set, so - # clear the latter when the recipe provides an explicit PE mapping. - if [[ -n "${NVSHMEM_HCA_PE_MAPPING:-}" ]]; then - unset NVSHMEM_HCA_LIST 2>/dev/null || true - fi -fi - -if [[ -n "${KV_ROLE_OVERRIDE:-}" ]]; then - KV_ROLE="$KV_ROLE_OVERRIDE" -elif [[ "$ROLE" == "prefill" ]]; then - KV_ROLE="kv_producer" -else - KV_ROLE="kv_consumer" -fi -KV_TRANSFER_CONFIG="{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"$KV_ROLE\",\"kv_load_failure_policy\":\"fail\"}" - -COMMON_ARGS=( - --port "$VLLM_PORT" - --served-model-name "$MODEL_NAME" - --trust-remote-code - --disable-access-log-for-endpoints=/health,/metrics - --tensor-parallel-size "$TP_SIZE" - --kv_transfer_config "$KV_TRANSFER_CONFIG" -) -# One frontend (HTTP + tokenize + DP load-balance) is CPU-bound and caps throughput, -# so run several (LLMD_API_SERVER_COUNT). Incompatible with --headless, so the -# headless-worker branch below drops it. With --data-parallel-hybrid-lb each node's -# api-server balances its local DP ranks, so VLLM_PORT is also the health port. -HEALTH_PORT="$VLLM_PORT" -API_SERVER_COUNT="${LLMD_API_SERVER_COUNT}" -# Multiple frontends only help the DP (wide-EP) path. A pure-TP engine has a single -# core with one frontend, so it keeps the default (and avoids --api-server-count -# interacting with the --headless multi-node TP launch below). -if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=(--api-server-count "$API_SERVER_COUNT") -fi -# Set to 1 by the pure-TP multi-node branch below on --headless followers, which -# run no local api-server; gates the post-launch health wait. -IS_HEADLESS_FOLLOWER=0 -# --moe-backend is model-specific (DSR1-FP8 wants deep_gemm, gpt-oss-MXFP4 -# rejects it), so each recipe sets its own via extra-args. - -# EP/DP knobs only when the recipe enables EP. Pure tensor-parallel roles skip -# them (vLLM rejects --data-parallel-size combined with TP>1). -if [[ "$ROLE_ENABLE_EP" == "true" ]]; then - COMMON_ARGS+=( - --enable-expert-parallel - --data-parallel-size "$DP_SIZE" - ) - if [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - COMMON_ARGS+=( - --data-parallel-hybrid-lb - --data-parallel-size-local "$DP_SIZE_LOCAL" - --data-parallel-address "$DP_ADDR" - --data-parallel-rpc-port 5555 - --data-parallel-start-rank "$START_RANK" - ) - fi -elif [[ "$LWS_GROUP_SIZE" -gt 1 ]]; then - # Pure TP spanning >1 node (e.g. DSV4-Pro decode TP=8 on GB200's 4-GPU - # nodes): use vLLM's headless multi-node API - leader binds --master-addr, - # followers join --headless with matching --nnodes/--node-rank. - COMMON_ARGS+=( - --master-addr "$DP_ADDR" - --nnodes "$LWS_GROUP_SIZE" - --node-rank "$LWS_WORKER_INDEX" - ) - if [[ "$LWS_WORKER_INDEX" -gt 0 ]]; then - COMMON_ARGS+=(--headless) - IS_HEADLESS_FOLLOWER=1 - fi -fi - -echo "Starting vLLM ($ROLE) DP=$DP_SIZE local=$DP_SIZE_LOCAL start_rank=$START_RANK group_size=$LWS_GROUP_SIZE" -# shellcheck disable=SC2086 -vllm serve "$MODEL" "${COMMON_ARGS[@]}" $ROLE_EXTRA_ARGS \ - > "$VLLM_LOG" 2>&1 & -VLLM_PID=$! - -# Each rank waits for its own engine /health before continuing (for wide-EP this -# blocks the bench until worker DP shards are up; a no-op for single-node). A -# pure-TP --headless follower runs no local api-server (only the TP-group leader -# binds a health port, and its /health only reports ready once every TP worker -# has joined), so it skips the wait and stays alive via the final `wait`. -if [[ "$IS_HEADLESS_FOLLOWER" -eq 1 ]]; then - echo "vLLM headless TP follower on rank $NODE_RANK (worker_index=$LWS_WORKER_INDEX): no local api-server, skipping health wait" -else - infx_bench wait --url "http://0.0.0.0:$HEALTH_PORT/health" --pid "$VLLM_PID" --log "$VLLM_LOG" - echo "vLLM ready on rank $NODE_RANK ($ROLE worker_index=$LWS_WORKER_INDEX, health port $HEALTH_PORT)" -fi - -# The sidecar forwards a prefill request, reads kv_transfer_params from vLLM's -# response, then hits its local decode vLLM, whose NIXLv2 connector pulls KV -# directly from prefill vLLM. -# -# DEP8 (EP on, hybrid-LB): every decode node runs an api-server for its local DP -# ranks, so every decode node runs a sidecar and endpoints.yaml lists one decode -# endpoint per node. Pure-TP: only the TP-group leader has an api-server -# (followers are --headless), so only the leader runs a sidecar and only leaders -# are listed as endpoints. -if [[ "$ROLE" == "decode" && ( "$ROLE_ENABLE_EP" == "true" || "$LWS_WORKER_INDEX" -eq 0 ) ]]; then - SIDECAR_CONNECTOR="nixlv2" - SIDECAR_FLAGS=(--port="$SIDECAR_PORT" --vllm-port="$VLLM_PORT" - --kv-connector="$SIDECAR_CONNECTOR" --secure-proxy=false - --enable-prefiller-sampling) - SIDECAR_HEALTH_PORT="$SIDECAR_PORT" - echo "Starting pd-sidecar (decode node_rank=$NODE_RANK worker_index=$LWS_WORKER_INDEX): ${SIDECAR_FLAGS[*]}" - pd-sidecar "${SIDECAR_FLAGS[@]}" > "$SIDECAR_LOG" 2>&1 & - SIDECAR_PID=$! - infx_bench wait --url "http://0.0.0.0:$SIDECAR_HEALTH_PORT/health" --pid "$SIDECAR_PID" \ - --log "$SIDECAR_LOG" - echo "pd-sidecar ready on $HOST_IP:$SIDECAR_HEALTH_PORT" -fi - -# Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT - - # namespace must match EPP's --pool-namespace (file-discovery filters by it; - # the schema default 'default' would drop every entry). See README.md. - python3 - < DEP8 hybrid-LB (an api-server per node); -# EP off => pure-TP (only each TP-group leader has an api-server). -decode_ep = ('$ROLE_ENABLE_EP' == 'true') -VLLM_PORT = int('$VLLM_PORT') -SIDECAR_PORT = int('$SIDECAR_PORT') -# ALL_IPS is rank-ordered: ranks [0:pn] are prefill nodes, [pn:pn+dn] decode. -prefill_ips = all_ips[:pn] or [os.environ['PREFILL_LEADER_IP']] -decode_ips = all_ips[pn:pn + dn] or [os.environ['DECODE_LEADER_IP']] -endpoints = [] - -def add_role(role, ips, base_port, group_size=1): - # group_size == 1: one endpoint per node (DEP8 hybrid-LB: each node's - # api-server / sidecar load-balances its local DP ranks). - # group_size > 1: one endpoint per TP-group leader (pure-TP: followers are - # --headless with no api-server), i.e. every group_size-th node IP. - serving_ips = ips[::group_size] if group_size > 1 else ips - for i, ip in enumerate(serving_ips): - endpoints.append({'name': f'{role}-{i}', 'namespace': NS, 'address': ip, - 'port': str(base_port), 'labels': {'llm-d.ai/role': role}}) - -# Prefill (DEP8 in every current recipe): one endpoint per node, EPP hits vLLM -# directly (VLLM_PORT). Decode: EPP hits the pd-sidecar (SIDECAR_PORT); one -# endpoint per node for DEP8, or one per TP-group leader for pure-TP. -add_role('prefill', prefill_ips, VLLM_PORT) -decode_group = 1 if decode_ep else max(1, dn // decode_workers) -add_role('decode', decode_ips, SIDECAR_PORT, group_size=decode_group) -yaml.safe_dump({'endpoints': endpoints}, open('/tmp/endpoints.yaml', 'w')) -print(f'endpoints.yaml ({len(endpoints)} endpoints):') -print(open('/tmp/endpoints.yaml').read()) -PY - - # EPP - # Config: when a recipe is set, project it down to the keys EPP's strict - # decoder accepts (it rejects the per-role vLLM / slurm keys); else use the - # default mounted at /etc/epp/config.yaml. - if [[ -n "$CONFIG_FILE" && -f "/etc/llmd-recipes/$CONFIG_FILE" ]]; then - EPP_CONFIG="/tmp/epp-config-from-recipe.yaml" - python3 - < ext_proc trips, Envoy 500s). - epp \ - --pool-name=epp \ - --pool-namespace=inferencex \ - --config-file="$EPP_CONFIG" \ - --grpc-port="$EPP_GRPC_PORT" \ - --grpc-health-port="$EPP_HEALTH_PORT" \ - --metrics-port="$EPP_METRICS_PORT" \ - --secure-serving=false \ - > "$EPP_LOG" 2>&1 & - EPP_PID=$! - - # Wait for EPP's gRPC listener before starting Envoy (Envoy's ext_proc dials - # it). gRPC has no plain HTTP /health, so probe the TCP listener directly. - echo "Waiting for EPP on 127.0.0.1:$EPP_GRPC_PORT" - EPP_WAIT_DEADLINE=$(( $(date +%s) + 60 )) - until (echo > "/dev/tcp/127.0.0.1/$EPP_GRPC_PORT") 2>/dev/null; do - if ! kill -0 "$EPP_PID" 2>/dev/null; then - echo "ERROR: EPP died before binding $EPP_GRPC_PORT" >&2 - exit 1 - fi - if [[ "$(date +%s)" -ge "$EPP_WAIT_DEADLINE" ]]; then - echo "ERROR: EPP did not bind $EPP_GRPC_PORT within 60s" >&2 - exit 1 - fi - sleep 1 - done - echo "EPP listening on $EPP_GRPC_PORT" - - # Envoy - envoy -c /etc/envoy/envoy.yaml > "$ENVOY_LOG" 2>&1 & - ENVOY_PID=$! - - # Probe admin /ready (9901); /health on :8080 routes through ext_proc -> EPP - # and needs request routing metadata, so it would 503 until traffic flows. - echo "Waiting for envoy admin on 127.0.0.1:9901/ready" - ENVOY_WAIT_DEADLINE=$(( $(date +%s) + 120 )) - until [[ "$(curl --output /dev/null --silent --write-out '%{http_code}' \ - "http://127.0.0.1:9901/ready" 2>/dev/null)" == "200" ]]; do - if ! kill -0 "$ENVOY_PID" 2>/dev/null; then - echo "ERROR: envoy died before admin /ready returned 200" >&2 - tail -n 80 "$ENVOY_LOG" >&2 || true - exit 1 - fi - if [[ "$(date +%s)" -ge "$ENVOY_WAIT_DEADLINE" ]]; then - echo "ERROR: envoy admin /ready did not return 200 within 120s" >&2 - tail -n 80 "$ENVOY_LOG" >&2 || true - exit 1 - fi - sleep 2 - done - echo "Envoy admin ready; listener should be on $ENVOY_PORT" - - # Gate on ALL prefill vLLM /health endpoints. Prefill ranks only wait on their own - # local /health, and with PREFILL_WORKERS>1 every prefill node must be probed, not - # just IPS[0]. curl gets explicit connect/max timeouts so a blackholed endpoint - # trips the deadline instead of hanging the run (a timeout-less curl once wedged - # a 2P run for 7h). - _prefill_ips=( "${_ALL_IPS[@]:0:${PREFILL_NODES}}" ) - [[ ${#_prefill_ips[@]} -gt 0 ]] || _prefill_ips=( "$PREFILL_LEADER_IP" ) - - # On failure, dump enough to tell a server-not-ready problem (TCP connects but - # /health is slow) apart from a network/subnet problem (TCP connect refused or - # times out, host unreachable). Uses only bash builtins + coreutils: the arm64 - # serving image has no iproute2 / nc. - _diag_prefill_endpoint() { - local ip="$1" port="$2" - { - echo "=== NET DIAG: decode -> prefill ${ip}:${port} ===" - echo "[diag] decode node: $(hostname -f 2>/dev/null || hostname) local-ips: $(hostname -I 2>/dev/null)" - echo "[diag] ifaces: DEFAULT_IFACE=${DEFAULT_IFACE:-} NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-} GLOO_SOCKET_IFNAME=${GLOO_SOCKET_IFNAME:-}" - # Local source address the kernel would pick to reach ip: reveals which - # subnet/interface the route uses, without needing iproute2. - python3 - "$ip" <<'PY' 2>&1 || true -import socket, struct, sys -ip = sys.argv[1] -# Source address the kernel picks to reach ip (which local iface/subnet). -try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect((ip, 80)) - print(f"[diag] local source IP toward {ip}: {s.getsockname()[0]}") - s.close() -except Exception as e: - print(f"[diag] no route to {ip}: {e}") -# Subnet check via /proc/net/route (no iproute2 needed). Fields are little-endian -# hex. On-link route (gateway 0.0.0.0) == same subnet; a non-zero gateway means the -# target is reached across a router == DIFFERENT subnet. -def _ntoa(v): # little-endian int -> dotted quad - return socket.inet_ntoa(struct.pack(' best[0]: - best = (plen, iface, gw, dest) - if best is None: - print(f"[diag] {ip}: NO matching route -> unreachable") - else: - plen, iface, gw, dest = best - if gw == 0: - print(f"[diag] {ip}: ON-LINK via {iface}, subnet {_ntoa(dest)}/{plen} -> SAME subnet (no router hop)") - else: - print(f"[diag] {ip}: via GATEWAY {_ntoa(gw)} on {iface}, subnet {_ntoa(dest)}/{plen} -> DIFFERENT subnet (crosses a router)") -except Exception as e: - print(f"[diag] subnet classification failed: {e}") -PY - # L4: raw TCP connect (bash /dev/tcp, 5s cap) - port reachable-but-slow - # vs unreachable/filtered. - if timeout 5 bash -c "exec 3<>/dev/tcp/${ip}/${port}" 2>/dev/null; then - echo "[diag] TCP connect ${ip}:${port} OK -> port open; server up but /health slow/not-ready (NOT a network issue)" - else - echo "[diag] TCP connect ${ip}:${port} FAILED/timed out -> closed, filtered, or unreachable (LIKELY network/subnet/firewall issue)" - fi - if command -v ping >/dev/null 2>&1; then - ping -c 2 -W 2 "$ip" 2>&1 || echo "[diag] ping ${ip} failed (ICMP blocked or host down)" - fi - curl -v --connect-timeout 5 --max-time 8 "http://${ip}:${port}/health" 2>&1 || true - echo "=== END NET DIAG ${ip}:${port} ===" - } >&2 - } - - # Log the decode->prefill target layout up front so a subnet/interface - # mismatch is visible even on a run that eventually succeeds. Every prefill - # node serves on VLLM_PORT (hybrid LB). - echo "[diag] decode-leader $(hostname 2>/dev/null) local-ips: $(hostname -I 2>/dev/null); prefill targets: ${_prefill_ips[*]}" - echo "Waiting for prefill vLLM /health on ${#_prefill_ips[@]} node(s): ${_prefill_ips[*]}" - PREFILL_WAIT_DEADLINE=$(( $(date +%s) + 300 )) - for _pidx in "${!_prefill_ips[@]}"; do - _pip="${_prefill_ips[$_pidx]}" - _pport="$VLLM_PORT" - until curl --output /dev/null --silent --fail \ - --connect-timeout 5 --max-time 10 \ - "http://$_pip:$_pport/health"; do - if [[ "$(date +%s)" -ge "$PREFILL_WAIT_DEADLINE" ]]; then - echo "ERROR: prefill vLLM at $_pip:$_pport not ready within 5 min" >&2 - _diag_prefill_endpoint "$_pip" "$_pport" - exit 1 - fi - sleep 5 - done - echo "Prefill vLLM at $_pip:$_pport is ready" - done - echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" - - # Benchmark sweep. BENCH_MAX_CONCURRENCY is 'x'-delimited from submit.sh (e.g. "1024x512"). - IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" - # GPU counts are embedded in the result filename as _gpus_/_ctx_/_gen_ so the CI - # "Process result" step can parse them. - # ctx = prefill GPUs, gen = decode GPUs. - _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) - _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) - _bench_total_gpus=$(( _bench_prefill_gpus + _bench_decode_gpus )) - if [[ "${EVAL_ONLY}" != "true" ]]; then - for max_concurrency in "${CONCURRENCIES[@]}"; do - num_prompts=$(( max_concurrency * BENCH_NUM_PROMPTS_MULTIPLIER )) - [[ "$num_prompts" -lt 16 ]] && num_prompts=16 - # Bench against Envoy (EPP routes to decode; the sidecar pulls from prefill via - # NIXL). The tokenizer is the checkpoint: served-model-name is not an HF repo id. - # Non-fatal: a failed or timed-out conc point must not abort the sweep or (under - # set -e) skip the allocation release below. - infx_bench fixed-seq point \ - --base-url "http://0.0.0.0:$ENVOY_PORT" \ - --model "$MODEL_NAME" \ - --backend openai \ - --tokenizer "$MODEL" \ - --isl "$BENCH_INPUT_LEN" \ - --osl "$BENCH_OUTPUT_LEN" \ - --random-range-ratio "$BENCH_RANDOM_RANGE_RATIO" \ - --conc "$max_concurrency" \ - --num-prompts "$num_prompts" \ - --result "$BENCHMARK_LOGS_DIR/${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}.json" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" - done - fi - - # Eval (optional) - if [[ "${RUN_EVAL}" == "true" ]]; then - if [[ -n "${EVAL_CONC:-}" ]]; then - EVAL_CONCURRENT_REQUESTS="$EVAL_CONC" - else - EVAL_CONCURRENT_REQUESTS=$(printf '%s' "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - # /workspace is the runner checkout, where the workflow looks for results*.json. - infx_bench eval --endpoint "http://127.0.0.1:$ENVOY_PORT" \ - --concurrency "$EVAL_CONCURRENT_REQUESTS" --stage-to /workspace - fi - - # job.slurm stops the srun step once this marker exists. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" -else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait -fi diff --git a/inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh b/inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh deleted file mode 100755 index ee3b986981..0000000000 --- a/inferencex-e2e/benchmarks/multi_node/llm-d/submit.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash -# -# Submit a multi-node llmd-vllm wide-EP P/D disagg benchmark job to SLURM. -# Submits the standalone llm-d Slurm job and prints JOB_ID on -# stdout so the runner can poll for completion. -# -# Topology (matches the llm-d wide-EP guide reference): -# 1 prefill instance with DP=PREFILL_NODES * GPUS_PER_NODE -# 1 decode instance with DP=DECODE_NODES * GPUS_PER_NODE -# each instance spans PREFILL_NODES / DECODE_NODES nodes via vLLM -# --data-parallel-hybrid-lb. Total nodes = PREFILL_NODES + DECODE_NODES. - -set -eo pipefail - -# Repo root resolved from this script's location, so paths below are -# independent of the caller's $PWD (the wrapper cd's into llm-d/ before -# invoking this script). -REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -source "$REPO_ROOT/benchmarks/check_env.sh" -check_env_vars \ - SLURM_ACCOUNT SLURM_PARTITION TIME_LIMIT MODEL_PATH MODEL_NAME \ - CONTAINER_IMAGE RUNNER_NAME BENCHMARK_LOGS_DIR GPUS_PER_NODE PREFILL_WORKERS \ - DECODE_WORKERS BENCH_NUM_PROMPTS_MULTIPLIER RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK \ - IS_AGENTIC FRAMEWORK SPEC_DECODING IS_MULTINODE -if [[ $# -ne 7 ]]; then - echo "Usage: submit.sh prefill_nodes decode_nodes isl osl concurrencies request_rate random_range_ratio" >&2 - exit 1 -fi - -PREFILL_NODES=$1 -DECODE_NODES=$2 -ISL=$3 -OSL=$4 -CONCURRENCIES=$5 -REQUEST_RATE=${6} -RANDOM_RANGE_RATIO=${7} - -NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) - -export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE -export MODEL_DIR=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export NUM_NODES=$NUM_NODES -export PREFILL_NODES=$PREFILL_NODES -export DECODE_NODES=$DECODE_NODES -export GPUS_PER_NODE=$GPUS_PER_NODE -# Each role's nodes split into this many INDEPENDENT DP/EP engines (default 1 = one -# engine over all role nodes), so DP_SIZE is PER-ENGINE. Matches how dynamo/AMD and -# upstream oci-high-tpt run 2P high-tpt (2 prefill : 1 decode). -export PREFILL_WORKERS -export DECODE_WORKERS -if (( PREFILL_NODES % PREFILL_WORKERS != 0 )); then - echo "Error: PREFILL_NODES ($PREFILL_NODES) not divisible by PREFILL_WORKERS ($PREFILL_WORKERS)" >&2 - exit 1 -fi -if (( DECODE_NODES % DECODE_WORKERS != 0 )); then - echo "Error: DECODE_NODES ($DECODE_NODES) not divisible by DECODE_WORKERS ($DECODE_WORKERS)" >&2 - exit 1 -fi -export PREFILL_DP_SIZE=$(( PREFILL_NODES / PREFILL_WORKERS * GPUS_PER_NODE )) -export DECODE_DP_SIZE=$(( DECODE_NODES / DECODE_WORKERS * GPUS_PER_NODE )) -export BENCH_INPUT_LEN=$ISL -export BENCH_OUTPUT_LEN=$OSL -export BENCH_MAX_CONCURRENCY=$CONCURRENCIES -export BENCH_REQUEST_RATE=$REQUEST_RATE -export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO -export BENCH_NUM_PROMPTS_MULTIPLIER - -export RUN_EVAL -export EVAL_ONLY -export EVAL_CONC="${EVAL_CONC:-}" -export EVAL_FRAMEWORK -export EVAL_LIMIT="${EVAL_LIMIT:-}" -export EVAL_SUITE="${EVAL_SUITE:-}" -export IS_AGENTIC -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export FRAMEWORK -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING -export IS_MULTINODE -export CONFIG_FILE="${CONFIG_FILE:-}" - -# Recipe may override SLURM time limit (longer topologies need more wall time). -if [[ -n "$CONFIG_FILE" ]]; then - RECIPE_PATH="${REPO_ROOT}/benchmarks/multi_node/llm-d-recipes/${CONFIG_FILE}" - if [[ -f "$RECIPE_PATH" ]]; then - RECIPE_TIME=$(python3 -c " -import yaml, sys -r = yaml.safe_load(open('$RECIPE_PATH')) -t = r.get('slurm', {}).get('time_limit', '') -print(t) -" 2>/dev/null || true) - [[ -n "$RECIPE_TIME" ]] && TIME_LIMIT="$RECIPE_TIME" - fi -fi - -mkdir -p "$BENCHMARK_LOGS_DIR" - -JOB_ID=$(sbatch \ - --parsable \ - --exclusive \ - -N "$NUM_NODES" \ - -n "$NUM_NODES" \ - --ntasks-per-node=1 \ - --gres=gpu:"$GPUS_PER_NODE" \ - --time "$TIME_LIMIT" \ - --partition "$SLURM_PARTITION" \ - --account "$SLURM_ACCOUNT" \ - --job-name "$RUNNER_NAME" \ - --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" \ - --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" \ - "$(dirname "$0")/job.slurm") - -if [[ -z "$JOB_ID" ]]; then - echo "Error: sbatch failed" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh b/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh index 36366b2696..87ea9cb7fc 100644 --- a/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh +++ b/inferencex-e2e/benchmarks/multi_node/runtime_settings.sh @@ -19,11 +19,4 @@ case "$FRAMEWORK" in fi fi ;; - llmd-vllm) - export BENCH_NUM_PROMPTS_MULTIPLIER=10 - export LLMD_CONTAINER_ENGINE=docker VLLM_RANDOMIZE_DP_DUMMY_INPUTS=1 - export VLLM_ENGINE_READY_TIMEOUT_S=1800 VLLM_LOGGING_LEVEL=INFO UCX_TLS=cuda_copy,cuda_ipc,rc - export NVSHMEM_REMOTE_TRANSPORT=ibgda NVSHMEM_IB_ENABLE_IBGDA=true NVSHMEM_SYMMETRIC_SIZE=16G - export LLMD_API_SERVER_COUNT=4 - ;; esac diff --git a/inferencex-e2e/infx/launch/drivers/__init__.py b/inferencex-e2e/infx/launch/drivers/__init__.py index e377318a3c..bb49084a0d 100644 --- a/inferencex-e2e/infx/launch/drivers/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/__init__.py @@ -13,7 +13,7 @@ from infx.launch import policy from infx.launch.backends import backend_class from infx.launch.context import Launch, LaunchError -from infx.launch.drivers import llmd, script, srt +from infx.launch.drivers import script, srt from infx.launch.policy import LaunchPath, launch_path if TYPE_CHECKING: @@ -36,7 +36,6 @@ class Route: LaunchPath.SRT_MULTI: Route("slurm", srt.run_multinode), LaunchPath.SRT_NATIVE: Route("slurm", srt.run_multinode), LaunchPath.SCRIPT: Route(None, script.run), - LaunchPath.LLMD: Route("slurm", llmd.run), } diff --git a/inferencex-e2e/infx/launch/drivers/llmd.py b/inferencex-e2e/infx/launch/drivers/llmd.py deleted file mode 100644 index 95b6f636be..0000000000 --- a/inferencex-e2e/infx/launch/drivers/llmd.py +++ /dev/null @@ -1,161 +0,0 @@ -"""llm-d vLLM multinode jobs submitted through benchmarks/multi_node/llm-d/submit.sh.""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import sys -from pathlib import Path - -from infx.bench.proc import echo -from infx.launch import artifacts, policy -from infx.launch.backends.base import BackendError -from infx.launch.backends.slurm import cli -from infx.launch.context import Launch, LaunchError -from infx.launch.drivers.srt import models -from infx.launch.drivers.srt.run import slurm_backend -from infx.launch.request import LlmdRequest, RequestError - -CANCEL_TIMEOUT_S = 600.0 -DEFAULT_TIME_LIMIT = "08:00:00" -LLMD_DIR = "benchmarks/multi_node/llm-d" - - -def _find_eval_dir(logs_dir: Path) -> Path | None: - for root, dirs, _files in os.walk(logs_dir): - if "eval_results" in dirs: - return Path(root) / "eval_results" - return None - - -def _stage_agentic(logs_dir: Path, workspace: Path) -> None: - agentic = logs_dir / "agentic" - if not agentic.is_dir(): - return - staged = workspace / "LOGS" / "agentic" - staged.mkdir(parents=True, exist_ok=True) - for entry in agentic.iterdir(): - destination = staged / entry.name - if entry.is_dir(): - shutil.copytree(entry, destination, dirs_exist_ok=True) - elif entry.is_file(): - shutil.copy2(entry, destination) - - -def run(launch: Launch) -> int: - """Submit the llm-d Slurm job, follow its log, and stage benchmark artifacts.""" - event = launch.life.event - backend = slurm_backend(launch) - request = LlmdRequest.from_env(launch.request.env) - if backend.settings.squash is None: - raise LaunchError(f"llmd-vllm: cluster {launch.cluster.id!r} has no slurm.squash") - if not request.disagg: - raise LaunchError("llmd-vllm supports only P/D disaggregated points") - - checkpoint = models.checkpoint(launch.cluster, request) - if checkpoint is None: - raise LaunchError( - f"cluster {launch.cluster.id!r} stages no checkpoint for MODEL={request.model}" - ) - model_path = models.host_path(launch.cluster, checkpoint) - if not checkpoint.node_local and not (model_path / "config.json").is_file(): - raise LaunchError(f"model checkpoint is unavailable: {model_path / 'config.json'}") - - with event.stage("prepare"): - squash = backend.prepare_image(request.image) - logs_dir = request.workspace / "benchmark_logs" - logs_dir.mkdir(parents=True, exist_ok=True) - - account = backend.settings.account or cli.default_account() - if not account: - raise RequestError.missing("SLURM_ACCOUNT") - - env = policy.runtime_env( - launch.cluster, - request, - models.job_env(launch.cluster, request, str(model_path)), - { - "SLURM_PARTITION": backend.settings.partition, - "SLURM_ACCOUNT": account, - "MODEL_PATH": str(model_path), - "MODEL_NAME": request.model, - "CONTAINER_IMAGE": request.image, - "GPUS_PER_NODE": str(launch.cluster.gpus_per_node), - "TIME_LIMIT": request.env.get("TIME_LIMIT") or DEFAULT_TIME_LIMIT, - "PREFILL_WORKERS": str(request.prefill_num_workers), - "DECODE_WORKERS": str(request.decode_num_workers), - "LLMD_CONTAINER_ENGINE": "pyxis", - "LLMD_SQUASH_FILE": squash.reference, - "BENCHMARK_LOGS_DIR": str(logs_dir), - }, - ) - - argv = [ - "bash", - "submit.sh", - str(request.prefill_nodes), - str(request.decode_nodes), - str(request.isl), - str(request.osl), - "x".join(map(str, request.conc_list)), - "inf", - request.random_range_ratio, - ] - with event.stage("submit"): - echo(argv, env) - submitted = subprocess.run( - argv, - stdout=subprocess.PIPE, - stderr=sys.stderr, - text=True, - env=env, - cwd=request.workspace / LLMD_DIR, - check=False, - ) - job_id = submitted.stdout.strip() - if submitted.returncode != 0 or not job_id: - event.fail("SubmitFailed", "llm-d submit.sh failed before returning a Slurm job id") - return 1 - if not (job_id.isascii() and job_id.isdigit()): - message = f"llm-d submit.sh printed {job_id!r} instead of a Slurm job id" - event.fail("SubmitFailed", message) - return 1 - - log_file = logs_dir / f"slurm_job-{job_id}.out" - job = backend.attach(job_id, log=log_file, outputs=logs_dir) - print(f"Submitted llm-d job: {job_id}", flush=True) - - launch.life.callback( - artifacts.bundle_server_logs, logs_dir, request.workspace / "multinode_server_logs.tar.gz" - ) - launch.life.callback(backend.cancel, job, wait_s=CANCEL_TIMEOUT_S) - - try: - backend.stream_logs(job) - except BackendError: - return 1 - - status = backend.state(job) - rc = 0 if status.succeeded else 1 - - with event.stage("collect"): - for result_file in sorted(logs_dir.glob(f"{request.result_filename}*.json")): - try: - artifacts.copy_to_workspace(result_file, request.workspace / result_file.name) - except artifacts.ArtifactError as error: - event.error(error) - rc = 1 - - if request.is_agentic and not request.eval_only: - _stage_agentic(logs_dir, request.workspace) - - if request.run_eval: - eval_dir = _find_eval_dir(logs_dir) or logs_dir / "eval_results" - try: - artifacts.copy_eval_artifacts(eval_dir, request.workspace) - except artifacts.ArtifactError as error: - event.error(error) - rc = 1 - - return rc diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index da8abdb6cb..5f0e9b33cb 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -44,11 +44,6 @@ class Override: Override(Match(model_glob="*/DeepSeek-V4-Pro-0813"), entry="DeepSeek-V4-Pro-0813"), ), "gb200-nv": ( - Override( - Match(any_of("dsv4"), any_of("fp4"), any_of("llmd-vllm")), - entry="DeepSeek-V4-Pro@numa1", - served_name="deepseek-ai/DeepSeek-V4-Pro", - ), Override( Match(any_of("dsr1"), any_of("fp4"), any_of("dynamo-sglang")), entry="deepseek-r1-0528-fp4-v2", diff --git a/inferencex-e2e/infx/launch/policy.py b/inferencex-e2e/infx/launch/policy.py index 3a6ed913e8..d6bf0edba4 100644 --- a/inferencex-e2e/infx/launch/policy.py +++ b/inferencex-e2e/infx/launch/policy.py @@ -57,7 +57,6 @@ class LaunchPath(StrEnum): SRT_NATIVE = "srt-native" SRT_BATCH = "srt-batch" SCRIPT = "script" - LLMD = "llmd" NATIVE_SRT_LANES: dict[str, tuple[Match, ...]] = { @@ -77,8 +76,6 @@ class LaunchPath(StrEnum): def launch_path(cluster_id: str, request: LaunchRequest) -> LaunchPath: if request.is_multinode: - if request.framework == "llmd-vllm": - return LaunchPath.LLMD if any(lane(request) for lane in NATIVE_SRT_LANES.get(cluster_id, ())): return LaunchPath.SRT_NATIVE return LaunchPath.SRT_MULTI diff --git a/inferencex-e2e/infx/launch/request.py b/inferencex-e2e/infx/launch/request.py index 88150eaa61..2c463a062d 100644 --- a/inferencex-e2e/infx/launch/request.py +++ b/inferencex-e2e/infx/launch/request.py @@ -181,17 +181,3 @@ class ScriptRequest(LaunchRequest): model: str = Field(alias="MODEL") gpu_count: int = Field(alias="GPU_COUNT") salloc_time_limit: int = Field(alias="SALLOC_TIME_LIMIT") - - -class LlmdRequest(SrtRequest): - """An llm-d vLLM multinode job submitted through benchmarks/multi_node/llm-d.""" - - model: str = Field(alias="MODEL") - disagg: TrueFlag = Field(alias="DISAGG") - prefill_nodes: int = Field(alias="PREFILL_NODES") - decode_nodes: int = Field(alias="DECODE_NODES") - prefill_num_workers: int = Field(1, alias="PREFILL_NUM_WORKERS") - decode_num_workers: int = Field(1, alias="DECODE_NUM_WORKERS") - isl: int = Field(alias="ISL") - osl: int = Field(alias="OSL") - random_range_ratio: str = Field(alias="RANDOM_RANGE_RATIO") diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index a8dbbd7bc2..ea040e79c1 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -20,7 +20,6 @@ DEFAULT_AGENTIC_DURATION_SECONDS = 3600 SINGLE_NODE_RECIPES = "benchmarks/single_node/srt-slurm-recipes" MULTI_NODE_RECIPES = "benchmarks/multi_node/srt-slurm-recipes" -LLMD_FRAMEWORK = "llmd-vllm" RECIPE_SELECTOR = re.compile(r"base|override_[\w-]+|zip_override_[\w-]+\[\d+\]") @@ -872,12 +871,6 @@ def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: def _validate_srt_recipes(self: Any) -> Any: entries = _master_search_space_entries(self) - if self.multinode and self.framework == LLMD_FRAMEWORK: - if self.srt_recipe_dir is not None or any( - entry.srt_recipe or entry.eval_srt_recipe for entry in entries - ): - raise ValueError(f"{LLMD_FRAMEWORK} selects recipes with CONFIG_FILE, not srt-recipe") - return self if self.srt_recipe_dir is None: raise ValueError(f"srt-slurm configs require {Fields.SRT_RECIPE_DIR.value}") for entry in entries: diff --git a/inferencex-e2e/infx/tests/launch/test_llmd_driver.py b/inferencex-e2e/infx/tests/launch/test_llmd_driver.py deleted file mode 100644 index 1123286573..0000000000 --- a/inferencex-e2e/infx/tests/launch/test_llmd_driver.py +++ /dev/null @@ -1,137 +0,0 @@ -"""The llm-d driver: run submit.sh, attach, stage artifacts.""" - -import json -import subprocess -import tarfile -from pathlib import Path - -import pytest - -from infx.tests.launch.fake_slurm import ( - base_env, - install_fakes, - launch, - make_workspace, - runner_for, - sandbox_runner_config, -) - -LLMD_SUBMIT = """#!/usr/bin/env bash -set -e -env > "$GITHUB_WORKSPACE/submitted.env" -echo "$PWD $*" > "$GITHUB_WORKSPACE/submitted.args" -logs="$BENCHMARK_LOGS_DIR" -job="$logs/slurm_job-4299" -mkdir -p "$logs/agentic/conc_128" "$job/eval_results" -echo '{"conc": 128}' > "$logs/point-identity_conc128.json" -echo trace > "$logs/agentic/conc_128/profile.json" -echo '{"score": 1}' > "$job/eval_results/results_gsm8k.json" -echo 'server log' > "$logs/server.log" -echo 'benchmark done' > "$logs/slurm_job-4299.out" -echo 'worker warning' > "$logs/slurm_job-4299.err" -echo 'submitting' >&2 -[[ "${NO_JOB_ID:-}" == 1 ]] && exit 1 -echo 4299 -""" - - -@pytest.fixture -def harness(tmp_path): - """Sandboxed gb200-nv cluster, fake Slurm binaries, and a workspace with a fake submit.sh.""" - sandbox = tmp_path / "sandbox" - sandbox.mkdir() - config = sandbox_runner_config(sandbox) - workspace = make_workspace(tmp_path / "workspace") - wrapper = workspace / "benchmarks/multi_node/llm-d/submit.sh" - wrapper.parent.mkdir(parents=True, exist_ok=True) - wrapper.write_text(LLMD_SUBMIT) - wrapper.chmod(0o755) - - model_root = sandbox / "mnt/numa1/models/DeepSeek-V4-Pro" - model_root.mkdir(parents=True) - (model_root / "config.json").write_text("{}\n") - - logs = tmp_path / "logs" - env = base_env( - fakes=install_fakes(tmp_path / "bin"), logs=logs, workspace=workspace, sandbox=sandbox - ) - env.update( - RUNNER_NAME=runner_for("gb200-nv"), - IS_MULTINODE="true", - IS_AGENTIC="1", - RUN_EVAL="true", - EVAL_ONLY="false", - FRAMEWORK="llmd-vllm", - MODEL="deepseek-ai/DeepSeek-V4-Pro", - MODEL_PREFIX="dsv4", - PRECISION="fp4", - SPEC_DECODING="mtp", - THINKING_MODE="thinking_on", - PREFILL_NODES="2", - DECODE_NODES="2", - ISL="8192", - OSL="1024", - RANDOM_RANGE_RATIO="0.8", - CONC="256", - CONC_LIST="256", - DISAGG="true", - IMAGE="vllm/vllm-openai:v0.21.0", - RESULT_FILENAME="point-identity", - ENROOT_IMPORT_TIME_LIMIT="10", - ) - return config, workspace, env - - -def run_launch(harness) -> subprocess.CompletedProcess[str]: - config, workspace, env = harness - return launch(env, config, workspace) - - -def test_llmd_driver_submits_the_wrapper_and_stages_artifacts(harness): - result = run_launch(harness) - config, workspace, env = harness - - assert result.returncode == 0, result.stdout + result.stderr - submitted = dict( - line.split("=", 1) for line in (workspace / "submitted.env").read_text().splitlines() if "=" in line - ) - model_path = f"{config.parent}/mnt/numa1/models/DeepSeek-V4-Pro" - assert submitted["MODEL_PATH"] == model_path - assert submitted["MODEL_NAME"] == env["MODEL"] - assert submitted["LLMD_CONTAINER_ENGINE"] == "pyxis" - assert submitted["LLMD_SQUASH_FILE"] - assert submitted["BENCHMARK_LOGS_DIR"] == f"{workspace}/benchmark_logs" - assert submitted["SLURM_PARTITION"] == "batch" - assert submitted["SLURM_ACCOUNT"] == "benchmark" - assert submitted["GPUS_PER_NODE"] == "4" - assert submitted["CONTAINER_IMAGE"] == env["IMAGE"] - assert (workspace / "submitted.args").read_text().split() == [ - str(workspace / "benchmarks/multi_node/llm-d"), - *"2 2 8192 1024 256 inf 0.8".split(), - ] - - assert json.loads((workspace / "point-identity_conc128.json").read_text()) == {"conc": 128} - assert (workspace / "LOGS/agentic/conc_128/profile.json").read_text() == "trace\n" - assert json.loads((workspace / "results_gsm8k.json").read_text()) == {"score": 1} - with tarfile.open(workspace / "multinode_server_logs.tar.gz") as bundle: - assert "./server.log" in bundle.getnames() - assert "submitting" in result.stderr - - -def test_llmd_driver_fails_when_the_wrapper_prints_no_job_id(harness): - _, workspace, env = harness - env["NO_JOB_ID"] = "1" - - result = launch(env, harness[0], workspace) - - assert result.returncode == 1 - assert "submit.sh failed before returning a Slurm job id" in result.stderr - - -def test_llmd_driver_leaves_node_local_checkpoints_to_the_job(harness): - config, workspace, env = harness - (config.parent / "mnt/numa1/models/DeepSeek-V4-Pro/config.json").unlink() - - result = launch(env, config, workspace) - - assert result.returncode == 0, result.stdout + result.stderr diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index a6e52eee69..6f8e47c0e3 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -55,7 +55,6 @@ def cluster(tmp_path, single_node_models: str = "staged") -> Cluster: ("b200-nscale", dict(MULTI, MODEL_PREFIX="glm5.3", PRECISION="fp8", FRAMEWORK="tilert", SPEC_DECODING="mtp"), LaunchPath.SRT_MULTI), ("mi355x-amds", dict(IS_MULTINODE="true", FRAMEWORK="atom-disagg"), LaunchPath.SRT_MULTI), ("mi355x-amds", dict(MULTI, FRAMEWORK="sglang-disagg"), LaunchPath.SRT_MULTI), - ("gb200-nv", dict(MULTI, FRAMEWORK="llmd-vllm", MODEL_PREFIX="dsv4", PRECISION="fp4", SPEC_DECODING="mtp"), LaunchPath.LLMD), ("gb200-nv", dict(MULTI, FRAMEWORK="tilert"), LaunchPath.SRT_MULTI), ("b300-dsxe", dict(SINGLE, MODEL_PREFIX="dsv41flash", FRAMEWORK="sglang", IS_AGENTIC="1"), LaunchPath.SRT_BATCH), ("b300-dsxe", dict(SINGLE, MODEL_PREFIX="dsv41flash", FRAMEWORK="sglang", IS_AGENTIC="1", INFX_BATCH_REENTRY="1"), LaunchPath.SRT_SINGLE), diff --git a/inferencex-e2e/infx/tests/matrix/test_validation.py b/inferencex-e2e/infx/tests/matrix/test_validation.py index a46620c758..54413d0a28 100644 --- a/inferencex-e2e/infx/tests/matrix/test_validation.py +++ b/inferencex-e2e/infx/tests/matrix/test_validation.py @@ -1060,18 +1060,6 @@ def test_recipe_settings_are_not_additional_settings( with pytest.raises(ValidationError, match=replacement): MultiNodeMasterConfigEntry(**valid_multinode_master_config) - def test_llmd_configs_keep_config_file(self, valid_multinode_master_config): - config = valid_multinode_master_config - config["framework"] = "llmd-vllm" - del config["srt-recipe-dir"] - row = config["scenarios"]["fixed-seq-len"][0]["search-space"][0] - del row["srt-recipe"] - row["prefill"]["additional-settings"].append("CONFIG_FILE=llmd.yaml") - assert validate_master_config({"llmd": config}, Path()) == {"llmd": config} - row["srt-recipe"] = "recipe.yaml" - with pytest.raises(ValidationError, match="llmd-vllm selects recipes with CONFIG_FILE"): - MultiNodeMasterConfigEntry(**config) - def test_eval_srt_recipe_is_multinode_only(self, valid_single_node_master_config): config = valid_single_node_master_config config["runner"] = "cluster:b200-nscale"